ЛР 6 — БВТ2301 — Курлова — на 5 #18
+43
-7
@@ -1,30 +1,66 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Values below are the ones used for evidence/ (see REPORT.md for the mapping).
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
GRACEFUL_TIMEOUT=8
|
||||
MAX_INFLIGHT_REQUESTS=64
|
||||
MAX_BODY_BYTES=65536
|
||||
MAX_RSS_MB=0
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_TIMEOUT=3
|
||||
DB_CONNECT_TIMEOUT=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
DB_OP_TIMEOUT=6
|
||||
REQUEST_TIMEOUT=30
|
||||
DB_READY_TIMEOUT=1.5
|
||||
READY_CHECK_SECONDS=1
|
||||
READY_FAILURES=2
|
||||
SESSION_CACHE_TTL_SECONDS=30
|
||||
SESSION_SWEEP_SECONDS=60
|
||||
CACHE_TIMEOUT=0.2
|
||||
SESSION_REVOKED_TTL_SECONDS=60
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
UPLOAD_CONCURRENCY=2
|
||||
RECONCILE_INTERVAL_SECONDS=300
|
||||
RECONCILE_GRACE_SECONDS=900
|
||||
# --- lab5: broker, jobs, worker
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=20
|
||||
JOB_BACKOFF_BASE_SECONDS=2
|
||||
JOB_BACKOFF_MAX_SECONDS=30
|
||||
JOB_SWEEP_SECONDS=5
|
||||
WORKER_CHILD_MEMORY_MB=1024
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing.
|
||||
MAX_PENDING_JOBS=200
|
||||
JPEG_QUALITY=85
|
||||
WEBP_QUALITY=80
|
||||
# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else.
|
||||
FAULT_INJECTION=1
|
||||
# --- lab6: OpenTelemetry. Compose sets the OTEL_* variables for api/worker (x-otel-env); shell values override:
|
||||
# service.name is fixed in code: messenger-api / messenger-worker.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# Track 1 knobs (default = telemetry on, 100% head sampling, collector profile "default"):
|
||||
# OTEL_SDK_DISABLED=false
|
||||
# OTEL_TRACES_SAMPLER_ARG=1.0
|
||||
# COLLECTOR_PROFILE=default # or "tail" (balanced tail sampling)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
name: Application stand
|
||||
# Student job: builds the application, starts the whole Compose stand and runs the public smoke and the
|
||||
# observability check. Credentials are random, generated inside the job, and thrown away with it.
|
||||
# Triggers deliberately exclude pull_request_target: code from a fork never gets repository secrets.
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
stand:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Create local .env and random credentials
|
||||
run: sh scripts/init-secrets.sh
|
||||
- name: Validate Compose configuration
|
||||
run: docker compose --env-file .env config --quiet
|
||||
- name: Build and start the stand, wait for readiness
|
||||
run: docker compose --env-file .env up -d --build --wait --wait-timeout 300
|
||||
- name: Contract structure
|
||||
run: make check
|
||||
- name: Public API smoke with telemetry enabled
|
||||
run: make test
|
||||
- name: Traces, logs, metrics correlation and privacy
|
||||
run: python3 student-tests/test_observability.py
|
||||
- name: Service state and application logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
docker compose ps -a
|
||||
docker compose logs --tail 80 api-1 api-2 worker otel-collector tempo loki prometheus
|
||||
- name: Remove the stand
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=120 \
|
||||
PIP_RETRIES=10
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-deps -r requirements.txt
|
||||
|
||||
COPY app/ app/
|
||||
COPY migrations/ migrations/
|
||||
COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
CMD ["sh", "-c", "exec uvicorn app.main:app --no-access-log --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""]
|
||||
@@ -1,37 +1,312 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Алёна Курлова, БВТ2301
|
||||
- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена наблюдаемости»**
|
||||
- Commit/PR: TODO после публикации (ветка `solution/bvt2301-kurlova`)
|
||||
- Источники, библиотеки, помощь ИИ: OpenTelemetry Python SDK 1.30.0 / instrumentation 0.51b0 (FastAPI, botocore), OpenTelemetry Collector contrib 0.120.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.1, Grafana 11.5.2; документация [Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), [tail sampling processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor). Код lab5 перенесён без изменения API. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; все числа ниже получены на локальном стенде и сохранены в `evidence/`.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Desktop (Compose v2.20+, нужен `--wait`), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7.2, Docker VM: 10 CPU / 7.7 GiB.
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh # .env и случайные credentials; существующие файлы не перезаписываются
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait # 15 сервисов, включая collector/prometheus/loki/tempo/grafana
|
||||
make check && make test
|
||||
```
|
||||
|
||||
Bootstrap выполняется автоматически при каждом `up` и безопасен при повторе: `migrate` (SQL-миграции с checksum и advisory lock), `minio-init` (bucket, политика, пользователь приложения), топология RabbitMQ объявляется кодом. **Lab6:** Grafana при старте импортирует datasources (`infra/grafana/provisioning/datasources`) и dashboard (`infra/grafana/dashboards/messenger.json`, папка «Messenger»), Prometheus читает правила из `infra/prometheus/rules/`. Ручного набора dashboard нет.
|
||||
|
||||
Интерфейсы (только `127.0.0.1`): API `:8080` (nginx), Grafana `:3000` (admin / `secrets/grafana_admin_password`), query API Prometheus `:9090`, Loki `:3100`, Tempo `:3200` — нужны скриптам доказательств. OTLP-порты collector (4317/4318) и его метрики (8888) наружу не публикуются.
|
||||
|
||||
Проверки (стенд запущен, `FAULT_INJECTION=1` только на учебном стенде):
|
||||
|
||||
```sh
|
||||
make test # публичный smoke при включённой телеметрии
|
||||
python3 student-tests/test_observability.py # уровни 3-4: trace/log/metrics одного upload и одной 500, приватность
|
||||
python3 student-tests/check_dashboard.py # все PromQL-запросы dashboard валидны и с данными
|
||||
caffeinate -i python3 student-tests/test_incidents.py # уровень 4: инциденты A/B/C, firing -> resolved (~15 мин)
|
||||
caffeinate -i python3 student-tests/test_worker_saturation.py # алерт насыщения worker (~10 мин)
|
||||
caffeinate -i python3 student-tests/test_exporter_queue_full.py # очередь exporter до предела (~5 мин)
|
||||
caffeinate -i python3 student-tests/track1.py run --repeats 3 # трек 5.1 (~35 мин), затем: python3 student-tests/track1.py summary
|
||||
python3 student-tests/test_worker_down.py; python3 student-tests/test_failure_windows.py # регрессия lab5
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # затем docker compose start api-1 api-2
|
||||
```
|
||||
|
||||
`caffeinate -i` (macOS) нужен для длинных прогонов: в первой попытке Mac ушёл в сон посреди инцидента, весь стенд «замёрз» на 18 минут (виден скачок gauge 94 → 1176 s), см. раздел «Ограничения».
|
||||
|
||||
Безопасная остановка с сохранением данных: `docker compose stop`. Намеренное удаление учебных данных (включая историю метрик/логов/трасс): `docker compose down -v`. `secrets/` и `.env` в git не попадают.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
client ─► nginx ─► api-1, api-2 ──(outbox)──► RabbitMQ ─► worker ×2
|
||||
│ │ PostgreSQL / Valkey / MinIO │
|
||||
│ └──── OTLP/HTTP :4318 ◄─────────────┘ (traces, metrics, logs; service.name
|
||||
▼ messenger-api / messenger-worker)
|
||||
otel-collector memory_limiter → attributes/scrub → [tail_sampling] → batch
|
||||
├─ otlp/tempo ─► Tempo (трассы, volume tempodata)
|
||||
├─ otlphttp/prometheus ─► Prometheus /api/v1/otlp (метрики, volume promdata, правила/алерты)
|
||||
└─ otlphttp/loki ─► Loki /otlp (логи, volume lokidata)
|
||||
Prometheus также скрейпит self-metrics collector :8888, Tempo, Loki.
|
||||
Grafana (volume grafanadata) — только интерфейс и запросы к трём источникам.
|
||||
```
|
||||
|
||||
Сеть у всех сервисов одна: внутренняя сеть Compose `default`. Порты в столбце «Порт» — внутренние; наружу опубликованы только отмеченные `127.0.0.1`. Credentials передаются Docker secrets из `secrets/` (создаёт `scripts/init-secrets.sh`), приложение читает их через переменные `*_FILE`. Digest образов указан в `compose.yaml`.
|
||||
|
||||
| Роль | Сервис | Версия | Health / readiness | Volume | Порт | Credentials (Docker secrets) | Назначение |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| API ×2 | `api-1`, `api-2` | сборка из `Dockerfile`, python:3.12.8-slim-bookworm + digest | `GET /health/ready` | нет | 8080, напрямую не опубликован | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key`, `broker_url` | REST API, outbox relay, gauges очереди |
|
||||
| Балансировщик | `nginx` | nginx:1.27.3-alpine | `GET /health/live`; проверка реплик в `infra/nginx-health.sh` | нет | `127.0.0.1:8080` | нет | единая точка входа |
|
||||
| Worker ×2 | `worker` | тот же образ | свежесть файла heartbeat, 15 с | нет | нет | `database_url`, `s3_access_key_id`, `s3_secret_access_key`, `broker_url` | обработка изображений |
|
||||
| БД | `postgres` | postgres:17.7-alpine | `pg_isready` | `pgdata` | 5432 | `postgres_password` | источник истины |
|
||||
| Кэш сессий | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | нет, кэш без персистентности | 6379 | нет, доступ только из внутренней сети | ускоряет проверку сессий; при сбое API читает PostgreSQL |
|
||||
| S3 | `minio` | madlabcg/minio по digest | `mc ready local` | `miniodata` | 9000 | `minio_root_user`, `minio_root_password` | вложения, приватный bucket |
|
||||
| Брокер | `rabbitmq` | rabbitmq:4.0.5-alpine | `check_port_connectivity` | `rabbitmqdata` | 5672 | `rabbitmq_password` | durable очередь заданий |
|
||||
| Bootstrap БД | `migrate` | тот же образ | разовый, успешное завершение | нет | нет | `database_url` | SQL-миграции |
|
||||
| Bootstrap S3 | `minio-init` | madlabcg/minio по digest | разовый, успешное завершение | нет | нет | `minio_root_user`, `minio_root_password`, `s3_access_key_id`, `s3_secret_access_key` | bucket, политика, пользователь приложения |
|
||||
| Collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.120.0 | Docker healthcheck нет: в образе нет shell; расширение `health_check` слушает 13133 | нет, это не хранилище | 4317, 4318, 8888 | нет, доступ только из внутренней сети | приём OTLP, ограничение памяти, очистка атрибутов, батчи, tail sampling, экспорт с ограниченной очередью |
|
||||
| Метрики | `prometheus` | prom/prometheus:v3.2.1 | `/-/ready` | `promdata`, 15 дней | `127.0.0.1:9090` | нет, только localhost | числовые ряды: RPS, ошибки, гистограммы, gauges; SLI, recording rules и алерты |
|
||||
| Логи | `loki` | grafana/loki:3.4.2 | `/ready` | `lokidata`, 7 дней | `127.0.0.1:3100` | нет, только localhost | события с текстом; индекс только по низкокардинальным labels, `event`/`trace_id`/`span_id` — structured metadata |
|
||||
| Трассы | `tempo` | grafana/tempo:2.7.1 | `/ready` | `tempodata`, 7 дней | `127.0.0.1:3200` | нет, только localhost | spans с причинной связью, поиск TraceQL |
|
||||
| UI | `grafana` | grafana/grafana:11.5.2 | `/api/health` | `grafanadata` | `127.0.0.1:3000` | `grafana_admin_password` | dashboard, Explore, переходы лог → трасса и трасса → логи |
|
||||
|
||||
Почему три хранилища: Grafana сама сигналы не хранит; метрика отвечает «сколько и как часто» (дёшево, агрегируется, но теряет конкретный запрос), лог — «что произошло в событии» (текст и технические ID), трасса — «где в цепочке процессов ушло время/возникла ошибка».
|
||||
|
||||
Что изменено относительно lab5 (бизнес-API и контракт не менялись):
|
||||
|
||||
- `app/telemetry.py` — настройка SDK (resource, sampler, views, exporters), JSON-логи в stdout с `trace_id`/`span_id`, OTLP-логи, инструменты метрик, перенос контекста;
|
||||
- `app/main.py` — инструментирование FastAPI (после регистрации middleware, иначе ответы 429/503/500 самого guard не попадали в spans и метрики — найдено проверкой), `messenger.request_id` в server span, gauges очереди из PostgreSQL, тестовая инъекция `X-Fault: slow|error` (только `FAULT_INJECTION=1`);
|
||||
- `app/db.py` — собственные CLIENT spans PostgreSQL и span ожидания пула. Автоинструментация `opentelemetry-instrumentation-psycopg 0.51b0` для async-курсоров закрывала span до завершения `await` (все db-spans были 0.0 ms) — с ней нельзя было отличить медленную БД от handler, поэтому она убрана;
|
||||
- `app/jobs.py`, `app/outbox.py`, `app/broker.py`, `app/worker.py` — W3C trace context в сообщении задания, producer/consumer spans, метрики jobs и time-to-ready;
|
||||
- `compose.yaml` — collector, Prometheus, Loki, Tempo, Grafana, volumes; `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` — конфиги и provisioning;
|
||||
- `.github/workflows/stand.yml` — CI: сборка, запуск стенда, `make test`, проверка наблюдаемости; `scripts/init-secrets.sh` — локальные credentials;
|
||||
- `docs/runbooks.md` — runbook каждого алерта; `student-tests/` — проверки lab6 (lab5-тесты сохранены).
|
||||
|
||||
Соответствие переменных `.env.example`: `OTEL_SERVICE_NAME` не используется — `service.name` задаётся кодом (`messenger-api`/`messenger-worker`), чтобы worker не мог унаследовать имя API из общего `.env`. `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, `OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab` — как в примере (задаются в `x-otel-env` compose). Дополнительно: `OTEL_SDK_DISABLED`, `OTEL_TRACES_SAMPLER_ARG`, `OTEL_METRIC_EXPORT_INTERVAL=10000`, `OTEL_BSP_MAX_QUEUE_SIZE=2048`, `OTEL_BLRP_MAX_QUEUE_SIZE=2048`, `OTEL_EXPORTER_OTLP_TIMEOUT=5`, `OTEL_SEMCONV_STABILITY_OPT_IN=http`, `COLLECTOR_PROFILE=default|tail`.
|
||||
|
||||
Источник истины и транзакционные границы (из lab5, не менялись): PostgreSQL хранит пользователей, чаты, сообщения, метаданные вложений и задания. Upload фиксирует в одной транзакции attachment `queued`, job и строку outbox, и только после COMMIT отвечает 202. Relay публикует outbox в RabbitMQ с publisher confirm. Worker одной транзакцией публикует результат: варианты, `ready`, job `succeeded`. S3 хранит байты, брокер — только ссылки на задания. Telemetry в эти транзакции не входит: trace context пишется в ту же outbox-строку, а spans и метрики экспортируются асинхронно и не могут задержать или отменить COMMIT.
|
||||
|
||||
Критичные зависимости `/health/ready` (как в lab5): PostgreSQL и S3. Collector/Tempo/Loki/Prometheus/Grafana **не** критичны — проверено инцидентом C.
|
||||
|
||||
## Схема сигналов
|
||||
|
||||
Версии: OpenTelemetry SDK 1.30.0, instrumentation 0.51b0, semantic conventions 0.51b0; HTTP-атрибуты по стабильной конвенции (`OTEL_SEMCONV_STABILITY_OPT_IN=http`: `http.request.method`, `http.route`, `http.response.status_code`, метрика `http.server.request.duration` в секундах). Collector — дистрибутив contrib 0.120.0 (нужны `tail_sampling` и `attributes`).
|
||||
|
||||
Resource attributes: `service.name` (messenger-api / messenger-worker), `service.namespace=messenger`, `service.version=1.6.0`, `service.instance.id` (api-1/api-2/hostname контейнера worker), `deployment.environment.name=lab`. В Prometheus: `service_name`, `instance`, `service_version`, `deployment_environment_name` (promote_resource_attributes; `job` = `messenger/<service.name>`).
|
||||
|
||||
| Сигнал контракта | Инструмент OTel → имя в Prometheus | Labels (все низкокардинальные) | Единица |
|
||||
| --- | --- | --- | --- |
|
||||
| HTTP requests (counter) | `http.server.request.duration` → `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`; класс статуса = `label_replace(..., "(\\d)..")` в запросе | запросы |
|
||||
| HTTP duration | та же гистограмма, `_bucket`; buckets 5 ms … 30 s (View) | те же | s |
|
||||
| Jobs processed/failed | `messenger.jobs.processed` → `messenger_jobs_processed_total` | `job_type`, `outcome` (succeeded/retry/failed/noop/lost/dlq/requeued) | jobs |
|
||||
| Job duration | `messenger.job.duration` → `messenger_job_duration_seconds_*`; buckets 50 ms … 120 s | `job_type`, `outcome` | s |
|
||||
| Queue depth / oldest age | `messenger.queue.depth`, `messenger.queue.oldest_age` → `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | `queue` (attachments.process / outbox), `state` | jobs / s |
|
||||
| DB pool | `db.client.connection.count`, `.pending_requests`, `.max` → `db_client_connection_*` | `db_client_connection_state` (used/idle) | соединения |
|
||||
| Worker busy/capacity | `messenger.worker.busy`, `messenger.worker.capacity` | — | jobs |
|
||||
| Process CPU/RAM | `process.cpu.time` → `process_cpu_time_seconds_total`, `process.memory.usage` → `process_memory_usage_bytes` | `process_cpu_owner` (self/children) | s / bytes |
|
||||
| End-to-end time to ready | `messenger.attachment.time_to_ready` → `messenger_attachment_time_to_ready_seconds_*`; buckets 0.25 s … 600 s | `job_type` | s |
|
||||
| Самонаблюдение телеметрии | `messenger.telemetry.sampler.decisions`, `messenger.telemetry.spans.exported`; collector `otelcol_*` | `decision`, `rule`, `outcome`, `exporter` | spans |
|
||||
| Логи | OTLP log records → Loki | stream labels: `service_name`, `service_instance_id`, `service_namespace`, `deployment_environment_name`; metadata: `event`, `severity_text`, `trace_id`, `span_id`, `code_*` | — |
|
||||
| Трассы | SERVER (FastAPI), PRODUCER `attachments.process publish`, CONSUMER `attachments.process process`, CLIENT `S3.*` (botocore), CLIENT `INSERT messages` и т.п. (свои), INTERNAL `imageproc`, `db.pool.acquire` | атрибуты: `messenger.request_id`, `messenger.attachment_id`, `messaging.message.id` (= job_id), `messenger.job.attempt/outcome`, `messenger.fault` | — |
|
||||
|
||||
Границы buckets подобраны под ожидаемые задержки. HTTP: 5–150 ms — обычные запросы (p50 около 7 ms, p99 около 35–50 ms под нагрузкой); 250 ms – 1 s — граница SLO 500 ms и порог алерта p95 1 s; 2.5–10 s — `lock_timeout` 5 s и `DB_OP_TIMEOUT` 6 s; 30 s — `REQUEST_TIMEOUT`. Job: 50 ms – 3 s — обычная обработка (около 0.15 с, 1 с при ограниченном CPU); 5–120 s — таймаут дочернего процесса 60 s и hard timeout 120 s. Time to ready: 0.25 s – 10 min — от мгновенной обработки до ожидания в очереди с backoff до 30 s и пятью попытками.
|
||||
|
||||
Технические ID (`request_id`, `attachment_id`, `job_id`) есть только в span-атрибутах и тексте логов (обоснование: по ним оператор находит трассу конкретной загрузки/ошибки; данные синтетические), никогда в labels метрик — проверено тестом: ни одного UUID в значениях labels, список labels фиксирован. `http.route` — шаблон (`/api/v1/chats/{chat_id}/messages`).
|
||||
|
||||
Запрещённое в telemetry: тело сообщения, пароль, Authorization/токен, изображение, filename, signed URL. Защита в два слоя: (1) SDK — заголовки не захватываются, атрибуты HTTP-гистограммы ограничены View, SQL параметризован (`db.query.text` содержит только `%s`), логи не печатают тело/пароль; (2) collector `attributes/scrub` удаляет `url.query` (там был бы `?filename=`), `url.full`, `http.target`, `http.url`, заголовки authorization/cookie. Проверка: `test_observability.py` ищет пароль, токен, текст-маркер, filename, `Authorization`, `Bearer`, `filename=`, `X-Amz-Signature` в трассах и логах — не найдено.
|
||||
|
||||
### Перенос контекста API → broker → worker (уровень 4)
|
||||
|
||||
1. Upload: server span `POST /api/v1/chats/{chat_id}/attachments`. В той же транзакции, что attachment `queued` и job, в `outbox.payload` записываются `traceparent`/`tracestate` текущего span (поля разрешены `contracts/jobs.schema.json`).
|
||||
2. Relay outbox (фоновый цикл API) создаёт PRODUCER span **дочерним** контексту из payload и публикует сообщение уже со своим `traceparent` (в теле и дублем в AMQP headers).
|
||||
3. Worker извлекает контекст из тела (fallback — headers) и создаёт CONSUMER span — дочерний producer. Внутри: `S3.GetObject`, `imageproc`, `S3.CopyObject`, `S3.PutObject`×3, SQL, `S3.DeleteObject`.
|
||||
4. Retry: `fail_attempt` пишет новую outbox-строку внутри consumer span → следующий PRODUCER — дочерний упавшему CONSUMER. Вся история upload, включая повторы, — **одна трасса**.
|
||||
5. Sampler `ParentBased(root=KindFilter(TraceIdRatio))`: корнем трассы может быть только SERVER/CONSUMER/PRODUCER, поэтому одиночные SQL-spans фоновых циклов (outbox polling, sweeper, readiness) не создают тысячи мусорных трасс; дети следуют решению родителя, поэтому upload и его job сэмплируются вместе.
|
||||
Ограничение: восстановление job по истёкшему lease (sweeper) начинает новую трассу (нет активного контекста); связь видна по `messenger.attachment_id`.
|
||||
|
||||
Пример (вывод `test_observability.py`, `evidence/observability.txt`; upload с `fault-transient1` — первая попытка падает временно):
|
||||
|
||||
```
|
||||
api SERVER POST /api/v1/chats/{chat_id}/attachments 9.0 ms
|
||||
api CLIENT S3.PutObject 3.4 ms
|
||||
api PRODUCER attachments.process publish 1.7 ms attempt=1
|
||||
worker CONSUMER attachments.process process 6.1 ms ERROR outcome=retry
|
||||
api PRODUCER attachments.process publish 2.9 ms attempt=2
|
||||
worker CONSUMER attachments.process process 168.6 ms outcome=succeeded
|
||||
worker CLIENT S3.GetObject 3.7 ms
|
||||
worker INTERNAL imageproc 139.6 ms
|
||||
worker CLIENT S3.CopyObject 4.4 ms
|
||||
worker CLIENT S3.PutObject (x3) 2.0-2.4 ms
|
||||
worker CLIENT S3.DeleteObject 1.0 ms
|
||||
trace 40391c8b40fc76e7f27913c3d84421a4, 42 spans (SQL-spans в выводе опущены). В последнем consumer результат публикуется
|
||||
SQL-spans: UPDATE attachment_jobs, INSERT attachment_variants ×3, UPDATE attachments, INSERT job_audit. В Loki по этому trace_id 6 записей:
|
||||
api: attachment_queued; worker: job_claimed, job_attempt_failed, job_finished, job_claimed, job_finished
|
||||
```
|
||||
|
||||
### SLI / SLO и алерты (уровень 4)
|
||||
|
||||
- **SLI доступности** = 1 − (5xx / все бизнес-запросы messenger-api); health не инструментирован, 4xx — исход клиента, не отказ. **SLO 99%** на скользящем окне 30 мин (учебное окно; в production те же формулы на 28–30 днях). Recording rule `messenger:slo_availability:ratio30m`.
|
||||
- **SLI латентности** = доля запросов ≤ 500 ms (bucket `le="0.5"`), **SLO 95%** / 30 мин, `messenger:slo_latency:ratio30m`.
|
||||
- Перцентили на dashboard — `histogram_quantile` по `sum by (le)` buckets **всех** реплик, а не среднее и не усреднение p99 реплик.
|
||||
- SLO проверяется значением recording rule в Prometheus. В инциденте B: перед блокировкой доступность 1.0000 и латентность 0.9825; в момент firing 0.6894 и 0.6933; через 2 минуты после снятия блокировки 0.7697 и 0.7616. Оба SLO нарушены, бюджет ошибок окна (1%) исчерпан. Значения: `messenger:slo_availability:ratio30m` и `messenger:slo_latency:ratio30m` на моменты 14:34:58, 14:36:53 и 14:39:09 UTC 7 ноября.
|
||||
|
||||
Ключевые панели dashboard (полный перечень 32 панелей с запросами и единицами — `docs/dashboard.md`):
|
||||
|
||||
| Панель | Запрос | Единица |
|
||||
| --- | --- | --- |
|
||||
| Латентность API p50/p95/p99 | `histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval])))` (и 0.5, 0.95) | секунды |
|
||||
| Доля ошибок 5xx | `(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]))` | доля |
|
||||
| RPS по классу статуса | `sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{...}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d).."))` | запросы/с |
|
||||
| Jobs по исходу | `sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))` | jobs/с |
|
||||
| Длительность job p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval])))` | секунды |
|
||||
| Глубина очереди | `max by (queue, state) (messenger_queue_depth)` | jobs |
|
||||
| Возраст старейшей job | `max(messenger_queue_oldest_age_seconds{queue="attachments.process"})` | секунды |
|
||||
| Время до ready p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))` | секунды |
|
||||
| Пул БД | `sum by (service_name, db_client_connection_state) (db_client_connection_count)`, `sum by (service_name) (db_client_connection_pending_requests)` | соединения, запросы |
|
||||
| Насыщение worker | `messenger:worker_saturation:ratio` = busy / capacity по свежим сериям | доля |
|
||||
| CPU / RSS процессов | `sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))`, `process_memory_usage_bytes` | ядра, байты |
|
||||
|
||||
| Alert | Условие / окно / порог / for | Проверен |
|
||||
| --- | --- | --- |
|
||||
| MessengerApiHighErrorRate | 5xx/все за 2m > 5% при > 0.2 rps, for 1m | инцидент B: firing и resolved |
|
||||
| MessengerQueueLagGrowing | max(oldest job age) > 60 s, for 1m | инцидент A: firing и resolved |
|
||||
| MessengerApiLatencyHigh | p95 (5m) > 1 s, for 2m | инцидент B: firing 14:38:00–14:41:45 UTC, затем resolved (`evidence/alerts_history.txt`) |
|
||||
| MessengerDbPoolWaiting | waiting > 0, for 1m | инцидент B: firing 14:36:30–14:37:00 UTC, затем resolved (`evidence/alerts_history.txt`) |
|
||||
| MessengerWorkerSaturated | busy/capacity > 0.9 (только свежие серии), for 5m | `test_worker_saturation.py`: firing и resolved |
|
||||
| TelemetryExportFailing | заполнение очереди exporter > 20% или отбросы/исчерпанные ретраи, for 1m | инцидент C1: firing и resolved |
|
||||
|
||||
Runbooks: `docs/runbooks.md`. Состояние — Prometheus `/api/v1/alerts` и панель «Алерты Prometheus» на dashboard.
|
||||
|
||||
### Collector: память и недоступный backend
|
||||
|
||||
`memory_limiter` первым в каждом pipeline (limit 180 MiB, spike 40 MiB при лимите контейнера 256 MiB): при превышении receiver отвечает отказом, SDK ретраит/отбрасывает. Каждый exporter: `sending_queue` в памяти на 100 батчей (≤ 2048 элементов) и `retry_on_failure` с `max_elapsed_time: 120s` — ни бесконечного буфера, ни бесконечных повторов. На стороне SDK: BatchSpanProcessor/BatchLogRecordProcessor с очередью 2048, при переполнении — отброс; экспорт идёт в отдельном потоке, запросы API не ждут telemetry.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke при включённой телеметрии | `make check`, `make validate`, `make test` | все PASS | `evidence/smoke.txt` |
|
||||
| Воспроизводимость из чистого clone | копия только файлов git, шаги job `.github/workflows/stand.yml`, отдельный compose-проект | все сервисы healthy, smoke PASS, dashboard импортирован, 13 правил, метрики идут | `evidence/clean_clone.txt` |
|
||||
| Минимум lab5 сохранён | `test_worker_down.py`, `test_failure_windows.py` (A–I) | PASS | `evidence/lab5_worker_down.txt`, `evidence/lab5_failure_windows.txt` |
|
||||
| Сохранность данных, реальный рестарт | `make test-persistence` | PASS; uptime всех хранилищ, включая Prometheus, Loki, Tempo, сброшен | `evidence/persistence.txt` |
|
||||
| Failover каждой реплики | `make test-failover` ×2 | PASS; 0 неудачных попыток, восстановление 2.0 и 0.9 с | `evidence/failover.txt` |
|
||||
| 3: Collector, 3 хранилища, Grafana в общем стенде; конфиги в Git | `compose.yaml`, `infra/` | стенд поднимается одной командой | `evidence/clean_clone.txt` |
|
||||
| 3: volumes сохраняют историю | рестарт Prometheus, Loki, Tempo, Grafana | трасса, логи и алерты 7 ноября доступны 9 ноября после рестарта | `evidence/telemetry_persistence.txt` |
|
||||
| 3: RPS/errors/duration, jobs, duration worker в Grafana | `check_dashboard.py` | 32 панели; все 44 запроса Prometheus, запросы Loki и Tempo выполнены через саму Grafana с данными; Grafana видит 6 правил алертов | `evidence/dashboard_check.txt` |
|
||||
| 3: контролируемый запрос → trace → лог; job → worker trace → лог | `test_observability.py` | PASS | `evidence/observability.txt` |
|
||||
| 3: нет секретов, изображений и ID в метках | `test_observability.py` | пароль, токен, текст, filename, Authorization, signed URL и байты PNG (в сыром виде и base64) не найдены; UUID в labels нет | `evidence/observability.txt` |
|
||||
| 4: W3C API → broker → worker, цепочка до S3/DB с retry | `test_observability.py` | одна трасса, 42 spans, consumer — дочерний producer, retry — дочерний упавшего consumer | `evidence/observability.txt` |
|
||||
| 4: p50/p95/p99, ошибки, очередь, DB pool, saturation | dashboard и `check_dashboard.py` | панели с запросами и единицами; перцентили из buckets | `infra/grafana/dashboards/messenger.json` |
|
||||
| 4: SLI/SLO и alert rules проверены отказом | `test_incidents.py`, `test_worker_saturation.py` | все 6 алертов: firing → resolved | `evidence/incidents.txt`, `evidence/alerts_history.txt`, `evidence/worker_saturation.txt` |
|
||||
| 4: memory limiter, batch, ограниченный буфер | Tempo остановлен при 100 RPS | очередь 100/100, отброшено 25147 и отказано 16802 spans, RSS collector до 206 MiB при лимите 256 MiB, API без ошибок | `evidence/exporter_queue_full.txt` |
|
||||
| 4: остановка telemetry не ломает сообщения | инцидент C | 0 неожиданных ошибок API при остановке backend и collector | `evidence/incidents.txt` |
|
||||
| 4: разбор двух инцидентов | разделы ниже | шаги диагностики и причины | `evidence/incidents.txt`, `docs/runbooks.md` |
|
||||
| 5, трек 1 | `track1.py run --repeats 3` | 4 конфигурации, по 3 прогона | `evidence/track1/results.jsonl`, `evidence/track1/summary.md`, `evidence/track1/matrix.log` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Перезапуск в `test-persistence` затрагивает API, worker, LB, PostgreSQL, Valkey, MinIO, RabbitMQ и, с lab6, Prometheus, Loki, Tempo, collector и Grafana (`scripts/restart.sh`), подтверждено состояниями контейнеров до и после.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Стенд: macOS arm64, Docker 29.7.2, VM 10 CPU / 7.7 GiB. Лимиты: API 0.5 CPU / 512 MiB ×2, worker 2 CPU / 1.5 GiB ×2, PostgreSQL 1 CPU / 512 MiB, collector 1 CPU / 256 MiB, Tempo 1 CPU / 1 GiB. Фоновая нагрузка в инцидентах: open-loop 10 RPS (75% POST message, 25% history, 2% `X-Fault: slow`). Времена отсчитываются от внесения отказа; полный вывод в `evidence/incidents.txt`, сводка в `evidence/incidents.json`.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| A. Остановка worker | рост возраста задания, `MessengerQueueLagGrowing` | pending +71 с, firing +131 с при возрасте 126 с; глубина 3 | backlog разобран за 3.6 с, resolved через 14 с; потерь заданий нет |
|
||||
| B. `LOCK TABLE messages` на 106 с | 5xx, ожидание в БД видно в трассе | pending +45 с, firing +105 с; 5xx 70.7%, p95 6.92 с, пул 10/10, ждут 19 | resolved через 135 с после снятия блокировки (окно 2m); подтверждённые данные не теряются |
|
||||
| C1. Tempo и Loki остановлены на 192 с | API работает, очередь ограничена | 1703 запроса, 0 неожиданных ошибок; очередь Tempo 79/100, `TelemetryExportFailing` firing | экспорт возобновился, resolved через 35 с; потеряно 387 записей telemetry (ретраи исчерпаны) |
|
||||
| C2. Collector остановлен на 60 с | API работает, очереди SDK ограничены | 740 запросов, 0 неожиданных ошибок; RSS api-1 108.6 → 109–119 MiB | telemetry за время простоя теряется сверх очереди SDK (2048) |
|
||||
| Tempo остановлен при 100 RPS, 223 с | очередь упирается в предел, отбросы считаются | очередь 100/100, отброшено 25147, отказано memory_limiter 16802 spans; RSS до 206 MiB | API: 24686 запросов, 0 ошибок, p99 55.8 ms; потерянные spans не восстанавливаются. После возврата Tempo сброс очереди превысил его лимит 15 MB/с: ещё 32410 spans отклонены (`RATE_LIMITED`) и посчитаны в `send_failed` |
|
||||
| Насыщение worker (1 слот ×2, 0.25 CPU), 4 upload/с | saturation 1.0 дольше `for: 5m` | firing +340 с; saturation 1.00, 194 в очереди, возраст 63 с, 3.16 job/с, p95 job 0.98 с | после возврата настроек resolved через 17 с, backlog разобран за 22 с |
|
||||
|
||||
### Инцидент 1: задания не обрабатываются
|
||||
|
||||
1. Сработал `MessengerQueueLagGrowing`. Метрики: возраст старейшего задания 126 с, 3 задания в `queued`, `rate(messenger_jobs_processed_total[1m]) = 0`. Это не медленная обработка, задания не берутся вообще.
|
||||
2. Логи worker: после остановки 0 событий `job_*`, последние события `worker_stopped`.
|
||||
3. Трасса upload: есть SERVER и PRODUCER, нет CONSUMER. Сообщение дошло до брокера, потребителя нет.
|
||||
4. Причина: остановлен worker. Действие: `docker compose start worker`. Задания сохранены в PostgreSQL и durable-очереди, поэтому все три стали ready за 3.6 с.
|
||||
|
||||
### Инцидент 2: медленная БД или медленный handler
|
||||
|
||||
1. Сработал `MessengerApiHighErrorRate`: 5xx 70.7%, p95 6.92 с. Ошибки 503 приходятся на `/api/v1/chats/{chat_id}/messages`.
|
||||
2. Метрики пула: занято 10 из 10, ожидают 19, поэтому подозрение падает на БД.
|
||||
3. Трасса упавшего запроса: `INSERT messages` занял 5001 ms из 5004 ms и завершился `QueryCanceled` (срабатывает `lock_timeout`). Время потрачено в PostgreSQL.
|
||||
4. Контраст: запрос с `X-Fault: slow` длится 615 ms, а его самый долгий DB-span — 1.1 ms, все DB-spans вместе — 8.3 ms. Так выглядит медленный handler.
|
||||
5. Шаг оператора: `pg_stat_activity` показывает блокирующий `LOCK TABLE messages`; после `pg_terminate_backend` ошибки прекращаются.
|
||||
|
||||
Побочный эффект, найденный в этом инциденте: исчерпание пула делает `/health/ready` неуспешным (событие `not_ready`), поэтому блокировка одной таблицы отключает обе реплики целиком. За время инцидента клиент получил 816 ответов 503 и 214 ответов 502 от nginx.
|
||||
|
||||
## Трек 5.1: цена наблюдаемости
|
||||
|
||||
**Гипотезы.** H1: стоимость телеметрии в приложении пропорциональна числу записанных spans, поэтому head sampling снижает CPU, а tail sampling — нет. H2: head sampling 10% теряет около 90% ошибок и редких медленных запросов, потому что решение принимается до их исхода. H3: tail sampling сохраняет все такие трассы и резко сокращает объём в хранилище. H4: метрики от сэмплирования не зависят.
|
||||
|
||||
**Методика.** Конфигурации: `off` (`OTEL_SDK_DISABLED=true`), `on100` (100%), `head10` (head sampling 10% корневых spans), `tail` (SDK 100%; collector оставляет ошибки, трассы > 500 ms, конвейер вложений и 10% остального; `decision_wait` 10 с). Перед каждым прогоном пересоздаются API, worker и collector. Профиль: open-loop 60 RPS (75% POST message, 25% GET history), 1 upload/с, 1% запросов с `+600 ms`, 1% запросов с ответом 500; прогрев 20 с, пауза 20 с, замер 90 с, пауза 30 с. Три прогона каждой конфигурации, порядок циклический. Объём данных: перед экспериментом в БД 16 739 сообщений, 38 чатов, 89 пользователей, 87 вложений; после — 75 179, 130, 273 и 1 277 (каждый прогон создаёт 4 новых чата, история самого большого чата 2 701 сообщение). GET history читает первую страницу (limit=20), поэтому рост БД на latency чтения почти не влияет. Latency считается от запланированного момента отправки (без coordinated omission), только для успешных обычных запросов. CPU — точная разность cgroup-счётчика за окно; для collector сверена с его собственной метрикой (0.52 и 0.54% ядра). Запуск: `evidence/track1/matrix.log`.
|
||||
|
||||
Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.
|
||||
|
||||
| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| off | 3 | 60.4 | 0 | 7.1 (6.4…8.3) | 11.0 (10.3…12.2) | 32.7 (31.7…33.9) | 17.8 (17.0…18.3) | 15.2 (14.8…15.3) |
|
||||
| on100 | 3 | 60.5 | 0 | 7.8 (6.6…8.6) | 14.3 (11.8…16.3) | 47.2 (43.4…54.4) | 22.7 (22.3…22.9) | 15.6 (15.3…15.9) |
|
||||
| head10 | 3 | 60.4 | 0 | 7.2 (6.6…8.3) | 11.2 (10.3…13.0) | 37.5 (36.6…39.2) | 19.7 (19.6…20.0) | 15.0 (14.9…15.1) |
|
||||
| tail | 3 | 60.4 | 0 | 7.9 (6.7…8.7) | 13.7 (11.9…14.7) | 44.5 (42.0…48.2) | 22.3 (22.1…22.5) | 15.3 (15.2…15.3) |
|
||||
|
||||
| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| off | 0.07 (0.06…0.08) | 0.70 (0.43…0.86) | 0.57 (0.54…0.60) | 0.34 (0.30…0.36) | 98 | 36 | 470 |
|
||||
| on100 | 0.57 (0.54…0.60) | 1.13 (1.03…1.29) | 0.57 (0.55…0.58) | 0.36 (0.33…0.38) | 113 | 73 | 571 |
|
||||
| head10 | 0.23 (0.23…0.24) | 0.41 (0.35…0.44) | 0.54 (0.50…0.58) | 0.34 (0.33…0.34) | 105 | 71 | 423 |
|
||||
| tail | 0.53 (0.52…0.54) | 0.64 (0.50…0.80) | 0.52 (0.50…0.53) | 0.33 (0.32…0.34) | 116 | 81 | 493 |
|
||||
|
||||
| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |
|
||||
|---|---|---|---|---|---|---|
|
||||
| off | — | 0 | 0 | — | 0 | 0 |
|
||||
| on100 | 57059 | 57059 | 0 | 21030 | 20547 | 132 |
|
||||
| head10 | 5560 | 5560 | 0 | 2496 | 2031 | 134 |
|
||||
| tail | 57106 | 9746 | 0 | 21062 | 3877 | 135 |
|
||||
|
||||
Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.
|
||||
|
||||
| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |
|
||||
|---|---|---|---|---|---|
|
||||
| off | 0 / 16470 | 0 / 164 | 0 / 163 | 0 / 270 | — / 163 |
|
||||
| on100 | 16530 / 16470 | 167 / 167 | 146 / 146 | 270 / 270 | 146 / 146 |
|
||||
| head10 | 1630 / 16470 | 18 / 198 | 15 / 166 | 25 / 270 | 166 / 166 |
|
||||
| tail | 2269 / 16470 | 180 / 180 | 164 / 164 | 270 / 270 | 164 / 164 |
|
||||
|
||||
**Результаты по гипотезам.**
|
||||
|
||||
- **H1 подтверждена.** CPU API относительно `off`: `on100` +4.9 п.п. (+28%, около 0.8 ms CPU на запрос), `head10` +1.9 п.п. (+11%), `tail` +4.5 п.п. (+25%). Диапазоны между прогонами не пересекаются. CPU worker не изменился (15.0–15.6%): его определяет обработка изображений.
|
||||
- **Latency.** p99: `off` 32.7, `head10` 37.5, `tail` 44.5, `on100` 47.2 ms; диапазоны `off` и `on100` не пересекаются. p50 различается меньше, чем прогоны одной конфигурации (6.4–8.7 ms), поэтому разницы по p50 заявить нельзя. p95 у `on100` и `tail` выше, но диапазоны частично пересекаются с `off`.
|
||||
- **H2 подтверждена.** При `head10` сохранилось 18/198 медленных запросов (9%), 15/166 ошибок (9%) и 25/270 конвейеров вложений (9%).
|
||||
- **H3 подтверждена.** `tail` сохранил 180/180, 164/164 и 270/270 при сокращении ingest Tempo на 81% (20547 → 3877 KiB) и числа spans на 83%. Обычных трасс осталось около 14%. Экономии между SDK и collector нет: 21 MiB за окно, как в `on100`.
|
||||
- **H4 подтверждена.** Счётчик HTTP 500 совпал с числом отправленных ошибок во всех режимах: 146/146, 166/166, 164/164. При `head10` метрика видит все 166 ошибок, а разобрать по трассе можно только 15.
|
||||
- **Потери при передаче** отсутствуют: `Spans → collector` равно `Spans → Tempo` в `on100` и `head10`, отказов и отбросов collector нет, SDK не сообщил об ошибках экспорта.
|
||||
|
||||
**Что теряется.** `head10`: около 90% любых трасс, включая ошибки и редкие медленные запросы. `tail`: 90% обычных успешных запросов; запросы медленнее обычного, но быстрее порога 500 ms; ответы 4xx (их server span не имеет статуса ERROR). Части трассы, пришедшие позже `decision_wait` (10 с) после решения об отбросе, тоже будут потеряны; для конвейера вложений это закрыто политикой по `messaging.system`.
|
||||
|
||||
**Сбалансированная конфигурация: `tail`** (`COLLECTOR_PROFILE=tail`). Её цена: те же +25% CPU API, что у полной трассировки, и +12 ms к p99, зато сохраняются все диагностически важные трассы при объёме хранения на 81% меньше. `head10` разумен, только если критичен CPU приложения и достаточно метрик и логов. По умолчанию в репозитории оставлен `on100`: на защите нужно найти любую контрольную трассу, а нагрузка учебная.
|
||||
|
||||
**Ограничения измерения.** Один Docker-host и генератор на той же машине, API далёк от насыщения (около 11% ядра на реплику), поэтому поведение под перегрузкой здесь не измерено. Три прогона по 90 с дают ориентировочные оценки. RSS Tempo между конфигурациями не сравним: контейнер не пересоздавался. Колонка SDK-решений из сводки исключена: счётчики пересозданных контейнеров выпадали из окна запроса и давали неверные разности. Tail sampling требует, чтобы все spans трассы приходили в один collector; при нескольких collector нужна маршрутизация по trace_id.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Выдерживается: отказ одной реплики API, остановка и перезапуск worker и брокера с volumes, недоступность Tempo, Loki или Collector без влияния на бизнес-API (данные telemetry при этом теряются сверх ограниченных очередей, потери считаются).
|
||||
|
||||
Не выдерживается и не заявляется: потеря Docker-host, отказ единственных PostgreSQL, nginx или collector как высокодоступных компонентов. Хранилища telemetry работают в одном экземпляре.
|
||||
|
||||
Известные ограничения и упрощения:
|
||||
|
||||
- Блокировка БД исчерпывает пул и переводит реплики в not ready, поэтому отказ одной таблицы останавливает весь API. Улучшение: отдельное соединение для readiness. Пока это гипотеза, не реализовано.
|
||||
- OTLP-метрики в Prometheus не получают staleness-маркеров: серии остановленного процесса видны ещё 5 минут. Правило насыщения worker это учитывает; в остальных запросах значения исчезают с задержкой.
|
||||
- `MessengerApiHighErrorRate` срабатывает и на 503 от admission control очереди (видно в тесте насыщения): это реальный отказ клиенту, но причину надо смотреть по маршруту и логам.
|
||||
- Восстановление задания по истёкшему lease начинает новую трассу; связь видна по `messenger.attachment_id`.
|
||||
- Метрики CPU worker учитывают дочерние процессы только после их завершения; RSS дочерних процессов не учитывается.
|
||||
- Первый прогон инцидентов сорван сном Mac: стенд замер на 18 минут. Длинные прогоны нужно запускать под `caffeinate -i`.
|
||||
- `FAULT_INJECTION=1` и заголовок `X-Fault` допустимы только на учебном стенде. HTTP без TLS — только локально, TLS и mTLS относятся к lab7.
|
||||
- После долгого простоя Tempo сброс накопленной очереди упирается в лимит ingest: часть данных теряется уже после восстановления, алерт остаётся firing ещё около 2 минут (окно rate).
|
||||
- Окно SLO 30 минут учебное.
|
||||
|
||||
Следующие шаги: отдельный пул для readiness, TLS в lab7, проверка tail sampling под нагрузкой, близкой к насыщению.
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""RabbitMQ topology and confirmed publishing (aio-pika).
|
||||
|
||||
Topology (declared idempotently by every publisher and consumer):
|
||||
exchange attachments direct, durable
|
||||
queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit
|
||||
exchange attachments.dlx fanout, durable
|
||||
queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator)
|
||||
Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker
|
||||
has taken responsibility for the message (written to the quorum log).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType, Message
|
||||
|
||||
from app import config
|
||||
|
||||
log = logging.getLogger("messenger.broker")
|
||||
ROUTING_KEY = "attachment.process.v1"
|
||||
|
||||
|
||||
async def connect():
|
||||
return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, client_properties={"connection_name": config.INSTANCE_ID})
|
||||
|
||||
|
||||
async def declare(channel):
|
||||
dlx = await channel.declare_exchange(config.BROKER_DLX, ExchangeType.FANOUT, durable=True)
|
||||
dlq = await channel.declare_queue(config.BROKER_DLQ, durable=True, arguments={"x-queue-type": "quorum"})
|
||||
await dlq.bind(dlx)
|
||||
exchange = await channel.declare_exchange(config.BROKER_EXCHANGE, ExchangeType.DIRECT, durable=True)
|
||||
queue = await channel.declare_queue(
|
||||
config.BROKER_QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-queue-type": "quorum",
|
||||
"x-dead-letter-exchange": config.BROKER_DLX,
|
||||
"x-delivery-limit": config.BROKER_DELIVERY_LIMIT,
|
||||
},
|
||||
)
|
||||
await queue.bind(exchange, ROUTING_KEY)
|
||||
return exchange, queue
|
||||
|
||||
|
||||
class Publisher:
|
||||
def __init__(self) -> None:
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
self.exchange = None
|
||||
|
||||
async def ensure(self) -> None:
|
||||
if self.channel is not None and not self.channel.is_closed:
|
||||
return
|
||||
if self.connection is None or self.connection.is_closed:
|
||||
self.connection = await connect()
|
||||
self.channel = await self.connection.channel(publisher_confirms=True)
|
||||
self.exchange, _ = await declare(self.channel)
|
||||
|
||||
async def publish(self, body: dict) -> None:
|
||||
await self.ensure()
|
||||
message = Message(
|
||||
json.dumps(body, separators=(",", ":")).encode(),
|
||||
content_type="application/json",
|
||||
delivery_mode=DeliveryMode.PERSISTENT,
|
||||
message_id=body["job_id"],
|
||||
type=body["type"],
|
||||
# Native AMQP headers duplicate the body carrier (same W3C values) for broker-side tooling.
|
||||
headers={"attempt": body["attempt"], **{k: body[k] for k in ("traceparent", "tracestate") if k in body}},
|
||||
)
|
||||
# mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack.
|
||||
await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5)
|
||||
|
||||
async def close(self) -> None:
|
||||
try:
|
||||
if self.connection is not None:
|
||||
await self.connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.connection = self.channel = self.exchange = None
|
||||
@@ -0,0 +1,98 @@
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
DATABASE_URL = _secret("DATABASE_URL")
|
||||
CACHE_URL = _secret("CACHE_URL")
|
||||
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
|
||||
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
|
||||
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
|
||||
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
|
||||
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
|
||||
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
|
||||
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
|
||||
READY_FAILURES = _int("READY_FAILURES", 2)
|
||||
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
|
||||
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
|
||||
PASSWORD_MIN = 12
|
||||
PASSWORD_MAX = 128
|
||||
SESSION_REVOKED_TTL_SECONDS = max(
|
||||
_int("SESSION_REVOKED_TTL_SECONDS", 60),
|
||||
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
|
||||
)
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY")
|
||||
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
|
||||
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
|
||||
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
|
||||
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
|
||||
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
|
||||
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
|
||||
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
|
||||
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
|
||||
|
||||
# --- lab5: broker, jobs, worker ---
|
||||
BROKER_URL = _secret("BROKER_URL")
|
||||
BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments")
|
||||
BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
||||
BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx")
|
||||
BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq")
|
||||
BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20)
|
||||
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1)
|
||||
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2)
|
||||
WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY
|
||||
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5)
|
||||
JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60)
|
||||
JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20)
|
||||
JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0)
|
||||
JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0)
|
||||
JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0)
|
||||
WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768)
|
||||
WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25)
|
||||
WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat")
|
||||
OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0)
|
||||
OUTBOX_BATCH = _int("OUTBOX_BATCH", 50)
|
||||
OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400)
|
||||
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400)
|
||||
# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables).
|
||||
MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200)
|
||||
PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0)
|
||||
THUMBNAIL_SIZE = 256
|
||||
CROP_SIZE = 128
|
||||
OPTIMIZED_SIZE = 1280
|
||||
JPEG_QUALITY = _int("JPEG_QUALITY", 85)
|
||||
WEBP_QUALITY = _int("WEBP_QUALITY", 80)
|
||||
# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py).
|
||||
FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1"
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
|
||||
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
|
||||
class BadCursor(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fmt(ts: datetime) -> str:
|
||||
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
|
||||
|
||||
|
||||
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
|
||||
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(scope: str, cursor: str):
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
raise BadCursor() from None
|
||||
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
|
||||
raise BadCursor()
|
||||
ts, item_id = data["t"], data["i"]
|
||||
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
|
||||
raise BadCursor()
|
||||
try:
|
||||
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
|
||||
if str(uuid.UUID(item_id)) != item_id:
|
||||
raise BadCursor()
|
||||
except ValueError:
|
||||
raise BadCursor() from None
|
||||
if encode_cursor(scope, ts, item_id) != cursor:
|
||||
raise BadCursor()
|
||||
return parsed, item_id
|
||||
@@ -0,0 +1,134 @@
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool, PoolTimeout
|
||||
|
||||
from app import config, telemetry
|
||||
from app.errors import unavailable
|
||||
|
||||
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
|
||||
OPERATION_RE = re.compile(r"^\s*([A-Za-z]+)")
|
||||
TABLE_RE = re.compile(r"\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-z_]+)", re.IGNORECASE)
|
||||
POOL_WAIT_SPAN_SECONDS = 0.005
|
||||
|
||||
|
||||
def statement(query) -> tuple:
|
||||
"""Span name and attributes for a parametrised SQL statement (values are never part of the text)."""
|
||||
text = query if isinstance(query, str) else str(query)
|
||||
op = OPERATION_RE.match(text)
|
||||
table = TABLE_RE.search(text)
|
||||
operation = op.group(1).upper() if op else "SQL"
|
||||
name = operation + (" " + table.group(1) if table else "")
|
||||
return name, {"db.system": "postgresql", "db.operation.name": operation, "db.namespace": "messenger",
|
||||
"db.collection.name": table.group(1) if table else "", "db.query.text": text[:400]}
|
||||
|
||||
|
||||
async def discard(conn) -> None:
|
||||
try:
|
||||
await conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class Guarded:
|
||||
def __init__(self, conn, timeout: float):
|
||||
self.conn = conn
|
||||
self.timeout = timeout
|
||||
|
||||
async def execute(self, *args, **kwargs):
|
||||
# Own CLIENT span around the awaited statement: the time really spent in PostgreSQL (lock waits included).
|
||||
# (opentelemetry-instrumentation-psycopg 0.51b0 ends async spans before the await completes -> 0 ms spans.)
|
||||
name, attrs = statement(args[0] if args else "")
|
||||
with telemetry.span(name, kind=SpanKind.CLIENT, attributes=attrs) as span:
|
||||
try:
|
||||
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(self.conn)
|
||||
span.set_status(Status(StatusCode.ERROR, "client timeout %.1fs" % self.timeout))
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
def transaction(self):
|
||||
return self.conn.transaction()
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self) -> None:
|
||||
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS * 2,
|
||||
)
|
||||
self.pool = AsyncConnectionPool(
|
||||
conninfo=config.DATABASE_URL,
|
||||
min_size=config.DB_POOL_MIN,
|
||||
max_size=config.DB_POOL_MAX,
|
||||
timeout=config.DB_POOL_TIMEOUT,
|
||||
max_lifetime=600.0,
|
||||
open=False,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": config.DB_CONNECT_TIMEOUT,
|
||||
"options": options,
|
||||
},
|
||||
)
|
||||
|
||||
async def open(self) -> None:
|
||||
await self.pool.open(wait=False)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.pool.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connection(self):
|
||||
started, acquired = time.time_ns(), False
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
|
||||
acquired = True
|
||||
waited = (time.time_ns() - started) / 1e9
|
||||
if waited >= POOL_WAIT_SPAN_SECONDS: # only noticeable waits become spans
|
||||
telemetry.tracer.start_span("db.pool.acquire", start_time=started,
|
||||
attributes={"db.system": "postgresql",
|
||||
"db.client.connection.wait_s": round(waited, 4)}).end()
|
||||
yield Guarded(conn, config.DB_OP_TIMEOUT)
|
||||
except BROKEN as exc:
|
||||
if not acquired:
|
||||
span = telemetry.tracer.start_span("db.pool.acquire", start_time=started,
|
||||
attributes={"db.system": "postgresql"})
|
||||
span.record_exception(exc)
|
||||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
span.end()
|
||||
raise unavailable("Хранилище недоступно") from None
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
async with self.connection() as conn:
|
||||
async with conn.transaction():
|
||||
yield conn
|
||||
|
||||
async def in_transaction(self, action):
|
||||
async def run():
|
||||
async with self.transaction() as conn:
|
||||
return await action(conn)
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
|
||||
try:
|
||||
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(conn)
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,37 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> ApiError:
|
||||
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
|
||||
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
|
||||
|
||||
|
||||
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": rid}},
|
||||
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off).
|
||||
|
||||
A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`.
|
||||
Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata.
|
||||
Crash faults fire only on the first attempt and first delivery, so recovery can be observed.
|
||||
api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish
|
||||
worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT
|
||||
worker_crash_after_commit worker exits after the result COMMIT, before broker ack
|
||||
worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1
|
||||
transientN attempts 1..N fail with a transient error (N = 1..9)
|
||||
permanent permanent error on the first attempt -> failed without retries
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from app import config
|
||||
|
||||
log = logging.getLogger("messenger.faults")
|
||||
TRANSIENT_RE = re.compile(r"fault-transient([1-9])")
|
||||
|
||||
|
||||
def active(filename: Optional[str], name: str) -> bool:
|
||||
return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename
|
||||
|
||||
|
||||
def transient_until(filename: Optional[str]) -> int:
|
||||
if not config.FAULT_INJECTION or not filename:
|
||||
return 0
|
||||
match = TRANSIENT_RE.search(filename)
|
||||
return int(match.group(1)) if match else 0
|
||||
|
||||
|
||||
def crash(point: str) -> None:
|
||||
log.error("FAULT INJECTION: process exits at %s", point)
|
||||
logging.shutdown()
|
||||
os._exit(70)
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
|
||||
|
||||
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
|
||||
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy
|
||||
stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is
|
||||
only a warning. Both finish before anything is stored.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app import config
|
||||
from app.errors import ApiError, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.imagecheck")
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
|
||||
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
||||
FULL_DECODE_MAX_PIXELS = 4000000
|
||||
SCAN_SECONDS = 10.0
|
||||
OUT_STEP = 1 << 20
|
||||
# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit.
|
||||
DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null")
|
||||
jpeg_slot = threading.Lock()
|
||||
|
||||
|
||||
class Invalid(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def invalid() -> ApiError:
|
||||
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
|
||||
|
||||
|
||||
class RowPlan:
|
||||
def __init__(self, segments):
|
||||
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
|
||||
self.seg = 0
|
||||
self.done = 0
|
||||
self.pos = 0
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
i, n = 0, len(data)
|
||||
while i < n:
|
||||
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
||||
self.seg += 1
|
||||
self.done = 0
|
||||
if self.seg >= len(self.segments):
|
||||
raise Invalid("data after the last row")
|
||||
length, rows = self.segments[self.seg]
|
||||
if self.pos > 0:
|
||||
take = min(length - self.pos, n - i)
|
||||
i += take
|
||||
self.pos += take
|
||||
if self.pos == length:
|
||||
self.pos = 0
|
||||
self.done += 1
|
||||
continue
|
||||
full = min((n - i) // length, rows - self.done)
|
||||
if full > 0:
|
||||
if max(data[i:i + full * length:length]) > 4:
|
||||
raise Invalid("bad filter type")
|
||||
i += full * length
|
||||
self.done += full
|
||||
else:
|
||||
if data[i] > 4:
|
||||
raise Invalid("bad filter type")
|
||||
self.pos = n - i
|
||||
i = n
|
||||
|
||||
def complete(self) -> bool:
|
||||
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
||||
self.seg += 1
|
||||
self.done = 0
|
||||
return self.seg >= len(self.segments) and self.pos == 0
|
||||
|
||||
|
||||
def png_plan(width, height, depth, color, interlace):
|
||||
bits = CHANNELS[color] * depth
|
||||
if not interlace:
|
||||
return RowPlan([(1 + (width * bits + 7) // 8, height)])
|
||||
segments = []
|
||||
for x0, y0, dx, dy in ADAM7:
|
||||
w = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
h = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if w and h:
|
||||
segments.append((1 + (w * bits + 7) // 8, h))
|
||||
return RowPlan(segments)
|
||||
|
||||
|
||||
def scan_png(fileobj) -> None:
|
||||
deadline = time.monotonic() + SCAN_SECONDS
|
||||
fileobj.seek(0)
|
||||
if fileobj.read(8) != PNG_SIGNATURE:
|
||||
raise Invalid("signature")
|
||||
plan = decomp = None
|
||||
palette = seen_idat = idat_closed = False
|
||||
color = None
|
||||
first = True
|
||||
while True:
|
||||
if time.monotonic() > deadline:
|
||||
raise Invalid("scan too slow")
|
||||
head = fileobj.read(8)
|
||||
if len(head) < 8:
|
||||
raise Invalid("truncated before IEND")
|
||||
length, kind = struct.unpack(">I4s", head)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise Invalid("chunk length")
|
||||
if first and kind != b"IHDR":
|
||||
raise Invalid("IHDR must be first")
|
||||
crc = zlib.crc32(kind)
|
||||
if kind == b"IHDR":
|
||||
body = fileobj.read(length)
|
||||
if not first or length != 13 or len(body) != 13:
|
||||
raise Invalid("IHDR")
|
||||
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
|
||||
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
|
||||
raise Invalid("IHDR fields")
|
||||
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
|
||||
raise Invalid("color type / depth")
|
||||
plan = png_plan(width, height, depth, color, interlace)
|
||||
decomp = zlib.decompressobj()
|
||||
crc = zlib.crc32(body, crc)
|
||||
else:
|
||||
remaining = length
|
||||
while remaining:
|
||||
buf = fileobj.read(min(remaining, 65536))
|
||||
if not buf:
|
||||
raise Invalid("truncated chunk")
|
||||
remaining -= len(buf)
|
||||
crc = zlib.crc32(buf, crc)
|
||||
if kind == b"IDAT":
|
||||
if idat_closed or (color == 3 and not palette):
|
||||
raise Invalid("IDAT order")
|
||||
seen_idat = True
|
||||
out = decomp.decompress(buf, OUT_STEP)
|
||||
while True:
|
||||
plan.feed(out)
|
||||
if not decomp.unconsumed_tail:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise Invalid("scan too slow")
|
||||
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
|
||||
if kind == b"PLTE":
|
||||
palette = True
|
||||
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
||||
raise Invalid("PLTE")
|
||||
elif kind != b"IDAT" and seen_idat:
|
||||
idat_closed = True
|
||||
tail = fileobj.read(4)
|
||||
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
|
||||
raise Invalid("crc")
|
||||
first = False
|
||||
if kind == b"IEND":
|
||||
if length or not seen_idat:
|
||||
raise Invalid("IEND")
|
||||
break
|
||||
if not decomp.eof or decomp.unused_data or not plan.complete():
|
||||
raise Invalid("pixel data does not match IHDR")
|
||||
|
||||
|
||||
def check_jpeg(fileobj) -> None:
|
||||
fileobj.flush()
|
||||
descriptor = fileobj.fileno()
|
||||
os.lseek(descriptor, 0, os.SEEK_SET)
|
||||
try:
|
||||
with jpeg_slot:
|
||||
done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=SCAN_SECONDS, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("jpeg_check_timeout")
|
||||
raise unavailable("Проверка изображения не успела завершиться") from None
|
||||
# A signal means the checker died (e.g. OOM killer), which says nothing about the file.
|
||||
if done.returncode < 0:
|
||||
log.error("jpeg_check_killed signal=%d", -done.returncode)
|
||||
raise unavailable("Проверка изображения прервана")
|
||||
if done.returncode != 0:
|
||||
raise invalid()
|
||||
|
||||
|
||||
def inspect_image(fileobj, size: int, content_type: str) -> None:
|
||||
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
img = Image.open(fileobj)
|
||||
width, height = img.size
|
||||
if img.format != FORMATS[content_type]:
|
||||
raise invalid()
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
|
||||
if content_type == "image/png":
|
||||
scan_png(fileobj)
|
||||
if width * height <= FULL_DECODE_MAX_PIXELS:
|
||||
fileobj.seek(0)
|
||||
Image.open(fileobj).load()
|
||||
else:
|
||||
check_jpeg(fileobj)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise invalid() from None
|
||||
finally:
|
||||
fileobj.seek(0)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a
|
||||
wall-clock timeout, so a hostile or huge image can only kill this process, never the worker.
|
||||
|
||||
usage: python -m app.imageproc <source> <out_dir> <max_pixels>
|
||||
stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex.
|
||||
|
||||
Rules (contracts/README.md "Фоновая обработка"):
|
||||
thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px
|
||||
crop JPEG, centred square scaled to 128x128 (small sources are upscaled)
|
||||
optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4
|
||||
EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha).
|
||||
EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB).
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageFile, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
THUMB, CROP, OPTIMIZED = 256, 128, 1280
|
||||
JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85"))
|
||||
WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80"))
|
||||
ORIENTATION_TAG = 0x0112
|
||||
EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5
|
||||
|
||||
|
||||
def fit(width: int, height: int, bound: int):
|
||||
"""Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales."""
|
||||
if width <= bound and height <= bound:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return bound, max(1, height * bound // width)
|
||||
return max(1, width * bound // height), bound
|
||||
|
||||
|
||||
def has_alpha(img) -> bool:
|
||||
return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info)
|
||||
|
||||
|
||||
def to_8bit(img):
|
||||
if img.mode in ("I;16", "I;16B", "I;16L", "I"):
|
||||
return img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
|
||||
return img
|
||||
|
||||
|
||||
def on_white(img):
|
||||
if has_alpha(img):
|
||||
rgba = img.convert("RGBA")
|
||||
flat = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
flat.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return flat, rgba
|
||||
return img.convert("RGB"), None
|
||||
|
||||
|
||||
def process(source: str, out_dir: str, max_pixels: int) -> list:
|
||||
img = Image.open(source)
|
||||
if img.format not in ("PNG", "JPEG"):
|
||||
raise ValueError("format")
|
||||
width, height = img.size
|
||||
if width * height > max_pixels:
|
||||
sys.exit(EXIT_TOO_LARGE)
|
||||
orientation = img.getexif().get(ORIENTATION_TAG, 1)
|
||||
upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height)
|
||||
thumb_size = fit(*upright, THUMB)
|
||||
opt_size = fit(*upright, OPTIMIZED)
|
||||
if img.format == "JPEG":
|
||||
# DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept).
|
||||
need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0])
|
||||
img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1])))
|
||||
img.load()
|
||||
img = ImageOps.exif_transpose(img)
|
||||
img = to_8bit(img)
|
||||
flat, rgba = on_white(img)
|
||||
del img
|
||||
|
||||
results = []
|
||||
|
||||
def save(name, image, ext, fmt, content_type, **params):
|
||||
path = os.path.join(out_dir, name + "." + ext)
|
||||
image.save(path, fmt, **params)
|
||||
results.append({"name": name, "file": path, "content_type": content_type,
|
||||
"width": image.size[0], "height": image.size[1]})
|
||||
|
||||
jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False}
|
||||
save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0),
|
||||
"jpg", "JPEG", "image/jpeg", **jpeg)
|
||||
w, h = flat.size
|
||||
side = min(w, h)
|
||||
left, top = (w - side) // 2, (h - side) // 2
|
||||
save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)),
|
||||
"jpg", "JPEG", "image/jpeg", **jpeg)
|
||||
source_for_webp = rgba if rgba is not None else flat
|
||||
optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize(
|
||||
opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
||||
try:
|
||||
results = process(source, out_dir, max_pixels)
|
||||
except MemoryError:
|
||||
sys.exit(EXIT_TOO_COMPLEX)
|
||||
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
sys.exit(EXIT_INVALID)
|
||||
json.dump(results, sys.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker.
|
||||
|
||||
State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it
|
||||
(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes
|
||||
`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT.
|
||||
|
||||
enqueue nothing -> queued (API, same txn as attachment 'queued')
|
||||
claim queued(due) -> processing, lease (worker; also takes over an expired lease)
|
||||
heartbeat processing: lease_until += lease (only while owner is still alive)
|
||||
complete processing -> succeeded + variants (fenced by lease_owner)
|
||||
fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts)
|
||||
sweep_leases processing(expired lease) -> queued | failed
|
||||
replay failed -> queued (explicit operator action, audited)
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
from app import config, telemetry
|
||||
from app.cursor import fmt
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
SCHEMA_VERSION = 1
|
||||
VARIANT_ORDER = ("thumbnail", "crop", "optimized")
|
||||
REQUIRED_VARIANTS = VARIANT_ORDER
|
||||
|
||||
|
||||
def backoff_seconds(attempt: int) -> float:
|
||||
"""Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd)."""
|
||||
ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)))
|
||||
return ceiling / 2 + random.uniform(0, ceiling / 2)
|
||||
|
||||
|
||||
def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict:
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"job_id": str(job_id),
|
||||
"type": JOB_TYPE,
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": attempt,
|
||||
"created_at": fmt(created_at),
|
||||
}
|
||||
|
||||
|
||||
def parse_message(body: bytes) -> Optional[dict]:
|
||||
"""Strict check against jobs.schema.json; anything else goes to the DLQ untouched."""
|
||||
try:
|
||||
data = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
return None
|
||||
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}:
|
||||
return None
|
||||
if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE:
|
||||
return None
|
||||
if "traceparent" in data and not (isinstance(data["traceparent"], str)
|
||||
and telemetry.TRACEPARENT_RE.match(data["traceparent"])):
|
||||
return None
|
||||
if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512):
|
||||
return None
|
||||
for key in ("pipeline_version", "attempt"):
|
||||
if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1:
|
||||
return None
|
||||
try:
|
||||
data["job_id"] = str(uuid.UUID(data["job_id"]))
|
||||
data["attachment_id"] = str(uuid.UUID(data["attachment_id"]))
|
||||
except (ValueError, AttributeError, TypeError):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None):
|
||||
await conn.execute(
|
||||
"INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())",
|
||||
(job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})),
|
||||
)
|
||||
|
||||
|
||||
async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0):
|
||||
body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"])
|
||||
# W3C trace context of the transaction that schedules the job (upload request span or the failed consumer
|
||||
# span for a retry) travels in the job message (jobs.schema.json traceparent/tracestate).
|
||||
body.update(telemetry.inject())
|
||||
await conn.execute(
|
||||
"INSERT INTO outbox (job_id, payload, available_at, created_at)"
|
||||
" VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())",
|
||||
(job["id"], json.dumps(body), delay),
|
||||
)
|
||||
|
||||
|
||||
async def enqueue(conn, attachment_id: str, actor: str) -> dict:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts,"
|
||||
" next_attempt_at, created_at, updated_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())"
|
||||
" RETURNING id, attachment_id, pipeline_version, attempt, created_at",
|
||||
(attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS),
|
||||
)
|
||||
job = await cur.fetchone()
|
||||
await _outbox(conn, job, 1)
|
||||
await audit(conn, job["id"], attachment_id, "enqueued", actor, 1)
|
||||
return job
|
||||
|
||||
|
||||
async def pending_count(conn) -> int:
|
||||
cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')")
|
||||
return (await cur.fetchone())["n"]
|
||||
|
||||
|
||||
async def queue_stats(conn) -> dict:
|
||||
"""Backlog for metrics: queued / processing jobs, age of the oldest unfinished job, unpublished outbox rows."""
|
||||
cur = await conn.execute(
|
||||
"SELECT count(*) FILTER (WHERE state = 'queued') AS queued,"
|
||||
" count(*) FILTER (WHERE state = 'processing') AS processing,"
|
||||
" coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)::float8 AS oldest_age"
|
||||
" FROM attachment_jobs WHERE state IN ('queued', 'processing')"
|
||||
)
|
||||
row = dict(await cur.fetchone())
|
||||
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
|
||||
row["outbox"] = (await cur.fetchone())["n"]
|
||||
return row
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- worker side
|
||||
|
||||
JOB_COLUMNS = (
|
||||
"j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until,"
|
||||
" j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired,"
|
||||
" (j.next_attempt_at <= clock_timestamp()) AS due"
|
||||
)
|
||||
|
||||
|
||||
async def claim(conn, msg: dict, owner: str):
|
||||
"""Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version."""
|
||||
cur = await conn.execute(
|
||||
"SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status"
|
||||
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
|
||||
" WHERE j.id = %s FOR UPDATE OF j",
|
||||
(msg["job_id"],),
|
||||
)
|
||||
job = await cur.fetchone()
|
||||
if job is None or str(job["attachment_id"]) != msg["attachment_id"]:
|
||||
return "missing", None
|
||||
if job["state"] in ("succeeded", "failed"):
|
||||
return "done", job
|
||||
if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]:
|
||||
return "newer_version", job
|
||||
if msg["attempt"] != job["attempt"]:
|
||||
return "stale", job
|
||||
if job["state"] == "processing" and not job["lease_expired"]:
|
||||
return "busy", job
|
||||
if job["state"] == "queued" and not job["due"]:
|
||||
return "early", job
|
||||
takeover = job["state"] == "processing"
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'processing', lease_owner = %s,"
|
||||
" lease_until = clock_timestamp() + make_interval(secs => %s),"
|
||||
" started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp()),"
|
||||
" updated_at = clock_timestamp() WHERE id = %s",
|
||||
(owner, config.JOB_LEASE_SECONDS, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status IN ('queued', 'processing')",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"])
|
||||
return "claimed", job
|
||||
|
||||
|
||||
async def heartbeat(conn, job_id, owner: str) -> bool:
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
|
||||
(config.JOB_LEASE_SECONDS, job_id, owner),
|
||||
)
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
async def attachment_source(conn, attachment_id) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status"
|
||||
" FROM attachments WHERE id = %s",
|
||||
(attachment_id,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool:
|
||||
"""Atomically publish the result. False when the lease was lost: someone else owns the effect now."""
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL,"
|
||||
" lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
|
||||
(job["id"], owner),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return False
|
||||
for v in variants:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,"
|
||||
" size_bytes, width, height, sha256, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())"
|
||||
" ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
|
||||
(job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"],
|
||||
v["size_bytes"], v["width"], v["height"], v["sha256"]),
|
||||
)
|
||||
# Never move a newer pipeline_version back, never touch an attachment that is already ready.
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s",
|
||||
(job["pipeline_version"], job["attachment_id"], job["pipeline_version"]),
|
||||
)
|
||||
await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"],
|
||||
{"variants": [v["name"] for v in variants]})
|
||||
return True
|
||||
|
||||
|
||||
async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str:
|
||||
attempt = job["attempt"]
|
||||
if transient and attempt < job["max_attempts"]:
|
||||
delay = backoff_seconds(attempt)
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL,"
|
||||
" next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s,"
|
||||
" updated_at = clock_timestamp() WHERE id = %s",
|
||||
(delay, detail[:500], error_code, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status IN ('queued', 'processing')",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
await _outbox(conn, job, attempt + 1, delay=delay)
|
||||
await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt,
|
||||
{"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)})
|
||||
return "retry"
|
||||
final_code = error_code if not transient else "retry_exhausted"
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(),"
|
||||
" last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(detail[:500], final_code, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status IN ('queued', 'processing')",
|
||||
(final_code, job["attachment_id"]),
|
||||
)
|
||||
await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt,
|
||||
{"error_code": final_code, "cause": error_code, "detail": detail[:200]})
|
||||
return "failed"
|
||||
|
||||
|
||||
async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str:
|
||||
"""Returns retry | failed | lost (lease is not ours any more: the new owner decides)."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs"
|
||||
" WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE",
|
||||
(job["id"], owner),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "lost"
|
||||
return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient)
|
||||
|
||||
|
||||
async def sweep_leases(conn, actor: str) -> List[str]:
|
||||
"""Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner"
|
||||
" FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()"
|
||||
" ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED",
|
||||
)
|
||||
outcomes = []
|
||||
for row in await cur.fetchall():
|
||||
outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired",
|
||||
"lease of %s expired" % (row["lease_owner"] or "?"), True)
|
||||
outcomes.append("%s:%s" % (row["id"], outcome))
|
||||
return outcomes
|
||||
|
||||
|
||||
async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict:
|
||||
"""failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed."""
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at,"
|
||||
" a.status AS attachment_status, a.staging_key"
|
||||
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
|
||||
" WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a",
|
||||
(attachment_id,),
|
||||
)
|
||||
job = await cur.fetchone()
|
||||
if job is None:
|
||||
return {"result": "refused", "reason": "no job for attachment"}
|
||||
if job["state"] != "failed" or job["attachment_status"] != "failed":
|
||||
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"],
|
||||
{"state": job["state"], "attachment_status": job["attachment_status"]})
|
||||
return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])}
|
||||
if not job["staging_key"] or not await staging_exists(job["staging_key"]):
|
||||
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"})
|
||||
return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"}
|
||||
attempt = job["attempt"] + 1
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp(),"
|
||||
" finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status = 'failed'",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
await _outbox(conn, job, attempt)
|
||||
await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt)
|
||||
return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt}
|
||||
|
||||
|
||||
async def failed_jobs(conn, limit: int = 50) -> List[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at"
|
||||
" FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s",
|
||||
(limit,),
|
||||
)
|
||||
return [
|
||||
{"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"],
|
||||
"max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"],
|
||||
"failed_at": fmt(r["updated_at"])}
|
||||
for r in await cur.fetchall()
|
||||
]
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def older_than(seconds: int) -> datetime:
|
||||
return utcnow() - timedelta(seconds=seconds)
|
||||
+721
@@ -0,0 +1,721 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import signal
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import sys
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from app import config, faults, jobs, passwords, reconcile, repo, telemetry
|
||||
from app.cursor import BadCursor, decode_cursor, encode_cursor
|
||||
from app.db import Database
|
||||
from app.imagecheck import inspect_image
|
||||
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
|
||||
from app.broker import Publisher
|
||||
from app.outbox import Relay
|
||||
from app.s3 import S3Storage
|
||||
from app.sessions import SessionStore
|
||||
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
ZERO_WIDTH = str.maketrans({char: None for char in ""})
|
||||
EXEMPT_PATHS = ("/health/live", "/health/ready")
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||||
background = set()
|
||||
db = Database()
|
||||
sessions = SessionStore(db)
|
||||
s3 = S3Storage()
|
||||
publisher = Publisher()
|
||||
relay = Relay(db, publisher)
|
||||
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0,
|
||||
"queue": None}
|
||||
VARIANTS = ("original", "thumbnail", "crop", "optimized")
|
||||
|
||||
|
||||
def rss_mb() -> float:
|
||||
try:
|
||||
with open("/proc/self/statm") as handle:
|
||||
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
|
||||
except OSError:
|
||||
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
|
||||
|
||||
|
||||
def count_rejection(code: str) -> None:
|
||||
state["rejected"][code] = state["rejected"].get(code, 0) + 1
|
||||
|
||||
|
||||
async def sweeper() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
|
||||
try:
|
||||
removed = await sessions.sweep()
|
||||
if removed:
|
||||
log.info("sessions_expired removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("session_sweep_skipped: storage unavailable")
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
|
||||
if removed:
|
||||
log.info("idempotency_cleanup removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("idempotency_cleanup_skipped")
|
||||
try:
|
||||
removed = await relay.cleanup()
|
||||
if removed:
|
||||
log.info("outbox_cleanup removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("outbox_cleanup_skipped")
|
||||
|
||||
|
||||
async def pending_watch() -> None:
|
||||
"""Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second."""
|
||||
while True:
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
queue = await jobs.queue_stats(conn)
|
||||
state["pending_jobs"] = queue["queued"] + queue["processing"]
|
||||
state["queue"] = queue
|
||||
except Exception:
|
||||
state["queue"] = None # unknown is not zero: the gauge disappears instead of lying
|
||||
await asyncio.sleep(config.PENDING_CHECK_SECONDS)
|
||||
|
||||
|
||||
async def reconciler_loop() -> None:
|
||||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
|
||||
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
|
||||
log.info("reconcile %s", result)
|
||||
except Exception:
|
||||
log.warning("reconcile_skipped: dependency unavailable")
|
||||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def readiness_watch() -> None:
|
||||
failures = 0
|
||||
while True:
|
||||
if await db.ready():
|
||||
failures = 0
|
||||
if not state["ready"]:
|
||||
state["ready"] = True
|
||||
log.info("ready_again: database reachable")
|
||||
else:
|
||||
failures += 1
|
||||
if failures >= config.READY_FAILURES and state["ready"]:
|
||||
state["ready"] = False
|
||||
log.warning("not_ready: database unreachable, business requests get 503")
|
||||
s3_ok = await run_in_threadpool(s3.probe)
|
||||
if s3_ok != state["s3_ok"]:
|
||||
state["s3_ok"] = s3_ok
|
||||
log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
|
||||
await asyncio.sleep(config.READY_CHECK_SECONDS)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
previous = {}
|
||||
|
||||
def on_signal(signum, frame):
|
||||
if not state["draining"]:
|
||||
state["draining"] = True
|
||||
log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name)
|
||||
handler = previous.get(signum)
|
||||
if callable(handler):
|
||||
handler(signum, frame)
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
previous[sig] = signal.getsignal(sig)
|
||||
signal.signal(sig, on_signal)
|
||||
await db.open()
|
||||
telemetry.register_pool(db)
|
||||
telemetry.register_queue(state)
|
||||
await sessions.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
state["s3_ok"] = await run_in_threadpool(s3.probe)
|
||||
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()),
|
||||
asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())]
|
||||
log.info(
|
||||
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
|
||||
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
|
||||
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
|
||||
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
|
||||
)
|
||||
log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s",
|
||||
config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION)
|
||||
yield
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await publisher.close()
|
||||
await sessions.close()
|
||||
await db.close()
|
||||
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def guard(request: Request, call_next):
|
||||
rid = uuid.uuid4().hex
|
||||
request.state.rid = rid
|
||||
# X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request.
|
||||
telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID)
|
||||
exempt = request.url.path in EXEMPT_PATHS
|
||||
if state["draining"] and request.url.path != "/health/live":
|
||||
count_rejection("draining")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
|
||||
if not exempt and not state["ready"]:
|
||||
count_rejection("not_ready")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
|
||||
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
|
||||
count_rejection("inflight")
|
||||
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
|
||||
state["inflight"] += 1
|
||||
try:
|
||||
fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else ""
|
||||
if fault:
|
||||
await inject_http_fault(fault)
|
||||
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
count_rejection("request_timeout")
|
||||
log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path)
|
||||
response = error_body(
|
||||
rid, config.INSTANCE_ID, 503, "unavailable",
|
||||
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
|
||||
)
|
||||
except Exception as exc:
|
||||
telemetry.record_exception(exc)
|
||||
log.exception("unhandled_error request_id=%s", rid)
|
||||
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
finally:
|
||||
state["inflight"] -= 1
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
async def inject_http_fault(fault: str) -> None:
|
||||
"""Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error`
|
||||
raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose
|
||||
trace survival can be counted (span attribute messenger.fault)."""
|
||||
telemetry.set_attributes(messenger__fault=fault[:16])
|
||||
if fault == "slow":
|
||||
await asyncio.sleep(0.6)
|
||||
elif fault == "error":
|
||||
raise RuntimeError("injected handler failure")
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
if exc.status in (413, 429, 503):
|
||||
count_rejection(exc.code)
|
||||
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
def parse_uuid(value: object) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def non_blank(value: object, max_len: int) -> bool:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
|
||||
return False
|
||||
return value.translate(ZERO_WIDTH).strip() != ""
|
||||
|
||||
|
||||
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
|
||||
declared = request.headers.get("content-length")
|
||||
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
raw += chunk
|
||||
if len(raw) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
try:
|
||||
body = json.loads(bytes(raw).decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Некорректный JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
allowed = set(required) | set(optional)
|
||||
if set(body) - allowed:
|
||||
raise bad("Лишние поля в запросе")
|
||||
if set(required) - set(body):
|
||||
raise bad("Отсутствуют обязательные поля")
|
||||
return body
|
||||
|
||||
|
||||
def ensure_capacity() -> None:
|
||||
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
|
||||
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
|
||||
|
||||
|
||||
def bearer_token(request: Request) -> str:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
token = value.strip()
|
||||
if scheme.lower() != "bearer" or not token:
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
async def actor(request: Request) -> dict:
|
||||
user = await sessions.resolve(bearer_token(request))
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(request: Request) -> Tuple[str, str]:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized("Basic")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized("Basic") from None
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator:
|
||||
raise unauthorized("Basic")
|
||||
return username, password
|
||||
|
||||
|
||||
async def member_chat(user: dict, chat_id: str) -> dict:
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
chat = await repo.chat_for_member(conn, cid, user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
|
||||
def page_params(request: Request) -> Tuple[int, Optional[str]]:
|
||||
query = request.query_params
|
||||
limit = 50
|
||||
if "limit" in query:
|
||||
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
|
||||
raise bad("limit должен быть целым 1..100")
|
||||
limit = int(query["limit"])
|
||||
cursor = query.get("cursor")
|
||||
if cursor is not None and cursor == "":
|
||||
raise bad("Некорректный cursor")
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def keyset(scope: str, cursor: Optional[str]):
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
return decode_cursor(scope, cursor)
|
||||
except BadCursor:
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
|
||||
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
|
||||
return JSONResponse({"items": items, "next_cursor": next_cursor})
|
||||
|
||||
|
||||
def detect_content_type(head: bytes) -> Optional[str]:
|
||||
if head[:8] == PNG_MAGIC:
|
||||
return "image/png"
|
||||
if head[:3] == JPEG_MAGIC:
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
|
||||
raise bad("filename: 1..255 символов, не только пробелы")
|
||||
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
|
||||
raise bad("filename содержит недопустимые символы")
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
async def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready():
|
||||
if not await db.ready():
|
||||
raise unavailable("База данных недоступна")
|
||||
if not state["s3_ok"]:
|
||||
raise unavailable("Хранилище файлов недоступно")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
body = await read_json(request, ("username", "display_name", "password"))
|
||||
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
if not non_blank(body["display_name"], 100):
|
||||
raise bad("display_name: 1..100 символов, не только пробелы")
|
||||
password = body["password"]
|
||||
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
|
||||
ensure_capacity()
|
||||
password_hash = await run_in_threadpool(passwords.hash_password, password)
|
||||
async with db.connection() as conn:
|
||||
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
|
||||
if user is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
log.info("user_created id=%s", user["id"])
|
||||
return JSONResponse(user, status_code=201)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def login(request: Request):
|
||||
if await request.body():
|
||||
raise bad("POST /auth/sessions не принимает тело запроса")
|
||||
username, password = basic_credentials(request)
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
async with db.connection() as conn:
|
||||
row = await repo.credentials(conn, username)
|
||||
stored = row["password_hash"] if row else passwords.DUMMY_HASH
|
||||
matches = await run_in_threadpool(passwords.verify_password, stored, password)
|
||||
if row is None or not matches:
|
||||
log.info("login_failed username_known=%s", row is not None)
|
||||
raise unauthorized("Basic")
|
||||
user = repo.public_user(row)
|
||||
token, expires_at = await sessions.create(user)
|
||||
log.info("session_created user=%s", user["id"])
|
||||
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
async def logout(request: Request):
|
||||
token = bearer_token(request)
|
||||
user = await sessions.resolve(token)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
await sessions.revoke(token)
|
||||
log.info("session_revoked user=%s", user["id"])
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def me(request: Request):
|
||||
return await actor(request)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user = await actor(request)
|
||||
body = await read_json(request, ("title", "member_ids"))
|
||||
if not non_blank(body["title"], 100):
|
||||
raise bad("title: 1..100 символов, не только пробелы")
|
||||
raw_ids = body["member_ids"]
|
||||
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
|
||||
raise bad("member_ids: массив из 1..100 UUID")
|
||||
ids: List[str] = []
|
||||
for value in raw_ids:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("member_ids содержит некорректный UUID")
|
||||
ids.append(parsed)
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user["id"] not in ids:
|
||||
raise bad("Создатель должен входить в member_ids")
|
||||
ensure_capacity()
|
||||
|
||||
async def insert(conn):
|
||||
return await repo.create_chat(conn, body["title"], ids)
|
||||
|
||||
try:
|
||||
chat = await db.in_transaction(insert)
|
||||
except repo.UnknownMember:
|
||||
raise bad("member_ids содержит неизвестного пользователя") from None
|
||||
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
|
||||
return JSONResponse(chat, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(request: Request):
|
||||
user = await actor(request)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "chats:" + user["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(chat_id: str, request: Request):
|
||||
return await member_chat(await actor(request), chat_id)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def create_message(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
|
||||
idem_key = request.headers.get("idempotency-key", "")
|
||||
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
|
||||
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
body = await read_json(request, ("text",), ("attachment_ids",))
|
||||
|
||||
text = body["text"]
|
||||
attachment_ids_raw = body.get("attachment_ids", [])
|
||||
|
||||
if not isinstance(attachment_ids_raw, list):
|
||||
raise bad("attachment_ids должен быть массивом UUID")
|
||||
if len(attachment_ids_raw) > 10:
|
||||
raise bad("attachment_ids: максимум 10 вложений")
|
||||
|
||||
attachment_ids: List[str] = []
|
||||
for value in attachment_ids_raw:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("attachment_ids содержит некорректный UUID")
|
||||
attachment_ids.append(parsed)
|
||||
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise bad("attachment_ids содержит дубликаты")
|
||||
|
||||
has_text = isinstance(text, str) and non_blank(text, 4000)
|
||||
has_attachments = len(attachment_ids) > 0
|
||||
|
||||
if not has_text and not has_attachments:
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
|
||||
if isinstance(text, str) and len(text) > 4000:
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise bad("text должен быть строкой")
|
||||
|
||||
ensure_capacity()
|
||||
|
||||
normalized = {"text": text}
|
||||
if attachment_ids:
|
||||
normalized["attachment_ids"] = attachment_ids
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
async def do(conn):
|
||||
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
|
||||
|
||||
status, message = await db.in_transaction(do)
|
||||
|
||||
if status == "conflict":
|
||||
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
|
||||
if status == "not_member":
|
||||
raise not_found()
|
||||
if status == "attachment_not_found":
|
||||
raise not_found()
|
||||
if status == "attachment_failed":
|
||||
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
|
||||
return JSONResponse(message, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def list_messages(chat_id: str, request: Request):
|
||||
chat = await member_chat(await actor(request), chat_id)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "messages:" + chat["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
chat = await member_chat(user, chat_id)
|
||||
|
||||
filename = request.query_params.get("filename", "")
|
||||
validate_filename(filename)
|
||||
|
||||
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if declared_ct not in ALLOWED_CONTENT_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
|
||||
|
||||
declared_length = request.headers.get("content-length")
|
||||
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
|
||||
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
ensure_capacity()
|
||||
# Backpressure: do not accept more work than the pipeline can hold; the client retries later.
|
||||
if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS:
|
||||
count_rejection("queue_full")
|
||||
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
|
||||
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
|
||||
head = b""
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
if len(head) < 8:
|
||||
head += chunk[: 8 - len(head)]
|
||||
if len(head) == 8 and detect_content_type(head) is None:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||||
sha.update(chunk)
|
||||
spool.write(chunk)
|
||||
if size == 0:
|
||||
raise bad("Пустой файл")
|
||||
actual_ct = detect_content_type(head)
|
||||
if actual_ct is None:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||||
if actual_ct != declared_ct:
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
|
||||
async with upload_slots:
|
||||
await run_in_threadpool(inspect_image, spool, size, actual_ct)
|
||||
|
||||
attachment_id = str(uuid.uuid4())
|
||||
staging_key = "staging/" + attachment_id
|
||||
object_key = "attachments/%s/original" % attachment_id
|
||||
async with db.connection() as conn:
|
||||
await repo.begin_attachment(
|
||||
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(),
|
||||
staging_key, object_key,
|
||||
)
|
||||
try:
|
||||
async with upload_slots:
|
||||
await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct)
|
||||
except BaseException:
|
||||
defer(compensate(attachment_id, staging_key))
|
||||
raise
|
||||
|
||||
# Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox.
|
||||
try:
|
||||
att = await db.in_transaction(lambda conn: repo.queue_attachment(
|
||||
conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS))
|
||||
except BaseException:
|
||||
await compensate(attachment_id, staging_key)
|
||||
raise
|
||||
if att == repo.QUEUE_FULL:
|
||||
defer(compensate(attachment_id, staging_key))
|
||||
count_rejection("queue_full")
|
||||
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
|
||||
if att is None:
|
||||
raise unavailable("Загрузка отменена сверкой, повторите запрос")
|
||||
telemetry.set_attributes(messenger__attachment_id=att["id"])
|
||||
if faults.active(filename, "api_crash_after_commit"):
|
||||
faults.crash("API after COMMIT of job/outbox, before publish")
|
||||
relay.notify()
|
||||
log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size)
|
||||
return JSONResponse(att, status_code=202)
|
||||
|
||||
|
||||
def defer(coro) -> None:
|
||||
task = asyncio.ensure_future(coro)
|
||||
background.add(task)
|
||||
task.add_done_callback(background.discard)
|
||||
|
||||
|
||||
async def compensate(attachment_id: str, object_key: str) -> None:
|
||||
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
await repo.abort_attachment(conn, attachment_id)
|
||||
await run_in_threadpool(s3.delete, object_key)
|
||||
except BaseException:
|
||||
log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id)
|
||||
|
||||
|
||||
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
|
||||
user = await actor(request)
|
||||
aid = parse_uuid(attachment_id)
|
||||
if aid is None:
|
||||
raise bad("attachment_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
row = await repo.get_attachment(conn, aid)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def get_attachment(attachment_id: str, request: Request):
|
||||
row = await accessible_attachment(request, attachment_id)
|
||||
variants = []
|
||||
if row["status"] == "ready":
|
||||
async with db.connection() as conn:
|
||||
variants = await repo.get_variants(conn, row["id"], row["pipeline_version"])
|
||||
return repo._attachment(row, variants)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download_attachment(attachment_id: str, request: Request):
|
||||
row = await accessible_attachment(request, attachment_id)
|
||||
variant = request.query_params.get("variant", "original")
|
||||
if variant not in VARIANTS:
|
||||
raise bad("variant: original, thumbnail, crop или optimized")
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing")
|
||||
else "Обработка вложения завершилась ошибкой")
|
||||
key, content_type = row["object_key"], row["content_type"]
|
||||
if variant != "original":
|
||||
async with db.connection() as conn:
|
||||
found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant]
|
||||
if not found:
|
||||
raise not_found()
|
||||
key, content_type = found[0]["object_key"], found[0]["content_type"]
|
||||
opened = await run_in_threadpool(s3.open_stream, key)
|
||||
if opened is None:
|
||||
log.error("object_missing attachment=%s variant=%s", row["id"], variant)
|
||||
raise unavailable("Файл временно недоступен")
|
||||
chunks, length = opened
|
||||
return StreamingResponse(
|
||||
iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)},
|
||||
)
|
||||
|
||||
|
||||
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def fallback(request: Request):
|
||||
raise not_found()
|
||||
|
||||
|
||||
# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel
|
||||
# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics.
|
||||
telemetry.setup("messenger-api", app)
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work).
|
||||
|
||||
Rows are locked while they are published; `published_at` is set in the same transaction only after the
|
||||
broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are
|
||||
expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish
|
||||
leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import config, telemetry
|
||||
|
||||
log = logging.getLogger("messenger.outbox")
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, db, publisher) -> None:
|
||||
self.db = db
|
||||
self.publisher = publisher
|
||||
self.wake = asyncio.Event()
|
||||
self.broker_ok = False
|
||||
self.published = 0
|
||||
|
||||
def notify(self) -> None:
|
||||
self.wake.set()
|
||||
|
||||
async def run_once(self) -> int:
|
||||
sent, failure = 0, None
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
cur = await conn.execute(
|
||||
"SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()"
|
||||
" ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED",
|
||||
(config.OUTBOX_BATCH,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
try:
|
||||
await self.publish(row["payload"])
|
||||
except Exception as exc:
|
||||
# Keep what was already confirmed: commit those rows, record the error, stop the batch.
|
||||
await conn.execute(
|
||||
"UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s",
|
||||
(type(exc).__name__, row["id"]),
|
||||
)
|
||||
failure = exc
|
||||
break
|
||||
await conn.execute(
|
||||
"UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1"
|
||||
" WHERE id = %s",
|
||||
(row["id"],),
|
||||
)
|
||||
sent += 1
|
||||
self.published += sent
|
||||
if failure is not None:
|
||||
raise failure
|
||||
return sent
|
||||
|
||||
async def publish(self, payload: dict) -> None:
|
||||
"""PRODUCER span, child of the context stored with the job (upload request or failed attempt);
|
||||
the message then carries the producer's own traceparent, so the consumer becomes its child."""
|
||||
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "publish",
|
||||
"messaging.destination.name": config.BROKER_EXCHANGE, "messaging.message.id": payload["job_id"],
|
||||
"messenger.job.attempt": payload["attempt"], "messenger.attachment_id": payload["attachment_id"]}
|
||||
with telemetry.span(config.BROKER_QUEUE + " publish", kind=SpanKind.PRODUCER,
|
||||
context=telemetry.extract(payload), attributes=attrs):
|
||||
body = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")}
|
||||
body.update(telemetry.inject())
|
||||
await self.publisher.publish(body)
|
||||
|
||||
async def loop(self) -> None:
|
||||
delay = config.OUTBOX_POLL_SECONDS
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(self.wake.wait(), delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
self.wake.clear()
|
||||
try:
|
||||
while await self.run_once() == config.OUTBOX_BATCH:
|
||||
pass
|
||||
if not self.broker_ok:
|
||||
log.info("outbox_relay_resumed: broker reachable")
|
||||
self.broker_ok = True
|
||||
delay = config.OUTBOX_POLL_SECONDS
|
||||
except Exception as exc:
|
||||
if self.broker_ok:
|
||||
log.warning("outbox_relay_paused: %s; jobs stay in outbox", type(exc).__name__)
|
||||
self.broker_ok = False
|
||||
await self.publisher.close()
|
||||
delay = min(10.0, max(delay * 2, 1.0))
|
||||
|
||||
async def cleanup(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)",
|
||||
(config.OUTBOX_RETENTION_SECONDS,),
|
||||
)
|
||||
return cur.rowcount
|
||||
@@ -0,0 +1,16 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import Argon2Error, VerifyMismatchError
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return HASHER.hash(password)
|
||||
|
||||
|
||||
def verify_password(stored: str, password: str) -> bool:
|
||||
try:
|
||||
return HASHER.verify(stored, password)
|
||||
except (VerifyMismatchError, Argon2Error, ValueError):
|
||||
return False
|
||||
@@ -0,0 +1,155 @@
|
||||
"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout).
|
||||
|
||||
Key layout (all keys are generated by the server):
|
||||
staging/<attachment_id> written by the API before 202, read by the worker
|
||||
attachments/<attachment_id>/original server-side copy made by the worker
|
||||
attachments/<attachment_id>/v<N>/<variant>.* variants of pipeline version N
|
||||
Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row.
|
||||
|
||||
Cases (all gated by a grace period so in-flight work is never touched):
|
||||
stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both
|
||||
orphan_object object without any row/variant row, older than grace -> delete object
|
||||
staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete
|
||||
staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete
|
||||
(after that a replay is refused; the attachment stays failed and visible)
|
||||
missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing
|
||||
Queued/processing work is never touched: its staging object is the input of a live job.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from app import config
|
||||
|
||||
log = logging.getLogger("messenger.reconcile")
|
||||
|
||||
LOCK_ID = 2026_0104
|
||||
PREFIXES = ("staging/", "attachments/")
|
||||
BATCH = 500
|
||||
|
||||
|
||||
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
|
||||
started = time.monotonic()
|
||||
result = {
|
||||
"dry_run": dry_run, "grace_seconds": grace_seconds,
|
||||
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0,
|
||||
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
|
||||
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
|
||||
}
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
|
||||
if not (await cur.fetchone())["got"]:
|
||||
return {"skipped": "another reconciler is running"}
|
||||
try:
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=grace_seconds)
|
||||
await _stale_uploading(conn, s3, cutoff, dry_run, result)
|
||||
await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result)
|
||||
await _orphan_objects(conn, s3, cutoff, dry_run, result)
|
||||
await _missing_objects(conn, s3, cutoff, dry_run, result)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
result["seconds"] = round(time.monotonic() - started, 3)
|
||||
return result
|
||||
|
||||
|
||||
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, coalesce(staging_key, object_key) AS key FROM attachments"
|
||||
" WHERE status = 'uploading' AND created_at < %s",
|
||||
(cutoff,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
modified = await run_in_threadpool(s3.head, row["key"])
|
||||
if modified is not None and modified >= cutoff:
|
||||
result["protected_fresh_objects"] += 1
|
||||
continue
|
||||
result["stale_uploading"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
if modified is not None:
|
||||
await run_in_threadpool(s3.delete, row["key"])
|
||||
result["deleted_objects"] += 1
|
||||
done = await conn.execute(
|
||||
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
|
||||
)
|
||||
result["removed_rows"] += done.rowcount
|
||||
|
||||
|
||||
async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL"
|
||||
" AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
|
||||
(retention_cutoff,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
if await run_in_threadpool(s3.head, row["staging_key"]) is None:
|
||||
continue
|
||||
result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
again = await conn.execute(
|
||||
"SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
|
||||
(row["id"], retention_cutoff),
|
||||
)
|
||||
if await again.fetchone() is not None:
|
||||
await run_in_threadpool(s3.delete, row["staging_key"])
|
||||
result["deleted_objects"] += 1
|
||||
|
||||
|
||||
async def _known_keys(conn, keys):
|
||||
cur = await conn.execute(
|
||||
"SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)"
|
||||
" UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)"
|
||||
" UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)",
|
||||
(keys, keys, keys),
|
||||
)
|
||||
return {row["k"] for row in await cur.fetchall()}
|
||||
|
||||
|
||||
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
|
||||
for prefix in PREFIXES:
|
||||
pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix)))
|
||||
while True:
|
||||
page = await run_in_threadpool(next, pages, None)
|
||||
if page is None:
|
||||
break
|
||||
result["objects_scanned"] += len(page)
|
||||
keys = [o["Key"] for o in page]
|
||||
if not keys:
|
||||
continue
|
||||
known = await _known_keys(conn, keys)
|
||||
for obj in page:
|
||||
if obj["Key"] in known or obj["LastModified"] >= cutoff:
|
||||
continue
|
||||
result["orphan_objects"] += 1
|
||||
if not dry_run and not await _known_keys(conn, [obj["Key"]]):
|
||||
await run_in_threadpool(s3.delete, obj["Key"])
|
||||
result["deleted_objects"] += 1
|
||||
|
||||
|
||||
async def _missing_objects(conn, s3, cutoff, dry_run, result):
|
||||
last = None
|
||||
while True:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
|
||||
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
|
||||
(cutoff, last, last, BATCH),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
result["rows_scanned"] += 1
|
||||
if await run_in_threadpool(s3.exists, row["object_key"]):
|
||||
continue
|
||||
result["missing_objects"] += 1
|
||||
if not dry_run:
|
||||
done = await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
|
||||
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
|
||||
)
|
||||
result["marked_failed"] += done.rowcount
|
||||
last = str(rows[-1]["id"])
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
import hashlib
|
||||
import json
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import psycopg
|
||||
|
||||
from app import config, jobs
|
||||
from app.cursor import fmt
|
||||
|
||||
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
|
||||
CHAT_QUERY = (
|
||||
"SELECT c.id, c.title, c.created_at,"
|
||||
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
|
||||
)
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def decoded(value) -> str:
|
||||
return bytes(value).decode("utf-8")
|
||||
|
||||
|
||||
def public_user(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": decoded(row["display_name"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _chat(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": decoded(row["title"]),
|
||||
"member_ids": [str(value) for value in row["member_ids"]],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": decoded(row["text"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
|
||||
msg = _message(row)
|
||||
msg["attachment_ids"] = attachment_ids
|
||||
return msg
|
||||
|
||||
|
||||
VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2}
|
||||
|
||||
|
||||
def _variant(row) -> dict:
|
||||
return {
|
||||
"name": row["name"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"width": row["width"],
|
||||
"height": row["height"],
|
||||
"sha256": row["sha256"],
|
||||
}
|
||||
|
||||
|
||||
def _attachment(row, variants=None) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
"updated_at": fmt(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
|
||||
try:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
|
||||
" RETURNING id, username, display_name, created_at",
|
||||
(username, display_name.encode("utf-8"), password_hash),
|
||||
)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
return None
|
||||
return public_user(await cur.fetchone())
|
||||
|
||||
|
||||
async def credentials(conn, username: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
|
||||
" RETURNING id, title, created_at",
|
||||
(title.encode("utf-8"),),
|
||||
)
|
||||
chat = await cur.fetchone()
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
|
||||
(chat["id"], member_ids),
|
||||
)
|
||||
if cur.rowcount != len(member_ids):
|
||||
raise UnknownMember()
|
||||
chat["member_ids"] = list(member_ids)
|
||||
return _chat(chat)
|
||||
|
||||
|
||||
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
||||
(user_id, chat_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _chat(row) if row else None
|
||||
|
||||
|
||||
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY
|
||||
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
|
||||
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY c.created_at, c.id LIMIT %s",
|
||||
(user_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [_chat(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
|
||||
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _message(row) if row else None
|
||||
|
||||
|
||||
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
|
||||
cur = await conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
|
||||
(message_id,),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [str(row["attachment_id"]) for row in rows]
|
||||
|
||||
|
||||
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
|
||||
for pos, aid in enumerate(attachment_ids):
|
||||
await conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
(message_id, aid, pos),
|
||||
)
|
||||
|
||||
|
||||
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
|
||||
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY created_at, id LIMIT %s",
|
||||
(chat_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
messages = []
|
||||
for row in rows[:limit]:
|
||||
aids = await _fetch_message_attachment_ids(conn, row["id"])
|
||||
messages.append(_message_with_attachments(row, aids))
|
||||
return messages, len(rows) > limit
|
||||
|
||||
|
||||
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
|
||||
raw = "%s:%s:%s" % (user_id, chat_id, key)
|
||||
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
|
||||
|
||||
|
||||
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
|
||||
attachment_ids: List[str], idem_key: str, body_hash: str):
|
||||
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
)
|
||||
if await cur.fetchone() is None:
|
||||
return "not_member", None
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT request_body_hash, response_json FROM idempotency_keys"
|
||||
" WHERE user_id = %s AND chat_id = %s AND key = %s",
|
||||
(sender_id, chat_id, idem_key),
|
||||
)
|
||||
existing = await cur.fetchone()
|
||||
|
||||
if existing:
|
||||
if existing["request_body_hash"] != body_hash:
|
||||
return "conflict", None
|
||||
return "replay", json.loads(existing["response_json"])
|
||||
|
||||
if attachment_ids:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
|
||||
(attachment_ids,),
|
||||
)
|
||||
att_rows = await cur.fetchall()
|
||||
found_ids = {str(row["id"]) for row in att_rows}
|
||||
for aid in attachment_ids:
|
||||
if aid not in found_ids:
|
||||
return "attachment_not_found", None
|
||||
for row in att_rows:
|
||||
if str(row["chat_id"]) != chat_id:
|
||||
return "attachment_not_found", None
|
||||
if row["status"] == "failed":
|
||||
return "attachment_failed", None
|
||||
|
||||
text_bytes = text.encode("utf-8") if text else b""
|
||||
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text_bytes, chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "not_member", None
|
||||
|
||||
if attachment_ids:
|
||||
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
|
||||
|
||||
message = _message_with_attachments(row, list(attachment_ids))
|
||||
|
||||
await conn.execute(
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
|
||||
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
return "created", message
|
||||
|
||||
|
||||
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
|
||||
(ttl_seconds,),
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
|
||||
content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
||||
" staging_key, status, error_code, pipeline_version, created_at, updated_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key,
|
||||
config.PIPELINE_VERSION),
|
||||
)
|
||||
|
||||
|
||||
ATTACHMENT_COLUMNS = (
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status,"
|
||||
" error_code, pipeline_version, created_at, updated_at"
|
||||
)
|
||||
|
||||
|
||||
QUEUE_FULL = "queue_full"
|
||||
ADMISSION_LOCK = 2026_0105
|
||||
|
||||
|
||||
async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0):
|
||||
"""uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff).
|
||||
|
||||
With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock,
|
||||
so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight.
|
||||
"""
|
||||
if max_pending:
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,))
|
||||
if await jobs.pending_count(conn) >= max_pending:
|
||||
return QUEUE_FULL
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS,
|
||||
(attachment_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
await jobs.enqueue(conn, attachment_id, actor)
|
||||
return _attachment(row)
|
||||
|
||||
|
||||
async def abort_attachment(conn, attachment_id: str) -> None:
|
||||
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
async def get_attachment(conn, attachment_id: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'",
|
||||
(attachment_id,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants"
|
||||
" WHERE attachment_id = %s AND pipeline_version = %s",
|
||||
(attachment_id, pipeline_version),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99))
|
||||
@@ -0,0 +1,122 @@
|
||||
import logging
|
||||
from typing import Iterator, Optional, Tuple
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app import config
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.s3")
|
||||
|
||||
MISSING = ("NoSuchKey", "404", "NotFound")
|
||||
|
||||
|
||||
class S3Storage:
|
||||
def __init__(self) -> None:
|
||||
self.client = None
|
||||
self.probe_client = None
|
||||
|
||||
def open(self) -> None:
|
||||
self.probe_client = self._client(1.0, 1.5)
|
||||
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
|
||||
|
||||
def _client(self, connect: float, read: float):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
|
||||
config=BotoConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=connect,
|
||||
read_timeout=read,
|
||||
retries={"max_attempts": 0},
|
||||
),
|
||||
)
|
||||
|
||||
def probe(self) -> bool:
|
||||
try:
|
||||
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
self.client.put_object(
|
||||
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
|
||||
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return None
|
||||
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
except BotoCoreError as exc:
|
||||
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
body = response["Body"]
|
||||
return body.iter_chunks(65536), int(response["ContentLength"])
|
||||
|
||||
def head(self, key: str):
|
||||
"""LastModified of the object, or None when it does not exist."""
|
||||
try:
|
||||
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return None
|
||||
raise
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self.head(key) is not None
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
|
||||
def list_pages(self, prefix: str):
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
|
||||
yield page.get("Contents", [])
|
||||
|
||||
# ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient)
|
||||
|
||||
def put_file(self, key: str, path: str, content_type: str) -> None:
|
||||
with open(path, "rb") as handle:
|
||||
self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type)
|
||||
|
||||
def download(self, key: str, path: str) -> bool:
|
||||
"""False when the object does not exist."""
|
||||
try:
|
||||
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return False
|
||||
raise
|
||||
with open(path, "wb") as handle:
|
||||
for chunk in response["Body"].iter_chunks(1 << 20):
|
||||
handle.write(chunk)
|
||||
return True
|
||||
|
||||
def copy(self, source_key: str, target_key: str) -> None:
|
||||
"""Server-side copy: the original bytes never pass through the worker again."""
|
||||
self.client.copy_object(
|
||||
Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key},
|
||||
)
|
||||
|
||||
def ready(self) -> bool:
|
||||
try:
|
||||
self.client.head_bucket(Bucket=config.S3_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import valkey.asyncio as valkey
|
||||
|
||||
from app import config, repo
|
||||
from app.cursor import fmt
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, db) -> None:
|
||||
self.db = db
|
||||
self.client = None
|
||||
self.stats = {"hits": 0, "misses": 0, "errors": 0}
|
||||
|
||||
async def open(self) -> None:
|
||||
if not config.CACHE_URL:
|
||||
log.info("session cache disabled: CACHE_URL is empty")
|
||||
return
|
||||
self.client = valkey.Valkey.from_url(
|
||||
config.CACHE_URL,
|
||||
socket_timeout=config.CACHE_TIMEOUT,
|
||||
socket_connect_timeout=config.CACHE_TIMEOUT,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
self.client = None
|
||||
|
||||
@staticmethod
|
||||
def digest(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session:" + digest.hex()
|
||||
|
||||
async def create(self, user: dict) -> tuple:
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = self.digest(token)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
|
||||
" VALUES (%s, %s, clock_timestamp(), %s)",
|
||||
(digest, user["id"], expires_at),
|
||||
)
|
||||
await self._cache_put(digest, user, expires_at)
|
||||
return token, fmt(expires_at)
|
||||
|
||||
async def resolve(self, token: str) -> Optional[dict]:
|
||||
digest = self.digest(token)
|
||||
cached = await self._cache_get(digest)
|
||||
if cached == REVOKED:
|
||||
self.stats["hits"] += 1
|
||||
return None
|
||||
if cached is not None:
|
||||
self.stats["hits"] += 1
|
||||
return cached
|
||||
self.stats["misses"] += 1
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
|
||||
" FROM sessions s JOIN users u ON u.id = s.user_id"
|
||||
" WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(digest,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = repo.public_user(row)
|
||||
await self._cache_put(digest, user, row["expires_at"])
|
||||
return user
|
||||
|
||||
async def revoke(self, token: str) -> None:
|
||||
digest = self.digest(token)
|
||||
await self._cache_revoke(digest)
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
|
||||
except Exception:
|
||||
await self._cache_release(digest)
|
||||
raise
|
||||
|
||||
async def sweep(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
return cur.rowcount
|
||||
|
||||
def _cache_ttl(self, expires_at: datetime) -> int:
|
||||
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
|
||||
|
||||
async def _cache_get(self, digest: bytes) -> Optional[dict]:
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
raw = await self.client.get(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("get")
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
if raw == REVOKED:
|
||||
return REVOKED
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
ttl = self._cache_ttl(expires_at)
|
||||
if ttl <= 0:
|
||||
return
|
||||
payload = json.dumps(user)
|
||||
try:
|
||||
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
|
||||
except Exception:
|
||||
self._cache_failed("set")
|
||||
|
||||
async def _cache_release(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.delete(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("release")
|
||||
|
||||
async def _cache_revoke(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
|
||||
except Exception:
|
||||
self._cache_failed("revoke")
|
||||
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
|
||||
|
||||
def _cache_failed(self, operation: str) -> None:
|
||||
self.stats["errors"] += 1
|
||||
if self.stats["errors"] % 50 == 1:
|
||||
log.warning("session_cache_failed %s, falling back to postgres (errors=%d)", operation, self.stats["errors"])
|
||||
@@ -0,0 +1,393 @@
|
||||
"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector.
|
||||
|
||||
Everything is configured from the standard OTEL_* environment variables plus a few of our own:
|
||||
|
||||
OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline)
|
||||
OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision)
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs)
|
||||
OTEL_METRIC_EXPORT_INTERVAL metric push period, ms
|
||||
OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request
|
||||
|
||||
Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed
|
||||
URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job
|
||||
type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
|
||||
from opentelemetry import context as otel_context
|
||||
from opentelemetry import metrics, propagate, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
from app import config
|
||||
|
||||
ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
|
||||
SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0")
|
||||
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
QUEUE = config.BROKER_QUEUE
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
|
||||
# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md.
|
||||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30)
|
||||
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120)
|
||||
READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600)
|
||||
|
||||
tracer = trace.get_tracer("messenger")
|
||||
meter = metrics.get_meter("messenger")
|
||||
_state = {"service": "messenger", "providers": []}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- logging
|
||||
|
||||
class EventFilter(logging.Filter):
|
||||
"""Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word."""
|
||||
|
||||
EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)")
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not getattr(record, "event", None):
|
||||
match = self.EVENT_RE.match(str(record.msg))
|
||||
record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log"
|
||||
return True
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line on stdout, with trace_id/span_id of the active span (container logs)."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
out = {
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs,
|
||||
"level": record.levelname,
|
||||
"service": _state["service"],
|
||||
"event": getattr(record, "event", ""),
|
||||
"logger": record.name,
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
if ctx.is_valid:
|
||||
out["trace_id"] = format(ctx.trace_id, "032x")
|
||||
out["span_id"] = format(ctx.span_id, "016x")
|
||||
if record.exc_info:
|
||||
out["exc"] = self.formatException(record.exc_info)
|
||||
return json.dumps(out, ensure_ascii=False)
|
||||
|
||||
|
||||
def _setup_logging(logger_provider=None) -> None:
|
||||
root = logging.getLogger()
|
||||
for handler in list(root.handlers):
|
||||
root.removeHandler(handler)
|
||||
root.setLevel(logging.INFO)
|
||||
stdout = logging.StreamHandler(sys.stdout)
|
||||
stdout.setFormatter(JsonFormatter())
|
||||
stdout.addFilter(EventFilter())
|
||||
root.addHandler(stdout)
|
||||
# The exporter's own warnings (backend down) must not be fed back into the exporter.
|
||||
sdk_log = logging.getLogger("opentelemetry")
|
||||
sdk_log.propagate = False
|
||||
sdk_log.addHandler(stdout)
|
||||
for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"):
|
||||
logging.getLogger(noisy).setLevel(logging.WARNING)
|
||||
if logger_provider is not None:
|
||||
from opentelemetry.sdk._logs import LoggingHandler
|
||||
|
||||
otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otlp.addFilter(EventFilter())
|
||||
root.addHandler(otlp)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- sampling
|
||||
|
||||
def _build_sampler():
|
||||
"""ParentBased(root=KindFilter(TraceIdRatio)).
|
||||
|
||||
Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling,
|
||||
lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces.
|
||||
Children always follow the parent's decision, so an upload and its worker job are sampled together.
|
||||
"""
|
||||
from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult,
|
||||
TraceIdRatioBased)
|
||||
|
||||
ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
||||
decisions = meter.create_counter(
|
||||
"messenger.telemetry.sampler.decisions", unit="{span}",
|
||||
description="Head sampler decisions for spans (sample/drop) and the rule that decided")
|
||||
|
||||
class KindFilter(Sampler):
|
||||
def __init__(self, delegate):
|
||||
self.delegate = delegate
|
||||
|
||||
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
||||
trace_state=None):
|
||||
if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER):
|
||||
decisions.add(1, {"decision": "drop", "rule": "root_kind"})
|
||||
return SamplingResult(Decision.DROP)
|
||||
result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links,
|
||||
trace_state)
|
||||
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"})
|
||||
return result
|
||||
|
||||
def get_description(self):
|
||||
return "KindFilter{%s}" % self.delegate.get_description()
|
||||
|
||||
class CountingParentBased(ParentBased):
|
||||
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
||||
trace_state=None):
|
||||
result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
|
||||
parent = trace.get_current_span(parent_context).get_span_context()
|
||||
if parent.is_valid:
|
||||
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop",
|
||||
"rule": "parent"})
|
||||
return result
|
||||
|
||||
return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio)))
|
||||
|
||||
|
||||
class _CountingExporter:
|
||||
"""Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side)."""
|
||||
|
||||
def __init__(self, delegate):
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
|
||||
self.delegate = delegate
|
||||
self.ok = SpanExportResult.SUCCESS
|
||||
self.counter = meter.create_counter(
|
||||
"messenger.telemetry.spans.exported", unit="{span}",
|
||||
description="Spans handed to the OTLP exporter by outcome (success/failure)")
|
||||
|
||||
def export(self, spans):
|
||||
result = self.delegate.export(spans)
|
||||
self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"})
|
||||
return result
|
||||
|
||||
def shutdown(self):
|
||||
return self.delegate.shutdown()
|
||||
|
||||
def force_flush(self, timeout_millis: int = 30000):
|
||||
return self.delegate.force_flush(timeout_millis)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- setup
|
||||
|
||||
def setup(service_name: str, fastapi_app=None) -> None:
|
||||
_state["service"] = service_name
|
||||
if not ENABLED:
|
||||
_setup_logging(None)
|
||||
logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true")
|
||||
return
|
||||
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
# Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab).
|
||||
res = Resource.create({
|
||||
"service.name": service_name,
|
||||
"service.namespace": "messenger",
|
||||
"service.version": SERVICE_VERSION,
|
||||
"service.instance.id": config.INSTANCE_ID,
|
||||
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
||||
})
|
||||
|
||||
views = [
|
||||
# Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc.
|
||||
View(instrument_name="http.server.request.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
|
||||
attribute_keys={"http.request.method", "http.route", "http.response.status_code"}),
|
||||
View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}),
|
||||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.attachment.time_to_ready",
|
||||
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
|
||||
# No catch-all View("*"): an instrument matched by two views would be exported twice.
|
||||
# Instruments without a matching view get the SDK default aggregation.
|
||||
]
|
||||
meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
|
||||
views=views)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
tracer_provider = TracerProvider(resource=res, sampler=_build_sampler())
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter())))
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=res)
|
||||
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
_setup_logging(logger_provider)
|
||||
_state["providers"] = [tracer_provider, meter_provider, logger_provider]
|
||||
|
||||
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
||||
|
||||
# PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s
|
||||
# placeholders), so values, message bodies and hashes never reach db.query.text.
|
||||
BotocoreInstrumentor().instrument()
|
||||
if fastapi_app is not None:
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
|
||||
# Health probes (LB every second) are excluded from spans and from the HTTP SLI.
|
||||
FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready",
|
||||
exclude_spans=["receive", "send"])
|
||||
_process_metrics()
|
||||
logging.getLogger("messenger.telemetry").info(
|
||||
"telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name,
|
||||
os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
for provider in _state["providers"]:
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- context carrier
|
||||
|
||||
def inject() -> dict:
|
||||
"""W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}."""
|
||||
carrier: dict = {}
|
||||
if trace.get_current_span().get_span_context().is_valid:
|
||||
propagate.inject(carrier)
|
||||
return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")}
|
||||
|
||||
|
||||
def extract(carrier: Optional[dict]):
|
||||
"""Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context."""
|
||||
if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))):
|
||||
return otel_context.Context()
|
||||
clean = {"traceparent": str(carrier["traceparent"])}
|
||||
if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512:
|
||||
clean["tracestate"] = carrier["tracestate"]
|
||||
return propagate.extract(clean, context=otel_context.Context())
|
||||
|
||||
|
||||
def set_attributes(**attrs) -> None:
|
||||
span = trace.get_current_span()
|
||||
for key, value in attrs.items():
|
||||
if value is not None:
|
||||
span.set_attribute(key.replace("__", "."), value)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None):
|
||||
with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {},
|
||||
record_exception=True, set_status_on_exception=True) as current:
|
||||
yield current
|
||||
|
||||
|
||||
def record_exception(exc: BaseException) -> None:
|
||||
"""Exception event on the active span (type + stack, never request data) for errors handled by our code."""
|
||||
current = trace.get_current_span()
|
||||
current.record_exception(exc)
|
||||
current.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
|
||||
|
||||
def mark_error(current, description: str) -> None:
|
||||
current.set_status(Status(StatusCode.ERROR, description))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- instruments
|
||||
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||||
|
||||
|
||||
def _rss_bytes() -> int:
|
||||
try:
|
||||
with open("/proc/self/statm") as handle:
|
||||
return int(handle.read().split()[1]) * PAGE_SIZE
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _process_metrics() -> None:
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def cpu(_options):
|
||||
own = resource.getrusage(resource.RUSAGE_SELF)
|
||||
kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit)
|
||||
yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"})
|
||||
yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"})
|
||||
|
||||
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)")
|
||||
meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By",
|
||||
description="Resident set size of the process")
|
||||
|
||||
|
||||
def register_pool(db) -> None:
|
||||
"""DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats)."""
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def count(_options):
|
||||
stats = db.pool.get_stats()
|
||||
size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0)
|
||||
yield Observation(max(0, size - idle), {"db.client.connection.state": "used"})
|
||||
yield Observation(idle, {"db.client.connection.state": "idle"})
|
||||
|
||||
meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}")
|
||||
meter.create_observable_up_down_counter(
|
||||
"db.client.connection.pending_requests",
|
||||
[lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}")
|
||||
meter.create_observable_up_down_counter(
|
||||
"db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}")
|
||||
|
||||
|
||||
def register_queue(state: dict) -> None:
|
||||
"""Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second.
|
||||
|
||||
Reported by every API replica (identical values): dashboards and alerts use max() across replicas.
|
||||
"""
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def depth(_options):
|
||||
queue = state.get("queue")
|
||||
if queue is None:
|
||||
return []
|
||||
return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}),
|
||||
Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}),
|
||||
Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})]
|
||||
|
||||
def oldest(_options):
|
||||
queue = state.get("queue")
|
||||
return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})]
|
||||
|
||||
meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}",
|
||||
description="Jobs accepted but not finished, by state")
|
||||
meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s",
|
||||
description="Age of the oldest queued/processing job since durable acceptance")
|
||||
|
||||
|
||||
def register_worker(worker) -> None:
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]],
|
||||
unit="{job}", description="Jobs being processed right now")
|
||||
meter.create_observable_gauge("messenger.worker.capacity",
|
||||
[lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}",
|
||||
description="Concurrent job slots of this worker")
|
||||
|
||||
|
||||
jobs_processed = meter.create_counter(
|
||||
"messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome")
|
||||
job_duration = meter.create_histogram(
|
||||
"messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome")
|
||||
time_to_ready = meter.create_histogram(
|
||||
"messenger.attachment.time_to_ready", unit="s",
|
||||
description="From durable acceptance (job COMMIT, 202) to the ready COMMIT")
|
||||
+367
@@ -0,0 +1,367 @@
|
||||
"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
|
||||
|
||||
Per delivery:
|
||||
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
|
||||
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
|
||||
3. download staging object, verify size+sha256 against DB metadata
|
||||
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
|
||||
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
|
||||
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
|
||||
7. delete staging (best effort, reconciler finishes it), then ack
|
||||
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
|
||||
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import config, faults, jobs, telemetry
|
||||
from app.broker import connect, declare
|
||||
from app.db import Database
|
||||
from app.s3 import S3Storage
|
||||
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
def __init__(self, code: str, detail: str = ""):
|
||||
super().__init__(code)
|
||||
self.code, self.detail = code, detail or code
|
||||
|
||||
|
||||
class Transient(Exception):
|
||||
def __init__(self, code: str, detail: str = ""):
|
||||
super().__init__(code)
|
||||
self.code, self.detail = code, detail or code
|
||||
|
||||
|
||||
class LeaseLost(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def child_limits() -> None:
|
||||
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
cpu = config.JOB_TIMEOUT_SECONDS
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
||||
os.nice(5)
|
||||
|
||||
|
||||
def sha256_file(path: str):
|
||||
digest, size = hashlib.sha256(), 0
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
return digest.hexdigest(), size
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self) -> None:
|
||||
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
|
||||
self.db = Database()
|
||||
self.s3 = S3Storage()
|
||||
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
|
||||
self.stopping = asyncio.Event()
|
||||
self.inflight = 0
|
||||
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
|
||||
|
||||
# ------------------------------------------------------------------ lifecycle
|
||||
|
||||
async def run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, self.stopping.set)
|
||||
await self.db.open()
|
||||
telemetry.register_pool(self.db)
|
||||
telemetry.register_worker(self)
|
||||
await asyncio.to_thread(self.s3.open)
|
||||
connection = await connect()
|
||||
channel = await connection.channel()
|
||||
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
|
||||
_, queue = await declare(channel)
|
||||
tag = await queue.consume(self.on_message, no_ack=False)
|
||||
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
|
||||
log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
|
||||
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
|
||||
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
|
||||
await self.stopping.wait()
|
||||
log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
|
||||
try:
|
||||
await queue.cancel(tag)
|
||||
except Exception:
|
||||
pass
|
||||
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
|
||||
while self.inflight and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.2)
|
||||
for task in background:
|
||||
task.cancel()
|
||||
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
|
||||
await connection.close()
|
||||
await self.db.close()
|
||||
log.info("worker_stopped stats=%s", self.stats)
|
||||
|
||||
async def alive(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
|
||||
handle.write(str(time.time()))
|
||||
except OSError:
|
||||
pass
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def sweeper(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
|
||||
try:
|
||||
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
|
||||
if swept:
|
||||
log.warning("lease_expired recovered=%s", swept)
|
||||
except Exception:
|
||||
log.warning("lease_sweep_skipped: database unavailable")
|
||||
|
||||
# ------------------------------------------------------------------ delivery
|
||||
|
||||
async def on_message(self, message) -> None:
|
||||
async with self.slots:
|
||||
self.inflight += 1
|
||||
started = time.monotonic()
|
||||
# CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier).
|
||||
try:
|
||||
carrier = json.loads(message.body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
carrier = None
|
||||
if not isinstance(carrier, dict) or "traceparent" not in carrier:
|
||||
carrier = dict(message.headers or {})
|
||||
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
|
||||
"messaging.destination.name": config.BROKER_QUEUE,
|
||||
"messaging.message.id": str(message.message_id or ""),
|
||||
"messaging.rabbitmq.message.redelivered": bool(message.redelivered)}
|
||||
outcome = "requeued"
|
||||
with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER,
|
||||
context=telemetry.extract(carrier), attributes=attrs) as current:
|
||||
try:
|
||||
outcome = await self.handle(message)
|
||||
except Exception:
|
||||
log.exception("unexpected_error, message returned to queue")
|
||||
await asyncio.sleep(1)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
finally:
|
||||
self.inflight -= 1
|
||||
current.set_attribute("messenger.job.outcome", outcome)
|
||||
if outcome in ("retry", "failed", "dlq", "requeued", "lost"):
|
||||
telemetry.mark_error(current, outcome)
|
||||
labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome}
|
||||
telemetry.jobs_processed.add(1, labels)
|
||||
telemetry.job_duration.record(time.monotonic() - started, labels)
|
||||
|
||||
@staticmethod
|
||||
async def settle(awaitable) -> None:
|
||||
try:
|
||||
await awaitable
|
||||
except Exception:
|
||||
log.warning("could not settle delivery (connection lost); broker will redeliver")
|
||||
|
||||
async def handle(self, message) -> str:
|
||||
"""Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued."""
|
||||
msg = jobs.parse_message(message.body)
|
||||
if msg is None:
|
||||
self.stats["dlq"] += 1
|
||||
log.error("job_invalid_message -> DLQ message_id=%s", message.message_id)
|
||||
await self.settle(message.reject(requeue=False))
|
||||
return "dlq"
|
||||
telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"])
|
||||
try:
|
||||
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
|
||||
except Exception:
|
||||
log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"])
|
||||
await asyncio.sleep(2)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
return "requeued"
|
||||
if decision == "done" and job["state"] == "succeeded":
|
||||
await self.cleanup_staging(job["attachment_id"])
|
||||
if decision != "claimed":
|
||||
self.stats["noop"] += 1
|
||||
telemetry.set_attributes(messenger__job__decision=decision)
|
||||
log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
|
||||
message.redelivered)
|
||||
await self.settle(message.ack())
|
||||
return "noop"
|
||||
started = time.monotonic()
|
||||
log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
|
||||
job["max_attempts"])
|
||||
lease_lost = asyncio.Event()
|
||||
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
|
||||
outcome = None
|
||||
work = asyncio.create_task(self.process(job, message))
|
||||
lost = asyncio.create_task(lease_lost.wait())
|
||||
try:
|
||||
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
|
||||
return_when=asyncio.FIRST_COMPLETED)
|
||||
if work not in done:
|
||||
work.cancel()
|
||||
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
|
||||
outcome = "succeeded" if work.result() else "lost"
|
||||
except LeaseLost:
|
||||
outcome = "lost"
|
||||
except Permanent as exc:
|
||||
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
|
||||
except Transient as exc:
|
||||
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
|
||||
except Exception as exc: # S3/DB/network errors
|
||||
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
|
||||
finally:
|
||||
keeper.cancel()
|
||||
lost.cancel()
|
||||
if outcome is None:
|
||||
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
|
||||
await asyncio.sleep(2)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
return "requeued"
|
||||
self.stats[outcome] = self.stats.get(outcome, 0) + 1
|
||||
log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome,
|
||||
time.monotonic() - started)
|
||||
await self.settle(message.ack())
|
||||
return outcome
|
||||
|
||||
async def cleanup_staging(self, attachment_id) -> None:
|
||||
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
src = await jobs.attachment_source(conn, attachment_id)
|
||||
if src and src["status"] == "ready" and src["staging_key"]:
|
||||
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
||||
except Exception:
|
||||
log.warning("staging_cleanup_deferred attachment=%s", attachment_id)
|
||||
|
||||
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
|
||||
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
|
||||
except Exception:
|
||||
continue # DB blip: keep trying until the lease really expires
|
||||
if not alive:
|
||||
log.warning("lease_lost job=%s: another worker owns it now", job_id)
|
||||
lease_lost.set()
|
||||
return
|
||||
|
||||
async def record_failure(self, job, code: str, detail: str, transient: bool):
|
||||
log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code,
|
||||
transient)
|
||||
try:
|
||||
return await self.db.in_transaction(
|
||||
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
|
||||
except Exception:
|
||||
log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"])
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------ the actual work
|
||||
|
||||
async def process(self, job, message) -> bool:
|
||||
async with self.db.connection() as conn:
|
||||
src = await jobs.attachment_source(conn, job["attachment_id"])
|
||||
if src is None or not src["staging_key"]:
|
||||
raise Permanent("staging_missing", "attachment has no staging object")
|
||||
filename = src["filename"]
|
||||
first_delivery = job["attempt"] == 1 and not message.redelivered
|
||||
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
|
||||
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
|
||||
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
|
||||
if faults.active(filename, "permanent"):
|
||||
raise Permanent("invalid_image", "injected permanent failure")
|
||||
if job["attempt"] <= faults.transient_until(filename):
|
||||
raise Transient("dependency_unavailable", "injected transient failure")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
|
||||
source = os.path.join(tmp, "source")
|
||||
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
|
||||
raise Permanent("staging_missing", "staging object not found")
|
||||
digest, size = await asyncio.to_thread(sha256_file, source)
|
||||
if digest != src["sha256"] or size != src["size_bytes"]:
|
||||
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
|
||||
out_dir = os.path.join(tmp, "out")
|
||||
os.mkdir(out_dir)
|
||||
with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}):
|
||||
produced = await self.run_child(source, out_dir)
|
||||
|
||||
variants = []
|
||||
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
|
||||
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
|
||||
for item in produced:
|
||||
digest, size = await asyncio.to_thread(sha256_file, item["file"])
|
||||
key = prefix + os.path.basename(item["file"])
|
||||
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
|
||||
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
|
||||
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
|
||||
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
|
||||
raise Permanent("processing_failed", "not all variants were produced")
|
||||
|
||||
if first_delivery and faults.active(filename, "worker_crash_after_put"):
|
||||
faults.crash("after S3 put, before result COMMIT")
|
||||
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
|
||||
if not committed:
|
||||
log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"])
|
||||
return False
|
||||
telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
|
||||
{"job.type": telemetry.JOB_TYPE})
|
||||
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
|
||||
faults.crash("after result COMMIT, before ack")
|
||||
try:
|
||||
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
||||
except Exception:
|
||||
log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"])
|
||||
return True
|
||||
|
||||
async def run_child(self, source: str, out_dir: str) -> list:
|
||||
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
|
||||
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
|
||||
)
|
||||
try:
|
||||
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
|
||||
except asyncio.CancelledError:
|
||||
proc.kill()
|
||||
raise
|
||||
code = proc.returncode
|
||||
if code == 0:
|
||||
return json.loads(stdout)
|
||||
if code in PERMANENT_EXIT:
|
||||
raise Permanent(PERMANENT_EXIT[code])
|
||||
if code == -signal.SIGXCPU:
|
||||
raise Permanent("image_too_complex", "CPU limit exceeded")
|
||||
if code == -signal.SIGKILL:
|
||||
raise Transient("worker_resource_exhausted", "child killed (OOM)")
|
||||
raise Permanent("processing_failed", "child exit %s" % code)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
telemetry.setup("messenger-worker")
|
||||
try:
|
||||
asyncio.run(Worker().run())
|
||||
finally:
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+460
@@ -0,0 +1,460 @@
|
||||
name: mtusi-lab6
|
||||
|
||||
# OpenTelemetry SDK settings shared by the API replicas and the worker (track 1 overrides them from the shell).
|
||||
x-otel-env: &otel-env
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||||
OTEL_SEMCONV_STABILITY_OPT_IN: http
|
||||
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-10000}
|
||||
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
|
||||
OTEL_BSP_SCHEDULE_DELAY: "2000"
|
||||
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
|
||||
SERVICE_VERSION: "1.6.0"
|
||||
|
||||
x-api: &api-defaults
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab6
|
||||
env_file:
|
||||
- .env
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:1.27.3-alpine
|
||||
command: ["/bin/sh", "/opt/nginx-health.sh"]
|
||||
environment:
|
||||
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
|
||||
HEALTH_INTERVAL: "1"
|
||||
HEALTH_FALL: "2"
|
||||
HEALTH_RISE: "2"
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
|
||||
depends_on:
|
||||
api-1:
|
||||
condition: service_healthy
|
||||
api-2:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
api-1:
|
||||
<<: *api-defaults
|
||||
environment:
|
||||
<<: *otel-env
|
||||
INSTANCE_ID: api-1
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
|
||||
|
||||
api-2:
|
||||
<<: *api-defaults
|
||||
environment:
|
||||
<<: *otel-env
|
||||
INSTANCE_ID: api-2
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab6
|
||||
command: ["python", "-m", "app.worker"]
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
<<: *otel-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
CACHE_URL_FILE: ""
|
||||
INSTANCE_ID: ""
|
||||
DB_POOL_MAX: "6"
|
||||
# Overridable from the shell for the track-1 experiment (shell env wins over .env interpolation).
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${WORKER_CPUS:-2.00}"
|
||||
memory: 1536M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32
|
||||
hostname: rabbitmq
|
||||
entrypoint: ["/bin/sh", "/opt/entrypoint.sh"]
|
||||
secrets:
|
||||
- rabbitmq_password
|
||||
volumes:
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
|
||||
- ./infra/rabbitmq/entrypoint.sh:/opt/entrypoint.sh:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
postgres:
|
||||
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_INITDB_ARGS: --data-checksums
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- synchronous_commit=on
|
||||
- -c
|
||||
- shared_buffers=128MB
|
||||
- -c
|
||||
- work_mem=8MB
|
||||
- -c
|
||||
- max_connections=50
|
||||
- -c
|
||||
- track_io_timing=on
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command:
|
||||
- valkey-server
|
||||
- --save
|
||||
- ""
|
||||
- --appendonly
|
||||
- "no"
|
||||
- --maxmemory
|
||||
- 64mb
|
||||
- --maxmemory-policy
|
||||
- allkeys-lru
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
minio:
|
||||
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
|
||||
command: ["server", "/data"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "mc ready local || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
minio-init:
|
||||
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
|
||||
entrypoint: ["/bin/sh", "/opt/init.sh"]
|
||||
environment:
|
||||
S3_BUCKET: messenger-attachments
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infra/minio/init.sh:/opt/init.sh:ro
|
||||
- ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab6
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
restart: "no"
|
||||
|
||||
# ------------------------------------------------------------------ observability (lab6)
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0@sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917
|
||||
command:
|
||||
- --config=/etc/otelcol/collector.yaml
|
||||
- --config=/etc/otelcol/profile-${COLLECTOR_PROFILE:-default}.yaml
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
||||
- ./infra/otel/profile-default.yaml:/etc/otelcol/profile-default.yaml:ro
|
||||
- ./infra/otel/profile-tail.yaml:/etc/otelcol/profile-tail.yaml:ro
|
||||
# OTLP 4317/4318 and self-metrics 8888 stay inside the compose network.
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1@sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --web.enable-otlp-receiver
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- promdata:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090" # query API for evidence scripts (localhost only)
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.4.2@sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- lokidata:/loki
|
||||
ports:
|
||||
- "127.0.0.1:3100:3100"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.1@sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
ports:
|
||||
- "127.0.0.1:3200:3200"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 1024M
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2@sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json
|
||||
secrets:
|
||||
- grafana_admin_password
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
miniodata:
|
||||
rabbitmqdata:
|
||||
promdata:
|
||||
lokidata:
|
||||
tempodata:
|
||||
grafanadata:
|
||||
|
||||
secrets:
|
||||
minio_root_user:
|
||||
file: ./secrets/minio_root_user
|
||||
minio_root_password:
|
||||
file: ./secrets/minio_root_password
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
rabbitmq_password:
|
||||
file: ./secrets/rabbitmq_password
|
||||
grafana_admin_password:
|
||||
file: ./secrets/grafana_admin_password
|
||||
@@ -0,0 +1,244 @@
|
||||
# Панели dashboard «Messenger — наблюдаемость (lab6)»
|
||||
|
||||
Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. `$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).
|
||||
|
||||
## SLO (окно 30 мин) и алерты
|
||||
|
||||
**Доступность (SLO 99%)** (stat, единица: доля (0..1))
|
||||
— 1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.
|
||||
|
||||
```promql
|
||||
messenger:slo_availability:ratio30m
|
||||
```
|
||||
|
||||
**Латентность ≤500 ms (SLO 95%)** (stat, единица: доля (0..1))
|
||||
— Доля запросов в bucket le=0.5 s за 30 мин.
|
||||
|
||||
```promql
|
||||
messenger:slo_latency:ratio30m
|
||||
```
|
||||
|
||||
**Error ratio 5xx (2m)** (stat, единица: доля (0..1))
|
||||
|
||||
```promql
|
||||
messenger:http_error_ratio:rate2m
|
||||
```
|
||||
|
||||
**Старейшее задание** (stat, единица: секунды)
|
||||
|
||||
```promql
|
||||
max(messenger_queue_oldest_age_seconds{queue="attachments.process"})
|
||||
```
|
||||
|
||||
**Алерты Prometheus (firing/pending)** (alertlist)
|
||||
|
||||
Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).
|
||||
|
||||
|
||||
**RPS API** (stat, единица: запросы/с)
|
||||
|
||||
```promql
|
||||
messenger:http_requests:rate2m
|
||||
```
|
||||
|
||||
**p95 API (5m)** (stat, единица: секунды)
|
||||
|
||||
```promql
|
||||
messenger:http_latency_p95:rate5m
|
||||
```
|
||||
|
||||
**Загрузка worker** (stat, единица: доля (0..1))
|
||||
|
||||
```promql
|
||||
messenger:worker_saturation:ratio
|
||||
```
|
||||
|
||||
**Ожидают пул БД** (stat, единица: штуки)
|
||||
|
||||
```promql
|
||||
max(db_client_connection_pending_requests)
|
||||
```
|
||||
|
||||
## API (messenger-api, все реплики)
|
||||
|
||||
**RPS по маршруту и методу** (timeseries, единица: запросы/с)
|
||||
— Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID.
|
||||
|
||||
```promql
|
||||
sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # {{http_request_method}} {{http_route}}
|
||||
```
|
||||
|
||||
**RPS по классу статуса** (timeseries, единица: запросы/с)
|
||||
|
||||
```promql
|
||||
sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d)..")) # {{status_class}}
|
||||
```
|
||||
|
||||
**Латентность API p50/p95/p99 (из histogram buckets)** (timeseries, единица: секунды)
|
||||
— histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик.
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p50
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p95
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p99
|
||||
```
|
||||
|
||||
**Доля ошибок 5xx** (timeseries, единица: доля (0..1))
|
||||
— Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом.
|
||||
|
||||
```promql
|
||||
(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # 5xx / все
|
||||
```
|
||||
|
||||
**p95 по маршруту** (timeseries, единица: секунды)
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # {{http_route}}
|
||||
```
|
||||
|
||||
**Запросы в обработке** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum by (instance) (http_server_active_requests{service_name="messenger-api"}) # {{instance}}
|
||||
```
|
||||
|
||||
## Очередь и worker
|
||||
|
||||
**Jobs по исходу (в сек.)** (timeseries, единица: операции/с)
|
||||
— succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1
|
||||
|
||||
```promql
|
||||
sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval])) # {{outcome}}
|
||||
```
|
||||
|
||||
**Длительность job (succeeded) p50/p95/p99** (timeseries, единица: секунды)
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p50
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p95
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p99
|
||||
```
|
||||
|
||||
**Глубина очереди (из PostgreSQL)** (timeseries, единица: штуки)
|
||||
— max() по репликам API: каждая публикует одно и то же значение из БД.
|
||||
|
||||
```promql
|
||||
max by (queue, state) (messenger_queue_depth) # {{queue}} {{state}}
|
||||
```
|
||||
|
||||
**Возраст старейшей job** (timeseries, единица: секунды)
|
||||
— От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.
|
||||
|
||||
```promql
|
||||
max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) # oldest age
|
||||
```
|
||||
|
||||
**Время до ready p50/p95/p99** (timeseries, единица: секунды)
|
||||
— messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready.
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p50
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p95
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p99
|
||||
```
|
||||
|
||||
**Насыщение worker: занято / ёмкость** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum(messenger_worker_busy) # busy
|
||||
```
|
||||
```promql
|
||||
sum(messenger_worker_capacity) # capacity
|
||||
```
|
||||
```promql
|
||||
messenger:worker_saturation:ratio # saturation
|
||||
```
|
||||
|
||||
## Ресурсы: пул БД, CPU, RAM
|
||||
|
||||
**Пул БД: used / idle / waiting** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum by (service_name, db_client_connection_state) (db_client_connection_count) # {{service_name}} {{db_client_connection_state}}
|
||||
```
|
||||
```promql
|
||||
sum by (service_name) (db_client_connection_pending_requests) # {{service_name}} waiting
|
||||
```
|
||||
```promql
|
||||
sum by (service_name) (db_client_connection_max) # {{service_name}} max
|
||||
```
|
||||
|
||||
**CPU процессов (ядра)** (timeseries, единица: доля (0..1))
|
||||
— worker: self + завершённые дочерние процессы обработки изображений.
|
||||
|
||||
```promql
|
||||
sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval])) # {{service_name}} {{instance}}
|
||||
```
|
||||
```promql
|
||||
rate(otelcol_process_cpu_seconds_total[$__rate_interval]) # otel-collector
|
||||
```
|
||||
|
||||
**RSS процессов** (timeseries, единица: байты)
|
||||
|
||||
```promql
|
||||
process_memory_usage_bytes # {{service_name}} {{instance}}
|
||||
```
|
||||
```promql
|
||||
otelcol_process_memory_rss_bytes # otel-collector
|
||||
```
|
||||
|
||||
**Telemetry pipeline: принято / отправлено / ошибки (items/s)** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval])) # accepted spans
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval])) # accepted log records
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval])) # accepted metric points
|
||||
```
|
||||
```promql
|
||||
sum by (exporter) (rate({__name__=~"otelcol_exporter_sent_(spans|log_records|metric_points)_total"}[$__rate_interval])) # sent {{exporter}}
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0) # send failed spans
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0) # dropped spans (queue full)
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0) # send failed logs
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0) # dropped logs (queue full)
|
||||
```
|
||||
```promql
|
||||
sum by (exporter) (otelcol_exporter_queue_size) # queue {{exporter}}
|
||||
```
|
||||
|
||||
## Логи и трассы
|
||||
|
||||
**Логи API и worker (warning/error)** (logs)
|
||||
|
||||
```
|
||||
{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"
|
||||
```
|
||||
|
||||
**Трассы с ошибками или медленнее 500 ms (TraceQL)** (table)
|
||||
|
||||
```
|
||||
{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}
|
||||
```
|
||||
@@ -0,0 +1,90 @@
|
||||
# Runbooks алертов (lab6)
|
||||
|
||||
Алерты вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`) каждые 15 с. Состояние смотрится в Grafana
|
||||
(dashboard «Messenger — наблюдаемость», панель «Алерты Prometheus») или командой:
|
||||
|
||||
```sh
|
||||
curl -s http://127.0.0.1:9090/api/v1/alerts | python3 -m json.tool
|
||||
```
|
||||
|
||||
Доставка во внешние каналы не настроена (по заданию достаточно локального firing/resolved).
|
||||
|
||||
Общий порядок диагностики: **метрика → трасса → лог**. Метрика говорит «что и сколько», трасса — «где в
|
||||
цепочке тратится время / где ошибка», лог — «какое событие и с какими техническими ID».
|
||||
Все шаги ниже используют только сохранённую telemetry.
|
||||
|
||||
---
|
||||
|
||||
## MessengerApiHighErrorRate
|
||||
|
||||
**Условие:** `messenger:http_error_ratio:rate2m > 0.05` при `messenger:http_requests:rate2m > 0.2`, `for: 1m`.
|
||||
Числитель — ответы 5xx, знаменатель — все бизнес-запросы messenger-api (health-пробы не инструментированы).
|
||||
4xx (неверный пароль, чужой чат, 409) в числитель не входят.
|
||||
|
||||
1. Dashboard → «RPS по классу статуса» и «Доля ошибок 5xx»: какой код (500 или 503) и с какого момента.
|
||||
2. Prometheus: на каком маршруте ошибки
|
||||
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))`.
|
||||
3. Отличить БД от handler:
|
||||
- пул: `max(db_client_connection_pending_requests)`, `db_client_connection_count{db_client_connection_state="used"}`.
|
||||
Ожидание пула > 0 → соединения заняты долгими запросами к БД;
|
||||
- Tempo (Explore → Tempo → TraceQL):
|
||||
`{resource.service.name="messenger-api" && span.db.system="postgresql" && status=error}` — если в трассе
|
||||
db-span (`INSERT`/`SELECT`) занимает почти всё время запроса и завершился исключением
|
||||
`LockNotAvailable`/`QueryCanceled`/`OperationalError` — причина в PostgreSQL (блокировка, перегрузка, недоступность);
|
||||
- если server span длинный, а db-spans короткие (единицы ms) — медленный/падающий handler
|
||||
(`{span.messenger.fault="slow"}` в учебном стенде даёт именно такую картину).
|
||||
4. Loki: `{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"` — события `unhandled_error`
|
||||
(500, есть stack trace в атрибуте), `not_ready` (БД недоступна), `request_timeout`.
|
||||
По `X-Request-Id`, который получил клиент: `{service_name="messenger-api"} |= "<request_id>"` → поле `trace_id`
|
||||
→ кнопка «Open trace».
|
||||
5. Если виновата БД: `docker compose exec postgres psql -U messenger -d messenger -c "SELECT pid, state,
|
||||
wait_event_type, now()-xact_start AS age, left(query,80) FROM pg_stat_activity WHERE state <> 'idle'"` —
|
||||
найти блокирующую транзакцию; согласовать и выполнить `SELECT pg_terminate_backend(<pid>)`.
|
||||
6. Восстановление: доля 5xx падает сразу, alert уходит в resolved, когда окно 2m очистится (~2–3 мин).
|
||||
|
||||
## MessengerQueueLagGrowing
|
||||
|
||||
**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60`, `for: 1m`.
|
||||
Возраст считается от COMMIT задания (момент ответа 202) по данным PostgreSQL; публикуют все реплики API,
|
||||
поэтому `max()`.
|
||||
|
||||
1. Dashboard → «Глубина очереди», «Jobs по исходу», «Насыщение worker».
|
||||
- `sum(rate(messenger_jobs_processed_total[1m])) == 0` при растущей глубине → worker не работает;
|
||||
- `messenger:worker_saturation:ratio ≈ 1` и jobs/s > 0 → worker не успевает (нужно больше реплик/конкурентности);
|
||||
- много `outcome="retry"` → ошибки зависимостей (S3/DB), см. логи `job_attempt_failed`.
|
||||
2. Tempo: найти upload по attachment_id `{span.messenger.attachment_id="<id>"}`. Если в трассе есть
|
||||
PRODUCER `attachments.process publish`, но нет CONSUMER — сообщение в брокере, worker его не забрал.
|
||||
Если нет и PRODUCER — outbox не опубликован: смотреть `max(messenger_queue_depth{queue="outbox"})` и лог
|
||||
`outbox_relay_paused` (брокер недоступен).
|
||||
3. Loki: `{service_name="messenger-worker"}` — последние события `worker_stopping`/`worker_stopped` или отсутствие
|
||||
`job_*` событий; `{service_name="messenger-worker"} | event="job_attempt_failed"` — причины retry.
|
||||
4. Действие: `docker compose ps worker`, `docker compose start worker` (или масштабирование
|
||||
`docker compose up -d --scale worker=3`). Задания не теряются: они в PostgreSQL/outbox и в durable очереди.
|
||||
5. Восстановление: backlog уходит за секунды, возраст падает до 0, alert resolved через ~15–30 с.
|
||||
|
||||
## MessengerApiLatencyHigh
|
||||
|
||||
**Условие:** p95 из buckets всех реплик за 5m > 1 s, `for: 2m`. Шаги — как в HighErrorRate, пункт 3:
|
||||
по трассам `{resource.service.name="messenger-api" && duration > 1s}` определить, какой span занимает время
|
||||
(db / S3 / handler).
|
||||
|
||||
## MessengerDbPoolWaiting
|
||||
|
||||
**Условие:** `max(db_client_connection_pending_requests) > 0`, `for: 1m`. Пул исчерпан: долгие запросы или
|
||||
блокировки в БД (см. HighErrorRate, п. 3 и 5) либо пул мал для нагрузки (`DB_POOL_MAX`).
|
||||
|
||||
## MessengerWorkerSaturated
|
||||
|
||||
**Условие:** занято > 90% слотов всех worker 5 минут. Если очередь при этом растёт — масштабировать worker;
|
||||
проверить `messenger_job_duration_seconds` p95 (не стали ли задания медленнее).
|
||||
|
||||
## TelemetryExportFailing
|
||||
|
||||
**Условие:** очередь exporter заполнена больше чем на 20% либо есть send_failed/enqueue_failed > 0, `for: 1m`.
|
||||
Бизнес-API **не** затронут: SDK и collector имеют ограниченные очереди.
|
||||
|
||||
1. `docker compose ps tempo loki prometheus` — какой backend недоступен.
|
||||
2. Панель «Telemetry pipeline»: `otelcol_exporter_queue_size` по exporter (ёмкость 100 батчей) и
|
||||
`otelcol_exporter_enqueue_failed_*` — сколько данных уже отброшено.
|
||||
3. `docker compose start tempo loki`. Данные, отброшенные при переполнении очереди, не восстанавливаются —
|
||||
их объём виден в счётчиках и фиксируется в отчёте инцидента.
|
||||
@@ -0,0 +1,6 @@
|
||||
ALERTS{alertstate=firing} во время финального прогона инцидентов A/B/C (источник: Prometheus TSDB), UTC
|
||||
MessengerApiHighErrorRate page ['14:37:00..14:39:00']
|
||||
MessengerApiLatencyHigh ticket ['14:38:00..14:41:45']
|
||||
MessengerDbPoolWaiting ticket ['14:36:30..14:37:00']
|
||||
MessengerQueueLagGrowing page ['14:35:00..14:35:00']
|
||||
TelemetryExportFailing ticket ['14:41:00..14:42:45']
|
||||
@@ -0,0 +1,9 @@
|
||||
# 2026-11-09T14:55:39Z clean copy of git files, steps of .github/workflows/stand.yml (-p lab6clean to keep the main stand volumes)
|
||||
secrets ready in secrets/ (.env: present)
|
||||
compose config OK
|
||||
Container lab6clean-tempo-1 Healthy
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)
|
||||
grafana dashboards: ['Messenger — наблюдаемость (lab6)']
|
||||
prometheus rules: 13
|
||||
@@ -0,0 +1,92 @@
|
||||
ok 1 series Доступность (SLO 99%) Доступность (SLO 99%)
|
||||
ok 1 series Латентность ≤500 ms (SLO 95%) Латентность ≤500 ms (SLO 95%)
|
||||
ok 1 series Error ratio 5xx (2m) Error ratio 5xx (2m)
|
||||
ok 1 series Старейшее задание Старейшее задание
|
||||
ok 1 series RPS API RPS API
|
||||
ok 1 series p95 API (5m) p95 API (5m)
|
||||
ok 1 series Загрузка worker Загрузка worker
|
||||
ok 1 series Ожидают пул БД Ожидают пул БД
|
||||
ok 6 series RPS по маршруту и методу {{http_request_method}} {{http_route}}
|
||||
ok 2 series RPS по классу статуса {{status_class}}
|
||||
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p50
|
||||
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p95
|
||||
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p99
|
||||
ok 1 series Доля ошибок 5xx 5xx / все
|
||||
ok 5 series p95 по маршруту {{http_route}}
|
||||
ok 2 series Запросы в обработке {{instance}}
|
||||
ok 1 series Jobs по исходу (в сек.) {{outcome}}
|
||||
ok 1 series Длительность job (succeeded) p50/p95/p99 p50
|
||||
ok 1 series Длительность job (succeeded) p50/p95/p99 p95
|
||||
ok 1 series Длительность job (succeeded) p50/p95/p99 p99
|
||||
ok 3 series Глубина очереди (из PostgreSQL) {{queue}} {{state}}
|
||||
ok 1 series Возраст старейшей job oldest age
|
||||
ok 1 series Время до ready p50/p95/p99 p50
|
||||
ok 1 series Время до ready p50/p95/p99 p95
|
||||
ok 1 series Время до ready p50/p95/p99 p99
|
||||
ok 1 series Насыщение worker: занято / ёмкость busy
|
||||
ok 1 series Насыщение worker: занято / ёмкость capacity
|
||||
ok 1 series Насыщение worker: занято / ёмкость saturation
|
||||
ok 4 series Пул БД: used / idle / waiting {{service_name}} {{db_client_connection_state}}
|
||||
ok 2 series Пул БД: used / idle / waiting {{service_name}} waiting
|
||||
ok 2 series Пул БД: used / idle / waiting {{service_name}} max
|
||||
ok 4 series CPU процессов (ядра) {{service_name}} {{instance}}
|
||||
ok 1 series CPU процессов (ядра) otel-collector
|
||||
ok 4 series RSS процессов {{service_name}} {{instance}}
|
||||
ok 1 series RSS процессов otel-collector
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted spans
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted log records
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted metric points
|
||||
ok 3 series Telemetry pipeline: принято / отправлено / ошибк sent {{exporter}}
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed spans
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped spans (queue full)
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed logs
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped logs (queue full)
|
||||
ok 3 series Telemetry pipeline: принято / отправлено / ошибк queue {{exporter}}
|
||||
panels: 32, prometheus queries without data: 0
|
||||
grafana prometheus ok 3 values Доступность (SLO 99%)
|
||||
grafana prometheus ok 3 values Латентность ≤500 ms (SLO 95%)
|
||||
grafana prometheus ok 3 values Error ratio 5xx (2m)
|
||||
grafana prometheus ok 9 values Старейшее задание
|
||||
grafana prometheus ok 3 values RPS API
|
||||
grafana prometheus ok 3 values p95 API (5m)
|
||||
grafana prometheus ok 9 values Загрузка worker
|
||||
grafana prometheus ok 9 values Ожидают пул БД
|
||||
grafana prometheus ok 18 values RPS по маршруту и методу
|
||||
grafana prometheus ok 5 values RPS по классу статуса
|
||||
grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket
|
||||
grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket
|
||||
grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket
|
||||
grafana prometheus ok 3 values Доля ошибок 5xx
|
||||
grafana prometheus ok 15 values p95 по маршруту
|
||||
grafana prometheus ok 6 values Запросы в обработке
|
||||
grafana prometheus ok 3 values Jobs по исходу (в сек.)
|
||||
grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99
|
||||
grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99
|
||||
grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99
|
||||
grafana prometheus ok 27 values Глубина очереди (из PostgreSQL)
|
||||
grafana prometheus ok 9 values Возраст старейшей job
|
||||
grafana prometheus ok 3 values Время до ready p50/p95/p99
|
||||
grafana prometheus ok 3 values Время до ready p50/p95/p99
|
||||
grafana prometheus ok 3 values Время до ready p50/p95/p99
|
||||
grafana prometheus ok 9 values Насыщение worker: занято / ёмкость
|
||||
grafana prometheus ok 9 values Насыщение worker: занято / ёмкость
|
||||
grafana prometheus ok 9 values Насыщение worker: занято / ёмкость
|
||||
grafana prometheus ok 36 values Пул БД: used / idle / waiting
|
||||
grafana prometheus ok 18 values Пул БД: used / idle / waiting
|
||||
grafana prometheus ok 18 values Пул БД: used / idle / waiting
|
||||
grafana prometheus ok 34 values CPU процессов (ядра)
|
||||
grafana prometheus ok 8 values CPU процессов (ядра)
|
||||
grafana prometheus ok 36 values RSS процессов
|
||||
grafana prometheus ok 9 values RSS процессов
|
||||
grafana prometheus ok 2 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 8 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 8 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 18 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 27 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana loki ok 7 values Логи API и worker (warning/error)
|
||||
grafana tempo ok 17 traces Трассы с ошибками или медленнее 500 ms (TraceQL) (datasource proxy)
|
||||
via Grafana: queries without data or with errors: 0
|
||||
@@ -0,0 +1,20 @@
|
||||
17:41:17 tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=100 batches
|
||||
17:41:31 + 14 s queue=9/100 collector_rss=68.7MiB dropped=0 refused=0
|
||||
17:41:44 + 27 s queue=24/100 collector_rss=81.4MiB dropped=0 refused=0
|
||||
17:41:58 + 41 s queue=39/100 collector_rss=102MiB dropped=0 refused=0
|
||||
17:42:12 + 55 s queue=39/100 collector_rss=123.9MiB dropped=0 refused=0
|
||||
17:42:26 + 69 s queue=55/100 collector_rss=133.1MiB dropped=0 refused=0
|
||||
17:42:40 + 83 s queue=70/100 collector_rss=160.9MiB dropped=0 refused=0
|
||||
17:42:54 + 97 s queue=85/100 collector_rss=161.1MiB dropped=0 refused=0
|
||||
17:43:08 +111 s queue=100/100 collector_rss=164.1MiB dropped=1062 refused=0
|
||||
17:43:22 +125 s queue=100/100 collector_rss=173.4MiB dropped=15664 refused=0
|
||||
17:43:36 +139 s queue=99/100 collector_rss=177.4MiB dropped=16214 refused=4248
|
||||
17:43:50 +153 s queue=99/100 collector_rss=184MiB dropped=16214 refused=5310
|
||||
17:44:04 +167 s queue=99/100 collector_rss=185.5MiB dropped=16214 refused=6372
|
||||
17:44:18 +181 s queue=99/100 collector_rss=205.9MiB dropped=16214 refused=6372
|
||||
17:44:32 +195 s queue=100/100 collector_rss=204.4MiB dropped=24597 refused=12004
|
||||
17:44:46 +209 s queue=100/100 collector_rss=204.1MiB dropped=25147 refused=16252
|
||||
17:45:00 +223 s queue=100/100 collector_rss=203.9MiB dropped=25147 refused=16802
|
||||
17:45:01 result: peak queue=100/100, dropped by full queue=25147 spans, refused by memory_limiter=16802 spans, peak RSS=205.9 MiB
|
||||
17:45:01 API: requests=24686 ok=24686 unexpected_errors=0 p99=55.79 ms
|
||||
PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected
|
||||
@@ -0,0 +1,16 @@
|
||||
# 2026-11-09T13:40:58Z
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 1)
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 1.997}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
mtusi-lab6-api-1-1 Up 47 hours (healthy)
|
||||
$ docker compose start api-1 api-2
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 2)
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 0.903}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
mtusi-lab6-api-2-1 Up 41 seconds (healthy)
|
||||
$ docker compose start api-1 api-2
|
||||
FAILOVER_DONE
|
||||
@@ -0,0 +1,195 @@
|
||||
{
|
||||
"A_worker_stopped": {
|
||||
"alert_pending_s": 70.62756085395813,
|
||||
"alert_firing_s": 131,
|
||||
"age_at_firing_s": 126,
|
||||
"drain_after_restart_s": 3.6,
|
||||
"resolved_after_restart_s": 14,
|
||||
"lost_jobs": 0
|
||||
},
|
||||
"B_db_lock": {
|
||||
"alert_pending_s": 45.0726158618927,
|
||||
"alert_firing_s": 105,
|
||||
"error_ratio_at_firing": 0.707,
|
||||
"p95_at_firing_s": 6.92,
|
||||
"db_span_ms": 5001,
|
||||
"request_ms": 5004,
|
||||
"cause": "QueryCanceled",
|
||||
"resolved_after_unlock_s": 135
|
||||
},
|
||||
"C1_backends_down": {
|
||||
"outage_s": 192,
|
||||
"api_requests": 1703,
|
||||
"api_unexpected_errors": 0,
|
||||
"collector_samples": [
|
||||
{
|
||||
"t": 14,
|
||||
"collector_mem": "190.5MiB",
|
||||
"cpu": "0.05%",
|
||||
"queue_tempo": 6.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"collector_mem": "190.5MiB",
|
||||
"cpu": "0.07%",
|
||||
"queue_tempo": 6.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"collector_mem": "190.5MiB",
|
||||
"cpu": "0.04%",
|
||||
"queue_tempo": 14.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"collector_mem": "191.5MiB",
|
||||
"cpu": "0.58%",
|
||||
"queue_tempo": 21.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 62,
|
||||
"collector_mem": "190.8MiB",
|
||||
"cpu": "0.31%",
|
||||
"queue_tempo": 29.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 74,
|
||||
"collector_mem": "190.7MiB",
|
||||
"cpu": "0.02%",
|
||||
"queue_tempo": 36.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 86,
|
||||
"collector_mem": "190.3MiB",
|
||||
"cpu": "0.13%",
|
||||
"queue_tempo": 36.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 98,
|
||||
"collector_mem": "190.3MiB",
|
||||
"cpu": "0.01%",
|
||||
"queue_tempo": 44.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 110,
|
||||
"collector_mem": "191.7MiB",
|
||||
"cpu": "0.02%",
|
||||
"queue_tempo": 51.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 122,
|
||||
"collector_mem": "192.7MiB",
|
||||
"cpu": "0.37%",
|
||||
"queue_tempo": 57.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 134,
|
||||
"collector_mem": "191.9MiB",
|
||||
"cpu": "0.10%",
|
||||
"queue_tempo": 64.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 146,
|
||||
"collector_mem": "192.4MiB",
|
||||
"cpu": "0.38%",
|
||||
"queue_tempo": 64.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 158,
|
||||
"collector_mem": "192.4MiB",
|
||||
"cpu": "0.22%",
|
||||
"queue_tempo": 72.0,
|
||||
"queue_loki": 0.0
|
||||
},
|
||||
{
|
||||
"t": 170,
|
||||
"collector_mem": "192MiB",
|
||||
"cpu": "0.03%",
|
||||
"queue_tempo": 79.0,
|
||||
"queue_loki": 0.0
|
||||
}
|
||||
],
|
||||
"alert": "firing",
|
||||
"dropped_items": 0.0,
|
||||
"send_failed_items": 387.0,
|
||||
"traces_after_restart": 20,
|
||||
"alert_resolved_after_restart_s": 35
|
||||
},
|
||||
"C2_collector_down": {
|
||||
"outage_s": 60,
|
||||
"api1_mem_before": "108.6MiB",
|
||||
"api1_mem_during": [
|
||||
"108.8MiB",
|
||||
"109.3MiB",
|
||||
"109.3MiB",
|
||||
"109.1MiB",
|
||||
"109.3MiB",
|
||||
"119.3MiB"
|
||||
],
|
||||
"api_requests": 740,
|
||||
"api_unexpected_errors": 0
|
||||
},
|
||||
"traffic": {
|
||||
"profile": {
|
||||
"rate_rps": 10,
|
||||
"duration_s": 3600,
|
||||
"upload_rate_rps": 0,
|
||||
"slow_pct": 2,
|
||||
"error_pct": 0,
|
||||
"threads": 48,
|
||||
"mix": "75% POST message / 25% GET history(limit=20); uploads separate"
|
||||
},
|
||||
"started_at": 1794061933.7705631,
|
||||
"finished_at": 1794062670.181464,
|
||||
"generator_backlog_at_end": 0,
|
||||
"requests": 7365,
|
||||
"ok": 6335,
|
||||
"successful_rps": 1.76,
|
||||
"status_counts": {
|
||||
"200": 1593,
|
||||
"201": 4742,
|
||||
"503": 816,
|
||||
"502": 214
|
||||
},
|
||||
"server_errors_unexpected": 1030,
|
||||
"latency_ms_plain_ok": {
|
||||
"p50": 14.87,
|
||||
"p95": 22.77,
|
||||
"p99": 27.78,
|
||||
"max": 8280.75
|
||||
},
|
||||
"faults_sent": {
|
||||
"slow": 157,
|
||||
"error": 0
|
||||
},
|
||||
"by_kind": {
|
||||
"post_message": {
|
||||
"n": 5510,
|
||||
"p50_ms": 14.32,
|
||||
"p99_ms": 25.09
|
||||
},
|
||||
"get_history": {
|
||||
"n": 1855,
|
||||
"p50_ms": 17.84,
|
||||
"p99_ms": 43.91
|
||||
},
|
||||
"upload": {
|
||||
"n": 0,
|
||||
"p50_ms": null,
|
||||
"p99_ms": null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
17:32:13 background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow)
|
||||
17:32:43 === A. worker stopped (queue lag)
|
||||
17:32:44 worker stopped at 17:32:44
|
||||
17:32:44 3 uploads accepted (202) while worker is down: ['62859e73', '99403faf', '4ad2b534']
|
||||
17:33:54 MessengerQueueLagGrowing pending (+71 s)
|
||||
17:34:54 MessengerQueueLagGrowing firing (+131 s)
|
||||
17:34:54 diagnosis from telemetry:
|
||||
17:34:54 metrics: oldest job age=126 s, depth={'processing': 0.0, 'queued': 3.0}, worker jobs/s over 1m=0.000
|
||||
17:34:54 logs: worker job_* events after stop: 0; last worker events: ['worker_stopped', 'worker_stopped']
|
||||
17:34:54 trace of upload 62859e73: span kinds ['PRODUCER', 'SERVER'] -> published, no CONSUMER: the job waits in the queue
|
||||
17:34:57 worker started (+131 s)
|
||||
17:34:58 backlog drained: all 3 ready 3.6 s after worker start
|
||||
17:35:08 MessengerQueueLagGrowing resolved (+14 s)
|
||||
17:35:08 === B. DB lock on messages (slow/unavailable DB for writes and history reads)
|
||||
17:35:08 ACCESS EXCLUSIVE lock taken on table messages at 17:35:08
|
||||
17:35:53 MessengerApiHighErrorRate pending (+45 s)
|
||||
17:36:53 MessengerApiHighErrorRate firing (+105 s)
|
||||
17:36:53 diagnosis from telemetry:
|
||||
17:36:53 metrics: 5xx ratio=70.7%, p95=6.92 s, pool used(max replica)=10 waiting=19
|
||||
17:36:53 5xx/s /api/v1/chats 500 0.00
|
||||
17:36:53 5xx/s /api/v1/chats/{chat_id}/messages 503 9.59
|
||||
17:36:53 trace 23982edb8bd1246afc6bd6522faaa524 (failed request):
|
||||
api SERVER POST /api/v1/chats/{chat_id}/messages 5004.2 ms ERROR {"messenger.request_id": "0f42d95e28624ecb9e443dc88c491177", "messenger.instance_id": "api-1", "http.response.status_code": "503"}
|
||||
api CLIENT SELECT 0.1 ms UNSET {"db.system": "postgresql"}
|
||||
api CLIENT SELECT chat_members 0.1 ms UNSET {"db.system": "postgresql"}
|
||||
api CLIENT SELECT idempotency_keys 0.1 ms UNSET {"db.system": "postgresql"}
|
||||
api CLIENT INSERT messages 5001.0 ms ERROR {"db.system": "postgresql"}
|
||||
17:36:53 => DB span INSERT messages took 5001 ms of 5004 ms request, status ERROR, exception QueryCanceled: the time is spent in PostgreSQL
|
||||
17:36:53 contrast: slow HANDLER trace 47ee6988aff06d8a0f5592a97fb9a967: request 615 ms, longest DB span 1.1 ms, all DB 8.3 ms
|
||||
17:36:53 logs: 29 API warning/error records in 5 min, events=['not_ready', 'outbox_relay_paused']
|
||||
17:36:53 operator step (pg_stat_activity) finds the blocker: 4246|active|Timeout|BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT
|
||||
17:36:53 lock released (+106 s)
|
||||
17:39:09 MessengerApiHighErrorRate resolved (+135 s)
|
||||
17:39:09 === C1. Tempo and Loki stopped (telemetry backends down)
|
||||
17:39:22 + 14 s collector mem=190.5MiB cpu=0.05% queue tempo=6 loki=0 (capacity 100 batches)
|
||||
17:39:35 + 26 s collector mem=190.5MiB cpu=0.07% queue tempo=6 loki=0 (capacity 100 batches)
|
||||
17:39:47 + 38 s collector mem=190.5MiB cpu=0.04% queue tempo=14 loki=0 (capacity 100 batches)
|
||||
17:39:59 + 50 s collector mem=191.5MiB cpu=0.58% queue tempo=21 loki=0 (capacity 100 batches)
|
||||
17:40:10 + 62 s collector mem=190.8MiB cpu=0.31% queue tempo=29 loki=0 (capacity 100 batches)
|
||||
17:40:23 + 74 s collector mem=190.7MiB cpu=0.02% queue tempo=36 loki=0 (capacity 100 batches)
|
||||
17:40:35 + 86 s collector mem=190.3MiB cpu=0.13% queue tempo=36 loki=0 (capacity 100 batches)
|
||||
17:40:47 + 98 s collector mem=190.3MiB cpu=0.01% queue tempo=44 loki=0 (capacity 100 batches)
|
||||
17:40:59 +110 s collector mem=191.7MiB cpu=0.02% queue tempo=51 loki=0 (capacity 100 batches)
|
||||
17:41:10 +122 s collector mem=192.7MiB cpu=0.37% queue tempo=57 loki=0 (capacity 100 batches)
|
||||
17:41:23 +134 s collector mem=191.9MiB cpu=0.10% queue tempo=64 loki=0 (capacity 100 batches)
|
||||
17:41:35 +146 s collector mem=192.4MiB cpu=0.38% queue tempo=64 loki=0 (capacity 100 batches)
|
||||
17:41:47 +158 s collector mem=192.4MiB cpu=0.22% queue tempo=72 loki=0 (capacity 100 batches)
|
||||
17:41:59 +170 s collector mem=192MiB cpu=0.03% queue tempo=79 loki=0 (capacity 100 batches)
|
||||
17:41:59 alert TelemetryExportFailing during outage: firing
|
||||
17:41:59 API during the outage: 1703 requests, unexpected errors=0; alert TelemetryExportFailing=firing
|
||||
17:42:20 tempo/loki started and ready (+192 s)
|
||||
17:42:30 new traces after restart visible: 20 (export resumed)
|
||||
17:42:56 TelemetryExportFailing resolved (+35 s)
|
||||
17:43:16 telemetry lost and counted: enqueue_failed(queue full)=0 items, send_failed(retries exhausted)=387 items
|
||||
17:43:16 === C2. Collector stopped (SDK side buffering)
|
||||
17:44:30 api-1 RSS before=108.6MiB during=['108.8MiB', '109.3MiB', '109.3MiB', '109.1MiB', '109.3MiB', '119.3MiB']; 740 requests, unexpected errors=0
|
||||
17:44:30 summary written to /Users/alenchha/mtusi2026lab6/evidence/incidents.json
|
||||
PASS: incidents A,B,C diagnosed from telemetry, alerts fired and resolved
|
||||
@@ -0,0 +1,80 @@
|
||||
17:36:11 scenario A: scenario_a
|
||||
17:36:11 client saw: missing X-Request-Id
|
||||
17:36:11 committed attachment 8f23222e-3018-4692-a4ce-f67bfcf43871 survived the API crash
|
||||
17:36:12 statuses: queued -> ready
|
||||
17:36:12 audit attempt=1 enqueued api:api-2
|
||||
17:36:12 audit attempt=1 claimed 832caf57a286:a39a11c5
|
||||
17:36:12 audit attempt=1 succeeded 832caf57a286:a39a11c5
|
||||
17:36:13 scenario A PASS (2.4s)
|
||||
17:36:13 scenario B: scenario_b
|
||||
17:36:37 statuses: queued -> processing -> queued -> processing -> ready
|
||||
17:36:37 audit attempt=1 enqueued api:api-1
|
||||
17:36:37 audit attempt=1 claimed 18bdb5d574fa:8873b063
|
||||
17:36:37 audit attempt=1 retry_scheduled sweeper:18bdb5d574fa:4f98f68 lease_expired delay=1.97s
|
||||
17:36:37 audit attempt=2 claimed 832caf57a286:a39a11c5
|
||||
17:36:37 audit attempt=2 succeeded 832caf57a286:a39a11c5
|
||||
17:36:38 scenario B PASS (24.9s)
|
||||
17:36:38 scenario C: scenario_c
|
||||
17:36:39 statuses: queued -> ready
|
||||
17:36:39 audit attempt=1 enqueued api:api-2
|
||||
17:36:39 audit attempt=1 claimed 18bdb5d574fa:4f98f687
|
||||
17:36:39 audit attempt=1 succeeded 18bdb5d574fa:4f98f687
|
||||
17:36:49 scenario C PASS (10.4s)
|
||||
17:36:49 scenario D: scenario_d
|
||||
17:36:58 statuses: processing -> queued -> ready
|
||||
17:36:58 audit attempt=1 enqueued api:api-1
|
||||
17:36:58 audit attempt=1 claimed 832caf57a286:a39a11c5
|
||||
17:36:58 audit attempt=1 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=1.25s
|
||||
17:36:58 audit attempt=2 claimed 18bdb5d574fa:79c71044
|
||||
17:36:58 audit attempt=2 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=2.58s
|
||||
17:36:58 audit attempt=3 claimed 832caf57a286:a39a11c5
|
||||
17:36:58 audit attempt=3 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=4.3s
|
||||
17:36:58 audit attempt=4 claimed 18bdb5d574fa:79c71044
|
||||
17:36:58 audit attempt=4 succeeded 18bdb5d574fa:79c71044
|
||||
17:36:59 backoff delays: [1.25, 2.58, 4.3]
|
||||
17:36:59 scenario D PASS (10.4s)
|
||||
17:36:59 scenario E: scenario_e
|
||||
17:37:00 audit attempt=1 enqueued api:api-1
|
||||
17:37:00 audit attempt=1 claimed 832caf57a286:a39a11c5
|
||||
17:37:00 audit attempt=1 failed 832caf57a286:a39a11c5 invalid_image
|
||||
17:37:00 scenario E PASS (0.7s)
|
||||
17:37:00 scenario F: scenario_f
|
||||
17:37:24 failed registry contains the job; replay
|
||||
17:37:47 statuses: queued -> processing -> ready
|
||||
17:37:47 audit attempt=1 enqueued api:api-2
|
||||
17:37:47 audit attempt=1 claimed 18bdb5d574fa:79c71044
|
||||
17:37:47 audit attempt=1 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=1.99s
|
||||
17:37:47 audit attempt=2 claimed 832caf57a286:a39a11c5
|
||||
17:37:47 audit attempt=2 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=2.59s
|
||||
17:37:47 audit attempt=3 claimed 18bdb5d574fa:79c71044
|
||||
17:37:47 audit attempt=3 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=4.09s
|
||||
17:37:47 audit attempt=4 claimed 832caf57a286:a39a11c5
|
||||
17:37:47 audit attempt=4 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=11.17s
|
||||
17:37:47 audit attempt=5 claimed 18bdb5d574fa:79c71044
|
||||
17:37:47 audit attempt=5 failed 18bdb5d574fa:79c71044 retry_exhausted
|
||||
17:37:47 audit attempt=6 replayed operator:test-suite
|
||||
17:37:47 audit attempt=6 claimed 832caf57a286:a39a11c5
|
||||
17:37:47 audit attempt=6 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=19.41s
|
||||
17:37:47 audit attempt=7 claimed 18bdb5d574fa:79c71044
|
||||
17:37:47 audit attempt=7 succeeded 18bdb5d574fa:79c71044
|
||||
17:37:49 second replay refused: job is succeeded, attachment is ready
|
||||
17:37:50 scenario F PASS (50.4s)
|
||||
17:37:50 scenario G: scenario_g
|
||||
17:38:17 statuses: processing -> queued -> processing -> ready
|
||||
17:38:17 audit attempt=1 enqueued api:api-1
|
||||
17:38:17 audit attempt=1 claimed 832caf57a286:a39a11c5
|
||||
17:38:17 audit attempt=1 retry_scheduled sweeper:18bdb5d574fa:79c7104 lease_expired delay=1.77s
|
||||
17:38:17 audit attempt=2 claimed 18bdb5d574fa:79c71044
|
||||
17:38:17 audit attempt=2 succeeded 18bdb5d574fa:79c71044
|
||||
17:38:49 scenario G PASS (59.1s)
|
||||
17:38:49 scenario H: scenario_h
|
||||
17:38:53 DLQ depth 0 -> 1
|
||||
17:38:53 scenario H PASS (3.8s)
|
||||
17:38:53 scenario I: scenario_i
|
||||
17:38:58 broker down: upload 202, outbox holds 1 unpublished job
|
||||
17:39:03 statuses: queued -> ready
|
||||
17:39:03 audit attempt=1 enqueued api:api-2
|
||||
17:39:03 audit attempt=1 claimed 832caf57a286:a39a11c5
|
||||
17:39:03 audit attempt=1 succeeded 832caf57a286:a39a11c5
|
||||
17:39:04 scenario I PASS (11.4s)
|
||||
PASS: failure windows A B C D E F G H I
|
||||
@@ -0,0 +1,14 @@
|
||||
17:26:48 stop all workers
|
||||
17:26:49 upload 202 queued id=bc973dcb-254c-4a56-8746-5accb5e85267
|
||||
17:26:49 message e459cc31-bf5f-431c-9c1d-e473e95a59a1 created with queued attachment
|
||||
17:26:54 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0}
|
||||
17:26:54 restart api-1 api-2 rabbitmq (volumes kept)
|
||||
17:26:59 started before: mtusi-lab6-api-1-1 2026-11-09T14:18:47.635837802Z; mtusi-lab6-api-2-1 2026-11-09T14:18:47.722431011Z; mtusi-lab6-rabbitmq-1 2026-11-07T14:44:49.033246884Z
|
||||
17:26:59 started after: mtusi-lab6-api-1-1 2026-11-09T14:26:55.137236792Z; mtusi-lab6-api-2-1 2026-11-09T14:26:55.0483775Z; mtusi-lab6-rabbitmq-1 2026-11-09T14:26:55.741984625Z
|
||||
17:27:03 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0}
|
||||
17:27:03 start workers
|
||||
17:27:06 status sequence observed: queued -> ready
|
||||
17:27:07 objects: ['attachments/bc973dcb-254c-4a56-8746-5accb5e85267/original', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/crop.jpg', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/optimized.webp', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/thumbnail.jpg']
|
||||
17:27:07 redelivery: republish the finished job through the outbox (duplicate message)
|
||||
17:27:14 duplicate acked as no-op: variants=3 successes=1
|
||||
PASS: worker down / API+broker restart / redelivery no-op
|
||||
@@ -0,0 +1,27 @@
|
||||
09:16:45 message sent request_id=07f7f48bf1c74c01b3f546bd98c19492 (text marker kept private)
|
||||
09:16:47 upload 5791d45a-87cb-465a-b8af-47a02fcfe006: queued -> queued -> ready
|
||||
09:16:51 Tempo: trace ef3b4c403f734bcb04ee9b5ce085aa68, 42 spans; tree (DB client spans omitted):
|
||||
api SERVER POST /api/v1/chats/{chat_id}/attachments 10.2 ms UNSET {"http.response.status_code": "202", "messenger.request_id": "f33563a1565d46bea878a83207f42bef", "messenger.instance_id": "api-1", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"}
|
||||
api CLIENT S3.PutObject 3.9 ms UNSET {"rpc.method": "PutObject"}
|
||||
api PRODUCER attachments.process publish 1.5 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.job.attempt": "1", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"}
|
||||
worker CONSUMER attachments.process process 4.9 ms ERROR {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006", "messenger.job.attempt": "1", "messenger.job.outcome": "retry"}
|
||||
api PRODUCER attachments.process publish 4.6 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.job.attempt": "2", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"}
|
||||
worker CONSUMER attachments.process process 169.0 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006", "messenger.job.attempt": "2", "messenger.job.outcome": "succeeded"}
|
||||
worker CLIENT S3.GetObject 6.8 ms UNSET {"rpc.method": "GetObject"}
|
||||
worker INTERNAL imageproc 137.9 ms UNSET {"messenger.pipeline_version": "1"}
|
||||
worker CLIENT S3.CopyObject 3.8 ms UNSET {"rpc.method": "CopyObject"}
|
||||
worker CLIENT S3.PutObject 2.2 ms UNSET {"rpc.method": "PutObject"}
|
||||
worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"}
|
||||
worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"}
|
||||
worker CLIENT S3.DeleteObject 1.0 ms UNSET {"rpc.method": "DeleteObject"}
|
||||
09:16:51 PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links
|
||||
09:16:51 Loki: 6 records with trace_id=ef3b4c403f734bcb04ee9b5ce085aa68: [('api', 'attachment_queued'), ('worker', 'job_claimed'), ('worker', 'job_attempt_failed'), ('worker', 'job_finished'), ('worker', 'job_claimed'), ('worker', 'job_finished')]
|
||||
09:16:51 Loki record fields: ['code_filepath', 'code_function', 'code_lineno', 'deployment_environment_name', 'detected_level', 'event', 'flags', 'observed_timestamp', 'scope_name', 'service_instance_id', 'service_name', 'service_namespace', 'service_version', 'severity_number', 'severity_text', 'span_id', 'telemetry_sdk_language', 'telemetry_sdk_name', 'telemetry_sdk_version', 'trace_id', 'ts']
|
||||
09:16:55 HTTP 500 X-Request-Id=9302b0182c0243df8acc92865370b939 -> Loki event=unhandled_error trace_id=6a254a70c77d5419df95d6f8d2e681ce
|
||||
09:16:55 Tempo: GET /api/v1/chats status=ERROR request_id matches; exception event recorded=True
|
||||
09:16:55 message POST trace 26d27e3a639403ae1576707024358b6e: ['POST /api/v1/chats/{chat_id}/messages', 'SELECT', 'SELECT chat_members', 'SELECT idempotency_keys', 'INSERT messages', 'INSERT idempotency_keys']
|
||||
09:16:55 PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from traces and logs
|
||||
09:16:55 PASS: 994 series, label names: ['__name__', 'db_client_connection_state', 'decision', 'deployment_environment_name', 'error_type', 'http_request_method', 'http_response_status_code', 'http_route', 'instance', 'job', 'job_type', 'le', 'network_protocol_version', 'outcome', 'process_cpu_owner', 'queue', 'rule', 'service_instance_id', 'service_name', 'service_version', 'state', 'url_scheme']
|
||||
09:16:55 http_route values (templates only): ['/api/v1/attachments/{attachment_id}', '/api/v1/auth/sessions', '/api/v1/chats', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users']
|
||||
09:17:07 PASS: required metrics present: http_server_request_duration_seconds_bucket, messenger_jobs_processed_total, messenger_job_duration_seconds_bucket, messenger_attachment_time_to_ready_seconds_bucket, messenger_queue_depth, messenger_queue_oldest_age_seconds, messenger_worker_busy, messenger_worker_capacity, db_client_connection_count, db_client_connection_pending_requests, process_cpu_time_seconds_total, process_memory_usage_bytes, otelcol_receiver_accepted_spans_total, otelcol_exporter_sent_spans_total, otelcol_exporter_queue_size
|
||||
PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)
|
||||
@@ -0,0 +1,32 @@
|
||||
# 2026-11-11T06:17:14Z scripts/restart.sh restarts business and telemetry stores, volumes kept
|
||||
$ docker compose ps (before)
|
||||
mtusi-lab6-api-1-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-api-2-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-grafana-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-loki-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-minio-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-otel-collector-1 Up 3 minutes
|
||||
mtusi-lab6-postgres-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-prometheus-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-rabbitmq-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-tempo-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-valkey-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-worker-1 Up 3 minutes (healthy)
|
||||
mtusi-lab6-worker-2 Up 3 minutes (healthy)
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ docker compose ps (after)
|
||||
mtusi-lab6-api-1-1 Up 10 seconds (healthy)
|
||||
mtusi-lab6-api-2-1 Up 10 seconds (healthy)
|
||||
mtusi-lab6-grafana-1 Up 21 seconds (healthy)
|
||||
mtusi-lab6-loki-1 Up 20 seconds (healthy)
|
||||
mtusi-lab6-minio-1 Up 23 seconds (healthy)
|
||||
mtusi-lab6-otel-collector-1 Up 21 seconds
|
||||
mtusi-lab6-postgres-1 Up 22 seconds (healthy)
|
||||
mtusi-lab6-prometheus-1 Up 21 seconds (healthy)
|
||||
mtusi-lab6-rabbitmq-1 Up 21 seconds (healthy)
|
||||
mtusi-lab6-tempo-1 Up 11 seconds (health: starting)
|
||||
mtusi-lab6-valkey-1 Up 22 seconds (healthy)
|
||||
mtusi-lab6-worker-1 Up 11 seconds (healthy)
|
||||
mtusi-lab6-worker-2 Up 11 seconds (healthy)
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-11-09T14:26:13Z
|
||||
$ make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
$ make validate
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
$ make test
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,16 @@
|
||||
# 2026-11-09T14:29:49Z restart of telemetry stores, volumes kept
|
||||
mtusi-lab6-grafana-1 Up 2 days (healthy)
|
||||
mtusi-lab6-loki-1 Up 2 days (healthy)
|
||||
mtusi-lab6-prometheus-1 Up 2 days (healthy)
|
||||
mtusi-lab6-tempo-1 Up 2 days (healthy)
|
||||
Container mtusi-lab6-prometheus-1 Started
|
||||
Container mtusi-lab6-grafana-1 Started
|
||||
Container mtusi-lab6-tempo-1 Started
|
||||
Container mtusi-lab6-loki-1 Started
|
||||
mtusi-lab6-grafana-1 Up 27 seconds (healthy)
|
||||
mtusi-lab6-loki-1 Up 25 seconds (healthy)
|
||||
mtusi-lab6-prometheus-1 Up 27 seconds (healthy)
|
||||
mtusi-lab6-tempo-1 Up 26 seconds (healthy)
|
||||
Tempo: trace of 2026-11-07 incident B 23982edb8bd1246afc6bd6522faaa524 spans: 5
|
||||
Loki: API warn/error records for that window: 48 streams
|
||||
Prometheus: firing samples of MessengerApiHighErrorRate in that window: 9
|
||||
@@ -0,0 +1,12 @@
|
||||
{"config": "off", "rep": 1, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794231942.6489458, "finished_at": 1794232032.6500669, "generator_backlog_at_end": 1, "requests": 5490, "ok": 5428, "successful_rps": 60.31, "status_counts": {"201": 4003, "202": 90, "200": 1335, "500": 62}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.43, "p95": 10.28, "p99": 31.67, "max": 77.52}, "faults_sent": {"slow": 56, "error": 62}, "by_kind": {"post_message": {"n": 4003, "p50_ms": 6.1, "p99_ms": 27.37}, "get_history": {"n": 1397, "p50_ms": 7.63, "p99_ms": 32.03}, "upload": {"n": 90, "p50_ms": 19.99, "p99_ms": 166.23}}}, "cpu_pct_avg": {"api-2": 9.01, "api-1": 9.09, "worker": 15.35, "otel-collector": 0.07, "tempo": 0.43, "nginx": 1.29, "grafana": 0.98, "postgres": 5.1, "rabbitmq": 6.69, "valkey": 0.55, "minio": 1.63, "loki": 0.57, "prometheus": 0.36}, "mem_mib_max": {"api-2": 102.9, "api-1": 93.6, "worker": 134.2, "otel-collector": 35.7, "tempo": 291.9, "nginx": 5.2, "grafana": 135.7, "postgres": 58.6, "rabbitmq": 153.1, "valkey": 12.4, "minio": 306.3, "loki": 82.9, "prometheus": 273.4}, "telemetry_delta": {"sdk_span_decisions_sample": -38.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -38.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 26288.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794231942.6489012, 1794232032.66341], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "on100", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232111.334157, "finished_at": 1794232201.333892, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5440, "successful_rps": 60.44, "status_counts": {"201": 4029, "202": 90, "200": 1321, "500": 50}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.63, "p95": 16.28, "p99": 54.35, "max": 484.0}, "faults_sent": {"slow": 60, "error": 50}, "by_kind": {"post_message": {"n": 4029, "p50_ms": 8.21, "p99_ms": 54.1}, "get_history": {"n": 1371, "p50_ms": 10.46, "p99_ms": 54.35}, "upload": {"n": 90, "p50_ms": 21.71, "p99_ms": 61.23}}}, "cpu_pct_avg": {"api-2": 11.5, "api-1": 11.33, "worker": 15.68, "otel-collector": 0.6, "tempo": 1.08, "nginx": 1.26, "grafana": 0.91, "postgres": 4.65, "rabbitmq": 6.94, "valkey": 0.49, "minio": 1.57, "loki": 0.55, "prometheus": 0.36}, "mem_mib_max": {"api-2": 111.2, "api-1": 115.5, "worker": 144.0, "otel-collector": 71.3, "tempo": 420.8, "nginx": 13.8, "grafana": 136.0, "postgres": 79.0, "rabbitmq": 141.6, "valkey": 12.0, "minio": 299.2, "loki": 92.9, "prometheus": 276.3}, "telemetry_delta": {"sdk_span_decisions_sample": 56361.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 56361.0, "sdk_spans_exported_failed": null, "col_spans_in": 57045.0, "col_logs_in": 340.0, "col_points_in": 1224.0, "col_spans_out": 57045.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21529255.0, "tempo_spans_in": 57045.0, "tempo_bytes_in": 21033821.0, "loki_bytes_in": 135765.0, "loki_lines_in": 340.0, "prom_samples_in": 35456.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 60, "error": 50, "pipeline_consumer": 90}, "window": [1794232111.3341079, 1794232201.347548], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "head10", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232280.328288, "finished_at": 1794232370.6738691, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5435, "successful_rps": 60.39, "status_counts": {"201": 4045, "202": 90, "200": 1300, "500": 55}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.29, "p95": 12.98, "p99": 39.25, "max": 71.67}, "faults_sent": {"slow": 68, "error": 55}, "by_kind": {"post_message": {"n": 4045, "p50_ms": 8.03, "p99_ms": 39.25}, "get_history": {"n": 1355, "p50_ms": 9.8, "p99_ms": 39.23}, "upload": {"n": 90, "p50_ms": 20.56, "p99_ms": 28.61}}}, "cpu_pct_avg": {"worker": 14.92, "api-2": 9.9, "api-1": 9.7, "otel-collector": 0.24, "tempo": 0.35, "nginx": 1.22, "grafana": 0.85, "postgres": 4.5, "rabbitmq": 6.47, "valkey": 0.51, "minio": 1.47, "loki": 0.5, "prometheus": 0.33}, "mem_mib_max": {"worker": 146.7, "api-2": 101.2, "api-1": 113.3, "otel-collector": 67.4, "tempo": 500.3, "nginx": 5.3, "grafana": 135.7, "postgres": 66.7, "rabbitmq": 91.2, "valkey": 11.9, "minio": 301.4, "loki": 87.8, "prometheus": 277.5}, "telemetry_delta": {"sdk_span_decisions_sample": 5553.0, "sdk_span_decisions_drop_ratio": 51184.0, "sdk_spans_exported_ok": 5553.0, "sdk_spans_exported_failed": null, "col_spans_in": 5553.0, "col_logs_in": 345.0, "col_points_in": 1290.0, "col_spans_out": 5553.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2546103.0, "tempo_spans_in": 5553.0, "tempo_bytes_in": 2070193.0, "loki_bytes_in": 137005.0, "loki_lines_in": 345.0, "prom_samples_in": 35514.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 530, "slow": 7, "error": 6, "pipeline_consumer": 6}, "window": [1794232280.3282502, 1794232370.689455], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "tail", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232449.187008, "finished_at": 1794232539.181068, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5454, "successful_rps": 60.6, "status_counts": {"201": 4070, "202": 90, "200": 1294, "500": 36}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.29, "p95": 14.45, "p99": 42.01, "max": 106.41}, "faults_sent": {"slow": 56, "error": 36}, "by_kind": {"post_message": {"n": 4070, "p50_ms": 7.89, "p99_ms": 42.6}, "get_history": {"n": 1330, "p50_ms": 9.98, "p99_ms": 41.92}, "upload": {"n": 90, "p50_ms": 19.19, "p99_ms": 77.61}}}, "cpu_pct_avg": {"api-1": 10.92, "api-2": 11.23, "worker": 15.35, "otel-collector": 0.54, "tempo": 0.5, "nginx": 1.18, "grafana": 0.83, "postgres": 4.47, "rabbitmq": 6.56, "valkey": 0.47, "minio": 1.45, "loki": 0.52, "prometheus": 0.34}, "mem_mib_max": {"api-1": 137.4, "api-2": 116.8, "worker": 143.7, "otel-collector": 82.1, "tempo": 504.9, "nginx": 5.2, "grafana": 135.9, "postgres": 70.0, "rabbitmq": 157.4, "valkey": 11.9, "minio": 302.9, "loki": 90.2, "prometheus": 277.9}, "telemetry_delta": {"sdk_span_decisions_sample": 54799.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 54799.0, "sdk_spans_exported_failed": null, "col_spans_in": 56699.0, "col_logs_in": 326.0, "col_points_in": 1224.0, "col_spans_out": 10097.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21397727.0, "tempo_spans_in": 9711.0, "tempo_bytes_in": 3949068.0, "loki_bytes_in": 126917.0, "loki_lines_in": 326.0, "prom_samples_in": 36752.0, "tail_policies": {"baseline-10pct/false": 5090.0, "baseline-10pct/true": 620.0, "keep-async-pipeline/false": 5619.0, "keep-async-pipeline/true": 91.0, "keep-errors/false": 5673.0, "keep-errors/true": 37.0, "keep-slow/false": 5649.0, "keep-slow/true": 61.0}}, "traces_in_tempo": {"server_api": 758, "slow": 56, "error": 36, "pipeline_consumer": 90}, "window": [1794232449.186968, 1794232539.195683], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "off", "rep": 2, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232618.428749, "finished_at": 1794232708.432197, "generator_backlog_at_end": 1, "requests": 5490, "ok": 5439, "successful_rps": 60.43, "status_counts": {"200": 1202, "202": 90, "201": 4147, "500": 51}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.33, "p95": 12.21, "p99": 32.69, "max": 68.21}, "faults_sent": {"slow": 56, "error": 51}, "by_kind": {"post_message": {"n": 4147, "p50_ms": 8.1, "p99_ms": 33.23}, "get_history": {"n": 1253, "p50_ms": 9.5, "p99_ms": 27.42}, "upload": {"n": 90, "p50_ms": 20.59, "p99_ms": 28.39}}}, "cpu_pct_avg": {"api-1": 8.49, "api-2": 8.47, "worker": 14.85, "otel-collector": 0.06, "tempo": 0.82, "nginx": 1.25, "grafana": 0.93, "postgres": 4.84, "rabbitmq": 6.54, "valkey": 0.49, "minio": 2.04, "loki": 0.54, "prometheus": 0.3}, "mem_mib_max": {"api-1": 93.7, "api-2": 93.3, "worker": 134.7, "otel-collector": 35.6, "tempo": 423.4, "nginx": 5.2, "grafana": 135.9, "postgres": 74.4, "rabbitmq": 154.2, "valkey": 11.9, "minio": 303.4, "loki": 89.5, "prometheus": 276.7}, "telemetry_delta": {"sdk_span_decisions_sample": -133.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -133.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 23378.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794232618.428705, 1794232708.44837], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "on100", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232786.969322, "finished_at": 1794232876.97309, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5439, "successful_rps": 60.43, "status_counts": {"202": 90, "201": 4040, "200": 1309, "500": 51}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.09, "p95": 14.67, "p99": 43.98, "max": 89.3}, "faults_sent": {"slow": 57, "error": 51}, "by_kind": {"post_message": {"n": 4040, "p50_ms": 7.49, "p99_ms": 43.73}, "get_history": {"n": 1360, "p50_ms": 10.06, "p99_ms": 45.13}, "upload": {"n": 90, "p50_ms": 20.2, "p99_ms": 53.57}}}, "cpu_pct_avg": {"worker": 15.89, "api-1": 11.4, "api-2": 11.46, "otel-collector": 0.56, "tempo": 1.03, "nginx": 1.22, "grafana": 0.93, "postgres": 4.6, "rabbitmq": 6.71, "valkey": 0.45, "minio": 1.71, "loki": 0.58, "prometheus": 0.33}, "mem_mib_max": {"worker": 155.8, "api-1": 121.5, "api-2": 107.1, "otel-collector": 74.9, "tempo": 715.3, "nginx": 5.5, "grafana": 136.0, "postgres": 84.5, "rabbitmq": 154.9, "valkey": 12.1, "minio": 304.7, "loki": 91.0, "prometheus": 281.9}, "telemetry_delta": {"sdk_span_decisions_sample": 55086.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 55086.0, "sdk_spans_exported_failed": null, "col_spans_in": 56986.0, "col_logs_in": 341.0, "col_points_in": 1224.0, "col_spans_out": 56986.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21513949.0, "tempo_spans_in": 56986.0, "tempo_bytes_in": 21018520.0, "loki_bytes_in": 136272.0, "loki_lines_in": 341.0, "prom_samples_in": 34616.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 57, "error": 51, "pipeline_consumer": 90}, "window": [1794232786.969286, 1794232876.987838], "stats_samples": 14, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "head10", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232959.123384, "finished_at": 1794233049.151171, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5442, "successful_rps": 60.47, "status_counts": {"200": 1281, "201": 4071, "202": 90, "500": 48}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.6, "p95": 10.27, "p99": 36.75, "max": 114.16}, "faults_sent": {"slow": 62, "error": 48}, "by_kind": {"post_message": {"n": 4071, "p50_ms": 6.31, "p99_ms": 33.63}, "get_history": {"n": 1329, "p50_ms": 8.13, "p99_ms": 38.58}, "upload": {"n": 90, "p50_ms": 16.04, "p99_ms": 74.32}}}, "cpu_pct_avg": {"api-1": 9.85, "api-2": 9.72, "worker": 15.02, "otel-collector": 0.23, "tempo": 0.44, "nginx": 1.23, "grafana": 0.91, "postgres": 4.53, "rabbitmq": 6.55, "valkey": 0.51, "minio": 1.56, "loki": 0.55, "prometheus": 0.34}, "mem_mib_max": {"api-1": 99.3, "api-2": 101.9, "worker": 156.9, "otel-collector": 75.6, "tempo": 408.7, "nginx": 5.2, "grafana": 136.0, "postgres": 90.7, "rabbitmq": 91.4, "valkey": 12.4, "minio": 305.8, "loki": 92.2, "prometheus": 282.7}, "telemetry_delta": {"sdk_span_decisions_sample": 5559.0, "sdk_span_decisions_drop_ratio": 50970.0, "sdk_spans_exported_ok": 5559.0, "sdk_spans_exported_failed": null, "col_spans_in": 5559.0, "col_logs_in": 338.0, "col_points_in": 1296.0, "col_spans_out": 5559.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2555586.0, "tempo_spans_in": 5559.0, "tempo_bytes_in": 2082660.0, "loki_bytes_in": 132915.0, "loki_lines_in": 338.0, "prom_samples_in": 35520.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 554, "slow": 3, "error": 4, "pipeline_consumer": 9}, "window": [1794232959.123342, 1794233049.1668348], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "tail", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233127.526659, "finished_at": 1794233217.5204532, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5424, "successful_rps": 60.27, "status_counts": {"201": 3988, "202": 90, "200": 1346, "500": 66}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.71, "p95": 11.88, "p99": 43.18, "max": 135.99}, "faults_sent": {"slow": 70, "error": 66}, "by_kind": {"post_message": {"n": 3988, "p50_ms": 6.4, "p99_ms": 42.94}, "get_history": {"n": 1412, "p50_ms": 8.66, "p99_ms": 59.68}, "upload": {"n": 90, "p50_ms": 17.45, "p99_ms": 108.82}}}, "cpu_pct_avg": {"api-1": 11.36, "api-2": 11.19, "worker": 15.2, "otel-collector": 0.53, "tempo": 0.63, "nginx": 1.22, "grafana": 0.84, "postgres": 4.41, "rabbitmq": 6.66, "valkey": 0.47, "minio": 1.52, "loki": 0.53, "prometheus": 0.32}, "mem_mib_max": {"api-1": 102.4, "api-2": 103.0, "worker": 148.8, "otel-collector": 77.9, "tempo": 486.8, "nginx": 5.4, "grafana": 135.6, "postgres": 96.5, "rabbitmq": 151.3, "valkey": 12.1, "minio": 307.6, "loki": 93.8, "prometheus": 282.2}, "telemetry_delta": {"sdk_span_decisions_sample": 55543.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 55543.0, "sdk_spans_exported_failed": null, "col_spans_in": 57443.0, "col_logs_in": 356.0, "col_points_in": 1326.0, "col_spans_out": 10531.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21716724.0, "tempo_spans_in": 9969.0, "tempo_bytes_in": 4062445.0, "loki_bytes_in": 145752.0, "loki_lines_in": 356.0, "prom_samples_in": 41766.0, "tail_policies": {"baseline-10pct/false": 5284.0, "baseline-10pct/true": 612.0, "keep-async-pipeline/false": 5804.0, "keep-async-pipeline/true": 92.0, "keep-errors/false": 5826.0, "keep-errors/true": 70.0, "keep-slow/false": 5824.0, "keep-slow/true": 72.0}}, "traces_in_tempo": {"server_api": 774, "slow": 70, "error": 66, "pipeline_consumer": 90}, "window": [1794233127.526627, 1794233217.535255], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "off", "rep": 3, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233296.514442, "finished_at": 1794233386.5104158, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5440, "successful_rps": 60.44, "status_counts": {"201": 4051, "202": 90, "200": 1299, "500": 50}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.56, "p95": 10.45, "p99": 33.86, "max": 79.09}, "faults_sent": {"slow": 52, "error": 50}, "by_kind": {"post_message": {"n": 4051, "p50_ms": 6.23, "p99_ms": 33.4}, "get_history": {"n": 1349, "p50_ms": 7.95, "p99_ms": 50.36}, "upload": {"n": 90, "p50_ms": 18.98, "p99_ms": 37.17}}}, "cpu_pct_avg": {"api-1": 9.13, "api-2": 9.18, "worker": 15.32, "otel-collector": 0.08, "tempo": 0.86, "nginx": 1.34, "grafana": 0.98, "postgres": 5.15, "rabbitmq": 6.66, "valkey": 0.57, "minio": 1.66, "loki": 0.6, "prometheus": 0.36}, "mem_mib_max": {"api-1": 105.8, "api-2": 124.5, "worker": 144.7, "otel-collector": 35.6, "tempo": 695.5, "nginx": 5.2, "grafana": 136.0, "postgres": 94.7, "rabbitmq": 157.3, "valkey": 12.6, "minio": 309.7, "loki": 95.0, "prometheus": 289.2}, "telemetry_delta": {"sdk_span_decisions_sample": -209.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -209.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 23378.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794233296.514399, 1794233386.524056], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "on100", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233465.4906292, "finished_at": 1794233555.486665, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5445, "successful_rps": 60.5, "status_counts": {"201": 4032, "202": 90, "200": 1323, "500": 45}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.63, "p95": 11.84, "p99": 43.42, "max": 76.29}, "faults_sent": {"slow": 50, "error": 45}, "by_kind": {"post_message": {"n": 4032, "p50_ms": 6.34, "p99_ms": 42.73}, "get_history": {"n": 1368, "p50_ms": 8.43, "p99_ms": 43.79}, "upload": {"n": 90, "p50_ms": 18.37, "p99_ms": 75.49}}}, "cpu_pct_avg": {"api-1": 11.12, "api-2": 11.21, "worker": 15.33, "otel-collector": 0.54, "tempo": 1.29, "nginx": 1.19, "grafana": 0.89, "postgres": 4.63, "rabbitmq": 6.56, "valkey": 0.49, "minio": 1.7, "loki": 0.57, "prometheus": 0.38}, "mem_mib_max": {"api-1": 102.6, "api-2": 107.7, "worker": 143.0, "otel-collector": 73.6, "tempo": 575.9, "nginx": 5.2, "grafana": 135.5, "postgres": 99.4, "rabbitmq": 92.2, "valkey": 12.3, "minio": 311.5, "loki": 94.7, "prometheus": 286.5}, "telemetry_delta": {"sdk_span_decisions_sample": 55247.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 55247.0, "sdk_spans_exported_failed": null, "col_spans_in": 57147.0, "col_logs_in": 335.0, "col_points_in": 1224.0, "col_spans_out": 57147.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21561062.0, "tempo_spans_in": 57147.0, "tempo_bytes_in": 21069123.0, "loki_bytes_in": 132480.0, "loki_lines_in": 335.0, "prom_samples_in": 35456.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 50, "error": 45, "pipeline_consumer": 90}, "window": [1794233465.490568, 1794233555.500811], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "head10", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233634.709635, "finished_at": 1794233725.1747258, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5427, "successful_rps": 60.3, "status_counts": {"201": 4006, "202": 90, "200": 1331, "500": 63}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.57, "p95": 10.44, "p99": 36.56, "max": 264.95}, "faults_sent": {"slow": 68, "error": 63}, "by_kind": {"post_message": {"n": 4006, "p50_ms": 6.25, "p99_ms": 29.64}, "get_history": {"n": 1394, "p50_ms": 8.14, "p99_ms": 44.81}, "upload": {"n": 90, "p50_ms": 18.47, "p99_ms": 63.61}}}, "cpu_pct_avg": {"api-1": 10.01, "api-2": 10.04, "worker": 15.08, "otel-collector": 0.23, "tempo": 0.43, "nginx": 1.25, "grafana": 0.88, "postgres": 4.63, "rabbitmq": 6.55, "valkey": 0.48, "minio": 1.74, "loki": 0.58, "prometheus": 0.34}, "mem_mib_max": {"api-1": 101.0, "api-2": 99.4, "worker": 148.2, "otel-collector": 69.7, "tempo": 359.2, "nginx": 5.1, "grafana": 136.2, "postgres": 108.8, "rabbitmq": 156.7, "valkey": 12.1, "minio": 314.6, "loki": 122.6, "prometheus": 284.1}, "telemetry_delta": {"sdk_span_decisions_sample": 5568.0, "sdk_span_decisions_drop_ratio": 51652.0, "sdk_spans_exported_ok": 5568.0, "sdk_spans_exported_failed": null, "col_spans_in": 5568.0, "col_logs_in": 353.0, "col_points_in": 1287.0, "col_spans_out": 5568.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2566444.0, "tempo_spans_in": 5568.0, "tempo_bytes_in": 2085852.0, "loki_bytes_in": 142091.0, "loki_lines_in": 353.0, "prom_samples_in": 39528.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 546, "slow": 8, "error": 5, "pipeline_consumer": 10}, "window": [1794233634.709598, 1794233725.183886], "stats_samples": 14, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
{"config": "tail", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233805.530024, "finished_at": 1794233895.525802, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5428, "successful_rps": 60.31, "status_counts": {"201": 4009, "202": 90, "200": 1329, "500": 62}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.69, "p95": 14.69, "p99": 48.23, "max": 153.66}, "faults_sent": {"slow": 54, "error": 62}, "by_kind": {"post_message": {"n": 4009, "p50_ms": 8.35, "p99_ms": 45.87}, "get_history": {"n": 1391, "p50_ms": 10.45, "p99_ms": 51.17}, "upload": {"n": 90, "p50_ms": 20.58, "p99_ms": 84.12}}}, "cpu_pct_avg": {"worker": 15.24, "api-1": 10.99, "api-2": 11.11, "otel-collector": 0.52, "tempo": 0.8, "nginx": 1.16, "grafana": 0.88, "postgres": 4.49, "rabbitmq": 6.64, "valkey": 0.46, "minio": 1.69, "loki": 0.5, "prometheus": 0.32}, "mem_mib_max": {"worker": 143.4, "api-1": 106.7, "api-2": 102.9, "otel-collector": 83.0, "tempo": 487.7, "nginx": 5.1, "grafana": 135.8, "postgres": 114.2, "rabbitmq": 157.5, "valkey": 11.9, "minio": 316.2, "loki": 114.4, "prometheus": 284.0}, "telemetry_delta": {"sdk_span_decisions_sample": 55277.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 55277.0, "sdk_spans_exported_failed": null, "col_spans_in": 57177.0, "col_logs_in": 352.0, "col_points_in": 1224.0, "col_spans_out": 9512.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21589232.0, "tempo_spans_in": 9559.0, "tempo_bytes_in": 3899582.0, "loki_bytes_in": 143224.0, "loki_lines_in": 352.0, "prom_samples_in": 36752.0, "tail_policies": {"baseline-10pct/false": 4962.0, "baseline-10pct/true": 548.0, "keep-async-pipeline/false": 5420.0, "keep-async-pipeline/true": 90.0, "keep-errors/false": 5448.0, "keep-errors/true": 62.0, "keep-slow/false": 5456.0, "keep-slow/true": 54.0}}, "traces_in_tempo": {"server_api": 737, "slow": 54, "error": 62, "pipeline_consumer": 90}, "window": [1794233805.529983, 1794233895.540648], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
|
||||
@@ -0,0 +1,31 @@
|
||||
Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.
|
||||
|
||||
| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| off | 3 | 60.4 | 0 | 7.1 (6.4…8.3) | 11.0 (10.3…12.2) | 32.7 (31.7…33.9) | 17.8 (17.0…18.3) | 15.2 (14.8…15.3) |
|
||||
| on100 | 3 | 60.5 | 0 | 7.8 (6.6…8.6) | 14.3 (11.8…16.3) | 47.2 (43.4…54.4) | 22.7 (22.3…22.9) | 15.6 (15.3…15.9) |
|
||||
| head10 | 3 | 60.4 | 0 | 7.2 (6.6…8.3) | 11.2 (10.3…13.0) | 37.5 (36.6…39.2) | 19.7 (19.6…20.0) | 15.0 (14.9…15.1) |
|
||||
| tail | 3 | 60.4 | 0 | 7.9 (6.7…8.7) | 13.7 (11.9…14.7) | 44.5 (42.0…48.2) | 22.3 (22.1…22.5) | 15.3 (15.2…15.3) |
|
||||
|
||||
| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| off | 0.07 (0.06…0.08) | 0.70 (0.43…0.86) | 0.57 (0.54…0.60) | 0.34 (0.30…0.36) | 98 | 36 | 470 |
|
||||
| on100 | 0.57 (0.54…0.60) | 1.13 (1.03…1.29) | 0.57 (0.55…0.58) | 0.36 (0.33…0.38) | 113 | 73 | 571 |
|
||||
| head10 | 0.23 (0.23…0.24) | 0.41 (0.35…0.44) | 0.54 (0.50…0.58) | 0.34 (0.33…0.34) | 105 | 71 | 423 |
|
||||
| tail | 0.53 (0.52…0.54) | 0.64 (0.50…0.80) | 0.52 (0.50…0.53) | 0.33 (0.32…0.34) | 116 | 81 | 493 |
|
||||
|
||||
| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |
|
||||
|---|---|---|---|---|---|---|
|
||||
| off | — | 0 | 0 | — | 0 | 0 |
|
||||
| on100 | 57059 | 57059 | 0 | 21030 | 20547 | 132 |
|
||||
| head10 | 5560 | 5560 | 0 | 2496 | 2031 | 134 |
|
||||
| tail | 57106 | 9746 | 0 | 21062 | 3877 | 135 |
|
||||
|
||||
Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.
|
||||
|
||||
| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |
|
||||
|---|---|---|---|---|---|
|
||||
| off | 0 / 16470 | 0 / 164 | 0 / 163 | 0 / 270 | — / 163 |
|
||||
| on100 | 16530 / 16470 | 167 / 167 | 146 / 146 | 270 / 270 | 146 / 146 |
|
||||
| head10 | 1630 / 16470 | 18 / 198 | 15 / 166 | 25 / 270 | 166 / 166 |
|
||||
| tail | 2269 / 16470 | 180 / 180 | 164 / 164 | 270 / 270 | 164 / 164 |
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"alert_pending_s": 60.01730489730835,
|
||||
"alert_firing_s": 340,
|
||||
"saturation_at_firing": 1.0,
|
||||
"queued_at_firing": 194.0,
|
||||
"oldest_age_s": 63,
|
||||
"jobs_per_s": 3.16,
|
||||
"job_p95_s": 0.98,
|
||||
"alerts_at_firing": {
|
||||
"MessengerApiHighErrorRate": "firing",
|
||||
"MessengerQueueLagGrowing": "firing",
|
||||
"MessengerWorkerSaturated": "firing"
|
||||
},
|
||||
"resolved_after_restore_s": 17,
|
||||
"drained_after_restore_s": 22,
|
||||
"uploads": {
|
||||
"202": 1255,
|
||||
"503": 108
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
17:19:44 lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas
|
||||
17:19:51 uploads: 4/s open-loop (sample.png)
|
||||
17:20:51 fresh capacity=2 slots (stale series of recreated containers are excluded)
|
||||
17:20:51 MessengerWorkerSaturated pending (+60 s)
|
||||
17:25:31 MessengerWorkerSaturated firing (+340 s)
|
||||
17:25:31 at firing: saturation=1.00, queued=194, oldest age=63 s, succeeded=3.16 jobs/s, job p95=0.98 s, alerts={'MessengerApiHighErrorRate': 'firing', 'MessengerQueueLagGrowing': 'firing', 'MessengerWorkerSaturated': 'firing'}
|
||||
17:25:31 restore default worker configuration
|
||||
17:25:49 MessengerWorkerSaturated resolved (+17 s)
|
||||
17:25:54 backlog drained: True (+22 s after restore)
|
||||
PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,44 @@
|
||||
# Grafana 11.5.2 datasources (fixed uids are referenced by the dashboard JSON). Grafana stores no signals itself.
|
||||
# "$$" escapes "$" from Grafana's env-variable interpolation in provisioning files.
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
prometheusType: Prometheus
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
derivedFields:
|
||||
# trace_id is OTLP structured metadata -> link "Open trace" to Tempo.
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: tempo
|
||||
url: "$${__value.raw}"
|
||||
urlDisplayLabel: Open trace
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: "-5m"
|
||||
spanEndTimeShift: "5m"
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"'
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,39 @@
|
||||
# Loki 3.4.2 single binary: log store. Logs arrive by OTLP (http://loki:3100/otlp) from the Collector.
|
||||
# service.name -> stream label service_name; log attributes (event, code.*) and trace_id/span_id -> structured metadata.
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
volume_enabled: true
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*", "arn:aws:s3:::messenger-attachments/staging/*"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API.
|
||||
set -eu
|
||||
BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
ROOT_USER="$(cat /run/secrets/minio_root_user)"
|
||||
ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
|
||||
APP_USER="$(cat /run/secrets/s3_access_key_id)"
|
||||
APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)"
|
||||
|
||||
until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
echo "waiting for minio"; sleep 2
|
||||
done
|
||||
mc mb --ignore-existing "local/$BUCKET"
|
||||
mc anonymous set none "local/$BUCKET"
|
||||
mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \
|
||||
|| mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true
|
||||
mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true
|
||||
mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true
|
||||
echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*"
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
|
||||
SERVERS="${API_UPSTREAMS:-api-1 api-2}"
|
||||
INTERVAL="${HEALTH_INTERVAL:-1}"
|
||||
FALL="${HEALTH_FALL:-2}"
|
||||
RISE="${HEALTH_RISE:-2}"
|
||||
OUT=/etc/nginx/upstream/servers.conf
|
||||
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
|
||||
var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; }
|
||||
|
||||
render() {
|
||||
any_up=0
|
||||
for s in $SERVERS; do
|
||||
eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1
|
||||
done
|
||||
for s in $SERVERS; do
|
||||
mark=""
|
||||
if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then
|
||||
mark=" down"
|
||||
fi
|
||||
echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;"
|
||||
done
|
||||
}
|
||||
|
||||
for s in $SERVERS; do
|
||||
v=$(var "$s")
|
||||
eval "up_$v=1 ok_$v=0 bad_$v=0"
|
||||
done
|
||||
render > "$OUT"
|
||||
|
||||
watch() {
|
||||
pending=0
|
||||
while sleep "$INTERVAL"; do
|
||||
changed=0
|
||||
for s in $SERVERS; do
|
||||
v=$(var "$s")
|
||||
if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then
|
||||
eval "ok_$v=\$((ok_$v + 1)) bad_$v=0"
|
||||
if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then
|
||||
eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream"
|
||||
fi
|
||||
else
|
||||
eval "bad_$v=\$((bad_$v + 1)) ok_$v=0"
|
||||
if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then
|
||||
eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$changed" = 1 ]; then
|
||||
render > "$OUT"
|
||||
pending=1
|
||||
fi
|
||||
if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then
|
||||
pending=0
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
watch &
|
||||
exec nginx -g 'daemon off;'
|
||||
@@ -0,0 +1,75 @@
|
||||
worker_processes 2;
|
||||
|
||||
events {
|
||||
worker_connections 256;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver 127.0.0.11 valid=5s ipv6=off;
|
||||
|
||||
upstream api {
|
||||
zone api 64k;
|
||||
include /etc/nginx/upstream/servers.conf;
|
||||
keepalive 32;
|
||||
keepalive_timeout 4s;
|
||||
}
|
||||
|
||||
log_format main '$remote_addr - $upstream_addr [$time_local] '
|
||||
'"$request" $status $body_bytes_sent '
|
||||
'"$upstream_response_time" "$request_time"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 64k;
|
||||
error_page 413 @payload_too_large;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 5s;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
client_max_body_size 10m;
|
||||
proxy_request_buffering off;
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream off;
|
||||
}
|
||||
|
||||
location @payload_too_large {
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
add_header X-Instance-Id lb always;
|
||||
add_header Connection close always;
|
||||
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
|
||||
}
|
||||
|
||||
location /health/ {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/attachments/ {
|
||||
proxy_max_temp_file_size 0;
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout http_502 http_503;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
# OpenTelemetry Collector (contrib 0.120.0): receiver/processor/exporter only, it stores nothing long-term.
|
||||
# API, worker --OTLP/HTTP:4318--> collector --> Prometheus (metrics, OTLP), Loki (logs, OTLP), Tempo (traces, OTLP/gRPC)
|
||||
# Ports are reachable only inside the compose network; nothing here is published to the host.
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
max_request_body_size: 4194304 # 4 MiB per OTLP request
|
||||
|
||||
processors:
|
||||
# First in every pipeline: refuse new data (receivers answer 503 -> SDK retries/drops) before the container OOMs.
|
||||
# Container limit is 256 MiB; soft limit = limit_mib - spike_limit_mib = 140 MiB.
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 180
|
||||
spike_limit_mib: 40
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
timeout: 2s
|
||||
# Defense in depth: even if an instrumentation adds them, these attributes never leave the collector.
|
||||
# url.query would contain ?filename=...; headers could contain Authorization.
|
||||
attributes/scrub:
|
||||
actions:
|
||||
- {key: url.query, action: delete}
|
||||
- {key: url.full, action: delete}
|
||||
- {key: http.target, action: delete}
|
||||
- {key: http.url, action: delete}
|
||||
- {key: http.request.header.authorization, action: delete}
|
||||
- {key: http.request.header.cookie, action: delete}
|
||||
- {key: db.query.parameters, action: delete}
|
||||
|
||||
exporters:
|
||||
# Every exporter: bounded in-memory queue (batches) + bounded retry. Backend down => queue fills, then new
|
||||
# batches are dropped and counted (otelcol_exporter_enqueue_failed_*); RAM stays bounded, apps are unaffected.
|
||||
otlphttp/prometheus:
|
||||
endpoint: http://prometheus:9090/api/v1/otlp
|
||||
compression: gzip
|
||||
timeout: 5s
|
||||
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
|
||||
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
compression: gzip
|
||||
timeout: 5s
|
||||
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
|
||||
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls: {insecure: true}
|
||||
compression: gzip
|
||||
timeout: 5s
|
||||
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
|
||||
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/scrub, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/scrub, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,5 @@
|
||||
# Default profile: no extra processing. All spans that the SDK head-sampled are exported.
|
||||
# Selected with COLLECTOR_PROFILE=default (see compose.yaml); merged over collector.yaml.
|
||||
processors:
|
||||
batch:
|
||||
timeout: 2s
|
||||
@@ -0,0 +1,31 @@
|
||||
# Track 1 "balanced" profile (COLLECTOR_PROFILE=tail): SDK keeps 100% head sampling, the collector decides per
|
||||
# whole trace after decision_wait. Keeps: every trace with an error, every trace slower than 500 ms, every trace of
|
||||
# the attachment pipeline (rare, async, valuable), and 10% of the remaining ordinary traffic.
|
||||
# Metrics are computed in the SDK BEFORE any sampling, so RPS/error rate/percentiles are unaffected.
|
||||
processors:
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 20000
|
||||
expected_new_traces_per_sec: 200
|
||||
decision_cache:
|
||||
sampled_cache_size: 50000
|
||||
policies:
|
||||
- name: keep-errors
|
||||
type: status_code
|
||||
status_code: {status_codes: [ERROR]}
|
||||
- name: keep-slow
|
||||
type: latency
|
||||
latency: {threshold_ms: 500}
|
||||
- name: keep-async-pipeline
|
||||
type: string_attribute
|
||||
string_attribute: {key: messaging.system, values: [rabbitmq]}
|
||||
- name: baseline-10pct
|
||||
type: probabilistic
|
||||
probabilistic: {sampling_percentage: 10}
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/scrub, tail_sampling, batch]
|
||||
exporters: [otlp/tempo]
|
||||
@@ -0,0 +1,31 @@
|
||||
# Prometheus v3.2.1: metrics store. Application metrics arrive by OTLP push from the Collector
|
||||
# (--web.enable-otlp-receiver, /api/v1/otlp); infrastructure self-metrics are scraped.
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
otlp:
|
||||
# job=service.name, instance=service.instance.id are set by Prometheus; these become extra labels.
|
||||
promote_resource_attributes:
|
||||
- service.name
|
||||
- service.instance.id
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
|
||||
storage:
|
||||
tsdb:
|
||||
# OTLP pushes from several SDKs/batches may arrive slightly out of order.
|
||||
out_of_order_time_window: 10m
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs: [{targets: ["otel-collector:8888"]}]
|
||||
- job_name: tempo
|
||||
static_configs: [{targets: ["tempo:3200"]}]
|
||||
- job_name: loki
|
||||
static_configs: [{targets: ["loki:3100"]}]
|
||||
- job_name: prometheus
|
||||
static_configs: [{targets: ["localhost:9090"]}]
|
||||
@@ -0,0 +1,111 @@
|
||||
# Recording rules (SLI/SLO) and alert rules. Evaluated by Prometheus every 15 s; state (pending/firing/inactive)
|
||||
# is visible in Prometheus /alerts, the API /api/v1/alerts and in Grafana (alert list panel). No external delivery.
|
||||
# Runbooks: docs/runbooks.md (anchors in runbook_url).
|
||||
#
|
||||
# Label mapping (OTLP -> Prometheus 3): service.name -> service_name, service.instance.id -> instance,
|
||||
# http.route -> http_route, http.response.status_code -> http_response_status_code, job.type -> job_type.
|
||||
# HTTP SLI denominator: every business request of messenger-api (health probes are not instrumented).
|
||||
# Numerator of errors: 5xx only. 4xx (wrong password, 404 of a foreign chat, 409) are client outcomes, not failures.
|
||||
groups:
|
||||
- name: messenger-sli
|
||||
interval: 15s
|
||||
rules:
|
||||
- record: messenger:http_requests:rate2m
|
||||
expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[2m]))
|
||||
- record: messenger:http_errors_5xx:rate2m
|
||||
expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[2m])) or vector(0)
|
||||
- record: messenger:http_error_ratio:rate2m
|
||||
expr: messenger:http_errors_5xx:rate2m / messenger:http_requests:rate2m
|
||||
- record: messenger:http_latency_p95:rate5m
|
||||
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[5m])))
|
||||
# SLO window: rolling 30 min (lab-sized; production would use 28-30 days with the same formulas).
|
||||
# Availability SLO: 99% of business requests without 5xx.
|
||||
- record: messenger:slo_availability:ratio30m
|
||||
expr: |
|
||||
1 - (
|
||||
(sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[30m])) or vector(0))
|
||||
/ sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m]))
|
||||
)
|
||||
# Latency SLO: 95% of business requests complete in <= 500 ms (bucket le="0.5").
|
||||
- record: messenger:slo_latency:ratio30m
|
||||
expr: |
|
||||
sum(increase(http_server_request_duration_seconds_bucket{service_name="messenger-api",le="0.5"}[30m]))
|
||||
/ sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m]))
|
||||
- record: messenger:worker_saturation:ratio
|
||||
# OTLP-pushed series have no staleness markers: a recreated worker would stay in the denominator for the
|
||||
# 5 min lookback. Only samples fresher than 30 s (3 export intervals) are counted.
|
||||
expr: |
|
||||
sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 30))
|
||||
/ sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))
|
||||
|
||||
- name: messenger-alerts
|
||||
rules:
|
||||
- alert: MessengerApiHighErrorRate
|
||||
expr: messenger:http_error_ratio:rate2m > 0.05 and messenger:http_requests:rate2m > 0.2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "API: доля 5xx {{ $value | humanizePercentage }} за 2 мин (порог 5%)"
|
||||
description: "Больше 5% бизнес-запросов messenger-api завершаются 5xx (окно 2m, держится 1m). Расходуется бюджет SLO доступности 99%."
|
||||
runbook_url: "docs/runbooks.md#messengerapihigherrorrate"
|
||||
|
||||
- alert: MessengerQueueLagGrowing
|
||||
expr: max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Очередь вложений: старейшее задание ждёт {{ $value | humanizeDuration }}"
|
||||
description: "Принятое задание не завершено дольше 60 с (порог), условие держится 1m: worker остановлен, перегружен или не может писать в S3/DB."
|
||||
runbook_url: "docs/runbooks.md#messengerqueuelaggrowing"
|
||||
|
||||
- alert: MessengerApiLatencyHigh
|
||||
expr: messenger:http_latency_p95:rate5m > 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: ticket
|
||||
annotations:
|
||||
summary: "API p95 = {{ $value | humanizeDuration }} (порог 1 s, окно 5m)"
|
||||
description: "p95 из histogram buckets всех реплик выше 1 s."
|
||||
runbook_url: "docs/runbooks.md#messengerapilatencyhigh"
|
||||
|
||||
- alert: MessengerDbPoolWaiting
|
||||
expr: max(db_client_connection_pending_requests) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: ticket
|
||||
annotations:
|
||||
summary: "Запросы ждут соединение из пула БД ({{ $value }} в очереди)"
|
||||
description: "Пул psycopg исчерпан дольше минуты: БД медленная/блокирована или пул мал."
|
||||
runbook_url: "docs/runbooks.md#messengerdbpoolwaiting"
|
||||
|
||||
- alert: MessengerWorkerSaturated
|
||||
expr: messenger:worker_saturation:ratio > 0.9
|
||||
for: 5m
|
||||
labels:
|
||||
severity: ticket
|
||||
annotations:
|
||||
summary: "Worker занят на {{ $value | humanizePercentage }} ёмкости 5 минут"
|
||||
runbook_url: "docs/runbooks.md#messengerworkersaturated"
|
||||
|
||||
- alert: TelemetryExportFailing
|
||||
# Each term has "or vector(0)": the *_failed_* series do not exist until the first failure, and a sum with an
|
||||
# empty operand is empty (the alert would never fire). Early signal: exporter queue filling up (backend
|
||||
# down, retries in progress); late signal: data dropped (queue full) or retries exhausted.
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(otelcol_exporter_send_failed_spans_total[2m])) or vector(0))
|
||||
+ (sum(rate(otelcol_exporter_send_failed_log_records_total[2m])) or vector(0))
|
||||
+ (sum(rate(otelcol_exporter_enqueue_failed_spans_total[2m])) or vector(0))
|
||||
+ (sum(rate(otelcol_exporter_enqueue_failed_log_records_total[2m])) or vector(0))
|
||||
) > 0
|
||||
or
|
||||
max(max by (exporter) (otelcol_exporter_queue_size) / max by (exporter) (otelcol_exporter_queue_capacity)) > 0.2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: ticket
|
||||
annotations:
|
||||
summary: "Collector не может доставить telemetry в backend"
|
||||
description: "Очередь exporter заполнена >20% или данные отбрасываются: Tempo/Loki/Prometheus недоступен. Бизнес-API не затронут, RAM collector ограничена очередью и memory_limiter."
|
||||
runbook_url: "docs/runbooks.md#telemetryexportfailing"
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Credentials come from a Docker secret; RabbitMQ's config cannot read files, so render a conf.d fragment at start.
|
||||
set -eu
|
||||
PASS="$(cat /run/secrets/rabbitmq_password)"
|
||||
mkdir -p /etc/rabbitmq/conf.d
|
||||
cat > /etc/rabbitmq/conf.d/20-credentials.conf <<CONF
|
||||
default_user = messenger
|
||||
default_pass = ${PASS}
|
||||
default_vhost = messenger
|
||||
default_permissions.configure = .*
|
||||
default_permissions.read = .*
|
||||
default_permissions.write = .*
|
||||
CONF
|
||||
exec docker-entrypoint.sh rabbitmq-server
|
||||
@@ -0,0 +1,10 @@
|
||||
# Durable broker for the attachment pipeline (see REPORT.md).
|
||||
loopback_users.guest = true
|
||||
listeners.tcp.default = 5672
|
||||
# A delivery not acked for 10 min is returned to the queue (worker hang guard on the broker side;
|
||||
# the application lease in PostgreSQL detects a hung job much earlier).
|
||||
consumer_timeout = 600000
|
||||
# Refuse publishes before the disk fills up (publisher sees the block and the outbox keeps the jobs).
|
||||
disk_free_limit.absolute = 200MB
|
||||
vm_memory_high_watermark.relative = 0.6
|
||||
log.console.level = warning
|
||||
@@ -0,0 +1,28 @@
|
||||
# Tempo 2.7.1 single binary: trace store (local blocks on a volume). Receives OTLP/gRPC from the Collector.
|
||||
stream_over_http_enabled: true
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
log_level: warn
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
query_frontend:
|
||||
search:
|
||||
max_result_limit: 10000
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name bytea NOT NULL
|
||||
CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title bytea NOT NULL
|
||||
CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX chats_created_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
text bytea NOT NULL
|
||||
CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
key text NOT NULL
|
||||
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'),
|
||||
request_body_hash text NOT NULL,
|
||||
response_json text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,28 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
filename text NOT NULL
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL
|
||||
CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL
|
||||
CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL,
|
||||
status text NOT NULL DEFAULT 'ready'
|
||||
CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_size;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000);
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed'));
|
||||
ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT;
|
||||
CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key);
|
||||
CREATE INDEX attachments_status_created_idx ON attachments (status, created_at);
|
||||
@@ -0,0 +1,75 @@
|
||||
-- lab5: upload -> staging -> job/outbox -> broker -> worker -> final prefix + variants.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('uploading', 'queued', 'processing', 'ready', 'failed'));
|
||||
-- staging_key: private S3 prefix written by the API before 202; object_key: final original written by the worker.
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text;
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
|
||||
CREATE UNIQUE INDEX attachments_staging_key_idx ON attachments (staging_key) WHERE staging_key IS NOT NULL;
|
||||
|
||||
-- Effect key of the pipeline: (attachment_id, pipeline_version, name). A repeated job cannot add a second set.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CONSTRAINT variants_ct CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
size_bytes integer NOT NULL CONSTRAINT variants_size CHECK (size_bytes > 0),
|
||||
width integer NOT NULL CONSTRAINT variants_width CHECK (width > 0),
|
||||
height integer NOT NULL CONSTRAINT variants_height CHECK (height > 0),
|
||||
sha256 text NOT NULL CONSTRAINT variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name)
|
||||
);
|
||||
|
||||
-- One logical job per (attachment, pipeline_version); job_id never changes on redelivery or retry.
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
state text NOT NULL
|
||||
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed')),
|
||||
attempt integer NOT NULL DEFAULT 1 CONSTRAINT jobs_attempt CHECK (attempt >= 1),
|
||||
max_attempts integer NOT NULL CONSTRAINT jobs_max_attempts CHECK (max_attempts >= 1),
|
||||
next_attempt_at timestamptz NOT NULL,
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
error_code text,
|
||||
last_error text,
|
||||
created_at timestamptz NOT NULL,
|
||||
first_started_at timestamptz,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
updated_at timestamptz NOT NULL,
|
||||
UNIQUE (attachment_id, pipeline_version)
|
||||
);
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE state = 'processing';
|
||||
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE state = 'failed';
|
||||
CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (state) WHERE state IN ('queued', 'processing');
|
||||
|
||||
-- Transactional outbox: written in the same transaction as the job; the relay publishes with confirms.
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz NOT NULL,
|
||||
published_at timestamptz,
|
||||
publish_attempts integer NOT NULL DEFAULT 0,
|
||||
last_error text,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX outbox_published_idx ON outbox (published_at) WHERE published_at IS NOT NULL;
|
||||
|
||||
-- Operator-visible history of every state change of a job (including explicit replay).
|
||||
CREATE TABLE job_audit (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL,
|
||||
action text NOT NULL,
|
||||
actor text NOT NULL,
|
||||
attempt integer NOT NULL,
|
||||
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX job_audit_job_idx ON job_audit (job_id, id);
|
||||
@@ -0,0 +1,60 @@
|
||||
aio-pika==9.5.5
|
||||
aiormq==6.8.1
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
asgiref==3.13.0
|
||||
boto3==1.35.81
|
||||
botocore==1.35.81
|
||||
certifi==2026.10.23
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.2
|
||||
click==8.5.0
|
||||
Deprecated==3.0.1
|
||||
exceptiongroup==1.2.2
|
||||
fastapi==0.115.6
|
||||
googleapis-common-protos==1.75.0
|
||||
h11==0.16.0
|
||||
httptools==0.9.0
|
||||
idna==3.20
|
||||
importlib_metadata==8.5.0
|
||||
jmespath==1.0.1
|
||||
multidict==6.1.0
|
||||
opentelemetry-api==1.30.0
|
||||
opentelemetry-exporter-otlp-proto-common==1.30.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.30.0
|
||||
opentelemetry-instrumentation==0.51b0
|
||||
opentelemetry-instrumentation-asgi==0.51b0
|
||||
opentelemetry-instrumentation-botocore==0.51b0
|
||||
opentelemetry-instrumentation-fastapi==0.51b0
|
||||
opentelemetry-propagator-aws-xray==1.0.2
|
||||
opentelemetry-proto==1.30.0
|
||||
opentelemetry-sdk==1.30.0
|
||||
opentelemetry-semantic-conventions==0.51b0
|
||||
opentelemetry-util-http==0.51b0
|
||||
packaging==26.3
|
||||
pamqp==3.3.0
|
||||
pillow==11.0.0
|
||||
propcache==0.2.1
|
||||
protobuf==5.29.6
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
pycparser==3.11
|
||||
pydantic==2.14.1
|
||||
pydantic_core==2.50.1
|
||||
python-dateutil==2.9.0.post0
|
||||
requests==2.34.2
|
||||
s3transfer==0.10.4
|
||||
six==1.17.0
|
||||
starlette==0.41.3
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.3.0
|
||||
uvicorn==0.34.0
|
||||
uvloop==0.23.0
|
||||
valkey==6.2.0
|
||||
wrapt==1.17.3
|
||||
yarl==1.18.3
|
||||
zipp==4.1.2
|
||||
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup).
|
||||
|
||||
The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py
|
||||
Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов").
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
|
||||
PROM = {"type": "prometheus", "uid": "prometheus"}
|
||||
LOKI = {"type": "loki", "uid": "loki"}
|
||||
TEMPO = {"type": "tempo", "uid": "tempo"}
|
||||
API = 'service_name="messenger-api"'
|
||||
HTTP = "http_server_request_duration_seconds"
|
||||
_ids = iter(range(1, 1000))
|
||||
|
||||
|
||||
def target(expr, legend="", ref="A", instant=False):
|
||||
return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant,
|
||||
"instant": instant}
|
||||
|
||||
|
||||
def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None):
|
||||
panel = {
|
||||
"id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM,
|
||||
"gridPos": {"x": x, "y": y, "w": w, "h": h},
|
||||
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
|
||||
"stacking": {"mode": "normal" if stack else "none"},
|
||||
"showPoints": "never"}},
|
||||
"overrides": []},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [target(*t) if isinstance(t, tuple) else t for t in targets],
|
||||
}
|
||||
if extra:
|
||||
panel["fieldConfig"]["defaults"].update(extra)
|
||||
return panel
|
||||
|
||||
|
||||
def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None):
|
||||
steps = thresholds or [{"color": "green", "value": None}]
|
||||
panel = {
|
||||
"id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM,
|
||||
"gridPos": {"x": x, "y": y, "w": w, "h": h},
|
||||
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}},
|
||||
"overrides": []},
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"colorMode": "value", "graphMode": "area", "textMode": "auto"},
|
||||
"targets": [target(expr, title)],
|
||||
}
|
||||
if decimals is not None:
|
||||
panel["fieldConfig"]["defaults"]["decimals"] = decimals
|
||||
return panel
|
||||
|
||||
|
||||
def row(title, y):
|
||||
return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1},
|
||||
"panels": []}
|
||||
|
||||
|
||||
def q(p, expr_inner):
|
||||
return "histogram_quantile(%s, %s)" % (p, expr_inner)
|
||||
|
||||
|
||||
panels = []
|
||||
y = 0
|
||||
# ------------------------------------------------------------------ SLO and alerts
|
||||
panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1
|
||||
red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad},
|
||||
{"color": "green", "value": warn}]
|
||||
panels += [
|
||||
stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y,
|
||||
desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.",
|
||||
thresholds=red_yellow(0.99, 0.995), decimals=2),
|
||||
stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y,
|
||||
desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2),
|
||||
stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2),
|
||||
stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]),
|
||||
{"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)",
|
||||
"gridPos": {"x": 16, "y": y, "w": 8, "h": 8},
|
||||
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [],
|
||||
"groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter":
|
||||
{"firing": True, "pending": True, "noData": False, "normal": False, "error": True},
|
||||
"viewMode": "list", "datasource": "Prometheus"}},
|
||||
]
|
||||
y += 4
|
||||
panels += [
|
||||
stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1),
|
||||
stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
|
||||
stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]),
|
||||
stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
|
||||
]
|
||||
y += 4
|
||||
# ------------------------------------------------------------------ API
|
||||
panels.append(row("API (messenger-api, все реплики)", y)); y += 1
|
||||
rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API)
|
||||
panels += [
|
||||
ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count,
|
||||
"{{http_request_method}} {{http_route}}")], "reqps", 0, y,
|
||||
desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."),
|
||||
ts("RPS по классу статуса", [(
|
||||
'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count,
|
||||
"{{status_class}}")], "reqps", 12, y, stack=True),
|
||||
]
|
||||
y += 8
|
||||
bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)
|
||||
panels += [
|
||||
ts("Латентность API p50/p95/p99 (из histogram buckets)", [
|
||||
(q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y,
|
||||
desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."),
|
||||
ts("Доля ошибок 5xx", [
|
||||
('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count),
|
||||
"5xx / все", "A")], "percentunit", 12, y,
|
||||
desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."),
|
||||
]
|
||||
y += 8
|
||||
panels += [
|
||||
ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)),
|
||||
"{{http_route}}")], "s", 0, y),
|
||||
ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")],
|
||||
"short", 12, y),
|
||||
]
|
||||
y += 8
|
||||
# ------------------------------------------------------------------ worker and queue
|
||||
panels.append(row("Очередь и worker", y)); y += 1
|
||||
jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))"
|
||||
rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))"
|
||||
panels += [
|
||||
ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
|
||||
"{{outcome}}")], "ops", 0, y, stack=True,
|
||||
desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"),
|
||||
ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"),
|
||||
(q(0.99, jb), "p99", "C")], "s", 12, y),
|
||||
]
|
||||
y += 8
|
||||
panels += [
|
||||
ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")],
|
||||
"short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."),
|
||||
ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")],
|
||||
"s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.",
|
||||
extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never",
|
||||
"thresholdsStyle": {"mode": "line"}},
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None},
|
||||
{"color": "red", "value": 60}]}}),
|
||||
]
|
||||
y += 8
|
||||
panels += [
|
||||
ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")],
|
||||
"s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."),
|
||||
ts("Насыщение worker: занято / ёмкость", [
|
||||
("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"),
|
||||
("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y),
|
||||
]
|
||||
y += 8
|
||||
# ------------------------------------------------------------------ resources
|
||||
panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1
|
||||
panels += [
|
||||
ts("Пул БД: used / idle / waiting", [
|
||||
("sum by (service_name, db_client_connection_state) (db_client_connection_count)",
|
||||
"{{service_name}} {{db_client_connection_state}}", "A"),
|
||||
("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"),
|
||||
("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y),
|
||||
ts("CPU процессов (ядра)", [
|
||||
("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))",
|
||||
"{{service_name}} {{instance}}", "A"),
|
||||
("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y,
|
||||
desc="worker: self + завершённые дочерние процессы обработки изображений."),
|
||||
]
|
||||
y += 8
|
||||
panels += [
|
||||
ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"),
|
||||
("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y),
|
||||
ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [
|
||||
("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"),
|
||||
("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"),
|
||||
("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"),
|
||||
("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))",
|
||||
"sent {{exporter}}", "B"),
|
||||
# Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist.
|
||||
("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"),
|
||||
("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"),
|
||||
("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"),
|
||||
("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"),
|
||||
("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y),
|
||||
]
|
||||
y += 8
|
||||
# ------------------------------------------------------------------ logs and traces
|
||||
panels.append(row("Логи и трассы", y)); y += 1
|
||||
panels += [
|
||||
{"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI,
|
||||
"gridPos": {"x": 0, "y": y, "w": 12, "h": 10},
|
||||
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True},
|
||||
"targets": [{"datasource": LOKI, "refId": "A",
|
||||
"expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]},
|
||||
{"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO,
|
||||
"gridPos": {"x": 12, "y": y, "w": 12, "h": 10},
|
||||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20,
|
||||
"query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]},
|
||||
]
|
||||
y += 10
|
||||
|
||||
dashboard = {
|
||||
"uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"],
|
||||
"timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s",
|
||||
"time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels,
|
||||
"annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"},
|
||||
"enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]},
|
||||
"templating": {"list": []},
|
||||
}
|
||||
OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels)))
|
||||
|
||||
# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script).
|
||||
UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки",
|
||||
"bytes": "байты"}
|
||||
DOC = OUT.parents[3] / "docs/dashboard.md"
|
||||
lines = ["# Панели dashboard «%s»" % dashboard["title"], "",
|
||||
"Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. "
|
||||
"`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""]
|
||||
for panel in panels:
|
||||
if panel["type"] == "row":
|
||||
lines += ["## " + panel["title"], ""]
|
||||
continue
|
||||
unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
|
||||
lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else ""))
|
||||
if panel.get("description"):
|
||||
lines.append("— " + panel["description"])
|
||||
lines.append("")
|
||||
if panel["type"] == "alertlist":
|
||||
lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""]
|
||||
for t in panel.get("targets", []):
|
||||
query = t.get("expr") or t.get("query")
|
||||
legend = t.get("legendFormat", "")
|
||||
lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""),
|
||||
query + (" # " + legend if legend and legend != panel["title"] else ""), "```"]
|
||||
lines.append("")
|
||||
DOC.write_text("\n".join(lines), encoding="utf-8")
|
||||
print("wrote docs/dashboard.md")
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# Creates a local .env and random credentials in secrets/ (both are git-ignored). Prints no secret values.
|
||||
# Safe to repeat: existing files are kept, so running it on a live stand does not change its passwords.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
[ -f .env ] || cp .env.example .env
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
cd secrets
|
||||
|
||||
put() { # put <file> <value>
|
||||
[ -f "$1" ] || printf '%s' "$2" > "$1"
|
||||
}
|
||||
random() { python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')"; }
|
||||
|
||||
put postgres_password "$(random)"
|
||||
put minio_root_user messenger_root
|
||||
put minio_root_password "$(random)"
|
||||
put s3_access_key_id messenger_app
|
||||
put s3_secret_access_key "$(random)"
|
||||
put rabbitmq_password "$(random)"
|
||||
put grafana_admin_password "$(random)"
|
||||
put cache_url valkey://valkey:6379/0
|
||||
put database_url "postgresql://messenger:$(cat postgres_password)@postgres:5432/messenger"
|
||||
put broker_url "amqp://messenger:$(cat rabbitmq_password)@rabbitmq:5672/messenger"
|
||||
# Containers run under other uids and read these through Docker secrets (file bind mounts in Compose).
|
||||
chmod 644 ./*
|
||||
echo "secrets ready in secrets/ (.env: $( [ -f ../.env ] && echo present ))"
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...).
|
||||
|
||||
failed registry of failed jobs (the "DLQ" of business failures)
|
||||
show <attachment_id> job state + audit trail
|
||||
replay <attachment_id> --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing
|
||||
stats counts by state and outbox backlog
|
||||
Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md).
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import jobs # noqa: E402
|
||||
from app.cursor import fmt # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
|
||||
def out(value) -> None:
|
||||
print(json.dumps(value, ensure_ascii=False, indent=2, default=str))
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("failed")
|
||||
sub.add_parser("stats")
|
||||
show = sub.add_parser("show")
|
||||
show.add_argument("attachment_id")
|
||||
rp = sub.add_parser("replay")
|
||||
rp.add_argument("attachment_id")
|
||||
rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)")
|
||||
args = parser.parse_args()
|
||||
|
||||
db = Database()
|
||||
await db.open()
|
||||
try:
|
||||
if args.cmd == "failed":
|
||||
async with db.connection() as conn:
|
||||
out(await jobs.failed_jobs(conn))
|
||||
elif args.cmd == "stats":
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state")
|
||||
states = {r["state"]: r["n"] for r in await cur.fetchall()}
|
||||
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
|
||||
backlog = (await cur.fetchone())["n"]
|
||||
out({"jobs": states, "outbox_unpublished": backlog})
|
||||
elif args.cmd == "show":
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until,"
|
||||
" a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
|
||||
" WHERE j.attachment_id = %s", (args.attachment_id,))
|
||||
job = await cur.fetchone()
|
||||
cur = await conn.execute(
|
||||
"SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id",
|
||||
(args.attachment_id,))
|
||||
trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()]
|
||||
cur = await conn.execute(
|
||||
"SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts"
|
||||
" FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id",
|
||||
(args.attachment_id,))
|
||||
outbox = await cur.fetchall()
|
||||
out({"job": job, "audit": trail, "outbox": outbox})
|
||||
elif args.cmd == "replay":
|
||||
s3 = S3Storage()
|
||||
await run_in_threadpool(s3.open)
|
||||
|
||||
async def exists(key):
|
||||
return await run_in_threadpool(s3.exists, key)
|
||||
|
||||
result = await db.in_transaction(
|
||||
lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists))
|
||||
out(result)
|
||||
return 0 if result["result"] == "queued" else 3
|
||||
finally:
|
||||
await db.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS = ROOT / "migrations"
|
||||
LOCK_ID = 2026_0004
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
path = os.environ.get("DATABASE_URL_FILE")
|
||||
if path:
|
||||
return pathlib.Path(path).read_text(encoding="utf-8").strip()
|
||||
url = os.environ.get("DATABASE_URL", "").strip()
|
||||
if not url:
|
||||
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
return url
|
||||
|
||||
|
||||
def main() -> None:
|
||||
files = sorted(MIGRATIONS.glob("*.sql"))
|
||||
if not files:
|
||||
sys.exit("no migration files found in %s" % MIGRATIONS)
|
||||
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY,"
|
||||
" checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
|
||||
applied = {row[0]: row[1] for row in rows}
|
||||
for path in files:
|
||||
version = path.stem
|
||||
body = path.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
sys.exit("migration %s changed after it was applied" % version)
|
||||
print("skip %s" % version)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(body)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
|
||||
(version, checksum),
|
||||
)
|
||||
print("apply %s" % version)
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
print("migrations complete: %d file(s)" % len(files))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import config, reconcile # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
|
||||
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
|
||||
args = parser.parse_args()
|
||||
db, s3 = Database(), S3Storage()
|
||||
await db.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
try:
|
||||
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# Restart API, LB, worker and every store keeping all volumes: business stores (postgres, valkey, minio, rabbitmq)
|
||||
# and, since lab6, the telemetry stores (prometheus, loki, tempo) plus collector and grafana.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart -t 10 postgres valkey minio rabbitmq
|
||||
docker compose restart -t 10 prometheus loki tempo otel-collector grafana
|
||||
docker compose restart -t 25 worker api-1 api-2 nginx
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Every Prometheus query of the provisioned dashboard must parse and return data (after some traffic).
|
||||
|
||||
python3 student-tests/check_dashboard.py
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
import obs
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
dash = json.loads((ROOT / "infra/grafana/dashboards/messenger.json").read_text(encoding="utf-8"))
|
||||
bad = 0
|
||||
for panel in dash["panels"]:
|
||||
for t in panel.get("targets", []):
|
||||
if t.get("datasource", {}).get("type") != "prometheus":
|
||||
continue
|
||||
expr = t["expr"].replace("$__rate_interval", "2m")
|
||||
try:
|
||||
rows = obs.prom(expr)
|
||||
status = "ok %3d series" % len(rows) if rows else "EMPTY"
|
||||
except urllib.error.HTTPError as exc:
|
||||
status, rows = "ERROR %s" % exc.read()[:120], None
|
||||
if not rows:
|
||||
bad += 1
|
||||
print("%-16s %-48s %s" % (status, panel["title"][:48], t.get("legendFormat", "")))
|
||||
print("panels: %d, prometheus queries without data: %d" % (len(dash["panels"]), bad))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the same panels executed BY GRAFANA (/api/ds/query)
|
||||
# Proves the provisioned datasources work from Grafana itself (Prometheus, Loki and Tempo), with Grafana macros.
|
||||
import base64 # noqa: E402
|
||||
import urllib.request # noqa: E402
|
||||
|
||||
password = (ROOT / "secrets/grafana_admin_password").read_text().strip()
|
||||
auth = "Basic " + base64.b64encode(("admin:" + password).encode()).decode()
|
||||
via_bad = 0
|
||||
for panel in dash["panels"]:
|
||||
for t in panel.get("targets", []):
|
||||
q = dict(t)
|
||||
q.setdefault("datasource", panel.get("datasource"))
|
||||
if q["datasource"]["type"] == "tempo":
|
||||
# Grafana 11 runs TraceQL search in the browser through the datasource proxy, not /api/ds/query
|
||||
# ("backend TraceQL search queries are not supported"): check exactly that proxy path.
|
||||
import time as _t
|
||||
import urllib.parse as _p
|
||||
url = ("http://127.0.0.1:3000/api/datasources/proxy/uid/tempo/api/search?" +
|
||||
_p.urlencode({"q": q["query"], "limit": q.get("limit", 20), "start": int(_t.time()) - 1800,
|
||||
"end": int(_t.time())}))
|
||||
req = urllib.request.Request(url, headers={"Authorization": auth})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
n = len(json.loads(response.read()).get("traces", []))
|
||||
status = "ok %5d traces" % n if n else "EMPTY"
|
||||
except urllib.error.HTTPError as exc:
|
||||
status = "ERROR HTTP %s" % exc.code
|
||||
if not status.startswith("ok"):
|
||||
via_bad += 1
|
||||
print("grafana %-8s %-18s %-48s (datasource proxy)" % ("tempo", status, panel["title"][:48]))
|
||||
continue
|
||||
body = json.dumps({"queries": [q], "from": "now-30m", "to": "now"}).encode()
|
||||
req = urllib.request.Request("http://127.0.0.1:3000/api/ds/query", data=body, method="POST",
|
||||
headers={"Content-Type": "application/json", "Authorization": auth})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
res = json.loads(response.read())["results"][q["refId"]]
|
||||
frames = res.get("frames", [])
|
||||
rows = sum(len((f.get("data") or {}).get("values", [[]])[0] or []) for f in frames)
|
||||
status = "ERROR %s" % res["error"][:100] if res.get("error") else ("ok %5d values" % rows if rows else "EMPTY")
|
||||
except urllib.error.HTTPError as exc:
|
||||
status = "ERROR HTTP %s %s" % (exc.code, exc.read()[:100])
|
||||
if not status.startswith("ok"):
|
||||
via_bad += 1
|
||||
print("grafana %-8s %-18s %-48s" % (q["datasource"]["type"], status, panel["title"][:48]))
|
||||
print("via Grafana: queries without data or with errors: %d" % via_bad)
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Query helpers for the telemetry backends (stdlib only). Backends listen on 127.0.0.1 (compose.yaml).
|
||||
|
||||
Never prints tokens or passwords; callers print only ids, counts, span names and attributes.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
PROM = os.environ.get("PROM_URL", "http://127.0.0.1:9090")
|
||||
LOKI = os.environ.get("LOKI_URL", "http://127.0.0.1:3100")
|
||||
TEMPO = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200")
|
||||
|
||||
|
||||
def _get(url, params=None, timeout=15):
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
with urllib.request.urlopen(url, timeout=timeout) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def prom(expr, at=None):
|
||||
"""Instant query -> list of (labels, float value)."""
|
||||
params = {"query": expr}
|
||||
if at:
|
||||
params["time"] = at
|
||||
data = _get(PROM + "/api/v1/query", params)["data"]["result"]
|
||||
return [(r["metric"], float(r["value"][1])) for r in data]
|
||||
|
||||
|
||||
def prom_scalar(expr, default=0.0, at=None):
|
||||
rows = prom(expr, at)
|
||||
return rows[0][1] if rows else default
|
||||
|
||||
|
||||
def prom_range(expr, start, end, step=5):
|
||||
data = _get(PROM + "/api/v1/query_range", {"query": expr, "start": start, "end": end, "step": step})
|
||||
return data["data"]["result"]
|
||||
|
||||
|
||||
def alerts():
|
||||
"""{alertname: state} for pending/firing alerts in Prometheus."""
|
||||
out = {}
|
||||
for a in _get(PROM + "/api/v1/alerts")["data"]["alerts"]:
|
||||
out[a["labels"]["alertname"]] = a["state"]
|
||||
return out
|
||||
|
||||
|
||||
def loki(query, minutes=15, limit=500):
|
||||
end = time.time_ns()
|
||||
data = _get(LOKI + "/loki/api/v1/query_range", {"query": query, "limit": limit, "direction": "forward",
|
||||
"start": end - minutes * 60 * 10**9, "end": end})
|
||||
lines = []
|
||||
for stream in data["data"]["result"]:
|
||||
for ts, line in stream["values"]:
|
||||
lines.append({"ts": int(ts), "line": line, **stream["stream"]})
|
||||
return sorted(lines, key=lambda item: item["ts"])
|
||||
|
||||
|
||||
def tempo_search(traceql, minutes=30, limit=100, start=None, end=None):
|
||||
end = int(end or time.time() + 60)
|
||||
start = int(start or end - 60 - minutes * 60)
|
||||
data = _get(TEMPO + "/api/search", {"q": traceql, "start": start, "end": end, "limit": limit, "spss": 50},
|
||||
timeout=60)
|
||||
return data.get("traces", [])
|
||||
|
||||
|
||||
def _hex(value):
|
||||
return base64.b64decode(value).hex() if value else None
|
||||
|
||||
|
||||
def _attr(value):
|
||||
for key in ("stringValue", "intValue", "boolValue", "doubleValue"):
|
||||
if key in value:
|
||||
return value[key]
|
||||
return None
|
||||
|
||||
|
||||
def tempo_trace(trace_id):
|
||||
"""Flat list of spans: service, name, kind, span_id, parent_id, duration_ms, status, attrs (dict)."""
|
||||
try:
|
||||
data = _get(TEMPO + "/api/traces/" + trace_id)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return []
|
||||
raise
|
||||
spans = []
|
||||
for batch in data.get("batches", []):
|
||||
service = {a["key"]: _attr(a["value"]) for a in batch["resource"]["attributes"]}.get("service.name")
|
||||
for scope in batch.get("scopeSpans", []):
|
||||
for s in scope["spans"]:
|
||||
spans.append({
|
||||
"service": service, "name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""),
|
||||
"span_id": _hex(s["spanId"]), "parent_id": _hex(s.get("parentSpanId")),
|
||||
"start": int(s["startTimeUnixNano"]),
|
||||
"duration_ms": (int(s["endTimeUnixNano"]) - int(s["startTimeUnixNano"])) / 1e6,
|
||||
"status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", "") or "UNSET",
|
||||
"attrs": {a["key"]: _attr(a["value"]) for a in s.get("attributes", [])},
|
||||
})
|
||||
return sorted(spans, key=lambda item: item["start"])
|
||||
|
||||
|
||||
def tempo_raw(trace_id):
|
||||
with urllib.request.urlopen(TEMPO + "/api/traces/" + trace_id, timeout=15) as response:
|
||||
return response.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def print_tree(spans, out=print, skip_db=False):
|
||||
children = {}
|
||||
ids = {s["span_id"] for s in spans}
|
||||
for s in spans:
|
||||
children.setdefault(s["parent_id"] if s["parent_id"] in ids else None, []).append(s)
|
||||
|
||||
def walk(parent, depth):
|
||||
for s in children.get(parent, []):
|
||||
if skip_db and s["kind"] == "CLIENT" and s["name"].split()[0] in ("SELECT", "UPDATE", "INSERT", "DELETE"):
|
||||
continue
|
||||
extra = {k: v for k, v in s["attrs"].items() if k.startswith(("messenger.", "messaging.message", "db.system",
|
||||
"http.response.status_code", "rpc.method"))}
|
||||
out("%s%-8s %-9s %-44s %8.1f ms %-5s %s" % (" " * depth, s["service"].replace("messenger-", ""), s["kind"],
|
||||
s["name"][:44], s["duration_ms"], s["status"],
|
||||
json.dumps(extra, ensure_ascii=False)))
|
||||
walk(s["span_id"], depth + 1)
|
||||
|
||||
walk(None, 0)
|
||||
|
||||
|
||||
def wait_for(predicate, seconds=60, interval=2):
|
||||
deadline = time.monotonic() + seconds
|
||||
value = None
|
||||
while time.monotonic() < deadline:
|
||||
value = predicate()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(interval)
|
||||
return value
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Open-loop load generator (stdlib only): fixed arrival rate, latency measured from the SCHEDULED send time,
|
||||
so a slow server cannot slow down the generator (no coordinated omission while threads are available).
|
||||
|
||||
Mix per arrival: 75% POST message, 25% GET history (limit=20); plus an independent upload stream (sample.png).
|
||||
Rare injected requests (FAULT_INJECTION=1 on the stand): `X-Fault: slow` (+600 ms) and `X-Fault: error` (500)
|
||||
are added to GET history with the given percentages, to measure whether such traces survive sampling.
|
||||
|
||||
python3 student-tests/openloop.py --rate 40 --duration 60 --upload-rate 1 --slow-pct 1 --error-pct 1 \
|
||||
--output evidence/x.json
|
||||
Tokens stay in memory; the summary contains counts, percentiles and timestamps only.
|
||||
"""
|
||||
import argparse
|
||||
import http.client
|
||||
import json
|
||||
import queue
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
k = max(0, min(len(values) - 1, int(round(p / 100.0 * len(values) + 0.5)) - 1))
|
||||
return values[k]
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base_url, users=4, threads=64, timeout=30.0):
|
||||
self.base = urlsplit(base_url)
|
||||
self.threads = threads
|
||||
self.timeout = timeout
|
||||
c = Client(base_url)
|
||||
self.actors = []
|
||||
for i in range(users):
|
||||
a, b = c.user("load%da" % i), c.user("load%db" % i)
|
||||
chat = c.call("POST", "/api/v1/chats", 201, actor=a,
|
||||
body={"title": "load %d" % i, "member_ids": [a["id"], b["id"]]}, schema="Chat")["json"]
|
||||
self.actors.append((a["token"], chat["id"]))
|
||||
self.lock = threading.Lock()
|
||||
self.results = []
|
||||
self.local = threading.local()
|
||||
|
||||
def conn(self):
|
||||
c = getattr(self.local, "conn", None)
|
||||
if c is None:
|
||||
c = http.client.HTTPConnection(self.base.hostname, self.base.port or 80, timeout=self.timeout)
|
||||
self.local.conn = c
|
||||
return c
|
||||
|
||||
def request(self, method, path, body=None, headers=None):
|
||||
for attempt in (1, 2): # second attempt only for a dropped keep-alive connection BEFORE sending
|
||||
conn = self.conn()
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=headers or {})
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
return response.status, response.getheader("X-Request-Id")
|
||||
except (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError) as exc:
|
||||
conn.close()
|
||||
self.local.conn = None
|
||||
if attempt == 2:
|
||||
return "conn_error:" + type(exc).__name__, None
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
conn.close()
|
||||
self.local.conn = None
|
||||
return "error:" + type(exc).__name__, None
|
||||
return "error", None
|
||||
|
||||
def one(self, kind, fault, scheduled):
|
||||
token, chat = random.choice(self.actors)
|
||||
headers = {"Authorization": "Bearer " + token}
|
||||
if fault:
|
||||
headers["X-Fault"] = fault
|
||||
if kind == "post_message":
|
||||
headers.update({"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex})
|
||||
status, rid = self.request("POST", "/api/v1/chats/%s/messages" % chat,
|
||||
json.dumps({"text": "нагрузка " + uuid.uuid4().hex[:8]}).encode(), headers)
|
||||
elif kind == "get_history":
|
||||
status, rid = self.request("GET", "/api/v1/chats/%s/messages?limit=20" % chat, None, headers)
|
||||
else:
|
||||
headers["Content-Type"] = "image/png"
|
||||
status, rid = self.request("POST", "/api/v1/chats/%s/attachments?filename=load.png" % chat, SAMPLE, headers)
|
||||
done = time.monotonic()
|
||||
with self.lock:
|
||||
self.results.append({"kind": kind, "fault": fault, "status": status, "latency": done - scheduled,
|
||||
"at": time.time(), "rid": rid})
|
||||
|
||||
def run(self, rate, duration, upload_rate=0.0, slow_pct=0.0, error_pct=0.0, stop_event=None):
|
||||
work = queue.Queue()
|
||||
start = time.monotonic()
|
||||
self.started_at = time.time()
|
||||
|
||||
def worker():
|
||||
while True:
|
||||
item = work.get()
|
||||
if item is None:
|
||||
return
|
||||
self.one(*item)
|
||||
|
||||
pool = [threading.Thread(target=worker, daemon=True) for _ in range(self.threads)]
|
||||
for t in pool:
|
||||
t.start()
|
||||
|
||||
def schedule(r, picker):
|
||||
if r <= 0:
|
||||
return
|
||||
n = 0
|
||||
while True:
|
||||
due = start + n / r
|
||||
if due - start >= duration or (stop_event and stop_event.is_set()):
|
||||
return
|
||||
delay = due - time.monotonic()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
work.put(picker() + (due,))
|
||||
n += 1
|
||||
|
||||
def pick_main():
|
||||
if random.random() < 0.75:
|
||||
return ("post_message", None)
|
||||
roll = random.random() * 100
|
||||
fault = "slow" if roll < slow_pct * 4 else "error" if roll < (slow_pct + error_pct) * 4 else None
|
||||
return ("get_history", fault) # 25% share * 4 = requested share of ALL requests
|
||||
|
||||
schedulers = [threading.Thread(target=schedule, args=(rate, pick_main), daemon=True),
|
||||
threading.Thread(target=schedule, args=(upload_rate, lambda: ("upload", None)), daemon=True)]
|
||||
for t in schedulers:
|
||||
t.start()
|
||||
for t in schedulers:
|
||||
t.join()
|
||||
backlog = work.qsize()
|
||||
for _ in pool:
|
||||
work.put(None)
|
||||
for t in pool:
|
||||
t.join(self.timeout + 5)
|
||||
self.finished_at = time.time()
|
||||
return self.summary(rate, duration, upload_rate, slow_pct, error_pct, backlog)
|
||||
|
||||
def summary(self, rate, duration, upload_rate, slow_pct, error_pct, backlog):
|
||||
res = self.results
|
||||
ok = [r for r in res if isinstance(r["status"], int) and r["status"] < 400]
|
||||
plain_ok = [r["latency"] for r in ok if r["fault"] is None and r["kind"] != "upload"]
|
||||
out = {
|
||||
"profile": {"rate_rps": rate, "duration_s": duration, "upload_rate_rps": upload_rate,
|
||||
"slow_pct": slow_pct, "error_pct": error_pct, "threads": self.threads,
|
||||
"mix": "75% POST message / 25% GET history(limit=20); uploads separate"},
|
||||
"started_at": self.started_at, "finished_at": self.finished_at, "generator_backlog_at_end": backlog,
|
||||
"requests": len(res), "ok": len(ok),
|
||||
"successful_rps": round(len(ok) / max(duration, 1e-9), 2),
|
||||
"status_counts": {},
|
||||
"server_errors_unexpected": sum(1 for r in res if r["fault"] != "error" and (
|
||||
not isinstance(r["status"], int) or r["status"] >= 500)),
|
||||
"latency_ms_plain_ok": {p: round(percentile(plain_ok, q) * 1000, 2) if plain_ok else None
|
||||
for p, q in (("p50", 50), ("p95", 95), ("p99", 99), ("max", 100))},
|
||||
"faults_sent": {"slow": sum(1 for r in res if r["fault"] == "slow"),
|
||||
"error": sum(1 for r in res if r["fault"] == "error")},
|
||||
"by_kind": {},
|
||||
}
|
||||
for r in res:
|
||||
key = str(r["status"])
|
||||
out["status_counts"][key] = out["status_counts"].get(key, 0) + 1
|
||||
for kind in ("post_message", "get_history", "upload"):
|
||||
lat = [r["latency"] for r in ok if r["kind"] == kind and r["fault"] is None]
|
||||
out["by_kind"][kind] = {"n": sum(1 for r in res if r["kind"] == kind),
|
||||
"p50_ms": round(percentile(lat, 50) * 1000, 2) if lat else None,
|
||||
"p99_ms": round(percentile(lat, 99) * 1000, 2) if lat else None}
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--base-url", default="http://localhost:8080")
|
||||
p.add_argument("--rate", type=float, default=40)
|
||||
p.add_argument("--duration", type=float, default=60)
|
||||
p.add_argument("--upload-rate", type=float, default=0)
|
||||
p.add_argument("--slow-pct", type=float, default=0)
|
||||
p.add_argument("--error-pct", type=float, default=0)
|
||||
p.add_argument("--threads", type=int, default=64)
|
||||
p.add_argument("--output")
|
||||
a = p.parse_args()
|
||||
load = Load(a.base_url, threads=a.threads)
|
||||
summary = load.run(a.rate, a.duration, a.upload_rate, a.slow_pct, a.error_pct)
|
||||
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
||||
print(text)
|
||||
if a.output:
|
||||
Path(a.output).write_text(text + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB.
|
||||
|
||||
Each run:
|
||||
1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH
|
||||
2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses)
|
||||
3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second
|
||||
4. waits until every accepted upload is ready/failed and computes from attachment_jobs:
|
||||
queue lag = first_started_at - created_at (time a job waited for a worker)
|
||||
time to ready = finished_at - attachment.created_at
|
||||
throughput = finished jobs / (last finish - first arrival)
|
||||
Rejections (503 queue_full from admission control) are counted, not retried.
|
||||
|
||||
python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2
|
||||
python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30
|
||||
Results are appended as JSON lines to evidence/track1/results.jsonl.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready
|
||||
|
||||
OUT = ROOT / "evidence/track1"
|
||||
SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")}
|
||||
|
||||
GEN = r'''
|
||||
import io, random, sys
|
||||
from PIL import Image, ImageChops, ImageDraw
|
||||
w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3]
|
||||
if fmt == "JPEG": # photo-like noise: realistic JPEG sizes
|
||||
chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)]
|
||||
grain = Image.effect_noise((w, h), 12)
|
||||
img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan])
|
||||
else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling)
|
||||
img = Image.linear_gradient("L").resize((w, h)).convert("RGB")
|
||||
draw, rnd = ImageDraw.Draw(img), random.Random(1)
|
||||
for _ in range(400):
|
||||
x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300)
|
||||
draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256)))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6}))
|
||||
sys.stdout.buffer.write(buf.getvalue())
|
||||
'''
|
||||
|
||||
|
||||
def image(size):
|
||||
w, h, fmt = SIZES[size]
|
||||
cache = OUT / ("fixture-%s.%s" % (size, fmt.lower()))
|
||||
if not cache.exists():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt],
|
||||
cwd=ROOT, capture_output=True, check=True).stdout
|
||||
cache.write_bytes(data)
|
||||
return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg"
|
||||
|
||||
|
||||
def set_worker(replicas, concurrency, prefetch, cpus="2.00"):
|
||||
env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency),
|
||||
WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus)
|
||||
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"],
|
||||
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
|
||||
|
||||
|
||||
def set_admission(max_pending):
|
||||
env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending))
|
||||
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"],
|
||||
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
|
||||
|
||||
|
||||
def parse_mem(text):
|
||||
value = text.split("/")[0].strip()
|
||||
for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)):
|
||||
if value.endswith(unit):
|
||||
return float(value[: -len(unit)]) * mult
|
||||
return 0.0
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
def __init__(self, tag):
|
||||
super().__init__(daemon=True)
|
||||
self.tag = tag
|
||||
self.stop = threading.Event()
|
||||
self.rss, self.pending, self.cpu = [], [], []
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
ids = compose("ps", "-q", "worker", check_rc=False).split()
|
||||
if ids:
|
||||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids],
|
||||
capture_output=True, text=True).stdout
|
||||
rows = [line.split("|") for line in out.splitlines() if "|" in line]
|
||||
self.rss.append(sum(parse_mem(r[0]) for r in rows))
|
||||
self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows))
|
||||
try:
|
||||
self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
|
||||
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')"
|
||||
% self.tag)[0][0]))
|
||||
except Exception:
|
||||
pass
|
||||
self.stop.wait(1.0)
|
||||
|
||||
|
||||
def pct(values, q):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
|
||||
|
||||
|
||||
def drive(c, actor, chat, body, ctype, rate, duration, tag):
|
||||
"""Open-loop: request i is sent at t0 + i/rate regardless of earlier responses."""
|
||||
total = int(rate * duration)
|
||||
results = {"accepted": 0, "rejected_503": 0, "other_errors": 0}
|
||||
lock = threading.Lock()
|
||||
|
||||
def one(i):
|
||||
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i)))
|
||||
try:
|
||||
r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype})
|
||||
key = "accepted" if r["status"] == 202 else "rejected_503"
|
||||
except Exception: # connection errors, timeouts: counted, never retried
|
||||
key = "other_errors"
|
||||
with lock:
|
||||
results[key] += 1
|
||||
|
||||
t0 = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=64) as pool:
|
||||
for i in range(total):
|
||||
delay = t0 + i / rate - time.monotonic()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
pool.submit(one, i)
|
||||
results["send_seconds"] = round(time.monotonic() - t0, 2)
|
||||
results["offered"] = total
|
||||
return results
|
||||
|
||||
|
||||
def wait_drained(tag, seconds):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
|
||||
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0])
|
||||
if left == 0:
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def job_metrics(tag):
|
||||
rows = sql(
|
||||
"SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at),"
|
||||
" j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
|
||||
" WHERE a.filename LIKE 'load-%s-%%'" % tag)
|
||||
done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"]
|
||||
lag = [s - a for a, s, f in done]
|
||||
ttr = [f - a for a, s, f in done]
|
||||
span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0
|
||||
return {
|
||||
"succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"),
|
||||
"retries": sum(int(r[4]) - 1 for r in rows),
|
||||
"throughput_jobs_s": round(len(done) / span, 3) if span else None,
|
||||
"lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0),
|
||||
"ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0),
|
||||
}
|
||||
|
||||
|
||||
def record(summary):
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
with open(OUT / "results.jsonl", "a") as handle:
|
||||
handle.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
||||
log(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
|
||||
def experiment(args, label, max_pending=None):
|
||||
c = Client(timeout=60)
|
||||
wait_http_ready(c)
|
||||
alice = c.user("load")
|
||||
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]},
|
||||
schema="Chat")["json"]
|
||||
body, ctype = image(args.size)
|
||||
set_worker(args.replicas, args.concurrency, args.prefetch)
|
||||
time.sleep(2)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
sampler = Sampler(tag)
|
||||
sampler.start()
|
||||
log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % (
|
||||
label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch))
|
||||
sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag)
|
||||
load_end = time.monotonic()
|
||||
drained = wait_drained(tag, args.drain_timeout)
|
||||
drain_seconds = round(time.monotonic() - load_end, 2)
|
||||
sampler.stop.set()
|
||||
sampler.join()
|
||||
summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate,
|
||||
"duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency,
|
||||
"prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained,
|
||||
"drain_after_load_s": drain_seconds, **job_metrics(tag),
|
||||
"pending_max": max(sampler.pending or [0]),
|
||||
"worker_mem_peak_mib": round(max(sampler.rss or [0]), 1),
|
||||
"worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None,
|
||||
"worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1),
|
||||
"at": time.strftime("%Y-%m-%dT%H:%M:%S")}
|
||||
return summary, c, alice, chat
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("mode", choices=["run", "overload"])
|
||||
parser.add_argument("--size", choices=sorted(SIZES), default="medium")
|
||||
parser.add_argument("--rate", type=float, default=4)
|
||||
parser.add_argument("--duration", type=float, default=30)
|
||||
parser.add_argument("--replicas", type=int, default=1)
|
||||
parser.add_argument("--concurrency", type=int, default=2)
|
||||
parser.add_argument("--prefetch", type=int, default=2)
|
||||
parser.add_argument("--max-pending", type=int, default=30)
|
||||
parser.add_argument("--drain-timeout", type=float, default=600)
|
||||
parser.add_argument("--label", default="")
|
||||
args = parser.parse_args()
|
||||
if args.mode == "run":
|
||||
summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch))
|
||||
record(summary)
|
||||
return
|
||||
# Overload: admission limit on, offered rate far above capacity, then recovery.
|
||||
set_admission(args.max_pending)
|
||||
try:
|
||||
summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending)
|
||||
# Recovery probe: after the backlog drained, a fresh upload must be ready quickly again.
|
||||
body, ctype = image("small")
|
||||
t = time.monotonic()
|
||||
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body,
|
||||
headers={"Content-Type": ctype}, schema="Attachment")["json"]
|
||||
while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready":
|
||||
time.sleep(0.2)
|
||||
summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2)
|
||||
record(summary)
|
||||
finally:
|
||||
set_admission(200)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
"""Helpers for student resilience tests: the public HTTP client plus docker compose / psql / mc access.
|
||||
|
||||
Standard library only (Python 3.9+). Images that need Pillow are generated inside the API container.
|
||||
Never prints tokens or secrets.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check, run # noqa: E402,F401
|
||||
|
||||
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def log(*parts):
|
||||
print(time.strftime("%H:%M:%S"), *parts, flush=True)
|
||||
|
||||
|
||||
def compose(*args, check_rc=True, capture=True, timeout=180):
|
||||
done = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=capture, text=True, timeout=timeout)
|
||||
if check_rc and done.returncode != 0:
|
||||
raise Failure("docker compose %s failed: %s" % (" ".join(args[:2]), (done.stderr or "")[-300:]))
|
||||
return done.stdout if capture else ""
|
||||
|
||||
|
||||
def started_at(*services):
|
||||
"""Container name -> State.StartedAt, proof that a restart really happened."""
|
||||
ids = compose("ps", "-q", *services).split()
|
||||
out = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids],
|
||||
capture_output=True, text=True).stdout
|
||||
return "; ".join(line.strip("/")[:60] for line in out.splitlines())
|
||||
|
||||
|
||||
def sql(query):
|
||||
"""Run a read query in postgres; returns rows as lists of strings (tab separated)."""
|
||||
out = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t", "-c", query)
|
||||
return [line.split("\t") for line in out.splitlines() if line]
|
||||
|
||||
|
||||
def s3_keys(prefix):
|
||||
"""Object keys starting with prefix, listed with the root credentials inside a throwaway minio-init container."""
|
||||
wanted, prefix = prefix, prefix[:prefix.rfind("/") + 1]
|
||||
script = ('mc alias set l http://minio:9000 "$(cat /run/secrets/minio_root_user)" '
|
||||
'"$(cat /run/secrets/minio_root_password)" >/dev/null && '
|
||||
'mc ls --recursive --json l/messenger-attachments/%s || true' % prefix)
|
||||
out = compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "minio-init", "-c", script)
|
||||
keys = []
|
||||
for line in out.splitlines():
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if item.get("key"):
|
||||
keys.append(prefix + item["key"])
|
||||
return sorted(k for k in keys if k.startswith(wanted))
|
||||
|
||||
|
||||
def queue_depth(queue="attachments.process"):
|
||||
out = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "--vhost", "messenger", "list_queues",
|
||||
"name", "messages", "messages_ready", "messages_unacknowledged")
|
||||
for line in out.splitlines():
|
||||
parts = line.split()
|
||||
if parts and parts[0] == queue:
|
||||
return {"messages": int(parts[1]), "ready": int(parts[2]), "unacked": int(parts[3])}
|
||||
return None
|
||||
|
||||
|
||||
def wait_queue(min_ready=1, seconds=30):
|
||||
"""Quorum queue counters are refreshed by the stats emitter (~5 s), so poll."""
|
||||
deadline = time.monotonic() + seconds
|
||||
depth = None
|
||||
while time.monotonic() < deadline:
|
||||
depth = queue_depth()
|
||||
if depth and depth["ready"] >= min_ready:
|
||||
return depth
|
||||
time.sleep(1)
|
||||
return depth
|
||||
|
||||
|
||||
def jobs_cli(*args):
|
||||
done = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "scripts/jobs.py", *args],
|
||||
cwd=ROOT, capture_output=True, text=True, timeout=60)
|
||||
return done.returncode, json.loads(done.stdout) if done.stdout.strip() else None
|
||||
|
||||
|
||||
def wait_http_ready(c, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure("API not ready after %ds" % seconds)
|
||||
|
||||
|
||||
def upload(c, actor, chat, body=SAMPLE, filename="sample.png", ctype="image/png", expected=202):
|
||||
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe=""))
|
||||
schema = "Attachment" if expected == 202 else None
|
||||
return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype}, schema=schema)["json"]
|
||||
|
||||
|
||||
def get(c, actor, att_id):
|
||||
return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"]
|
||||
|
||||
|
||||
def wait_status(c, actor, att_id, statuses, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
seen = []
|
||||
while time.monotonic() < deadline:
|
||||
att = get(c, actor, att_id)
|
||||
if not seen or seen[-1] != att["status"]:
|
||||
seen.append(att["status"])
|
||||
if att["status"] in statuses:
|
||||
return att, seen
|
||||
time.sleep(0.5)
|
||||
raise Failure("attachment %s did not reach %s in %ds (seen %s)" % (att_id, statuses, seconds, seen))
|
||||
|
||||
|
||||
def objects_of(att_id):
|
||||
return s3_keys("attachments/%s/" % att_id) + s3_keys("staging/%s" % att_id)
|
||||
|
||||
|
||||
def unique_name(fault=None):
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
return ("fault-%s-%s.png" % (fault, tag)) if fault else ("plain-%s.png" % tag)
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded buffering at the limit: Tempo stopped under 100 req/s until the exporter queue is FULL.
|
||||
Expected: queue_size stops at capacity, extra spans are dropped and counted, collector RSS stays under the container
|
||||
limit, API has no unexpected errors. python3 student-tests/test_exporter_queue_full.py (~5 min)
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
|
||||
import obs
|
||||
from openloop import Load
|
||||
from stand import check, compose, log, run
|
||||
from test_incidents import docker_mem
|
||||
|
||||
|
||||
def value(expr):
|
||||
return obs.prom_scalar("sum(%s)" % expr)
|
||||
|
||||
|
||||
def main():
|
||||
load = Load("http://localhost:8080", users=4, threads=96)
|
||||
stop, summary = threading.Event(), {}
|
||||
thread = threading.Thread(target=lambda: summary.update(load.run(100, 900, 1, stop_event=stop)), daemon=True)
|
||||
thread.start()
|
||||
time.sleep(20)
|
||||
q = 'otelcol_exporter_queue_size{exporter="otlp/tempo"}'
|
||||
capacity = value('otelcol_exporter_queue_capacity{exporter="otlp/tempo"}')
|
||||
dropped0, refused0 = value("otelcol_exporter_enqueue_failed_spans_total"), value("otelcol_receiver_refused_spans_total")
|
||||
compose("stop", "-t", "5", "tempo")
|
||||
t0 = time.time()
|
||||
log("tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=%.0f batches" % capacity)
|
||||
peak_q, mems = 0, []
|
||||
try:
|
||||
for _ in range(16):
|
||||
time.sleep(12)
|
||||
size, mem = value(q), docker_mem("otel-collector")[0]
|
||||
peak_q, _ = max(peak_q, size), mems.append(mem)
|
||||
log(" +%3d s queue=%.0f/%.0f collector_rss=%s dropped=%.0f refused=%.0f" % (
|
||||
time.time() - t0, size, capacity, mem, value("otelcol_exporter_enqueue_failed_spans_total") - dropped0,
|
||||
value("otelcol_receiver_refused_spans_total") - refused0))
|
||||
dropped = value("otelcol_exporter_enqueue_failed_spans_total") - dropped0
|
||||
refused = value("otelcol_receiver_refused_spans_total") - refused0
|
||||
finally:
|
||||
compose("start", "tempo")
|
||||
stop.set()
|
||||
thread.join(60)
|
||||
log("result: peak queue=%.0f/%.0f, dropped by full queue=%.0f spans, refused by memory_limiter=%.0f spans, peak RSS=%s MiB"
|
||||
% (peak_q, capacity, dropped, refused, max(float(m.replace("MiB", "")) for m in mems)))
|
||||
log("API: requests=%s ok=%s unexpected_errors=%s p99=%s ms" % (
|
||||
summary["requests"], summary["ok"], summary["server_errors_unexpected"], summary["latency_ms_plain_ok"]["p99"]))
|
||||
check(peak_q <= capacity, "queue must stay within capacity")
|
||||
check(peak_q >= capacity * 0.95 or refused > 0, "the limit must really be reached")
|
||||
check(dropped + refused > 0, "overflow must be counted")
|
||||
check(summary["server_errors_unexpected"] == 0, "API must be unaffected")
|
||||
print("PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Executable
+194
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade 4 fault-injection suite (needs FAULT_INJECTION=1 on the stand, see REPORT.md).
|
||||
|
||||
Each scenario uploads a file whose display name selects an injected fault (app/faults.py) and checks the
|
||||
recovery through the public API plus DB/S3/broker state:
|
||||
A api_crash_after_commit crash between COMMIT(job+outbox) and publish -> another replica's relay publishes
|
||||
B worker_crash_after_put crash after S3 writes, before result COMMIT -> lease expiry -> same keys, one result
|
||||
C worker_crash_after_commit crash after COMMIT, before ack -> redelivery is a no-op
|
||||
D transient3 3 transient errors -> retries with growing jittered backoff -> ready
|
||||
E permanent permanent error -> failed at attempt 1, no retry, content/attach 409
|
||||
F transient6 attempts exhausted -> failed/retry_exhausted -> registry -> explicit replay -> ready;
|
||||
replay of a ready job is refused
|
||||
G worker_hang hung worker (no heartbeats) -> other worker takes over, hung one is fenced
|
||||
H poison message invalid broker message -> DLQ, not retried forever
|
||||
I broker down RabbitMQ stopped: upload still 202 (job waits in outbox) -> broker back -> ready
|
||||
usage: python3 student-tests/test_failure_windows.py [A B C ...]
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
|
||||
from stand import (Client, Failure, check, compose, get, jobs_cli, log, objects_of, queue_depth, run, sql, upload,
|
||||
unique_name, wait_http_ready, wait_status)
|
||||
|
||||
|
||||
def audit(att_id):
|
||||
return sql("SELECT action, actor, attempt, coalesce(detail->>'error_code',''), coalesce(detail->>'delay_seconds','')"
|
||||
" FROM job_audit WHERE attachment_id='%s' ORDER BY id" % att_id)
|
||||
|
||||
|
||||
def show_audit(att_id):
|
||||
for action, actor, attempt, code, delay in audit(att_id):
|
||||
log(" audit attempt=%s %-16s %-28s %s %s" % (attempt, action, actor[:28], code, ("delay=%ss" % delay) if delay else ""))
|
||||
|
||||
|
||||
def one_result(att_id):
|
||||
variants = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att_id)[0][0]
|
||||
succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att_id)[0][0]
|
||||
keys = objects_of(att_id)
|
||||
check(variants == "3", "exactly one set of 3 variant rows, got %s" % variants)
|
||||
check(succeeded == "1", "exactly one committed success, got %s" % succeeded)
|
||||
check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys),
|
||||
"original + 3 variants, staging cleaned: %s" % keys)
|
||||
|
||||
|
||||
def ready_and_single(c, actor, att_id, seconds=120):
|
||||
att, seen = wait_status(c, actor, att_id, ("ready", "failed"), seconds)
|
||||
check(att["status"] == "ready", "expected ready, got %s/%s" % (att["status"], att["error_code"]))
|
||||
log(" statuses:", " -> ".join(seen))
|
||||
show_audit(att_id)
|
||||
one_result(att_id)
|
||||
return att
|
||||
|
||||
|
||||
def scenario_a(c, alice, bob, chat):
|
||||
name = unique_name("api_crash_after_commit")
|
||||
try:
|
||||
upload(c, alice, chat, filename=name, expected=[202, 500, 502, 503])
|
||||
except Failure as exc:
|
||||
log(" client saw:", exc)
|
||||
wait_http_ready(c)
|
||||
rows = sql("SELECT id FROM attachments WHERE filename='%s'" % name)
|
||||
check(len(rows) == 1, "committed attachment must exist after API crash")
|
||||
att_id = rows[0][0]
|
||||
log(" committed attachment %s survived the API crash" % att_id)
|
||||
ready_and_single(c, bob, att_id)
|
||||
pub = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id"
|
||||
" WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att_id)[0][0]
|
||||
check(pub == "1", "outbox row published exactly once by the relay")
|
||||
|
||||
|
||||
def scenario_b(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, filename=unique_name("worker_crash_after_put"))
|
||||
ready_and_single(c, bob, att["id"])
|
||||
actions = [r[0] for r in audit(att["id"])]
|
||||
check("retry_scheduled" in actions, "lease expiry must schedule a new attempt")
|
||||
|
||||
|
||||
def scenario_c(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, filename=unique_name("worker_crash_after_commit"))
|
||||
ready_and_single(c, bob, att["id"])
|
||||
time.sleep(8)
|
||||
logs = compose("logs", "--since", "60s", "worker")
|
||||
check("decision=done" in logs, "redelivered message after crash must be a no-op")
|
||||
one_result(att["id"])
|
||||
|
||||
|
||||
def scenario_d(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, filename=unique_name("transient3"))
|
||||
ready_and_single(c, bob, att["id"])
|
||||
delays = [float(r[4]) for r in audit(att["id"]) if r[0] == "retry_scheduled"]
|
||||
check(len(delays) == 3, "three retries expected, got %s" % delays)
|
||||
bounds = [(1, 2), (2, 4), (4, 8)]
|
||||
check(all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "equal-jitter backoff bounds %s" % delays)
|
||||
log(" backoff delays:", delays)
|
||||
|
||||
|
||||
def scenario_e(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, filename=unique_name("permanent"))
|
||||
failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 60)
|
||||
check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", "permanent -> failed/invalid_image")
|
||||
check(failed["variants"] == [], "failed attachment has no variants")
|
||||
show_audit(att["id"])
|
||||
check([r[0] for r in audit(att["id"])].count("claimed") == 1, "permanent errors are not retried")
|
||||
c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
|
||||
c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "k-" + att["id"]})
|
||||
|
||||
|
||||
def scenario_f(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, filename=unique_name("transient6"))
|
||||
failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 120)
|
||||
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "exhausted -> retry_exhausted")
|
||||
attempts = sql("SELECT attempt, max_attempts FROM attachment_jobs WHERE attachment_id='%s'" % att["id"])[0]
|
||||
check(attempts == ["5", "5"], "bounded: stopped at attempt 5 of 5, got %s" % attempts)
|
||||
rc, registry = jobs_cli("failed")
|
||||
check(any(r["attachment_id"] == att["id"] for r in registry), "failed job listed in operator registry")
|
||||
log(" failed registry contains the job; replay")
|
||||
rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite")
|
||||
check(rc == 0 and result["result"] == "queued", "replay must queue: %s" % result)
|
||||
ready_and_single(c, bob, att["id"], 120)
|
||||
rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite")
|
||||
check(rc == 3 and result["result"] == "refused", "replay of a ready job must be refused: %s" % result)
|
||||
log(" second replay refused:", result["reason"])
|
||||
one_result(att["id"])
|
||||
|
||||
|
||||
def scenario_g(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, filename=unique_name("worker_hang"))
|
||||
ready_and_single(c, bob, att["id"], 150)
|
||||
rows = audit(att["id"])
|
||||
check(any(r[0] == "retry_scheduled" and r[3] == "lease_expired" for r in rows), "lease expiry detected")
|
||||
time.sleep(30) # the hung worker wakes up and must be fenced
|
||||
logs = compose("logs", "--since", "180s", "worker")
|
||||
check("lease_lost" in logs or "job_result_not_committed" in logs, "hung worker must be fenced by the lease")
|
||||
one_result(att["id"])
|
||||
|
||||
|
||||
POISON = ("import asyncio, aio_pika\n"
|
||||
"from app import config, broker\n"
|
||||
"async def m():\n"
|
||||
" conn = await broker.connect(); ch = await conn.channel(publisher_confirms=True)\n"
|
||||
" ex, _ = await broker.declare(ch)\n"
|
||||
" await ex.publish(aio_pika.Message(b'{not json', delivery_mode=2), routing_key=broker.ROUTING_KEY)\n"
|
||||
" await conn.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
|
||||
|
||||
def scenario_h(c, alice, bob, chat):
|
||||
before = (queue_depth("attachments.dlq") or {}).get("messages", 0)
|
||||
compose("exec", "-T", "api-1", "python", "-c", POISON)
|
||||
deadline = time.monotonic() + 30
|
||||
after = before
|
||||
while time.monotonic() < deadline and after <= before:
|
||||
time.sleep(1)
|
||||
after = (queue_depth("attachments.dlq") or {}).get("messages", 0)
|
||||
check(after == before + 1, "poison message must land in attachments.dlq (%s -> %s)" % (before, after))
|
||||
log(" DLQ depth %s -> %s" % (before, after))
|
||||
|
||||
|
||||
def scenario_i(c, alice, bob, chat):
|
||||
compose("stop", "-t", "10", "rabbitmq")
|
||||
try:
|
||||
att = upload(c, alice, chat, filename=unique_name())
|
||||
check(att["status"] == "queued", "upload accepted while broker is down")
|
||||
c.message(alice, chat, "", attachments=[att["id"]])
|
||||
time.sleep(3)
|
||||
pending = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id"
|
||||
" WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0]
|
||||
check(pending == "1", "job must wait in the outbox, got %s" % pending)
|
||||
check(get(c, bob, att["id"])["status"] == "queued", "still queued while broker is down")
|
||||
log(" broker down: upload 202, outbox holds 1 unpublished job")
|
||||
finally:
|
||||
compose("start", "rabbitmq")
|
||||
ready_and_single(c, bob, att["id"], 120)
|
||||
|
||||
|
||||
SCENARIOS = {"A": scenario_a, "B": scenario_b, "C": scenario_c, "D": scenario_d, "E": scenario_e,
|
||||
"F": scenario_f, "G": scenario_g, "H": scenario_h, "I": scenario_i}
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
picked = sys.argv[1:] or list(SCENARIOS)
|
||||
for key in picked:
|
||||
log("scenario %s: %s" % (key, SCENARIOS[key].__name__))
|
||||
started = time.monotonic()
|
||||
SCENARIOS[key](c, alice, bob, chat)
|
||||
log("scenario %s PASS (%.1fs)" % (key, time.monotonic() - started))
|
||||
print("PASS: failure windows %s" % " ".join(picked))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 controlled incidents, diagnosed ONLY from stored telemetry (Prometheus / Loki / Tempo), with alert
|
||||
firing -> resolved timings. A background open-loop traffic (messages + history, 2% slow handler requests) runs all
|
||||
the time so that error ratios have a denominator.
|
||||
|
||||
A. worker stopped -> MessengerQueueLagGrowing firing; start worker -> backlog drains -> resolved
|
||||
B. DB table lock (slow DB) -> MessengerApiHighErrorRate firing; trace shows the INSERT span waiting 5 s on the lock
|
||||
(vs. an X-Fault: slow request: long handler, short DB spans); unlock -> resolved
|
||||
C. Tempo + Loki stopped -> API keeps working, collector queue bounded, drops counted, TelemetryExportFailing;
|
||||
collector stopped -> API keeps working, SDK queues bounded (RSS flat)
|
||||
|
||||
python3 student-tests/test_incidents.py [--only A,B,C] (stack running, FAULT_INJECTION=1; ~15 min)
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import obs
|
||||
from openloop import Load
|
||||
from stand import Client, check, compose, log, run, upload, wait_status
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RESULTS = {}
|
||||
|
||||
|
||||
def docker_mem(service):
|
||||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}"],
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
name, mem, cpu = line.split("\t")
|
||||
if name.startswith("mtusi-lab6-%s-" % service):
|
||||
return mem.split(" / ")[0], cpu
|
||||
return "stopped", "-"
|
||||
|
||||
|
||||
def wait_alert(name, want, seconds, t0):
|
||||
"""want='firing' waits for firing; want='resolved' waits until the alert is gone. Returns seconds since t0."""
|
||||
deadline = time.monotonic() + seconds
|
||||
pending_at = None
|
||||
while time.monotonic() < deadline:
|
||||
state = obs.alerts().get(name)
|
||||
if state == "pending" and pending_at is None:
|
||||
pending_at = time.time() - t0
|
||||
log(" %s pending (+%.0f s)" % (name, pending_at))
|
||||
if (want == "firing" and state == "firing") or (want == "resolved" and state is None):
|
||||
elapsed = time.time() - t0
|
||||
log(" %s %s (+%.0f s)" % (name, want, elapsed))
|
||||
return elapsed, pending_at
|
||||
time.sleep(5)
|
||||
raise AssertionError("%s did not become %s within %d s" % (name, want, seconds))
|
||||
|
||||
|
||||
def psql(query):
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ A
|
||||
def incident_worker(c, alice, chat):
|
||||
log("=== A. worker stopped (queue lag)")
|
||||
t0 = time.time()
|
||||
compose("stop", "-t", "20", "worker")
|
||||
log("worker stopped at %s" % time.strftime("%H:%M:%S"))
|
||||
atts = [upload(c, alice, chat, filename="incident-a.png") for _ in range(3)]
|
||||
log("3 uploads accepted (202) while worker is down: %s" % [a["id"][:8] for a in atts])
|
||||
firing, pending = wait_alert("MessengerQueueLagGrowing", "firing", 300, t0)
|
||||
|
||||
log("diagnosis from telemetry:")
|
||||
age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})')
|
||||
depth = {m["state"]: v for m, v in obs.prom('max by (state) (messenger_queue_depth{queue="attachments.process"})')}
|
||||
processed = obs.prom_scalar('sum(rate(messenger_jobs_processed_total[1m]))')
|
||||
log(" metrics: oldest job age=%.0f s, depth=%s, worker jobs/s over 1m=%.3f" % (age, depth, processed))
|
||||
since = int((time.time() - t0) / 60) + 1
|
||||
worker_lines = obs.loki('{service_name="messenger-worker"} | event=~"job_.*"', minutes=since)
|
||||
worker_lines = [l for l in worker_lines if l["ts"] / 1e9 > t0 + 25]
|
||||
stop_lines = obs.loki('{service_name="messenger-worker"} | event=~"worker_stop.*"', minutes=since + 1)
|
||||
log(" logs: worker job_* events after stop: %d; last worker events: %s" % (
|
||||
len(worker_lines), [l.get("event") for l in stop_lines[-2:]]))
|
||||
traces = obs.tempo_search('{span.messenger.attachment_id="%s"}' % atts[0]["id"], minutes=since + 1)
|
||||
spans = obs.tempo_trace(traces[0]["traceID"]) if traces else []
|
||||
kinds = sorted({s["kind"] for s in spans if s["kind"] != "CLIENT"})
|
||||
log(" trace of upload %s: span kinds %s -> published, no CONSUMER: the job waits in the queue" % (
|
||||
atts[0]["id"][:8], kinds))
|
||||
check("PRODUCER" in kinds and "CONSUMER" not in kinds, "trace must stop at the producer while worker is down")
|
||||
|
||||
t1 = time.time()
|
||||
compose("start", "worker")
|
||||
log("worker started (+%.0f s)" % (t1 - t0))
|
||||
for att in atts:
|
||||
wait_status(c, alice, att["id"], ("ready",), 120)
|
||||
drained = time.time() - t1
|
||||
log("backlog drained: all 3 ready %.1f s after worker start" % drained)
|
||||
resolved, _ = wait_alert("MessengerQueueLagGrowing", "resolved", 300, t1)
|
||||
RESULTS["A_worker_stopped"] = {"alert_pending_s": pending, "alert_firing_s": round(firing), "age_at_firing_s":
|
||||
round(age), "drain_after_restart_s": round(drained, 1),
|
||||
"resolved_after_restart_s": round(resolved), "lost_jobs": 0}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ B
|
||||
def incident_db(c):
|
||||
log("=== B. DB lock on messages (slow/unavailable DB for writes and history reads)")
|
||||
t0 = time.time()
|
||||
locker = subprocess.Popen(
|
||||
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
|
||||
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;"],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
log("ACCESS EXCLUSIVE lock taken on table messages at %s" % time.strftime("%H:%M:%S"))
|
||||
try:
|
||||
firing, pending = wait_alert("MessengerApiHighErrorRate", "firing", 300, t0)
|
||||
log("diagnosis from telemetry:")
|
||||
ratio = obs.prom_scalar("messenger:http_error_ratio:rate2m")
|
||||
p95 = obs.prom_scalar("messenger:http_latency_p95:rate5m")
|
||||
by_route = obs.prom('sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count'
|
||||
'{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))')
|
||||
waiting = obs.prom_scalar("max(db_client_connection_pending_requests)")
|
||||
used = obs.prom_scalar('max(db_client_connection_count{service_name="messenger-api",db_client_connection_state="used"})')
|
||||
log(" metrics: 5xx ratio=%.1f%%, p95=%.2f s, pool used(max replica)=%.0f waiting=%.0f" % (
|
||||
ratio * 100, p95, used, waiting))
|
||||
for m, v in by_route:
|
||||
log(" 5xx/s %-40s %s %.2f" % (m.get("http_route"), m.get("http_response_status_code"), v))
|
||||
db_err = obs.tempo_search('{resource.service.name="messenger-api" && span.db.operation.name="INSERT"'
|
||||
' && status=error}', minutes=5, limit=5)
|
||||
check(db_err, "no trace with a failed DB span")
|
||||
spans = obs.tempo_trace(db_err[0]["traceID"])
|
||||
log(" trace %s (failed request):" % db_err[0]["traceID"])
|
||||
obs.print_tree(spans, out=lambda line: print(" " + line))
|
||||
raw = obs.tempo_raw(db_err[0]["traceID"])
|
||||
cause = next((name for name in ("LockNotAvailable", "QueryCanceled", "PoolTimeout") if name in raw), "?")
|
||||
db_span = max((s for s in spans if s["attrs"].get("db.system") == "postgresql"), key=lambda s: s["duration_ms"])
|
||||
server = [s for s in spans if s["kind"] == "SERVER"][0]
|
||||
log(" => DB span %s took %.0f ms of %.0f ms request, status %s, exception %s: the time is spent in PostgreSQL"
|
||||
% (db_span["name"], db_span["duration_ms"], server["duration_ms"], db_span["status"], cause))
|
||||
slow = obs.tempo_search('{span.messenger.fault="slow" && span.http.route="/api/v1/chats/{chat_id}/messages"}',
|
||||
minutes=10, limit=50)
|
||||
contrast = None
|
||||
for item in slow:
|
||||
sp = obs.tempo_trace(item["traceID"])
|
||||
srv = [s for s in sp if s["kind"] == "SERVER"]
|
||||
dbs = [s["duration_ms"] for s in sp if s["attrs"].get("db.system") == "postgresql"]
|
||||
if srv and dbs and srv[0]["attrs"].get("http.response.status_code") in ("200", 200) and max(dbs) < 100:
|
||||
contrast = (item["traceID"], srv[0]["duration_ms"], max(dbs), sum(dbs))
|
||||
break
|
||||
if contrast:
|
||||
log(" contrast: slow HANDLER trace %s: request %.0f ms, longest DB span %.1f ms, all DB %.1f ms"
|
||||
% contrast)
|
||||
errors = obs.loki('{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"', minutes=5)
|
||||
log(" logs: %d API warning/error records in 5 min, events=%s" % (
|
||||
len(errors), sorted({l.get("event") for l in errors})))
|
||||
blocker = psql("SELECT pid, state, wait_event_type, left(query, 60) FROM pg_stat_activity "
|
||||
"WHERE query LIKE '%LOCK TABLE messages%' AND pid <> pg_backend_pid()").strip()
|
||||
log(" operator step (pg_stat_activity) finds the blocker: %s" % blocker)
|
||||
finally:
|
||||
t1 = time.time()
|
||||
psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%LOCK TABLE messages%'"
|
||||
" AND pid <> pg_backend_pid()")
|
||||
locker.wait(30)
|
||||
log("lock released (+%.0f s)" % (t1 - t0))
|
||||
resolved, _ = wait_alert("MessengerApiHighErrorRate", "resolved", 400, t1)
|
||||
RESULTS["B_db_lock"] = {"alert_pending_s": pending, "alert_firing_s": round(firing),
|
||||
"error_ratio_at_firing": round(ratio, 3), "p95_at_firing_s": round(p95, 2),
|
||||
"db_span_ms": round(db_span["duration_ms"]), "request_ms": round(server["duration_ms"]),
|
||||
"cause": cause, "resolved_after_unlock_s": round(resolved)}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ C
|
||||
def counter(expr):
|
||||
return obs.prom_scalar("sum(%s)" % expr)
|
||||
|
||||
|
||||
def incident_telemetry(load):
|
||||
log("=== C1. Tempo and Loki stopped (telemetry backends down)")
|
||||
before_res = len(load.results)
|
||||
dropped0 = counter('otelcol_exporter_enqueue_failed_spans_total') + counter(
|
||||
'otelcol_exporter_enqueue_failed_log_records_total')
|
||||
failed0 = counter('otelcol_exporter_send_failed_spans_total') + counter(
|
||||
'otelcol_exporter_send_failed_log_records_total')
|
||||
t0 = time.time()
|
||||
compose("stop", "-t", "5", "tempo", "loki")
|
||||
samples = []
|
||||
for i in range(14): # ~140 s
|
||||
time.sleep(10)
|
||||
mem, cpu = docker_mem("otel-collector")
|
||||
q_tempo = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlp/tempo"})')
|
||||
q_loki = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})')
|
||||
samples.append({"t": round(time.time() - t0), "collector_mem": mem, "cpu": cpu, "queue_tempo": q_tempo,
|
||||
"queue_loki": q_loki})
|
||||
log(" +%3d s collector mem=%s cpu=%s queue tempo=%.0f loki=%.0f (capacity 100 batches)" % (
|
||||
samples[-1]["t"], mem, cpu, q_tempo, q_loki))
|
||||
alert_state = obs.alerts().get("TelemetryExportFailing")
|
||||
log(" alert TelemetryExportFailing during outage: %s" % alert_state)
|
||||
check(alert_state == "firing", "TelemetryExportFailing must fire while backends are down")
|
||||
window = [r for r in load.results[before_res:]]
|
||||
bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)]
|
||||
log(" API during the outage: %d requests, unexpected errors=%d; alert TelemetryExportFailing=%s" % (
|
||||
len(window), len(bad), alert_state))
|
||||
check(not bad, "API must keep working while telemetry backends are down")
|
||||
compose("start", "tempo", "loki")
|
||||
obs.wait_for(lambda: subprocess.run(["curl", "-sf", obs.TEMPO + "/ready"], capture_output=True).returncode == 0
|
||||
and subprocess.run(["curl", "-sf", obs.LOKI + "/ready"], capture_output=True).returncode == 0, 120, 3)
|
||||
t1 = time.time()
|
||||
log(" tempo/loki started and ready (+%.0f s)" % (t1 - t0))
|
||||
fresh = obs.wait_for(lambda: obs.tempo_search('{resource.service.name="messenger-api"}', start=t1,
|
||||
end=time.time() + 5, limit=20), 90, 5)
|
||||
log(" new traces after restart visible: %d (export resumed)" % len(fresh or []))
|
||||
check(fresh, "traces must flow again after the backends are back")
|
||||
resolved, _ = wait_alert("TelemetryExportFailing", "resolved", 300, t1)
|
||||
time.sleep(20)
|
||||
dropped = counter('otelcol_exporter_enqueue_failed_spans_total') + counter(
|
||||
'otelcol_exporter_enqueue_failed_log_records_total') - dropped0
|
||||
failed = counter('otelcol_exporter_send_failed_spans_total') + counter(
|
||||
'otelcol_exporter_send_failed_log_records_total') - failed0
|
||||
log(" telemetry lost and counted: enqueue_failed(queue full)=%.0f items, send_failed(retries exhausted)=%.0f items"
|
||||
% (dropped, failed))
|
||||
RESULTS["C1_backends_down"] = {"outage_s": round(t1 - t0), "api_requests": len(window), "api_unexpected_errors":
|
||||
len(bad), "collector_samples": samples, "alert": alert_state,
|
||||
"dropped_items": dropped, "send_failed_items": failed,
|
||||
"traces_after_restart": len(fresh or []),
|
||||
"alert_resolved_after_restart_s": round(resolved)}
|
||||
|
||||
log("=== C2. Collector stopped (SDK side buffering)")
|
||||
before_res = len(load.results)
|
||||
api_before = docker_mem("api-1")[0]
|
||||
compose("stop", "-t", "5", "otel-collector")
|
||||
mems = []
|
||||
for _ in range(6):
|
||||
time.sleep(10)
|
||||
mems.append(docker_mem("api-1")[0])
|
||||
window = load.results[before_res:]
|
||||
bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)]
|
||||
compose("start", "otel-collector")
|
||||
log(" api-1 RSS before=%s during=%s; %d requests, unexpected errors=%d" % (api_before, mems, len(window), len(bad)))
|
||||
check(not bad, "API must keep working while the collector is down")
|
||||
RESULTS["C2_collector_down"] = {"outage_s": 60, "api1_mem_before": api_before, "api1_mem_during": mems,
|
||||
"api_requests": len(window), "api_unexpected_errors": len(bad)}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--only", default="A,B,C")
|
||||
p.add_argument("--output", default=str(ROOT / "evidence/incidents.json"))
|
||||
args = p.parse_args()
|
||||
c = Client()
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
load = Load("http://localhost:8080", users=2, threads=48)
|
||||
stop = threading.Event()
|
||||
traffic = threading.Thread(target=lambda: RESULTS.setdefault("traffic", load.run(
|
||||
10, 3600, 0, slow_pct=2, error_pct=0, stop_event=stop)), daemon=True)
|
||||
traffic.start()
|
||||
log("background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow)")
|
||||
time.sleep(30)
|
||||
try:
|
||||
if "A" in args.only:
|
||||
incident_worker(c, alice, chat)
|
||||
if "B" in args.only:
|
||||
incident_db(c)
|
||||
if "C" in args.only:
|
||||
incident_telemetry(load)
|
||||
finally:
|
||||
stop.set()
|
||||
traffic.join(60)
|
||||
Path(args.output).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2, default=str) + "\n")
|
||||
log("summary written to %s" % args.output)
|
||||
print("PASS: incidents %s diagnosed from telemetry, alerts fired and resolved" % args.only)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Levels 3-4: one controlled upload and one failed request are found in traces, logs and metrics; privacy checks.
|
||||
|
||||
1. upload with fault-transient1 (attempt 1 fails transiently, attempt 2 succeeds) -> ONE trace:
|
||||
API SERVER span -> PRODUCER -> CONSUMER (retry, ERROR) -> PRODUCER -> CONSUMER (succeeded) -> S3/DB spans
|
||||
2. the same trace_id in Loki for messenger-api and messenger-worker log records
|
||||
3. X-Request-Id of a 500 (X-Fault: error) -> Loki log line -> its trace_id -> Tempo server span with that request_id
|
||||
4. no password / token / message text / filename in logs or traces; no ids in metric labels
|
||||
5. all metric names used by the dashboard and alert rules exist and have samples
|
||||
|
||||
python3 student-tests/test_observability.py (stack from REPORT.md must be running; FAULT_INJECTION=1)
|
||||
"""
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import obs
|
||||
from stand import Client, check, log, run, unique_name, upload, wait_status
|
||||
|
||||
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
|
||||
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url_full", "url_query",
|
||||
"filename", "text", "http_target", "url_path"}
|
||||
REQUIRED_METRICS = [
|
||||
"http_server_request_duration_seconds_bucket", "messenger_jobs_processed_total",
|
||||
"messenger_job_duration_seconds_bucket", "messenger_attachment_time_to_ready_seconds_bucket",
|
||||
"messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_worker_busy",
|
||||
"messenger_worker_capacity", "db_client_connection_count", "db_client_connection_pending_requests",
|
||||
"process_cpu_time_seconds_total", "process_memory_usage_bytes", "otelcol_receiver_accepted_spans_total",
|
||||
"otelcol_exporter_sent_spans_total", "otelcol_exporter_queue_size",
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
marker = "secret-text-" + uuid.uuid4().hex
|
||||
sent = c.message(alice, chat, marker)
|
||||
msg_rid = sent["headers"]["X-Request-Id"]
|
||||
log("message sent request_id=%s (text marker kept private)" % msg_rid)
|
||||
|
||||
# ---------------------------------------------------------------- 1. upload -> retry -> ready, one trace
|
||||
filename = unique_name("transient1")
|
||||
att = upload(c, alice, chat, filename=filename)
|
||||
ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90)
|
||||
check(ready["status"] == "ready", "upload must become ready after one retry")
|
||||
log("upload %s: %s" % (att["id"], " -> ".join(["queued"] + seen)))
|
||||
|
||||
def find():
|
||||
found = obs.tempo_search('{span.messenger.attachment_id="%s"}' % att["id"])
|
||||
if not found:
|
||||
return None
|
||||
spans = obs.tempo_trace(found[0]["traceID"])
|
||||
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
|
||||
producers = [s for s in spans if s["kind"] == "PRODUCER"]
|
||||
# Spans of one trace arrive from 3 processes with different batch delays: wait until all are in.
|
||||
return (found, spans) if len(consumers) >= 2 and len(producers) >= 2 and any(
|
||||
s["attrs"].get("messenger.job.outcome") == "succeeded" for s in consumers) else None
|
||||
|
||||
result = obs.wait_for(find, 60)
|
||||
check(result, "trace of the upload with both attempts not found in Tempo")
|
||||
found, spans = result
|
||||
trace_ids = {t["traceID"] for t in found}
|
||||
check(len(trace_ids) == 1, "upload and its jobs must be ONE trace, got %s" % trace_ids)
|
||||
trace_id = found[0]["traceID"]
|
||||
log("Tempo: trace %s, %d spans; tree (DB client spans omitted):" % (trace_id, len(spans)))
|
||||
obs.print_tree(spans, out=lambda line: print(" " + line), skip_db=True)
|
||||
|
||||
by_id = {s["span_id"]: s for s in spans}
|
||||
server = [s for s in spans if s["kind"] == "SERVER" and s["service"] == "messenger-api"]
|
||||
producers = [s for s in spans if s["kind"] == "PRODUCER"]
|
||||
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
|
||||
check(len(server) == 1 and server[0]["name"] == "POST /api/v1/chats/{chat_id}/attachments", "API server span")
|
||||
check(len(producers) >= 2 and len(consumers) >= 2, "two publish/consume pairs (attempt 1 + retry)")
|
||||
for cons in consumers:
|
||||
parent = by_id.get(cons["parent_id"])
|
||||
check(parent is not None and parent["kind"] == "PRODUCER", "consumer must be a child of a producer span")
|
||||
check(by_id[producers[0]["parent_id"]]["kind"] == "SERVER", "first producer is a child of the upload request")
|
||||
retry_parent = by_id[producers[1]["parent_id"]]
|
||||
check(retry_parent["kind"] == "CONSUMER" and retry_parent["attrs"].get("messenger.job.outcome") == "retry",
|
||||
"retry producer must be a child of the failed consumer")
|
||||
check(consumers[0]["status"] == "ERROR" and consumers[-1]["attrs"].get("messenger.job.outcome") == "succeeded",
|
||||
"attempt 1 ERROR/retry, last attempt succeeded")
|
||||
worker_ops = {s["name"] for s in spans if s["service"] == "messenger-worker"}
|
||||
for op in ("S3.GetObject", "S3.CopyObject", "S3.PutObject", "imageproc"):
|
||||
check(op in worker_ops, "worker span %s missing" % op)
|
||||
check(any(s["service"] == "messenger-worker" and s["attrs"].get("db.system") == "postgresql" for s in spans),
|
||||
"worker DB spans missing")
|
||||
log("PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links")
|
||||
|
||||
# ---------------------------------------------------------------- 2. logs of both services with the trace_id
|
||||
lines = obs.wait_for(lambda: [l for l in obs.loki('{service_name=~"messenger-api|messenger-worker"} | trace_id="%s"'
|
||||
% trace_id) if True] or None, 30)
|
||||
check(lines, "no Loki records with the trace_id")
|
||||
services = {l["service_name"] for l in lines}
|
||||
events = [(l["service_name"].replace("messenger-", ""), l.get("event")) for l in lines]
|
||||
log("Loki: %d records with trace_id=%s: %s" % (len(lines), trace_id, events))
|
||||
check(services == {"messenger-api", "messenger-worker"}, "logs of both services must carry the trace_id")
|
||||
check({"attachment_queued", "job_attempt_failed", "job_finished"} <= {e for _, e in events},
|
||||
"expected events attachment_queued, job_attempt_failed, job_finished")
|
||||
sample = lines[0]
|
||||
log("Loki record fields: %s" % sorted(k for k in sample if k not in ("line",)))
|
||||
|
||||
# ---------------------------------------------------------------- 3. X-Request-Id of an HTTP error -> log -> trace
|
||||
err = c.call("GET", "/api/v1/chats", 500, actor=alice, headers={"X-Fault": "error"})
|
||||
rid = err["headers"]["X-Request-Id"]
|
||||
check(err["json"]["error"]["request_id"] == rid and "Traceback" not in err["bytes"].decode(), "500 body is clean")
|
||||
err_lines = obs.wait_for(lambda: obs.loki('{service_name="messenger-api"} |= "%s"' % rid, minutes=5) or None, 30)
|
||||
check(err_lines, "500 request_id not found in Loki")
|
||||
err_trace = err_lines[0].get("trace_id")
|
||||
log("HTTP 500 X-Request-Id=%s -> Loki event=%s trace_id=%s" % (rid, err_lines[0].get("event"), err_trace))
|
||||
err_spans = obs.wait_for(lambda: obs.tempo_trace(err_trace) or None, 30)
|
||||
check(err_spans, "trace of the 500 not in Tempo")
|
||||
srv = [s for s in err_spans if s["kind"] == "SERVER"][0]
|
||||
check(srv["attrs"].get("messenger.request_id") == rid and srv["status"] == "ERROR"
|
||||
and str(srv["attrs"].get("http.response.status_code")) == "500", "server span must carry request_id and ERROR")
|
||||
log("Tempo: %s status=%s request_id matches; exception event recorded=%s" % (
|
||||
srv["name"], srv["status"], "exception" in obs.tempo_raw(err_trace)))
|
||||
|
||||
msg_trace = obs.wait_for(lambda: obs.tempo_search('{span.messenger.request_id="%s"}' % msg_rid) or None, 30)
|
||||
check(msg_trace, "message request trace not found by request_id")
|
||||
msg_spans = obs.tempo_trace(msg_trace[0]["traceID"])
|
||||
log("message POST trace %s: %s" % (msg_trace[0]["traceID"], [s["name"] for s in msg_spans]))
|
||||
|
||||
# ---------------------------------------------------------------- 4. privacy
|
||||
secrets = {"password": alice["registration"]["password"], "token": alice["token"], "message text": marker,
|
||||
"filename": filename}
|
||||
raw = obs.tempo_raw(trace_id) + obs.tempo_raw(msg_trace[0]["traceID"]) + obs.tempo_raw(err_trace)
|
||||
logs_raw = "\n".join(str(l) for l in obs.loki('{service_name=~"messenger-.*"}', minutes=10, limit=5000))
|
||||
for what, value in secrets.items():
|
||||
check(value not in raw, "%s leaked into traces" % what)
|
||||
check(value not in logs_raw, "%s leaked into logs" % what)
|
||||
# Image bytes: PNG signature/IHDR chunk as raw, escaped or base64 text (sample.png starts with iVBORw0KGgo).
|
||||
for word in ("Authorization", "Bearer ", "filename=", "X-Amz-Signature", "iVBORw0KGgo", "IHDR", "\\x89PNG",
|
||||
"\\u0089PNG"):
|
||||
check(word not in raw and word not in logs_raw, "%r found in telemetry" % word)
|
||||
log("PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from "
|
||||
"traces and logs")
|
||||
|
||||
series = obs.prom('{__name__=~"messenger_.*|http_server_.*|db_client_.*|process_.*"}')
|
||||
labels = set()
|
||||
for metric, _ in series:
|
||||
labels |= set(metric)
|
||||
for name, value in metric.items():
|
||||
check(not UUID_RE.search(value), "UUID-like value in metric label %s=%s" % (name, value))
|
||||
check(not labels & FORBIDDEN_LABELS, "forbidden metric labels: %s" % (labels & FORBIDDEN_LABELS))
|
||||
log("PASS: %d series, label names: %s" % (len(series), sorted(labels)))
|
||||
routes = sorted({m.get("http_route") for m, _ in series if m.get("http_route")})
|
||||
log("http_route values (templates only): %s" % routes)
|
||||
|
||||
# ---------------------------------------------------------------- 5. metrics referenced by dashboard/alerts
|
||||
time.sleep(12) # one metric export interval
|
||||
for name in REQUIRED_METRICS:
|
||||
check(obs.prom(name), "metric %s has no samples" % name)
|
||||
log("PASS: required metrics present: %s" % ", ".join(REQUIRED_METRICS))
|
||||
print("PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade 3 demo: worker stopped -> upload 202 -> message with attachment -> restart API + broker (volumes kept)
|
||||
-> worker started -> ready + thumbnail. Then an explicit duplicate delivery of the finished job is a no-op.
|
||||
|
||||
python3 student-tests/test_worker_down.py (stack from REPORT.md must be running)
|
||||
"""
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
from stand import (Client, check, compose, started_at, get, log, objects_of, wait_queue, run, sql, upload, wait_http_ready,
|
||||
wait_status, unique_name, SAMPLE)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
log("stop all workers")
|
||||
compose("stop", "-t", "20", "worker")
|
||||
try:
|
||||
att = upload(c, alice, chat, filename=unique_name())
|
||||
check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload must be 202 queued")
|
||||
log("upload 202 queued id=%s" % att["id"])
|
||||
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment")
|
||||
log("message %s created with queued attachment" % msg["id"])
|
||||
c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
|
||||
c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob)
|
||||
c.message(bob, chat, "другие сообщения работают")
|
||||
depth = wait_queue()
|
||||
log("broker queue while worker is down:", depth)
|
||||
check(depth and depth["ready"] >= 1, "job must be waiting in the broker")
|
||||
unpublished = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id"
|
||||
" WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0]
|
||||
check(unpublished == "0", "outbox row must be published with confirm")
|
||||
|
||||
log("restart api-1 api-2 rabbitmq (volumes kept)")
|
||||
before = started_at("api-1", "api-2", "rabbitmq")
|
||||
compose("restart", "-t", "10", "rabbitmq", "api-1", "api-2")
|
||||
wait_http_ready(c)
|
||||
after = started_at("api-1", "api-2", "rabbitmq")
|
||||
log("started before:", before)
|
||||
log("started after: ", after)
|
||||
check(before != after, "containers must really restart")
|
||||
check(get(c, bob, att["id"])["status"] == "queued", "still queued after restart, nothing lost")
|
||||
depth = wait_queue()
|
||||
log("broker queue after broker restart:", depth)
|
||||
check(depth and depth["ready"] >= 1, "durable message must survive broker restart")
|
||||
finally:
|
||||
log("start workers")
|
||||
compose("start", "worker")
|
||||
ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90)
|
||||
check(ready["status"] == "ready", "expected ready, got %s" % ready["status"])
|
||||
log("status sequence observed:", " -> ".join(seen))
|
||||
names = sorted(v["name"] for v in ready["variants"])
|
||||
check(names == ["crop", "optimized", "thumbnail"], "expected thumbnail+crop+optimized, got %s" % names)
|
||||
thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0]
|
||||
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail 256x160")
|
||||
body = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"]
|
||||
check(hashlib.sha256(body).hexdigest() == thumb["sha256"], "thumbnail checksum")
|
||||
orig = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"]
|
||||
check(orig == SAMPLE, "original unchanged byte for byte")
|
||||
keys = objects_of(att["id"])
|
||||
log("objects:", keys)
|
||||
check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), "original + 3 variants, staging removed")
|
||||
|
||||
log("redelivery: republish the finished job through the outbox (duplicate message)")
|
||||
sql("INSERT INTO outbox (job_id, payload, available_at, created_at) SELECT o.job_id, o.payload, now(), now()"
|
||||
" FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='%s' ORDER BY o.id LIMIT 1"
|
||||
% att["id"])
|
||||
time.sleep(5)
|
||||
again = get(c, bob, att["id"])
|
||||
check(again["status"] == "ready" and again["variants"] == ready["variants"], "duplicate must not change the result")
|
||||
rows = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att["id"])[0][0]
|
||||
succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att["id"])[0][0]
|
||||
check(rows == "3" and succeeded == "1", "one set of variants and one success (rows=%s succeeded=%s)" % (rows, succeeded))
|
||||
check(len(objects_of(att["id"])) == 4, "no new objects after duplicate")
|
||||
noop = compose("logs", "--since", "30s", "worker")
|
||||
check("decision=done" in noop, "worker must log the duplicate as no-op")
|
||||
log("duplicate acked as no-op: variants=%s successes=%s" % (rows, succeeded))
|
||||
print("PASS: worker down / API+broker restart / redelivery no-op")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MessengerWorkerSaturated: capacity is lowered (2 workers x 1 slot, 0.25 CPU each), uploads exceed it for longer than
|
||||
`for: 5m`; the alert must fire, and resolve after the normal configuration is restored and the backlog drains.
|
||||
|
||||
python3 student-tests/test_worker_saturation.py (stack running; ~10 min). Restores WORKER_* defaults at the end.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import obs
|
||||
from openloop import Load
|
||||
from stand import check, log, run
|
||||
from test_incidents import wait_alert
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ALERT = "MessengerWorkerSaturated"
|
||||
|
||||
|
||||
def worker(env):
|
||||
done = subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--no-deps", "--force-recreate",
|
||||
"worker"], cwd=ROOT, env={**os.environ, **env}, capture_output=True, text=True, timeout=240)
|
||||
check(done.returncode == 0, "worker recreate failed: %s" % done.stderr[-300:])
|
||||
|
||||
|
||||
def main():
|
||||
result = {}
|
||||
log("lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas")
|
||||
worker({"WORKER_CONCURRENCY": "1", "WORKER_PREFETCH": "1", "WORKER_CPUS": "0.25"})
|
||||
load = Load("http://localhost:8080", users=2, threads=32)
|
||||
stop = threading.Event()
|
||||
summary = {}
|
||||
thread = threading.Thread(target=lambda: summary.update(load.run(0, 900, upload_rate=4, stop_event=stop)), daemon=True)
|
||||
t0 = time.time()
|
||||
thread.start()
|
||||
log("uploads: 4/s open-loop (sample.png)")
|
||||
try:
|
||||
time.sleep(60)
|
||||
capacity = obs.prom_scalar("sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))")
|
||||
log("fresh capacity=%.0f slots (stale series of recreated containers are excluded)" % capacity)
|
||||
check(capacity == 2, "expected 2 slots, got %s" % capacity)
|
||||
firing, pending = wait_alert(ALERT, "firing", 480, t0)
|
||||
ratio = obs.prom_scalar("messenger:worker_saturation:ratio")
|
||||
depth = obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})')
|
||||
age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})')
|
||||
rate = obs.prom_scalar('sum(rate(messenger_jobs_processed_total{outcome="succeeded"}[2m]))')
|
||||
p95 = obs.prom_scalar('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[5m])))')
|
||||
others = obs.alerts()
|
||||
log("at firing: saturation=%.2f, queued=%.0f, oldest age=%.0f s, succeeded=%.2f jobs/s, job p95=%.2f s, alerts=%s"
|
||||
% (ratio, depth, age, rate, p95, others))
|
||||
check(ratio > 0.9, "saturation must be above the threshold")
|
||||
finally:
|
||||
stop.set()
|
||||
thread.join(60)
|
||||
log("restore default worker configuration")
|
||||
t1 = time.time()
|
||||
worker({})
|
||||
resolved, _ = wait_alert(ALERT, "resolved", 240, t1)
|
||||
drained = obs.wait_for(lambda: obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})',
|
||||
default=-1) == 0, 240, 5)
|
||||
log("backlog drained: %s (+%.0f s after restore)" % (bool(drained), time.time() - t1))
|
||||
check(drained, "backlog must drain")
|
||||
result.update({"alert_pending_s": pending, "alert_firing_s": round(firing), "saturation_at_firing": round(ratio, 3),
|
||||
"queued_at_firing": depth, "oldest_age_s": round(age), "jobs_per_s": round(rate, 2),
|
||||
"job_p95_s": round(p95, 2), "alerts_at_firing": others, "resolved_after_restore_s": round(resolved),
|
||||
"drained_after_restore_s": round(time.time() - t1), "uploads": summary.get("status_counts")})
|
||||
(ROOT / "evidence/worker_saturation.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 5.1 "Цена наблюдаемости": the same open-loop load against several telemetry configurations.
|
||||
|
||||
Configurations (applied by recreating api-1, api-2, worker and otel-collector with shell env overrides):
|
||||
off OTEL_SDK_DISABLED=true no SDK, no instrumentation (baseline)
|
||||
on100 sampler ratio 1.0, collector profile default every span exported
|
||||
head10 sampler ratio 0.1, collector profile default head sampling of root spans (children follow)
|
||||
tail sampler ratio 1.0, collector profile tail balanced: errors + >500 ms + async pipeline + 10% rest
|
||||
|
||||
Per run: recreate -> ready -> warm-up 20 s -> idle 20 s -> counters A -> load (rate, duration) -> idle 30 s ->
|
||||
counters B. Delta(B-A) = telemetry produced by the measured load. CPU is the exact cgroup delta over the load window
|
||||
(Docker API); RSS is the maximum of `docker stats` samples taken every 5 s. Afterwards Tempo is searched in the load window for traces of injected slow/error requests and
|
||||
upload pipelines, to count what survives sampling.
|
||||
|
||||
python3 student-tests/track1.py run [--configs off,on100,head10,tail] [--repeats 2] [--rate 60] [--duration 90]
|
||||
python3 student-tests/track1.py summary # evidence/track1/results.jsonl -> evidence/track1/summary.md
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import obs
|
||||
from openloop import Load
|
||||
from stand import Client, log
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence/track1"
|
||||
CONFIGS = {
|
||||
"off": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"},
|
||||
"on100": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"},
|
||||
"head10": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"},
|
||||
"tail": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"},
|
||||
}
|
||||
SERVICES = ("api-1", "api-2", "worker", "otel-collector", "tempo", "loki", "prometheus", "postgres", "nginx")
|
||||
COUNTERS = {
|
||||
# SDK side (sum over all instances; absent when the SDK is off)
|
||||
"sdk_span_decisions_sample": 'sum(messenger_telemetry_sampler_decisions_total{decision="sample"})',
|
||||
"sdk_span_decisions_drop_ratio": 'sum(messenger_telemetry_sampler_decisions_total{decision="drop",rule=~"ratio|parent"})',
|
||||
"sdk_spans_exported_ok": 'sum(messenger_telemetry_spans_exported_total{outcome="success"})',
|
||||
"sdk_spans_exported_failed": 'sum(messenger_telemetry_spans_exported_total{outcome="failure"})',
|
||||
# Collector
|
||||
"col_spans_in": "sum(otelcol_receiver_accepted_spans_total)",
|
||||
"col_logs_in": "sum(otelcol_receiver_accepted_log_records_total)",
|
||||
"col_points_in": "sum(otelcol_receiver_accepted_metric_points_total)",
|
||||
"col_spans_out": 'sum(otelcol_exporter_sent_spans_total{exporter="otlp/tempo"})',
|
||||
"col_spans_refused": "sum(otelcol_receiver_refused_spans_total)",
|
||||
"col_spans_dropped": 'sum(otelcol_exporter_enqueue_failed_spans_total) or vector(0)',
|
||||
"col_otlp_bytes_in": 'sum({__name__="http.server.request.size_bytes_total"})',
|
||||
# Backends
|
||||
"tempo_spans_in": "sum(tempo_distributor_spans_received_total)",
|
||||
"tempo_bytes_in": "sum(tempo_distributor_bytes_received_total)",
|
||||
"loki_bytes_in": "sum(loki_distributor_bytes_received_total)",
|
||||
"loki_lines_in": "sum(loki_distributor_lines_received_total)",
|
||||
"prom_samples_in": "sum(prometheus_tsdb_head_samples_appended_total)",
|
||||
}
|
||||
|
||||
|
||||
def compose(*args, env=None, timeout=300):
|
||||
done = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True,
|
||||
text=True, timeout=timeout, env={**os.environ, **(env or {})})
|
||||
if done.returncode != 0:
|
||||
raise RuntimeError("compose %s: %s" % (args[:2], done.stderr[-400:]))
|
||||
return done.stdout
|
||||
|
||||
|
||||
def apply(name):
|
||||
env = CONFIGS[name]
|
||||
compose("up", "-d", "--wait", "--force-recreate", "--no-deps", "otel-collector", "api-1", "api-2", "worker",
|
||||
env=env)
|
||||
compose("restart", "nginx") # re-resolve recreated replicas
|
||||
c = Client()
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", 200)
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(1)
|
||||
out = compose("exec", "-T", "api-1", "printenv", "OTEL_SDK_DISABLED", "OTEL_TRACES_SAMPLER_ARG")
|
||||
log("config %s applied: api-1 env %s" % (name, out.split()))
|
||||
|
||||
|
||||
def counters():
|
||||
values = {}
|
||||
for key, expr in COUNTERS.items():
|
||||
try:
|
||||
rows = obs.prom(expr)
|
||||
values[key] = rows[0][1] if rows else None
|
||||
except Exception:
|
||||
values[key] = None
|
||||
tail = {}
|
||||
for metric, value in obs.prom('{__name__=~"otelcol_processor_tail_sampling_count_traces_sampled.*"}'):
|
||||
tail["%s/%s" % (metric.get("policy"), metric.get("sampled"))] = value
|
||||
values["tail_policies"] = tail
|
||||
return values
|
||||
|
||||
|
||||
def delta(a, b):
|
||||
out = {}
|
||||
for key in COUNTERS:
|
||||
if a.get(key) is not None and b.get(key) is not None:
|
||||
out[key] = b[key] - a[key]
|
||||
elif b.get(key) is not None:
|
||||
out[key] = b[key] # series appeared during the window (counter started at 0)
|
||||
else:
|
||||
out[key] = None
|
||||
out["tail_policies"] = {k: v - a["tail_policies"].get(k, 0) for k, v in b["tail_policies"].items()}
|
||||
return out
|
||||
|
||||
|
||||
def parse_mem(text):
|
||||
"""'123.4MiB' -> 123.4 (MiB)."""
|
||||
match = re.match(r"\s*([\d.]+)\s*([A-Za-z]*)", text)
|
||||
if not match:
|
||||
return 0.0
|
||||
scale = {"B": 1 / 1048576, "kB": 1 / 1024, "KiB": 1 / 1024, "MiB": 1, "MB": 1, "GiB": 1024, "GB": 1024}
|
||||
return float(match.group(1)) * scale.get(match.group(2), 1)
|
||||
|
||||
|
||||
def sample_stats(stop, samples):
|
||||
while not stop.is_set():
|
||||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
|
||||
capture_output=True, text=True).stdout
|
||||
row = {}
|
||||
for line in out.splitlines():
|
||||
name, cpu, mem = line.split("\t")
|
||||
if not name.startswith("mtusi-lab6-"):
|
||||
continue
|
||||
svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0]
|
||||
cur = row.setdefault(svc, [0.0, 0.0])
|
||||
cur[0] += float(cpu.rstrip("%") or 0)
|
||||
cur[1] += parse_mem(mem.split(" / ")[0])
|
||||
samples.append(row)
|
||||
stop.wait(5)
|
||||
|
||||
|
||||
def docker_socket():
|
||||
host = subprocess.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
return host.replace("unix://", "")
|
||||
|
||||
|
||||
def cpu_seconds():
|
||||
"""Cumulative container CPU seconds per service from the Docker API (exact over a window, unlike docker stats)."""
|
||||
sock = docker_socket()
|
||||
ids = subprocess.run(["docker", "ps", "--filter", "label=com.docker.compose.project=mtusi-lab6", "--format",
|
||||
"{{.ID}} {{.Names}}"], capture_output=True, text=True).stdout.splitlines()
|
||||
out = {}
|
||||
for line in ids:
|
||||
cid, name = line.split()
|
||||
svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0]
|
||||
raw = subprocess.run(["curl", "-s", "--unix-socket", sock,
|
||||
"http://localhost/containers/%s/stats?stream=false&one-shot=true" % cid],
|
||||
capture_output=True, text=True).stdout
|
||||
out[svc] = out.get(svc, 0.0) + json.loads(raw)["cpu_stats"]["cpu_usage"]["total_usage"] / 1e9
|
||||
return out
|
||||
|
||||
|
||||
def count_traces(query, start, end):
|
||||
try:
|
||||
return len(obs.tempo_search(query, limit=10000, start=start, end=end))
|
||||
except Exception as exc:
|
||||
return "error:%s" % type(exc).__name__
|
||||
|
||||
|
||||
def one_run(name, rep, rate, duration):
|
||||
apply(name)
|
||||
load = Load("http://localhost:8080", users=4, threads=96)
|
||||
load.run(rate, 20, 0.5, 1, 1) # warm-up, same mix
|
||||
time.sleep(20)
|
||||
a = counters()
|
||||
load = Load("http://localhost:8080", users=4, threads=96)
|
||||
stop, samples = threading.Event(), []
|
||||
sampler = threading.Thread(target=sample_stats, args=(stop, samples), daemon=True)
|
||||
sampler.start()
|
||||
cpu_a = cpu_seconds()
|
||||
t_start = time.time()
|
||||
summary = load.run(rate, duration, 1, 1, 1)
|
||||
t_end = time.time()
|
||||
cpu_b = cpu_seconds()
|
||||
stop.set()
|
||||
sampler.join(15)
|
||||
time.sleep(30) # SDK batch (2 s) + collector batch (2 s) + tail decision_wait (10 s) + scrape (15 s)
|
||||
b = counters()
|
||||
d = delta(a, b)
|
||||
# % of one core averaged over the load window (cumulative cgroup CPU; several replicas of a service are summed)
|
||||
cpu = {svc: round((cpu_b[svc] - cpu_a[svc]) / (t_end - t_start) * 100, 2) for svc in cpu_b if svc in cpu_a}
|
||||
mem = {svc: round(max(s[svc][1] for s in samples if svc in s), 1) for svc in samples[0]} if samples else {}
|
||||
w0, w1 = int(t_start) - 2, int(t_end) + 15
|
||||
traces = {
|
||||
"server_api": count_traces('{resource.service.name="messenger-api" && kind=server}', w0, w1),
|
||||
"slow": count_traces('{span.messenger.fault="slow"}', w0, w1),
|
||||
"error": count_traces('{span.messenger.fault="error"}', w0, w1),
|
||||
"pipeline_consumer": count_traces('{kind=consumer && span.messenger.job.outcome="succeeded"}', w0, w1),
|
||||
}
|
||||
result = {"config": name, "rep": rep, "env": CONFIGS[name], "load": summary, "cpu_pct_avg": cpu,
|
||||
"mem_mib_max": mem, "telemetry_delta": d, "traces_in_tempo": traces,
|
||||
"window": [t_start, t_end], "stats_samples": len(samples)}
|
||||
log("%s#%d: ok_rps=%s p50/p95/p99=%s/%s/%s ms api_cpu=%s+%s%% spans_in=%s tempo_spans=%s slow=%s/%d err=%s/%d" % (
|
||||
name, rep, summary["successful_rps"], summary["latency_ms_plain_ok"]["p50"],
|
||||
summary["latency_ms_plain_ok"]["p95"], summary["latency_ms_plain_ok"]["p99"], cpu.get("api-1"),
|
||||
cpu.get("api-2"), d.get("col_spans_in"), d.get("tempo_spans_in"), traces["slow"],
|
||||
summary["faults_sent"]["slow"], traces["error"], summary["faults_sent"]["error"]))
|
||||
return result
|
||||
|
||||
|
||||
def cmd_run(args):
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
path = OUT / "results.jsonl"
|
||||
names = args.configs.split(",")
|
||||
try:
|
||||
for rep in range(1, args.repeats + 1):
|
||||
for name in names:
|
||||
result = one_run(name, rep, args.rate, args.duration)
|
||||
result["hardware"] = HARDWARE
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(result, ensure_ascii=False) + "\n")
|
||||
finally:
|
||||
log("restore default configuration (on100)")
|
||||
apply("on100")
|
||||
|
||||
|
||||
def fmt(v, digits=0):
|
||||
if v is None:
|
||||
return "—"
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
return ("%%.%df" % digits) % v
|
||||
|
||||
|
||||
def metric_500s(run_result):
|
||||
"""Injected 500s as seen by METRICS for this run: metrics are computed in the SDK, before any trace sampling."""
|
||||
start, end = run_result["window"]
|
||||
if run_result["config"] == "off":
|
||||
return None
|
||||
# Exact counter difference at the window borders (increase() extrapolates and overstates by ~10% here).
|
||||
expr = 'sum(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code="500"})'
|
||||
try:
|
||||
return obs.prom_scalar(expr, at=end + 20) - obs.prom_scalar(expr, at=start - 5)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def cmd_summary(_args):
|
||||
rows = [json.loads(line) for line in (OUT / "results.jsonl").read_text(encoding="utf-8").splitlines() if line]
|
||||
by = {}
|
||||
for r in rows:
|
||||
by.setdefault(r["config"], []).append(r)
|
||||
|
||||
def values(items, fn):
|
||||
return [v for v in (fn(x) for x in items) if isinstance(v, (int, float))]
|
||||
|
||||
def avg(items, fn):
|
||||
vals = values(items, fn)
|
||||
return statistics.mean(vals) if vals else None
|
||||
|
||||
def spread(items, fn, digits=1):
|
||||
vals = values(items, fn)
|
||||
if not vals:
|
||||
return "—"
|
||||
return "%s (%s…%s)" % (fmt(statistics.mean(vals), digits), fmt(min(vals), digits), fmt(max(vals), digits))
|
||||
|
||||
lat = lambda q: (lambda x: x["load"]["latency_ms_plain_ok"][q]) # noqa: E731
|
||||
cpu = lambda *svc: (lambda x: sum(x["cpu_pct_avg"].get(s, 0) for s in svc)) # noqa: E731
|
||||
delta_of = lambda key: (lambda x: x["telemetry_delta"].get(key)) # noqa: E731
|
||||
traces = lambda key: (lambda x: x["traces_in_tempo"].get(key)) # noqa: E731
|
||||
faults = lambda key: (lambda x: x["load"]["faults_sent"][key]) # noqa: E731
|
||||
names = [n for n in CONFIGS if n in by]
|
||||
|
||||
lines = ["Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.", "",
|
||||
"| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |",
|
||||
"|---|---|---|---|---|---|---|---|---|"]
|
||||
for n in names:
|
||||
i = by[n]
|
||||
lines.append("| %s | %d | %s | %s | %s | %s | %s | %s | %s |" % (
|
||||
n, len(i), fmt(avg(i, lambda x: x["load"]["successful_rps"]), 1),
|
||||
fmt(avg(i, lambda x: x["load"]["server_errors_unexpected"])), spread(i, lat("p50")), spread(i, lat("p95")),
|
||||
spread(i, lat("p99")), spread(i, cpu("api-1", "api-2")), spread(i, cpu("worker"))))
|
||||
lines += ["", "| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |",
|
||||
"|---|---|---|---|---|---|---|---|"]
|
||||
for n in names:
|
||||
i = by[n]
|
||||
mem = lambda s: (lambda x: x["mem_mib_max"].get(s)) # noqa: E731
|
||||
lines.append("| %s | %s | %s | %s | %s | %s | %s | %s |" % (
|
||||
n, spread(i, cpu("otel-collector"), 2), spread(i, cpu("tempo"), 2), spread(i, cpu("loki"), 2),
|
||||
spread(i, cpu("prometheus"), 2), fmt(avg(i, mem("api-1"))), fmt(avg(i, mem("otel-collector"))),
|
||||
fmt(avg(i, mem("tempo")))))
|
||||
lines += ["", "| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |",
|
||||
"|---|---|---|---|---|---|---|"]
|
||||
for n in names:
|
||||
i = by[n]
|
||||
kib = lambda key: (avg(i, delta_of(key)) / 1024 if avg(i, delta_of(key)) is not None else None) # noqa: E731
|
||||
lost = avg(i, lambda x: (x["telemetry_delta"].get("col_spans_refused") or 0) + (x["telemetry_delta"].get("col_spans_dropped") or 0))
|
||||
lines.append("| %s | %s | %s | %s | %s | %s | %s |" % (
|
||||
n, fmt(avg(i, delta_of("col_spans_in"))), fmt(avg(i, delta_of("tempo_spans_in"))), fmt(lost),
|
||||
fmt(kib("col_otlp_bytes_in")), fmt(kib("tempo_bytes_in")), fmt(kib("loki_bytes_in"))))
|
||||
lines += ["", "Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.", "",
|
||||
"| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |",
|
||||
"|---|---|---|---|---|---|"]
|
||||
for n in names:
|
||||
i = by[n]
|
||||
total = lambda fn: sum(values(i, fn)) # noqa: E731
|
||||
m500 = [metric_500s(x) for x in i]
|
||||
lines.append("| %s | %d / %d | %d / %d | %d / %d | %d / %d | %s / %d |" % (
|
||||
n, total(traces("server_api")), total(lambda x: x["load"]["requests"]),
|
||||
total(traces("slow")), total(faults("slow")), total(traces("error")), total(faults("error")),
|
||||
total(traces("pipeline_consumer")), total(lambda x: x["load"]["by_kind"]["upload"]["n"]),
|
||||
"—" if None in m500 else "%.0f" % sum(m500), total(faults("error"))))
|
||||
text = "\n".join(lines) + "\n"
|
||||
(OUT / "summary.md").write_text(text, encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
HARDWARE = {}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
r = sub.add_parser("run")
|
||||
r.add_argument("--configs", default="off,on100,head10,tail")
|
||||
r.add_argument("--repeats", type=int, default=2)
|
||||
r.add_argument("--rate", type=float, default=60)
|
||||
r.add_argument("--duration", type=float, default=90)
|
||||
sub.add_parser("summary")
|
||||
args = p.parse_args()
|
||||
if args.cmd == "run":
|
||||
info = subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} {{.MemTotal}}"],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
HARDWARE.update({"docker": info[0], "vm_cpus": int(info[1]), "vm_mem_gib": round(int(info[2]) / 2**30, 1)})
|
||||
cmd_run(args)
|
||||
else:
|
||||
cmd_summary(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Variant rules and worker resource limits. Runs INSIDE the worker container (Pillow + app code):
|
||||
|
||||
docker compose exec -T --index 1 worker python - < student-tests/variants_check.py
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
from app import config # noqa: E402
|
||||
|
||||
FAILS = []
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
print(("ok " if cond else "FAIL ") + msg, flush=True)
|
||||
if not cond:
|
||||
FAILS.append(msg)
|
||||
|
||||
|
||||
def run_child(path, mem_mb=config.WORKER_CHILD_MEMORY_MB, cpu=config.JOB_TIMEOUT_SECONDS, max_pixels=config.MAX_IMAGE_PIXELS):
|
||||
out = tempfile.mkdtemp()
|
||||
|
||||
def limits():
|
||||
resource.setrlimit(resource.RLIMIT_AS, (mem_mb << 20, mem_mb << 20))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
||||
|
||||
start = time.monotonic()
|
||||
done = subprocess.run([sys.executable, "-m", "app.imageproc", path, out, str(max_pixels)], capture_output=True,
|
||||
preexec_fn=limits, timeout=120, env={"PATH": os.environ["PATH"], "MALLOC_ARENA_MAX": "2"})
|
||||
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
produced = {v["name"]: v for v in json.loads(done.stdout)} if done.returncode == 0 else {}
|
||||
return done.returncode, produced, time.monotonic() - start, usage.ru_maxrss / 1024
|
||||
|
||||
|
||||
def save(img, fmt, **kw):
|
||||
path = tempfile.mktemp(suffix="." + fmt.lower())
|
||||
img.save(path, fmt, **kw)
|
||||
return path
|
||||
|
||||
|
||||
def dims(v):
|
||||
return v["width"], v["height"]
|
||||
|
||||
|
||||
def real(v):
|
||||
with Image.open(v["file"]) as im:
|
||||
return im.format, im.size, "exif" in im.info, im
|
||||
|
||||
|
||||
# 1. fixture 320x200 -> 256x160, crop 128, optimized not upscaled
|
||||
rc, p, _, _ = run_child("/app/tests/fixtures/sample.png" if os.path.exists("/app/tests/fixtures/sample.png") else
|
||||
save(Image.new("RGB", (320, 200), "red"), "PNG"))
|
||||
check(rc == 0 and dims(p["thumbnail"]) == (256, 160), "320x200 -> thumbnail 256x160")
|
||||
check(dims(p["crop"]) == (128, 128), "crop 128x128")
|
||||
check(dims(p["optimized"]) == (320, 200) and real(p["optimized"])[0] == "WEBP", "optimized WebP 320x200, no upscale")
|
||||
check(real(p["thumbnail"])[0] == "JPEG" and real(p["crop"])[0] == "JPEG", "thumbnail/crop are JPEG")
|
||||
|
||||
# 2. small source: no upscale for thumbnail, upscale allowed for crop
|
||||
rc, p, _, _ = run_child(save(Image.new("RGB", (40, 10), "blue"), "PNG"))
|
||||
check(dims(p["thumbnail"]) == (40, 10) and dims(p["crop"]) == (128, 128), "40x10: thumbnail 40x10, crop 128x128")
|
||||
|
||||
# 3. extreme aspect ratio: floor rounding, minimum 1 px
|
||||
rc, p, _, _ = run_child(save(Image.new("RGB", (3000, 2), "blue"), "PNG"))
|
||||
check(dims(p["thumbnail"]) == (256, 1) and dims(p["optimized"]) == (1280, 1), "3000x2 -> 256x1 / 1280x1 (min 1px)")
|
||||
rc, p, _, _ = run_child(save(Image.new("RGB", (1001, 999), "blue"), "PNG"))
|
||||
check(dims(p["thumbnail"]) == (256, 255), "1001x999 -> 256x255 (floor)")
|
||||
|
||||
# 4. EXIF orientation 6 (rotate 90): stored 400x200 -> upright 200x400 -> thumbnail 128x256; EXIF stripped
|
||||
exif = Image.Exif()
|
||||
exif[0x0112] = 6
|
||||
img = Image.new("RGB", (400, 200), "white")
|
||||
img.paste((255, 0, 0), (0, 0, 200, 200)) # left half red
|
||||
rc, p, _, _ = run_child(save(img, "JPEG", exif=exif.tobytes(), quality=95))
|
||||
fmt, size, has_exif, im = real(p["thumbnail"])
|
||||
check(size == (128, 256), "EXIF orientation applied: 400x200 (o=6) -> 128x256")
|
||||
with Image.open(p["thumbnail"]["file"]) as t:
|
||||
top = t.convert("RGB").getpixel((64, 20))
|
||||
check(top[0] > 200 and top[1] < 80, "rotated content: red half on top after orientation 6")
|
||||
check(not has_exif and "exif" not in Image.open(p["optimized"]["file"]).info, "EXIF removed from thumbnail/optimized")
|
||||
|
||||
# 5. alpha composited on white for JPEG, kept for WebP
|
||||
rgba = Image.new("RGBA", (100, 100), (0, 0, 0, 0))
|
||||
rc, p, _, _ = run_child(save(rgba, "PNG"))
|
||||
with Image.open(p["thumbnail"]["file"]) as t:
|
||||
px = t.convert("RGB").getpixel((50, 50))
|
||||
check(min(px) >= 250, "transparent PNG -> white JPEG thumbnail")
|
||||
check(Image.open(p["optimized"]["file"]).mode == "RGBA", "WebP optimized keeps alpha")
|
||||
|
||||
# 6. 16-bit grayscale PNG, CMYK JPEG, palette PNG
|
||||
rc16, _, _, _ = run_child(save(Image.new("I;16", (300, 300), 40000), "PNG"))
|
||||
rcc, _, _, _ = run_child(save(Image.new("CMYK", (300, 300), (0, 255, 0, 0)), "JPEG"))
|
||||
rcp, _, _, _ = run_child(save(Image.new("P", (300, 300), 3), "PNG", transparency=3))
|
||||
check(rc16 == 0 and rcc == 0 and rcp == 0, "16-bit PNG, CMYK JPEG, palette+transparency PNG are processed")
|
||||
|
||||
# 7. limits: pixels (exit 4), not an image (exit 3), memory (exit 5), CPU (SIGXCPU)
|
||||
rc, _, _, _ = run_child(save(Image.new("RGB", (100, 100)), "PNG"), max_pixels=5000)
|
||||
check(rc == 4, "pixel limit enforced before decode -> image_too_large (exit 4)")
|
||||
junk = tempfile.mktemp()
|
||||
open(junk, "wb").write(b"\x89PNG\r\n\x1a\n" + os.urandom(2000))
|
||||
rc, _, _, _ = run_child(junk)
|
||||
check(rc == 3, "corrupted PNG -> invalid_image (exit 3)")
|
||||
big = save(Image.new("RGB", (6000, 6000), "green"), "PNG")
|
||||
rc, _, _, _ = run_child(big, mem_mb=160)
|
||||
check(rc == 5, "36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got %s" % rc)
|
||||
|
||||
# 8. largest allowed inputs under the real limits: time and child peak RSS
|
||||
for name, w, h, fmt in (("png", 7900, 5000, "PNG"), ("jpeg", 7900, 5000, "JPEG")):
|
||||
noise = Image.effect_noise((w // 10, h // 10), 60).convert("RGB").resize((w, h))
|
||||
path = save(noise, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 1}))
|
||||
rc, p, secs, rss = run_child(path)
|
||||
check(rc == 0 and dims(p["thumbnail"]) == (256, 162), "%.1f Mpx %s (%d MB file) ok in %.1fs, peak child RSS %.0f MiB"
|
||||
% (w * h / 1e6, name.upper(), os.path.getsize(path) >> 20, secs, rss))
|
||||
if fmt == "PNG":
|
||||
rc, _, _, _ = run_child(path, cpu=0)
|
||||
check(rc == -24, "CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got %s" % rc)
|
||||
|
||||
print("RESULT:", "PASS" if not FAILS else "FAIL %s" % FAILS)
|
||||
sys.exit(1 if FAILS else 0)
|
||||
Reference in New Issue
Block a user