From 06382630297b233542c301c2dde29facc0b86470 Mon Sep 17 00:00:00 2001 From: alenchha Date: Wed, 11 Nov 2026 09:19:17 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=206=20=E2=80=94=20=D0=91=D0=92?= =?UTF-8?q?=D0=A22301=20=E2=80=94=20=D0=9A=D1=83=D1=80=D0=BB=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=20=E2=80=94=20=D0=BD=D0=B0=205=20(=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BA=201=20=C2=AB=D0=A6=D0=B5=D0=BD=D0=B0=20=D0=BD=D0=B0?= =?UTF-8?q?=D0=B1=D0=BB=D1=8E=D0=B4=D0=B0=D0=B5=D0=BC=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B8=C2=BB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Перенесён минимум lab5 без изменения API и добавлена наблюдаемость: - OTel SDK в API и worker: traces, metrics, logs по OTLP; W3C trace context API -> outbox -> RabbitMQ -> worker (producer/consumer spans, retry в той же трассе); - OTel Collector (memory_limiter, scrub, batch, ограниченные очереди, профиль tail sampling), Prometheus, Loki, Tempo, Grafana с provisioning, dashboard и 6 alert rules с runbooks; - проверки: наблюдаемость и приватность, инциденты A/B/C, насыщение worker, переполнение очереди exporter, эксперимент трека 1 (4 конфигурации x 3 прогона); - CI job со сборкой и запуском стенда; REPORT.md с доказательствами в evidence/. --- .env.example | 50 +- .github/workflows/stand.yml | 36 + Dockerfile | 35 + REPORT.md | 303 ++- app/__init__.py | 0 app/broker.py | 79 + app/config.py | 98 + app/cursor.py | 43 + app/db.py | 134 ++ app/errors.py | 37 + app/faults.py | 38 + app/imagecheck.py | 213 ++ app/imageproc.py | 115 ++ app/jobs.py | 340 +++ app/main.py | 721 +++++++ app/outbox.py | 100 + app/passwords.py | 16 + app/reconcile.py | 155 ++ app/repo.py | 338 +++ app/s3.py | 122 ++ app/sessions.py | 154 ++ app/telemetry.py | 393 ++++ app/worker.py | 367 ++++ compose.yaml | 460 +++++ docs/dashboard.md | 244 +++ docs/runbooks.md | 90 + evidence/alerts_history.txt | 6 + evidence/clean_clone.txt | 9 + evidence/dashboard_check.txt | 92 + evidence/exporter_queue_full.txt | 20 + evidence/failover.txt | 16 + evidence/incidents.json | 195 ++ evidence/incidents.txt | 58 + evidence/lab5_failure_windows.txt | 80 + evidence/lab5_worker_down.txt | 14 + evidence/observability.txt | 27 + evidence/persistence.txt | 32 + evidence/smoke.txt | 7 + evidence/telemetry_persistence.txt | 16 + evidence/track1/results.jsonl | 12 + evidence/track1/summary.md | 31 + evidence/worker_saturation.json | 20 + evidence/worker_saturation.txt | 10 + infra/grafana/dashboards/messenger.json | 1823 +++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 9 + .../provisioning/datasources/datasources.yaml | 44 + infra/loki/loki.yaml | 39 + infra/minio/app-policy.json | 15 + infra/minio/init.sh | 19 + infra/nginx-health.sh | 63 + infra/nginx.conf | 75 + infra/otel/collector.yaml | 88 + infra/otel/profile-default.yaml | 5 + infra/otel/profile-tail.yaml | 31 + infra/prometheus/prometheus.yml | 31 + infra/prometheus/rules/messenger.yml | 111 + infra/rabbitmq/entrypoint.sh | 14 + infra/rabbitmq/rabbitmq.conf | 10 + infra/tempo/tempo.yaml | 28 + migrations/0001_init.sql | 46 + migrations/0002_idempotency.sql | 12 + migrations/0003_attachments.sql | 28 + migrations/0004_allow_empty_text.sql | 2 + migrations/0005_attachment_upload_state.sql | 5 + migrations/0006_async_processing.sql | 75 + requirements.txt | 60 + scripts/gen_dashboard.py | 244 +++ scripts/init-secrets.sh | 28 + scripts/jobs.py | 88 + scripts/migrate.py | 60 + scripts/reconcile.py | 32 + scripts/restart.sh | 8 + scripts/stop-one.sh | 5 + student-tests/check_dashboard.py | 77 + student-tests/obs.py | 139 ++ student-tests/openloop.py | 203 ++ student-tests/queue_load.py | 250 +++ student-tests/stand.py | 132 ++ student-tests/test_exporter_queue_full.py | 58 + student-tests/test_failure_windows.py | 194 ++ student-tests/test_incidents.py | 269 +++ student-tests/test_observability.py | 159 ++ student-tests/test_worker_down.py | 84 + student-tests/test_worker_saturation.py | 75 + student-tests/track1.py | 343 ++++ student-tests/variants_check.py | 126 ++ 86 files changed, 10512 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/stand.yml create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/broker.py create mode 100644 app/config.py create mode 100644 app/cursor.py create mode 100644 app/db.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/imagecheck.py create mode 100644 app/imageproc.py create mode 100644 app/jobs.py create mode 100644 app/main.py create mode 100644 app/outbox.py create mode 100644 app/passwords.py create mode 100644 app/reconcile.py create mode 100644 app/repo.py create mode 100644 app/s3.py create mode 100644 app/sessions.py create mode 100644 app/telemetry.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 docs/dashboard.md create mode 100644 docs/runbooks.md create mode 100644 evidence/alerts_history.txt create mode 100644 evidence/clean_clone.txt create mode 100644 evidence/dashboard_check.txt create mode 100644 evidence/exporter_queue_full.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/incidents.json create mode 100644 evidence/incidents.txt create mode 100644 evidence/lab5_failure_windows.txt create mode 100644 evidence/lab5_worker_down.txt create mode 100644 evidence/observability.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/telemetry_persistence.txt create mode 100644 evidence/track1/results.jsonl create mode 100644 evidence/track1/summary.md create mode 100644 evidence/worker_saturation.json create mode 100644 evidence/worker_saturation.txt create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/loki.yaml create mode 100644 infra/minio/app-policy.json create mode 100755 infra/minio/init.sh create mode 100644 infra/nginx-health.sh create mode 100644 infra/nginx.conf create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/profile-default.yaml create mode 100644 infra/otel/profile-tail.yaml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules/messenger.yml create mode 100644 infra/rabbitmq/entrypoint.sh create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_allow_empty_text.sql create mode 100644 migrations/0005_attachment_upload_state.sql create mode 100644 migrations/0006_async_processing.sql create mode 100644 requirements.txt create mode 100644 scripts/gen_dashboard.py create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/jobs.py create mode 100644 scripts/migrate.py create mode 100644 scripts/reconcile.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/check_dashboard.py create mode 100644 student-tests/obs.py create mode 100644 student-tests/openloop.py create mode 100644 student-tests/queue_load.py create mode 100755 student-tests/stand.py create mode 100644 student-tests/test_exporter_queue_full.py create mode 100755 student-tests/test_failure_windows.py create mode 100644 student-tests/test_incidents.py create mode 100644 student-tests/test_observability.py create mode 100755 student-tests/test_worker_down.py create mode 100644 student-tests/test_worker_saturation.py create mode 100644 student-tests/track1.py create mode 100644 student-tests/variants_check.py diff --git a/.env.example b/.env.example index 2b32655..0e41580 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,66 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Values below are the ones used for evidence/ (see REPORT.md for the mapping). APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +GRACEFUL_TIMEOUT=8 +MAX_INFLIGHT_REQUESTS=64 +MAX_BODY_BYTES=65536 +MAX_RSS_MB=0 +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT=3 +DB_CONNECT_TIMEOUT=3 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_OP_TIMEOUT=6 +REQUEST_TIMEOUT=30 +DB_READY_TIMEOUT=1.5 +READY_CHECK_SECONDS=1 +READY_FAILURES=2 +SESSION_CACHE_TTL_SECONDS=30 +SESSION_SWEEP_SECONDS=60 +CACHE_TIMEOUT=0.2 +SESSION_REVOKED_TTL_SECONDS=60 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +UPLOAD_CONCURRENCY=2 +RECONCILE_INTERVAL_SECONDS=300 +RECONCILE_GRACE_SECONDS=900 +# --- lab5: broker, jobs, worker BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=20 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=30 +JOB_SWEEP_SECONDS=5 +WORKER_CHILD_MEMORY_MB=1024 STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. +# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing. +MAX_PENDING_JOBS=200 +JPEG_QUALITY=85 +WEBP_QUALITY=80 +# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else. +FAULT_INJECTION=1 +# --- lab6: OpenTelemetry. Compose sets the OTEL_* variables for api/worker (x-otel-env); shell values override: +# service.name is fixed in code: messenger-api / messenger-worker. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# Track 1 knobs (default = telemetry on, 100% head sampling, collector profile "default"): +# OTEL_SDK_DISABLED=false +# OTEL_TRACES_SAMPLER_ARG=1.0 +# COLLECTOR_PROFILE=default # or "tail" (balanced tail sampling) diff --git a/.github/workflows/stand.yml b/.github/workflows/stand.yml new file mode 100644 index 0000000..ce454e4 --- /dev/null +++ b/.github/workflows/stand.yml @@ -0,0 +1,36 @@ +name: Application stand +# Student job: builds the application, starts the whole Compose stand and runs the public smoke and the +# observability check. Credentials are random, generated inside the job, and thrown away with it. +# Triggers deliberately exclude pull_request_target: code from a fork never gets repository secrets. +on: [push, pull_request] +permissions: + contents: read +jobs: + stand: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Create local .env and random credentials + run: sh scripts/init-secrets.sh + - name: Validate Compose configuration + run: docker compose --env-file .env config --quiet + - name: Build and start the stand, wait for readiness + run: docker compose --env-file .env up -d --build --wait --wait-timeout 300 + - name: Contract structure + run: make check + - name: Public API smoke with telemetry enabled + run: make test + - name: Traces, logs, metrics correlation and privacy + run: python3 student-tests/test_observability.py + - name: Service state and application logs on failure + if: failure() + run: | + docker compose ps -a + docker compose logs --tail 80 api-1 api-2 worker otel-collector tempo loki prometheus + - name: Remove the stand + if: always() + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fea64a3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-deps -r requirements.txt + +COPY app/ app/ +COPY migrations/ migrations/ +COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/ + +USER 10001:10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +STOPSIGNAL SIGTERM + +CMD ["sh", "-c", "exec uvicorn app.main:app --no-access-log --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""] diff --git a/REPORT.md b/REPORT.md index b97ae42..2965aa8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,312 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Алёна Курлова, БВТ2301 +- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена наблюдаемости»** +- Commit/PR: TODO после публикации (ветка `solution/bvt2301-kurlova`) +- Источники, библиотеки, помощь ИИ: OpenTelemetry Python SDK 1.30.0 / instrumentation 0.51b0 (FastAPI, botocore), OpenTelemetry Collector contrib 0.120.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.1, Grafana 11.5.2; документация [Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), [tail sampling processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor). Код lab5 перенесён без изменения API. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; все числа ниже получены на локальном стенде и сохранены в `evidence/`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Desktop (Compose v2.20+, нужен `--wait`), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7.2, Docker VM: 10 CPU / 7.7 GiB. + +```sh +sh scripts/init-secrets.sh # .env и случайные credentials; существующие файлы не перезаписываются +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait # 15 сервисов, включая collector/prometheus/loki/tempo/grafana +make check && make test +``` + +Bootstrap выполняется автоматически при каждом `up` и безопасен при повторе: `migrate` (SQL-миграции с checksum и advisory lock), `minio-init` (bucket, политика, пользователь приложения), топология RabbitMQ объявляется кодом. **Lab6:** Grafana при старте импортирует datasources (`infra/grafana/provisioning/datasources`) и dashboard (`infra/grafana/dashboards/messenger.json`, папка «Messenger»), Prometheus читает правила из `infra/prometheus/rules/`. Ручного набора dashboard нет. + +Интерфейсы (только `127.0.0.1`): API `:8080` (nginx), Grafana `:3000` (admin / `secrets/grafana_admin_password`), query API Prometheus `:9090`, Loki `:3100`, Tempo `:3200` — нужны скриптам доказательств. OTLP-порты collector (4317/4318) и его метрики (8888) наружу не публикуются. + +Проверки (стенд запущен, `FAULT_INJECTION=1` только на учебном стенде): + +```sh +make test # публичный smoke при включённой телеметрии +python3 student-tests/test_observability.py # уровни 3-4: trace/log/metrics одного upload и одной 500, приватность +python3 student-tests/check_dashboard.py # все PromQL-запросы dashboard валидны и с данными +caffeinate -i python3 student-tests/test_incidents.py # уровень 4: инциденты A/B/C, firing -> resolved (~15 мин) +caffeinate -i python3 student-tests/test_worker_saturation.py # алерт насыщения worker (~10 мин) +caffeinate -i python3 student-tests/test_exporter_queue_full.py # очередь exporter до предела (~5 мин) +caffeinate -i python3 student-tests/track1.py run --repeats 3 # трек 5.1 (~35 мин), затем: python3 student-tests/track1.py summary +python3 student-tests/test_worker_down.py; python3 student-tests/test_failure_windows.py # регрессия lab5 +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # затем docker compose start api-1 api-2 +``` + +`caffeinate -i` (macOS) нужен для длинных прогонов: в первой попытке Mac ушёл в сон посреди инцидента, весь стенд «замёрз» на 18 минут (виден скачок gauge 94 → 1176 s), см. раздел «Ограничения». + +Безопасная остановка с сохранением данных: `docker compose stop`. Намеренное удаление учебных данных (включая историю метрик/логов/трасс): `docker compose down -v`. `secrets/` и `.env` в git не попадают. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client ─► nginx ─► api-1, api-2 ──(outbox)──► RabbitMQ ─► worker ×2 + │ │ PostgreSQL / Valkey / MinIO │ + │ └──── OTLP/HTTP :4318 ◄─────────────┘ (traces, metrics, logs; service.name + ▼ messenger-api / messenger-worker) + otel-collector memory_limiter → attributes/scrub → [tail_sampling] → batch + ├─ otlp/tempo ─► Tempo (трассы, volume tempodata) + ├─ otlphttp/prometheus ─► Prometheus /api/v1/otlp (метрики, volume promdata, правила/алерты) + └─ otlphttp/loki ─► Loki /otlp (логи, volume lokidata) + Prometheus также скрейпит self-metrics collector :8888, Tempo, Loki. + Grafana (volume grafanadata) — только интерфейс и запросы к трём источникам. +``` + +Сеть у всех сервисов одна: внутренняя сеть Compose `default`. Порты в столбце «Порт» — внутренние; наружу опубликованы только отмеченные `127.0.0.1`. Credentials передаются Docker secrets из `secrets/` (создаёт `scripts/init-secrets.sh`), приложение читает их через переменные `*_FILE`. Digest образов указан в `compose.yaml`. + +| Роль | Сервис | Версия | Health / readiness | Volume | Порт | Credentials (Docker secrets) | Назначение | +| --- | --- | --- | --- | --- | --- | --- | --- | +| API ×2 | `api-1`, `api-2` | сборка из `Dockerfile`, python:3.12.8-slim-bookworm + digest | `GET /health/ready` | нет | 8080, напрямую не опубликован | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key`, `broker_url` | REST API, outbox relay, gauges очереди | +| Балансировщик | `nginx` | nginx:1.27.3-alpine | `GET /health/live`; проверка реплик в `infra/nginx-health.sh` | нет | `127.0.0.1:8080` | нет | единая точка входа | +| Worker ×2 | `worker` | тот же образ | свежесть файла heartbeat, 15 с | нет | нет | `database_url`, `s3_access_key_id`, `s3_secret_access_key`, `broker_url` | обработка изображений | +| БД | `postgres` | postgres:17.7-alpine | `pg_isready` | `pgdata` | 5432 | `postgres_password` | источник истины | +| Кэш сессий | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | нет, кэш без персистентности | 6379 | нет, доступ только из внутренней сети | ускоряет проверку сессий; при сбое API читает PostgreSQL | +| S3 | `minio` | madlabcg/minio по digest | `mc ready local` | `miniodata` | 9000 | `minio_root_user`, `minio_root_password` | вложения, приватный bucket | +| Брокер | `rabbitmq` | rabbitmq:4.0.5-alpine | `check_port_connectivity` | `rabbitmqdata` | 5672 | `rabbitmq_password` | durable очередь заданий | +| Bootstrap БД | `migrate` | тот же образ | разовый, успешное завершение | нет | нет | `database_url` | SQL-миграции | +| Bootstrap S3 | `minio-init` | madlabcg/minio по digest | разовый, успешное завершение | нет | нет | `minio_root_user`, `minio_root_password`, `s3_access_key_id`, `s3_secret_access_key` | bucket, политика, пользователь приложения | +| Collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.120.0 | Docker healthcheck нет: в образе нет shell; расширение `health_check` слушает 13133 | нет, это не хранилище | 4317, 4318, 8888 | нет, доступ только из внутренней сети | приём OTLP, ограничение памяти, очистка атрибутов, батчи, tail sampling, экспорт с ограниченной очередью | +| Метрики | `prometheus` | prom/prometheus:v3.2.1 | `/-/ready` | `promdata`, 15 дней | `127.0.0.1:9090` | нет, только localhost | числовые ряды: RPS, ошибки, гистограммы, gauges; SLI, recording rules и алерты | +| Логи | `loki` | grafana/loki:3.4.2 | `/ready` | `lokidata`, 7 дней | `127.0.0.1:3100` | нет, только localhost | события с текстом; индекс только по низкокардинальным labels, `event`/`trace_id`/`span_id` — structured metadata | +| Трассы | `tempo` | grafana/tempo:2.7.1 | `/ready` | `tempodata`, 7 дней | `127.0.0.1:3200` | нет, только localhost | spans с причинной связью, поиск TraceQL | +| UI | `grafana` | grafana/grafana:11.5.2 | `/api/health` | `grafanadata` | `127.0.0.1:3000` | `grafana_admin_password` | dashboard, Explore, переходы лог → трасса и трасса → логи | + +Почему три хранилища: Grafana сама сигналы не хранит; метрика отвечает «сколько и как часто» (дёшево, агрегируется, но теряет конкретный запрос), лог — «что произошло в событии» (текст и технические ID), трасса — «где в цепочке процессов ушло время/возникла ошибка». + +Что изменено относительно lab5 (бизнес-API и контракт не менялись): + +- `app/telemetry.py` — настройка SDK (resource, sampler, views, exporters), JSON-логи в stdout с `trace_id`/`span_id`, OTLP-логи, инструменты метрик, перенос контекста; +- `app/main.py` — инструментирование FastAPI (после регистрации middleware, иначе ответы 429/503/500 самого guard не попадали в spans и метрики — найдено проверкой), `messenger.request_id` в server span, gauges очереди из PostgreSQL, тестовая инъекция `X-Fault: slow|error` (только `FAULT_INJECTION=1`); +- `app/db.py` — собственные CLIENT spans PostgreSQL и span ожидания пула. Автоинструментация `opentelemetry-instrumentation-psycopg 0.51b0` для async-курсоров закрывала span до завершения `await` (все db-spans были 0.0 ms) — с ней нельзя было отличить медленную БД от handler, поэтому она убрана; +- `app/jobs.py`, `app/outbox.py`, `app/broker.py`, `app/worker.py` — W3C trace context в сообщении задания, producer/consumer spans, метрики jobs и time-to-ready; +- `compose.yaml` — collector, Prometheus, Loki, Tempo, Grafana, volumes; `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` — конфиги и provisioning; +- `.github/workflows/stand.yml` — CI: сборка, запуск стенда, `make test`, проверка наблюдаемости; `scripts/init-secrets.sh` — локальные credentials; +- `docs/runbooks.md` — runbook каждого алерта; `student-tests/` — проверки lab6 (lab5-тесты сохранены). + +Соответствие переменных `.env.example`: `OTEL_SERVICE_NAME` не используется — `service.name` задаётся кодом (`messenger-api`/`messenger-worker`), чтобы worker не мог унаследовать имя API из общего `.env`. `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, `OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab` — как в примере (задаются в `x-otel-env` compose). Дополнительно: `OTEL_SDK_DISABLED`, `OTEL_TRACES_SAMPLER_ARG`, `OTEL_METRIC_EXPORT_INTERVAL=10000`, `OTEL_BSP_MAX_QUEUE_SIZE=2048`, `OTEL_BLRP_MAX_QUEUE_SIZE=2048`, `OTEL_EXPORTER_OTLP_TIMEOUT=5`, `OTEL_SEMCONV_STABILITY_OPT_IN=http`, `COLLECTOR_PROFILE=default|tail`. + +Источник истины и транзакционные границы (из lab5, не менялись): PostgreSQL хранит пользователей, чаты, сообщения, метаданные вложений и задания. Upload фиксирует в одной транзакции attachment `queued`, job и строку outbox, и только после COMMIT отвечает 202. Relay публикует outbox в RabbitMQ с publisher confirm. Worker одной транзакцией публикует результат: варианты, `ready`, job `succeeded`. S3 хранит байты, брокер — только ссылки на задания. Telemetry в эти транзакции не входит: trace context пишется в ту же outbox-строку, а spans и метрики экспортируются асинхронно и не могут задержать или отменить COMMIT. + +Критичные зависимости `/health/ready` (как в lab5): PostgreSQL и S3. Collector/Tempo/Loki/Prometheus/Grafana **не** критичны — проверено инцидентом C. + +## Схема сигналов + +Версии: OpenTelemetry SDK 1.30.0, instrumentation 0.51b0, semantic conventions 0.51b0; HTTP-атрибуты по стабильной конвенции (`OTEL_SEMCONV_STABILITY_OPT_IN=http`: `http.request.method`, `http.route`, `http.response.status_code`, метрика `http.server.request.duration` в секундах). Collector — дистрибутив contrib 0.120.0 (нужны `tail_sampling` и `attributes`). + +Resource attributes: `service.name` (messenger-api / messenger-worker), `service.namespace=messenger`, `service.version=1.6.0`, `service.instance.id` (api-1/api-2/hostname контейнера worker), `deployment.environment.name=lab`. В Prometheus: `service_name`, `instance`, `service_version`, `deployment_environment_name` (promote_resource_attributes; `job` = `messenger/`). + +| Сигнал контракта | Инструмент OTel → имя в Prometheus | Labels (все низкокардинальные) | Единица | +| --- | --- | --- | --- | +| HTTP requests (counter) | `http.server.request.duration` → `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`; класс статуса = `label_replace(..., "(\\d)..")` в запросе | запросы | +| HTTP duration | та же гистограмма, `_bucket`; buckets 5 ms … 30 s (View) | те же | s | +| Jobs processed/failed | `messenger.jobs.processed` → `messenger_jobs_processed_total` | `job_type`, `outcome` (succeeded/retry/failed/noop/lost/dlq/requeued) | jobs | +| Job duration | `messenger.job.duration` → `messenger_job_duration_seconds_*`; buckets 50 ms … 120 s | `job_type`, `outcome` | s | +| Queue depth / oldest age | `messenger.queue.depth`, `messenger.queue.oldest_age` → `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | `queue` (attachments.process / outbox), `state` | jobs / s | +| DB pool | `db.client.connection.count`, `.pending_requests`, `.max` → `db_client_connection_*` | `db_client_connection_state` (used/idle) | соединения | +| Worker busy/capacity | `messenger.worker.busy`, `messenger.worker.capacity` | — | jobs | +| Process CPU/RAM | `process.cpu.time` → `process_cpu_time_seconds_total`, `process.memory.usage` → `process_memory_usage_bytes` | `process_cpu_owner` (self/children) | s / bytes | +| End-to-end time to ready | `messenger.attachment.time_to_ready` → `messenger_attachment_time_to_ready_seconds_*`; buckets 0.25 s … 600 s | `job_type` | s | +| Самонаблюдение телеметрии | `messenger.telemetry.sampler.decisions`, `messenger.telemetry.spans.exported`; collector `otelcol_*` | `decision`, `rule`, `outcome`, `exporter` | spans | +| Логи | OTLP log records → Loki | stream labels: `service_name`, `service_instance_id`, `service_namespace`, `deployment_environment_name`; metadata: `event`, `severity_text`, `trace_id`, `span_id`, `code_*` | — | +| Трассы | SERVER (FastAPI), PRODUCER `attachments.process publish`, CONSUMER `attachments.process process`, CLIENT `S3.*` (botocore), CLIENT `INSERT messages` и т.п. (свои), INTERNAL `imageproc`, `db.pool.acquire` | атрибуты: `messenger.request_id`, `messenger.attachment_id`, `messaging.message.id` (= job_id), `messenger.job.attempt/outcome`, `messenger.fault` | — | + +Границы buckets подобраны под ожидаемые задержки. HTTP: 5–150 ms — обычные запросы (p50 около 7 ms, p99 около 35–50 ms под нагрузкой); 250 ms – 1 s — граница SLO 500 ms и порог алерта p95 1 s; 2.5–10 s — `lock_timeout` 5 s и `DB_OP_TIMEOUT` 6 s; 30 s — `REQUEST_TIMEOUT`. Job: 50 ms – 3 s — обычная обработка (около 0.15 с, 1 с при ограниченном CPU); 5–120 s — таймаут дочернего процесса 60 s и hard timeout 120 s. Time to ready: 0.25 s – 10 min — от мгновенной обработки до ожидания в очереди с backoff до 30 s и пятью попытками. + +Технические ID (`request_id`, `attachment_id`, `job_id`) есть только в span-атрибутах и тексте логов (обоснование: по ним оператор находит трассу конкретной загрузки/ошибки; данные синтетические), никогда в labels метрик — проверено тестом: ни одного UUID в значениях labels, список labels фиксирован. `http.route` — шаблон (`/api/v1/chats/{chat_id}/messages`). + +Запрещённое в telemetry: тело сообщения, пароль, Authorization/токен, изображение, filename, signed URL. Защита в два слоя: (1) SDK — заголовки не захватываются, атрибуты HTTP-гистограммы ограничены View, SQL параметризован (`db.query.text` содержит только `%s`), логи не печатают тело/пароль; (2) collector `attributes/scrub` удаляет `url.query` (там был бы `?filename=`), `url.full`, `http.target`, `http.url`, заголовки authorization/cookie. Проверка: `test_observability.py` ищет пароль, токен, текст-маркер, filename, `Authorization`, `Bearer`, `filename=`, `X-Amz-Signature` в трассах и логах — не найдено. + +### Перенос контекста API → broker → worker (уровень 4) + +1. Upload: server span `POST /api/v1/chats/{chat_id}/attachments`. В той же транзакции, что attachment `queued` и job, в `outbox.payload` записываются `traceparent`/`tracestate` текущего span (поля разрешены `contracts/jobs.schema.json`). +2. Relay outbox (фоновый цикл API) создаёт PRODUCER span **дочерним** контексту из payload и публикует сообщение уже со своим `traceparent` (в теле и дублем в AMQP headers). +3. Worker извлекает контекст из тела (fallback — headers) и создаёт CONSUMER span — дочерний producer. Внутри: `S3.GetObject`, `imageproc`, `S3.CopyObject`, `S3.PutObject`×3, SQL, `S3.DeleteObject`. +4. Retry: `fail_attempt` пишет новую outbox-строку внутри consumer span → следующий PRODUCER — дочерний упавшему CONSUMER. Вся история upload, включая повторы, — **одна трасса**. +5. Sampler `ParentBased(root=KindFilter(TraceIdRatio))`: корнем трассы может быть только SERVER/CONSUMER/PRODUCER, поэтому одиночные SQL-spans фоновых циклов (outbox polling, sweeper, readiness) не создают тысячи мусорных трасс; дети следуют решению родителя, поэтому upload и его job сэмплируются вместе. + Ограничение: восстановление job по истёкшему lease (sweeper) начинает новую трассу (нет активного контекста); связь видна по `messenger.attachment_id`. + +Пример (вывод `test_observability.py`, `evidence/observability.txt`; upload с `fault-transient1` — первая попытка падает временно): + +``` +api SERVER POST /api/v1/chats/{chat_id}/attachments 9.0 ms + api CLIENT S3.PutObject 3.4 ms + api PRODUCER attachments.process publish 1.7 ms attempt=1 + worker CONSUMER attachments.process process 6.1 ms ERROR outcome=retry + api PRODUCER attachments.process publish 2.9 ms attempt=2 + worker CONSUMER attachments.process process 168.6 ms outcome=succeeded + worker CLIENT S3.GetObject 3.7 ms + worker INTERNAL imageproc 139.6 ms + worker CLIENT S3.CopyObject 4.4 ms + worker CLIENT S3.PutObject (x3) 2.0-2.4 ms + worker CLIENT S3.DeleteObject 1.0 ms +trace 40391c8b40fc76e7f27913c3d84421a4, 42 spans (SQL-spans в выводе опущены). В последнем consumer результат публикуется +SQL-spans: UPDATE attachment_jobs, INSERT attachment_variants ×3, UPDATE attachments, INSERT job_audit. В Loki по этому trace_id 6 записей: +api: attachment_queued; worker: job_claimed, job_attempt_failed, job_finished, job_claimed, job_finished +``` + +### SLI / SLO и алерты (уровень 4) + +- **SLI доступности** = 1 − (5xx / все бизнес-запросы messenger-api); health не инструментирован, 4xx — исход клиента, не отказ. **SLO 99%** на скользящем окне 30 мин (учебное окно; в production те же формулы на 28–30 днях). Recording rule `messenger:slo_availability:ratio30m`. +- **SLI латентности** = доля запросов ≤ 500 ms (bucket `le="0.5"`), **SLO 95%** / 30 мин, `messenger:slo_latency:ratio30m`. +- Перцентили на dashboard — `histogram_quantile` по `sum by (le)` buckets **всех** реплик, а не среднее и не усреднение p99 реплик. +- SLO проверяется значением recording rule в Prometheus. В инциденте B: перед блокировкой доступность 1.0000 и латентность 0.9825; в момент firing 0.6894 и 0.6933; через 2 минуты после снятия блокировки 0.7697 и 0.7616. Оба SLO нарушены, бюджет ошибок окна (1%) исчерпан. Значения: `messenger:slo_availability:ratio30m` и `messenger:slo_latency:ratio30m` на моменты 14:34:58, 14:36:53 и 14:39:09 UTC 7 ноября. + +Ключевые панели dashboard (полный перечень 32 панелей с запросами и единицами — `docs/dashboard.md`): + +| Панель | Запрос | Единица | +| --- | --- | --- | +| Латентность API p50/p95/p99 | `histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval])))` (и 0.5, 0.95) | секунды | +| Доля ошибок 5xx | `(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]))` | доля | +| RPS по классу статуса | `sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{...}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d).."))` | запросы/с | +| Jobs по исходу | `sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))` | jobs/с | +| Длительность job p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval])))` | секунды | +| Глубина очереди | `max by (queue, state) (messenger_queue_depth)` | jobs | +| Возраст старейшей job | `max(messenger_queue_oldest_age_seconds{queue="attachments.process"})` | секунды | +| Время до ready p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))` | секунды | +| Пул БД | `sum by (service_name, db_client_connection_state) (db_client_connection_count)`, `sum by (service_name) (db_client_connection_pending_requests)` | соединения, запросы | +| Насыщение worker | `messenger:worker_saturation:ratio` = busy / capacity по свежим сериям | доля | +| CPU / RSS процессов | `sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))`, `process_memory_usage_bytes` | ядра, байты | + +| Alert | Условие / окно / порог / for | Проверен | +| --- | --- | --- | +| MessengerApiHighErrorRate | 5xx/все за 2m > 5% при > 0.2 rps, for 1m | инцидент B: firing и resolved | +| MessengerQueueLagGrowing | max(oldest job age) > 60 s, for 1m | инцидент A: firing и resolved | +| MessengerApiLatencyHigh | p95 (5m) > 1 s, for 2m | инцидент B: firing 14:38:00–14:41:45 UTC, затем resolved (`evidence/alerts_history.txt`) | +| MessengerDbPoolWaiting | waiting > 0, for 1m | инцидент B: firing 14:36:30–14:37:00 UTC, затем resolved (`evidence/alerts_history.txt`) | +| MessengerWorkerSaturated | busy/capacity > 0.9 (только свежие серии), for 5m | `test_worker_saturation.py`: firing и resolved | +| TelemetryExportFailing | заполнение очереди exporter > 20% или отбросы/исчерпанные ретраи, for 1m | инцидент C1: firing и resolved | + +Runbooks: `docs/runbooks.md`. Состояние — Prometheus `/api/v1/alerts` и панель «Алерты Prometheus» на dashboard. + +### Collector: память и недоступный backend + +`memory_limiter` первым в каждом pipeline (limit 180 MiB, spike 40 MiB при лимите контейнера 256 MiB): при превышении receiver отвечает отказом, SDK ретраит/отбрасывает. Каждый exporter: `sending_queue` в памяти на 100 батчей (≤ 2048 элементов) и `retry_on_failure` с `max_elapsed_time: 120s` — ни бесконечного буфера, ни бесконечных повторов. На стороне SDK: BatchSpanProcessor/BatchLogRecordProcessor с очередью 2048, при переполнении — отброс; экспорт идёт в отдельном потоке, запросы API не ждут telemetry. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke при включённой телеметрии | `make check`, `make validate`, `make test` | все PASS | `evidence/smoke.txt` | +| Воспроизводимость из чистого clone | копия только файлов git, шаги job `.github/workflows/stand.yml`, отдельный compose-проект | все сервисы healthy, smoke PASS, dashboard импортирован, 13 правил, метрики идут | `evidence/clean_clone.txt` | +| Минимум lab5 сохранён | `test_worker_down.py`, `test_failure_windows.py` (A–I) | PASS | `evidence/lab5_worker_down.txt`, `evidence/lab5_failure_windows.txt` | +| Сохранность данных, реальный рестарт | `make test-persistence` | PASS; uptime всех хранилищ, включая Prometheus, Loki, Tempo, сброшен | `evidence/persistence.txt` | +| Failover каждой реплики | `make test-failover` ×2 | PASS; 0 неудачных попыток, восстановление 2.0 и 0.9 с | `evidence/failover.txt` | +| 3: Collector, 3 хранилища, Grafana в общем стенде; конфиги в Git | `compose.yaml`, `infra/` | стенд поднимается одной командой | `evidence/clean_clone.txt` | +| 3: volumes сохраняют историю | рестарт Prometheus, Loki, Tempo, Grafana | трасса, логи и алерты 7 ноября доступны 9 ноября после рестарта | `evidence/telemetry_persistence.txt` | +| 3: RPS/errors/duration, jobs, duration worker в Grafana | `check_dashboard.py` | 32 панели; все 44 запроса Prometheus, запросы Loki и Tempo выполнены через саму Grafana с данными; Grafana видит 6 правил алертов | `evidence/dashboard_check.txt` | +| 3: контролируемый запрос → trace → лог; job → worker trace → лог | `test_observability.py` | PASS | `evidence/observability.txt` | +| 3: нет секретов, изображений и ID в метках | `test_observability.py` | пароль, токен, текст, filename, Authorization, signed URL и байты PNG (в сыром виде и base64) не найдены; UUID в labels нет | `evidence/observability.txt` | +| 4: W3C API → broker → worker, цепочка до S3/DB с retry | `test_observability.py` | одна трасса, 42 spans, consumer — дочерний producer, retry — дочерний упавшего consumer | `evidence/observability.txt` | +| 4: p50/p95/p99, ошибки, очередь, DB pool, saturation | dashboard и `check_dashboard.py` | панели с запросами и единицами; перцентили из buckets | `infra/grafana/dashboards/messenger.json` | +| 4: SLI/SLO и alert rules проверены отказом | `test_incidents.py`, `test_worker_saturation.py` | все 6 алертов: firing → resolved | `evidence/incidents.txt`, `evidence/alerts_history.txt`, `evidence/worker_saturation.txt` | +| 4: memory limiter, batch, ограниченный буфер | Tempo остановлен при 100 RPS | очередь 100/100, отброшено 25147 и отказано 16802 spans, RSS collector до 206 MiB при лимите 256 MiB, API без ошибок | `evidence/exporter_queue_full.txt` | +| 4: остановка telemetry не ломает сообщения | инцидент C | 0 неожиданных ошибок API при остановке backend и collector | `evidence/incidents.txt` | +| 4: разбор двух инцидентов | разделы ниже | шаги диагностики и причины | `evidence/incidents.txt`, `docs/runbooks.md` | +| 5, трек 1 | `track1.py run --repeats 3` | 4 конфигурации, по 3 прогона | `evidence/track1/results.jsonl`, `evidence/track1/summary.md`, `evidence/track1/matrix.log` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Перезапуск в `test-persistence` затрагивает API, worker, LB, PostgreSQL, Valkey, MinIO, RabbitMQ и, с lab6, Prometheus, Loki, Tempo, collector и Grafana (`scripts/restart.sh`), подтверждено состояниями контейнеров до и после. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: macOS arm64, Docker 29.7.2, VM 10 CPU / 7.7 GiB. Лимиты: API 0.5 CPU / 512 MiB ×2, worker 2 CPU / 1.5 GiB ×2, PostgreSQL 1 CPU / 512 MiB, collector 1 CPU / 256 MiB, Tempo 1 CPU / 1 GiB. Фоновая нагрузка в инцидентах: open-loop 10 RPS (75% POST message, 25% history, 2% `X-Fault: slow`). Времена отсчитываются от внесения отказа; полный вывод в `evidence/incidents.txt`, сводка в `evidence/incidents.json`. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| A. Остановка worker | рост возраста задания, `MessengerQueueLagGrowing` | pending +71 с, firing +131 с при возрасте 126 с; глубина 3 | backlog разобран за 3.6 с, resolved через 14 с; потерь заданий нет | +| B. `LOCK TABLE messages` на 106 с | 5xx, ожидание в БД видно в трассе | pending +45 с, firing +105 с; 5xx 70.7%, p95 6.92 с, пул 10/10, ждут 19 | resolved через 135 с после снятия блокировки (окно 2m); подтверждённые данные не теряются | +| C1. Tempo и Loki остановлены на 192 с | API работает, очередь ограничена | 1703 запроса, 0 неожиданных ошибок; очередь Tempo 79/100, `TelemetryExportFailing` firing | экспорт возобновился, resolved через 35 с; потеряно 387 записей telemetry (ретраи исчерпаны) | +| C2. Collector остановлен на 60 с | API работает, очереди SDK ограничены | 740 запросов, 0 неожиданных ошибок; RSS api-1 108.6 → 109–119 MiB | telemetry за время простоя теряется сверх очереди SDK (2048) | +| Tempo остановлен при 100 RPS, 223 с | очередь упирается в предел, отбросы считаются | очередь 100/100, отброшено 25147, отказано memory_limiter 16802 spans; RSS до 206 MiB | API: 24686 запросов, 0 ошибок, p99 55.8 ms; потерянные spans не восстанавливаются. После возврата Tempo сброс очереди превысил его лимит 15 MB/с: ещё 32410 spans отклонены (`RATE_LIMITED`) и посчитаны в `send_failed` | +| Насыщение worker (1 слот ×2, 0.25 CPU), 4 upload/с | saturation 1.0 дольше `for: 5m` | firing +340 с; saturation 1.00, 194 в очереди, возраст 63 с, 3.16 job/с, p95 job 0.98 с | после возврата настроек resolved через 17 с, backlog разобран за 22 с | + +### Инцидент 1: задания не обрабатываются + +1. Сработал `MessengerQueueLagGrowing`. Метрики: возраст старейшего задания 126 с, 3 задания в `queued`, `rate(messenger_jobs_processed_total[1m]) = 0`. Это не медленная обработка, задания не берутся вообще. +2. Логи worker: после остановки 0 событий `job_*`, последние события `worker_stopped`. +3. Трасса upload: есть SERVER и PRODUCER, нет CONSUMER. Сообщение дошло до брокера, потребителя нет. +4. Причина: остановлен worker. Действие: `docker compose start worker`. Задания сохранены в PostgreSQL и durable-очереди, поэтому все три стали ready за 3.6 с. + +### Инцидент 2: медленная БД или медленный handler + +1. Сработал `MessengerApiHighErrorRate`: 5xx 70.7%, p95 6.92 с. Ошибки 503 приходятся на `/api/v1/chats/{chat_id}/messages`. +2. Метрики пула: занято 10 из 10, ожидают 19, поэтому подозрение падает на БД. +3. Трасса упавшего запроса: `INSERT messages` занял 5001 ms из 5004 ms и завершился `QueryCanceled` (срабатывает `lock_timeout`). Время потрачено в PostgreSQL. +4. Контраст: запрос с `X-Fault: slow` длится 615 ms, а его самый долгий DB-span — 1.1 ms, все DB-spans вместе — 8.3 ms. Так выглядит медленный handler. +5. Шаг оператора: `pg_stat_activity` показывает блокирующий `LOCK TABLE messages`; после `pg_terminate_backend` ошибки прекращаются. + +Побочный эффект, найденный в этом инциденте: исчерпание пула делает `/health/ready` неуспешным (событие `not_ready`), поэтому блокировка одной таблицы отключает обе реплики целиком. За время инцидента клиент получил 816 ответов 503 и 214 ответов 502 от nginx. + +## Трек 5.1: цена наблюдаемости + +**Гипотезы.** H1: стоимость телеметрии в приложении пропорциональна числу записанных spans, поэтому head sampling снижает CPU, а tail sampling — нет. H2: head sampling 10% теряет около 90% ошибок и редких медленных запросов, потому что решение принимается до их исхода. H3: tail sampling сохраняет все такие трассы и резко сокращает объём в хранилище. H4: метрики от сэмплирования не зависят. + +**Методика.** Конфигурации: `off` (`OTEL_SDK_DISABLED=true`), `on100` (100%), `head10` (head sampling 10% корневых spans), `tail` (SDK 100%; collector оставляет ошибки, трассы > 500 ms, конвейер вложений и 10% остального; `decision_wait` 10 с). Перед каждым прогоном пересоздаются API, worker и collector. Профиль: open-loop 60 RPS (75% POST message, 25% GET history), 1 upload/с, 1% запросов с `+600 ms`, 1% запросов с ответом 500; прогрев 20 с, пауза 20 с, замер 90 с, пауза 30 с. Три прогона каждой конфигурации, порядок циклический. Объём данных: перед экспериментом в БД 16 739 сообщений, 38 чатов, 89 пользователей, 87 вложений; после — 75 179, 130, 273 и 1 277 (каждый прогон создаёт 4 новых чата, история самого большого чата 2 701 сообщение). GET history читает первую страницу (limit=20), поэтому рост БД на latency чтения почти не влияет. Latency считается от запланированного момента отправки (без coordinated omission), только для успешных обычных запросов. CPU — точная разность cgroup-счётчика за окно; для collector сверена с его собственной метрикой (0.52 и 0.54% ядра). Запуск: `evidence/track1/matrix.log`. + +Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются. + +| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % | +|---|---|---|---|---|---|---|---|---| +| off | 3 | 60.4 | 0 | 7.1 (6.4…8.3) | 11.0 (10.3…12.2) | 32.7 (31.7…33.9) | 17.8 (17.0…18.3) | 15.2 (14.8…15.3) | +| on100 | 3 | 60.5 | 0 | 7.8 (6.6…8.6) | 14.3 (11.8…16.3) | 47.2 (43.4…54.4) | 22.7 (22.3…22.9) | 15.6 (15.3…15.9) | +| head10 | 3 | 60.4 | 0 | 7.2 (6.6…8.3) | 11.2 (10.3…13.0) | 37.5 (36.6…39.2) | 19.7 (19.6…20.0) | 15.0 (14.9…15.1) | +| tail | 3 | 60.4 | 0 | 7.9 (6.7…8.7) | 13.7 (11.9…14.7) | 44.5 (42.0…48.2) | 22.3 (22.1…22.5) | 15.3 (15.2…15.3) | + +| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB | +|---|---|---|---|---|---|---|---| +| off | 0.07 (0.06…0.08) | 0.70 (0.43…0.86) | 0.57 (0.54…0.60) | 0.34 (0.30…0.36) | 98 | 36 | 470 | +| on100 | 0.57 (0.54…0.60) | 1.13 (1.03…1.29) | 0.57 (0.55…0.58) | 0.36 (0.33…0.38) | 113 | 73 | 571 | +| head10 | 0.23 (0.23…0.24) | 0.41 (0.35…0.44) | 0.54 (0.50…0.58) | 0.34 (0.33…0.34) | 105 | 71 | 423 | +| tail | 0.53 (0.52…0.54) | 0.64 (0.50…0.80) | 0.52 (0.50…0.53) | 0.33 (0.32…0.34) | 116 | 81 | 493 | + +| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB | +|---|---|---|---|---|---|---| +| off | — | 0 | 0 | — | 0 | 0 | +| on100 | 57059 | 57059 | 0 | 21030 | 20547 | 132 | +| head10 | 5560 | 5560 | 0 | 2496 | 2031 | 134 | +| tail | 57106 | 9746 | 0 | 21062 | 3877 | 135 | + +Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна. + +| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено | +|---|---|---|---|---|---| +| off | 0 / 16470 | 0 / 164 | 0 / 163 | 0 / 270 | — / 163 | +| on100 | 16530 / 16470 | 167 / 167 | 146 / 146 | 270 / 270 | 146 / 146 | +| head10 | 1630 / 16470 | 18 / 198 | 15 / 166 | 25 / 270 | 166 / 166 | +| tail | 2269 / 16470 | 180 / 180 | 164 / 164 | 270 / 270 | 164 / 164 | + +**Результаты по гипотезам.** + +- **H1 подтверждена.** CPU API относительно `off`: `on100` +4.9 п.п. (+28%, около 0.8 ms CPU на запрос), `head10` +1.9 п.п. (+11%), `tail` +4.5 п.п. (+25%). Диапазоны между прогонами не пересекаются. CPU worker не изменился (15.0–15.6%): его определяет обработка изображений. +- **Latency.** p99: `off` 32.7, `head10` 37.5, `tail` 44.5, `on100` 47.2 ms; диапазоны `off` и `on100` не пересекаются. p50 различается меньше, чем прогоны одной конфигурации (6.4–8.7 ms), поэтому разницы по p50 заявить нельзя. p95 у `on100` и `tail` выше, но диапазоны частично пересекаются с `off`. +- **H2 подтверждена.** При `head10` сохранилось 18/198 медленных запросов (9%), 15/166 ошибок (9%) и 25/270 конвейеров вложений (9%). +- **H3 подтверждена.** `tail` сохранил 180/180, 164/164 и 270/270 при сокращении ingest Tempo на 81% (20547 → 3877 KiB) и числа spans на 83%. Обычных трасс осталось около 14%. Экономии между SDK и collector нет: 21 MiB за окно, как в `on100`. +- **H4 подтверждена.** Счётчик HTTP 500 совпал с числом отправленных ошибок во всех режимах: 146/146, 166/166, 164/164. При `head10` метрика видит все 166 ошибок, а разобрать по трассе можно только 15. +- **Потери при передаче** отсутствуют: `Spans → collector` равно `Spans → Tempo` в `on100` и `head10`, отказов и отбросов collector нет, SDK не сообщил об ошибках экспорта. + +**Что теряется.** `head10`: около 90% любых трасс, включая ошибки и редкие медленные запросы. `tail`: 90% обычных успешных запросов; запросы медленнее обычного, но быстрее порога 500 ms; ответы 4xx (их server span не имеет статуса ERROR). Части трассы, пришедшие позже `decision_wait` (10 с) после решения об отбросе, тоже будут потеряны; для конвейера вложений это закрыто политикой по `messaging.system`. + +**Сбалансированная конфигурация: `tail`** (`COLLECTOR_PROFILE=tail`). Её цена: те же +25% CPU API, что у полной трассировки, и +12 ms к p99, зато сохраняются все диагностически важные трассы при объёме хранения на 81% меньше. `head10` разумен, только если критичен CPU приложения и достаточно метрик и логов. По умолчанию в репозитории оставлен `on100`: на защите нужно найти любую контрольную трассу, а нагрузка учебная. + +**Ограничения измерения.** Один Docker-host и генератор на той же машине, API далёк от насыщения (около 11% ядра на реплику), поэтому поведение под перегрузкой здесь не измерено. Три прогона по 90 с дают ориентировочные оценки. RSS Tempo между конфигурациями не сравним: контейнер не пересоздавался. Колонка SDK-решений из сводки исключена: счётчики пересозданных контейнеров выпадали из окна запроса и давали неверные разности. Tail sampling требует, чтобы все spans трассы приходили в один collector; при нескольких collector нужна маршрутизация по trace_id. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Выдерживается: отказ одной реплики API, остановка и перезапуск worker и брокера с volumes, недоступность Tempo, Loki или Collector без влияния на бизнес-API (данные telemetry при этом теряются сверх ограниченных очередей, потери считаются). + +Не выдерживается и не заявляется: потеря Docker-host, отказ единственных PostgreSQL, nginx или collector как высокодоступных компонентов. Хранилища telemetry работают в одном экземпляре. + +Известные ограничения и упрощения: + +- Блокировка БД исчерпывает пул и переводит реплики в not ready, поэтому отказ одной таблицы останавливает весь API. Улучшение: отдельное соединение для readiness. Пока это гипотеза, не реализовано. +- OTLP-метрики в Prometheus не получают staleness-маркеров: серии остановленного процесса видны ещё 5 минут. Правило насыщения worker это учитывает; в остальных запросах значения исчезают с задержкой. +- `MessengerApiHighErrorRate` срабатывает и на 503 от admission control очереди (видно в тесте насыщения): это реальный отказ клиенту, но причину надо смотреть по маршруту и логам. +- Восстановление задания по истёкшему lease начинает новую трассу; связь видна по `messenger.attachment_id`. +- Метрики CPU worker учитывают дочерние процессы только после их завершения; RSS дочерних процессов не учитывается. +- Первый прогон инцидентов сорван сном Mac: стенд замер на 18 минут. Длинные прогоны нужно запускать под `caffeinate -i`. +- `FAULT_INJECTION=1` и заголовок `X-Fault` допустимы только на учебном стенде. HTTP без TLS — только локально, TLS и mTLS относятся к lab7. +- После долгого простоя Tempo сброс накопленной очереди упирается в лимит ingest: часть данных теряется уже после восстановления, алерт остаётся firing ещё около 2 минут (окно rate). +- Окно SLO 30 минут учебное. + +Следующие шаги: отдельный пул для readiness, TLS в lab7, проверка tail sampling под нагрузкой, близкой к насыщению. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 0000000..eaf1946 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,79 @@ +"""RabbitMQ topology and confirmed publishing (aio-pika). + +Topology (declared idempotently by every publisher and consumer): + exchange attachments direct, durable + queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit + exchange attachments.dlx fanout, durable + queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator) +Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker +has taken responsibility for the message (written to the quorum log). +""" +import json +import logging + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType, Message + +from app import config + +log = logging.getLogger("messenger.broker") +ROUTING_KEY = "attachment.process.v1" + + +async def connect(): + return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, client_properties={"connection_name": config.INSTANCE_ID}) + + +async def declare(channel): + dlx = await channel.declare_exchange(config.BROKER_DLX, ExchangeType.FANOUT, durable=True) + dlq = await channel.declare_queue(config.BROKER_DLQ, durable=True, arguments={"x-queue-type": "quorum"}) + await dlq.bind(dlx) + exchange = await channel.declare_exchange(config.BROKER_EXCHANGE, ExchangeType.DIRECT, durable=True) + queue = await channel.declare_queue( + config.BROKER_QUEUE, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": config.BROKER_DLX, + "x-delivery-limit": config.BROKER_DELIVERY_LIMIT, + }, + ) + await queue.bind(exchange, ROUTING_KEY) + return exchange, queue + + +class Publisher: + def __init__(self) -> None: + self.connection = None + self.channel = None + self.exchange = None + + async def ensure(self) -> None: + if self.channel is not None and not self.channel.is_closed: + return + if self.connection is None or self.connection.is_closed: + self.connection = await connect() + self.channel = await self.connection.channel(publisher_confirms=True) + self.exchange, _ = await declare(self.channel) + + async def publish(self, body: dict) -> None: + await self.ensure() + message = Message( + json.dumps(body, separators=(",", ":")).encode(), + content_type="application/json", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=body["job_id"], + type=body["type"], + # Native AMQP headers duplicate the body carrier (same W3C values) for broker-side tooling. + headers={"attempt": body["attempt"], **{k: body[k] for k in ("traceparent", "tracestate") if k in body}}, + ) + # mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack. + await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5) + + async def close(self) -> None: + try: + if self.connection is not None: + await self.connection.close() + except Exception: + pass + self.connection = self.channel = self.exchange = None diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..fc6a68d --- /dev/null +++ b/app/config.py @@ -0,0 +1,98 @@ +import os +import socket + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + path = os.environ.get(name + "_FILE") + if path: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + return os.environ.get(name, default) + + +DATABASE_URL = _secret("DATABASE_URL") +CACHE_URL = _secret("CACHE_URL") +SESSION_STORE = os.environ.get("SESSION_STORE", "postgres") +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30) +SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60) +DB_POOL_MIN = _int("DB_POOL_MIN", 1) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0) +DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3) +DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000) +DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0) +REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0) +DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5) +READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0) +READY_FAILURES = _int("READY_FAILURES", 2) +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2) +MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64) +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_RSS_MB = _int("MAX_RSS_MB", 0) +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 +SESSION_REVOKED_TTL_SECONDS = max( + _int("SESSION_REVOKED_TTL_SECONDS", 60), + SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5, +) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY") +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760) +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000) +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0) +UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576) +RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300) +RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2) + +# --- lab5: broker, jobs, worker --- +BROKER_URL = _secret("BROKER_URL") +BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments") +BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx") +BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq") +BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20) +PIPELINE_VERSION = _int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) +WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY +JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60) +JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20) +JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0) +JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0) +JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0) +WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768) +WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25) +WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat") +OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0) +OUTBOX_BATCH = _int("OUTBOX_BATCH", 50) +OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400) +STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400) +# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables). +MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200) +PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0) +THUMBNAIL_SIZE = 256 +CROP_SIZE = 128 +OPTIMIZED_SIZE = 1280 +JPEG_QUALITY = _int("JPEG_QUALITY", 85) +WEBP_QUALITY = _int("WEBP_QUALITY", 80) +# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py). +FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1" diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 0000000..113aabc --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +import uuid +from datetime import datetime, timezone + +TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") +TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class BadCursor(Exception): + pass + + +def fmt(ts: datetime) -> str: + return ts.astimezone(timezone.utc).strftime(TS_FORMAT) + + +def encode_cursor(scope: str, created_at: str, item_id: str) -> str: + raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(scope: str, cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + data = json.loads(raw) + except Exception: + raise BadCursor() from None + if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope: + raise BadCursor() + ts, item_id = data["t"], data["i"] + if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str): + raise BadCursor() + try: + parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc) + if str(uuid.UUID(item_id)) != item_id: + raise BadCursor() + except ValueError: + raise BadCursor() from None + if encode_cursor(scope, ts, item_id) != cursor: + raise BadCursor() + return parsed, item_id diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..212b775 --- /dev/null +++ b/app/db.py @@ -0,0 +1,134 @@ +import asyncio +import re +import time +from contextlib import asynccontextmanager + +import psycopg +from opentelemetry.trace import SpanKind, Status, StatusCode +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from app import config, telemetry +from app.errors import unavailable + +BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) +OPERATION_RE = re.compile(r"^\s*([A-Za-z]+)") +TABLE_RE = re.compile(r"\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-z_]+)", re.IGNORECASE) +POOL_WAIT_SPAN_SECONDS = 0.005 + + +def statement(query) -> tuple: + """Span name and attributes for a parametrised SQL statement (values are never part of the text).""" + text = query if isinstance(query, str) else str(query) + op = OPERATION_RE.match(text) + table = TABLE_RE.search(text) + operation = op.group(1).upper() if op else "SQL" + name = operation + (" " + table.group(1) if table else "") + return name, {"db.system": "postgresql", "db.operation.name": operation, "db.namespace": "messenger", + "db.collection.name": table.group(1) if table else "", "db.query.text": text[:400]} + + +async def discard(conn) -> None: + try: + await conn.close() + except Exception: + pass + + +class Guarded: + def __init__(self, conn, timeout: float): + self.conn = conn + self.timeout = timeout + + async def execute(self, *args, **kwargs): + # Own CLIENT span around the awaited statement: the time really spent in PostgreSQL (lock waits included). + # (opentelemetry-instrumentation-psycopg 0.51b0 ends async spans before the await completes -> 0 ms spans.) + name, attrs = statement(args[0] if args else "") + with telemetry.span(name, kind=SpanKind.CLIENT, attributes=attrs) as span: + try: + return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout) + except asyncio.TimeoutError: + await discard(self.conn) + span.set_status(Status(StatusCode.ERROR, "client timeout %.1fs" % self.timeout)) + raise unavailable("Хранилище не ответило за отведённое время") from None + + def transaction(self): + return self.conn.transaction() + + +class Database: + def __init__(self) -> None: + options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % ( + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS * 2, + ) + self.pool = AsyncConnectionPool( + conninfo=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_POOL_TIMEOUT, + max_lifetime=600.0, + open=False, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": config.DB_CONNECT_TIMEOUT, + "options": options, + }, + ) + + async def open(self) -> None: + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close() + + @asynccontextmanager + async def connection(self): + started, acquired = time.time_ns(), False + try: + async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn: + acquired = True + waited = (time.time_ns() - started) / 1e9 + if waited >= POOL_WAIT_SPAN_SECONDS: # only noticeable waits become spans + telemetry.tracer.start_span("db.pool.acquire", start_time=started, + attributes={"db.system": "postgresql", + "db.client.connection.wait_s": round(waited, 4)}).end() + yield Guarded(conn, config.DB_OP_TIMEOUT) + except BROKEN as exc: + if not acquired: + span = telemetry.tracer.start_span("db.pool.acquire", start_time=started, + attributes={"db.system": "postgresql"}) + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + span.end() + raise unavailable("Хранилище недоступно") from None + + @asynccontextmanager + async def transaction(self): + async with self.connection() as conn: + async with conn.transaction(): + yield conn + + async def in_transaction(self, action): + async def run(): + async with self.transaction() as conn: + return await action(conn) + + try: + return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT) + except asyncio.TimeoutError: + raise unavailable("Хранилище не ответило за отведённое время") from None + + async def ready(self) -> bool: + try: + async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn: + try: + await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT) + except asyncio.TimeoutError: + await discard(conn) + return False + return True + except Exception: + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 0000000..8110d27 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +from typing import Optional + +from fastapi.responses import JSONResponse + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(scheme: str = "Bearer") -> ApiError: + challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE + message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия" + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Объект не найден") + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message, {"Retry-After": "1"}) + + +def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})}, + ) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 0000000..d9a2dfc --- /dev/null +++ b/app/faults.py @@ -0,0 +1,38 @@ +"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off). + +A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`. +Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata. +Crash faults fire only on the first attempt and first delivery, so recovery can be observed. + api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish + worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT + worker_crash_after_commit worker exits after the result COMMIT, before broker ack + worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1 + transientN attempts 1..N fail with a transient error (N = 1..9) + permanent permanent error on the first attempt -> failed without retries +""" +import logging +import os +import re +from typing import Optional + +from app import config + +log = logging.getLogger("messenger.faults") +TRANSIENT_RE = re.compile(r"fault-transient([1-9])") + + +def active(filename: Optional[str], name: str) -> bool: + return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename + + +def transient_until(filename: Optional[str]) -> int: + if not config.FAULT_INJECTION or not filename: + return 0 + match = TRANSIENT_RE.search(filename) + return int(match.group(1)) if match else 0 + + +def crash(point: str) -> None: + log.error("FAULT INJECTION: process exits at %s", point) + logging.shutdown() + os._exit(70) diff --git a/app/imagecheck.py b/app/imagecheck.py new file mode 100644 index 0000000..fe698c5 --- /dev/null +++ b/app/imagecheck.py @@ -0,0 +1,213 @@ +"""Integrity checks for uploaded PNG/JPEG with bounded memory. + +PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in +0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy +stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is +only a warning. Both finish before anything is stored. +""" +import logging +import os +import struct +import subprocess +import threading +import time +import zlib + +from PIL import Image + +from app import config +from app.errors import ApiError, unavailable + +log = logging.getLogger("messenger.imagecheck") + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +Image.MAX_IMAGE_PIXELS = None +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +FULL_DECODE_MAX_PIXELS = 4000000 +SCAN_SECONDS = 10.0 +OUT_STEP = 1 << 20 +# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit. +DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null") +jpeg_slot = threading.Lock() + + +class Invalid(Exception): + pass + + +def invalid() -> ApiError: + return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG") + + +class RowPlan: + def __init__(self, segments): + self.segments = [s for s in segments if s[0] > 1 and s[1] > 0] + self.seg = 0 + self.done = 0 + self.pos = 0 + + def feed(self, data: bytes) -> None: + i, n = 0, len(data) + while i < n: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + if self.seg >= len(self.segments): + raise Invalid("data after the last row") + length, rows = self.segments[self.seg] + if self.pos > 0: + take = min(length - self.pos, n - i) + i += take + self.pos += take + if self.pos == length: + self.pos = 0 + self.done += 1 + continue + full = min((n - i) // length, rows - self.done) + if full > 0: + if max(data[i:i + full * length:length]) > 4: + raise Invalid("bad filter type") + i += full * length + self.done += full + else: + if data[i] > 4: + raise Invalid("bad filter type") + self.pos = n - i + i = n + + def complete(self) -> bool: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + return self.seg >= len(self.segments) and self.pos == 0 + + +def png_plan(width, height, depth, color, interlace): + bits = CHANNELS[color] * depth + if not interlace: + return RowPlan([(1 + (width * bits + 7) // 8, height)]) + segments = [] + for x0, y0, dx, dy in ADAM7: + w = (width - x0 + dx - 1) // dx if width > x0 else 0 + h = (height - y0 + dy - 1) // dy if height > y0 else 0 + if w and h: + segments.append((1 + (w * bits + 7) // 8, h)) + return RowPlan(segments) + + +def scan_png(fileobj) -> None: + deadline = time.monotonic() + SCAN_SECONDS + fileobj.seek(0) + if fileobj.read(8) != PNG_SIGNATURE: + raise Invalid("signature") + plan = decomp = None + palette = seen_idat = idat_closed = False + color = None + first = True + while True: + if time.monotonic() > deadline: + raise Invalid("scan too slow") + head = fileobj.read(8) + if len(head) < 8: + raise Invalid("truncated before IEND") + length, kind = struct.unpack(">I4s", head) + if length > 0x7FFFFFFF: + raise Invalid("chunk length") + if first and kind != b"IHDR": + raise Invalid("IHDR must be first") + crc = zlib.crc32(kind) + if kind == b"IHDR": + body = fileobj.read(length) + if not first or length != 13 or len(body) != 13: + raise Invalid("IHDR") + width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1): + raise Invalid("IHDR fields") + if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]: + raise Invalid("color type / depth") + plan = png_plan(width, height, depth, color, interlace) + decomp = zlib.decompressobj() + crc = zlib.crc32(body, crc) + else: + remaining = length + while remaining: + buf = fileobj.read(min(remaining, 65536)) + if not buf: + raise Invalid("truncated chunk") + remaining -= len(buf) + crc = zlib.crc32(buf, crc) + if kind == b"IDAT": + if idat_closed or (color == 3 and not palette): + raise Invalid("IDAT order") + seen_idat = True + out = decomp.decompress(buf, OUT_STEP) + while True: + plan.feed(out) + if not decomp.unconsumed_tail: + break + if time.monotonic() > deadline: + raise Invalid("scan too slow") + out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP) + if kind == b"PLTE": + palette = True + if color in (0, 4) or length % 3 or not 3 <= length <= 768: + raise Invalid("PLTE") + elif kind != b"IDAT" and seen_idat: + idat_closed = True + tail = fileobj.read(4) + if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF: + raise Invalid("crc") + first = False + if kind == b"IEND": + if length or not seen_idat: + raise Invalid("IEND") + break + if not decomp.eof or decomp.unused_data or not plan.complete(): + raise Invalid("pixel data does not match IHDR") + + +def check_jpeg(fileobj) -> None: + fileobj.flush() + descriptor = fileobj.fileno() + os.lseek(descriptor, 0, os.SEEK_SET) + try: + with jpeg_slot: + done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=SCAN_SECONDS, check=False) + except subprocess.TimeoutExpired: + log.error("jpeg_check_timeout") + raise unavailable("Проверка изображения не успела завершиться") from None + # A signal means the checker died (e.g. OOM killer), which says nothing about the file. + if done.returncode < 0: + log.error("jpeg_check_killed signal=%d", -done.returncode) + raise unavailable("Проверка изображения прервана") + if done.returncode != 0: + raise invalid() + + +def inspect_image(fileobj, size: int, content_type: str) -> None: + """Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type).""" + try: + fileobj.seek(0) + img = Image.open(fileobj) + width, height = img.size + if img.format != FORMATS[content_type]: + raise invalid() + if width * height > config.MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей") + if content_type == "image/png": + scan_png(fileobj) + if width * height <= FULL_DECODE_MAX_PIXELS: + fileobj.seek(0) + Image.open(fileobj).load() + else: + check_jpeg(fileobj) + except ApiError: + raise + except Exception: + raise invalid() from None + finally: + fileobj.seek(0) diff --git a/app/imageproc.py b/app/imageproc.py new file mode 100644 index 0000000..13842a6 --- /dev/null +++ b/app/imageproc.py @@ -0,0 +1,115 @@ +"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a +wall-clock timeout, so a hostile or huge image can only kill this process, never the worker. + +usage: python -m app.imageproc +stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex. + +Rules (contracts/README.md "Фоновая обработка"): + thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px + crop JPEG, centred square scaled to 128x128 (small sources are upscaled) + optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4 +EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha). +EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB). +""" +import json +import math +import os +import sys + +from PIL import Image, ImageFile, ImageOps + +Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding +ImageFile.LOAD_TRUNCATED_IMAGES = False + +THUMB, CROP, OPTIMIZED = 256, 128, 1280 +JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85")) +WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80")) +ORIENTATION_TAG = 0x0112 +EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5 + + +def fit(width: int, height: int, bound: int): + """Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales.""" + if width <= bound and height <= bound: + return width, height + if width >= height: + return bound, max(1, height * bound // width) + return max(1, width * bound // height), bound + + +def has_alpha(img) -> bool: + return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info) + + +def to_8bit(img): + if img.mode in ("I;16", "I;16B", "I;16L", "I"): + return img.convert("I").point(lambda v: v * (1 / 257)).convert("L") + return img + + +def on_white(img): + if has_alpha(img): + rgba = img.convert("RGBA") + flat = Image.new("RGB", rgba.size, (255, 255, 255)) + flat.paste(rgba, mask=rgba.getchannel("A")) + return flat, rgba + return img.convert("RGB"), None + + +def process(source: str, out_dir: str, max_pixels: int) -> list: + img = Image.open(source) + if img.format not in ("PNG", "JPEG"): + raise ValueError("format") + width, height = img.size + if width * height > max_pixels: + sys.exit(EXIT_TOO_LARGE) + orientation = img.getexif().get(ORIENTATION_TAG, 1) + upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height) + thumb_size = fit(*upright, THUMB) + opt_size = fit(*upright, OPTIMIZED) + if img.format == "JPEG": + # DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept). + need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0]) + img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1]))) + img.load() + img = ImageOps.exif_transpose(img) + img = to_8bit(img) + flat, rgba = on_white(img) + del img + + results = [] + + def save(name, image, ext, fmt, content_type, **params): + path = os.path.join(out_dir, name + "." + ext) + image.save(path, fmt, **params) + results.append({"name": name, "file": path, "content_type": content_type, + "width": image.size[0], "height": image.size[1]}) + + jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False} + save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0), + "jpg", "JPEG", "image/jpeg", **jpeg) + w, h = flat.size + side = min(w, h) + left, top = (w - side) // 2, (h - side) // 2 + save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)), + "jpg", "JPEG", "image/jpeg", **jpeg) + source_for_webp = rgba if rgba is not None else flat + optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize( + opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0) + save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4) + return results + + +def main() -> None: + source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3]) + try: + results = process(source, out_dir, max_pixels) + except MemoryError: + sys.exit(EXIT_TOO_COMPLEX) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): + sys.exit(EXIT_INVALID) + json.dump(results, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 0000000..2bfa24d --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,340 @@ +"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker. + +State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it +(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes +`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT. + + enqueue nothing -> queued (API, same txn as attachment 'queued') + claim queued(due) -> processing, lease (worker; also takes over an expired lease) + heartbeat processing: lease_until += lease (only while owner is still alive) + complete processing -> succeeded + variants (fenced by lease_owner) + fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts) + sweep_leases processing(expired lease) -> queued | failed + replay failed -> queued (explicit operator action, audited) +""" +import json +import random +import uuid +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +from app import config, telemetry +from app.cursor import fmt + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 +VARIANT_ORDER = ("thumbnail", "crop", "optimized") +REQUIRED_VARIANTS = VARIANT_ORDER + + +def backoff_seconds(attempt: int) -> float: + """Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd).""" + ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict: + return { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": fmt(created_at), + } + + +def parse_message(body: bytes) -> Optional[dict]: + """Strict check against jobs.schema.json; anything else goes to the DLQ untouched.""" + try: + data = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + return None + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}: + return None + if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE: + return None + if "traceparent" in data and not (isinstance(data["traceparent"], str) + and telemetry.TRACEPARENT_RE.match(data["traceparent"])): + return None + if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512): + return None + for key in ("pipeline_version", "attempt"): + if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1: + return None + try: + data["job_id"] = str(uuid.UUID(data["job_id"])) + data["attachment_id"] = str(uuid.UUID(data["attachment_id"])) + except (ValueError, AttributeError, TypeError): + return None + return data + + +async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None): + await conn.execute( + "INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)" + " VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())", + (job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})), + ) + + +async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0): + body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"]) + # W3C trace context of the transaction that schedules the job (upload request span or the failed consumer + # span for a retry) travels in the job message (jobs.schema.json traceparent/tracestate). + body.update(telemetry.inject()) + await conn.execute( + "INSERT INTO outbox (job_id, payload, available_at, created_at)" + " VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())", + (job["id"], json.dumps(body), delay), + ) + + +async def enqueue(conn, attachment_id: str, actor: str) -> dict: + cur = await conn.execute( + "INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts," + " next_attempt_at, created_at, updated_at)" + " VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())" + " RETURNING id, attachment_id, pipeline_version, attempt, created_at", + (attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS), + ) + job = await cur.fetchone() + await _outbox(conn, job, 1) + await audit(conn, job["id"], attachment_id, "enqueued", actor, 1) + return job + + +async def pending_count(conn) -> int: + cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')") + return (await cur.fetchone())["n"] + + +async def queue_stats(conn) -> dict: + """Backlog for metrics: queued / processing jobs, age of the oldest unfinished job, unpublished outbox rows.""" + cur = await conn.execute( + "SELECT count(*) FILTER (WHERE state = 'queued') AS queued," + " count(*) FILTER (WHERE state = 'processing') AS processing," + " coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)::float8 AS oldest_age" + " FROM attachment_jobs WHERE state IN ('queued', 'processing')" + ) + row = dict(await cur.fetchone()) + cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL") + row["outbox"] = (await cur.fetchone())["n"] + return row + + +# ---------------------------------------------------------------- worker side + +JOB_COLUMNS = ( + "j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until," + " j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired," + " (j.next_attempt_at <= clock_timestamp()) AS due" +) + + +async def claim(conn, msg: dict, owner: str): + """Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version.""" + cur = await conn.execute( + "SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status" + " FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.id = %s FOR UPDATE OF j", + (msg["job_id"],), + ) + job = await cur.fetchone() + if job is None or str(job["attachment_id"]) != msg["attachment_id"]: + return "missing", None + if job["state"] in ("succeeded", "failed"): + return "done", job + if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]: + return "newer_version", job + if msg["attempt"] != job["attempt"]: + return "stale", job + if job["state"] == "processing" and not job["lease_expired"]: + return "busy", job + if job["state"] == "queued" and not job["due"]: + return "early", job + takeover = job["state"] == "processing" + await conn.execute( + "UPDATE attachment_jobs SET state = 'processing', lease_owner = %s," + " lease_until = clock_timestamp() + make_interval(secs => %s)," + " started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp())," + " updated_at = clock_timestamp() WHERE id = %s", + (owner, config.JOB_LEASE_SECONDS, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (job["attachment_id"],), + ) + await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"]) + return "claimed", job + + +async def heartbeat(conn, job_id, owner: str) -> bool: + cur = await conn.execute( + "UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s", + (config.JOB_LEASE_SECONDS, job_id, owner), + ) + return cur.rowcount == 1 + + +async def attachment_source(conn, attachment_id) -> Optional[dict]: + cur = await conn.execute( + "SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status" + " FROM attachments WHERE id = %s", + (attachment_id,), + ) + return await cur.fetchone() + + +async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool: + """Atomically publish the result. False when the lease was lost: someone else owns the effect now.""" + cur = await conn.execute( + "UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL," + " lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s", + (job["id"], owner), + ) + if cur.rowcount != 1: + return False + for v in variants: + await conn.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type," + " size_bytes, width, height, sha256, created_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())" + " ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING", + (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"]), + ) + # Never move a newer pipeline_version back, never touch an attachment that is already ready. + await conn.execute( + "UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()" + " WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s", + (job["pipeline_version"], job["attachment_id"], job["pipeline_version"]), + ) + await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"], + {"variants": [v["name"] for v in variants]}) + return True + + +async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str: + attempt = job["attempt"] + if transient and attempt < job["max_attempts"]: + delay = backoff_seconds(attempt) + await conn.execute( + "UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL," + " next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s," + " updated_at = clock_timestamp() WHERE id = %s", + (delay, detail[:500], error_code, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (job["attachment_id"],), + ) + await _outbox(conn, job, attempt + 1, delay=delay) + await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt, + {"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)}) + return "retry" + final_code = error_code if not transient else "retry_exhausted" + await conn.execute( + "UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp()," + " last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s", + (detail[:500], final_code, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (final_code, job["attachment_id"]), + ) + await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt, + {"error_code": final_code, "cause": error_code, "detail": detail[:200]}) + return "failed" + + +async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str: + """Returns retry | failed | lost (lease is not ours any more: the new owner decides).""" + cur = await conn.execute( + "SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE", + (job["id"], owner), + ) + row = await cur.fetchone() + if row is None: + return "lost" + return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient) + + +async def sweep_leases(conn, actor: str) -> List[str]: + """Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt.""" + cur = await conn.execute( + "SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner" + " FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()" + " ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED", + ) + outcomes = [] + for row in await cur.fetchall(): + outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired", + "lease of %s expired" % (row["lease_owner"] or "?"), True) + outcomes.append("%s:%s" % (row["id"], outcome)) + return outcomes + + +async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict: + """failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed.""" + cur = await conn.execute( + "SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at," + " a.status AS attachment_status, a.staging_key" + " FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a", + (attachment_id,), + ) + job = await cur.fetchone() + if job is None: + return {"result": "refused", "reason": "no job for attachment"} + if job["state"] != "failed" or job["attachment_status"] != "failed": + await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], + {"state": job["state"], "attachment_status": job["attachment_status"]}) + return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])} + if not job["staging_key"] or not await staging_exists(job["staging_key"]): + await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"}) + return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"} + attempt = job["attempt"] + 1 + await conn.execute( + "UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp()," + " finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s", + (attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'failed'", + (job["attachment_id"],), + ) + await _outbox(conn, job, attempt) + await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt) + return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt} + + +async def failed_jobs(conn, limit: int = 50) -> List[dict]: + cur = await conn.execute( + "SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at" + " FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s", + (limit,), + ) + return [ + {"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"], + "max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"], + "failed_at": fmt(r["updated_at"])} + for r in await cur.fetchall() + ] + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def older_than(seconds: int) -> datetime: + return utcnow() - timedelta(seconds=seconds) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..8ef8a9f --- /dev/null +++ b/app/main.py @@ -0,0 +1,721 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import logging +import os +import re +import resource +import signal +import tempfile +import unicodedata +import sys +import uuid +from contextlib import asynccontextmanager +from typing import List, Optional, Tuple + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from starlette.concurrency import iterate_in_threadpool, run_in_threadpool +from starlette.responses import StreamingResponse + +from app import config, faults, jobs, passwords, reconcile, repo, telemetry +from app.cursor import BadCursor, decode_cursor, encode_cursor +from app.db import Database +from app.imagecheck import inspect_image +from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable +from app.broker import Publisher +from app.outbox import Relay +from app.s3 import S3Storage +from app.sessions import SessionStore + +log = logging.getLogger("messenger") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"}) +EXEMPT_PATHS = ("/health/live", "/health/ready") +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 +ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"} +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" + +upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) +background = set() +db = Database() +sessions = SessionStore(db) +s3 = S3Storage() +publisher = Publisher() +relay = Relay(db, publisher) +state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0, + "queue": None} +VARIANTS = ("original", "thumbnail", "crop", "optimized") + + +def rss_mb() -> float: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE / 1048576 + except OSError: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak / 1048576 if sys.platform == "darwin" else peak / 1024 + + +def count_rejection(code: str) -> None: + state["rejected"][code] = state["rejected"].get(code, 0) + 1 + + +async def sweeper() -> None: + while True: + await asyncio.sleep(config.SESSION_SWEEP_SECONDS) + try: + removed = await sessions.sweep() + if removed: + log.info("sessions_expired removed=%d", removed) + except Exception: + log.warning("session_sweep_skipped: storage unavailable") + try: + async with db.connection() as conn: + removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS) + if removed: + log.info("idempotency_cleanup removed=%d", removed) + except Exception: + log.warning("idempotency_cleanup_skipped") + try: + removed = await relay.cleanup() + if removed: + log.info("outbox_cleanup removed=%d", removed) + except Exception: + log.warning("outbox_cleanup_skipped") + + +async def pending_watch() -> None: + """Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second.""" + while True: + try: + async with db.connection() as conn: + queue = await jobs.queue_stats(conn) + state["pending_jobs"] = queue["queued"] + queue["processing"] + state["queue"] = queue + except Exception: + state["queue"] = None # unknown is not zero: the gauge disappears instead of lying + await asyncio.sleep(config.PENDING_CHECK_SECONDS) + + +async def reconciler_loop() -> None: + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + while True: + try: + result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False) + if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))): + log.info("reconcile %s", result) + except Exception: + log.warning("reconcile_skipped: dependency unavailable") + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + + +async def readiness_watch() -> None: + failures = 0 + while True: + if await db.ready(): + failures = 0 + if not state["ready"]: + state["ready"] = True + log.info("ready_again: database reachable") + else: + failures += 1 + if failures >= config.READY_FAILURES and state["ready"]: + state["ready"] = False + log.warning("not_ready: database unreachable, business requests get 503") + s3_ok = await run_in_threadpool(s3.probe) + if s3_ok != state["s3_ok"]: + state["s3_ok"] = s3_ok + log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503") + await asyncio.sleep(config.READY_CHECK_SECONDS) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + previous = {} + + def on_signal(signum, frame): + if not state["draining"]: + state["draining"] = True + log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name) + handler = previous.get(signum) + if callable(handler): + handler(signum, frame) + + for sig in (signal.SIGTERM, signal.SIGINT): + previous[sig] = signal.getsignal(sig) + signal.signal(sig, on_signal) + await db.open() + telemetry.register_pool(db) + telemetry.register_queue(state) + await sessions.open() + await run_in_threadpool(s3.open) + state["s3_ok"] = await run_in_threadpool(s3.probe) + tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()), + asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())] + log.info( + "started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s", + config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS, + config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES, + config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET, + ) + log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s", + config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION) + yield + for task in tasks: + task.cancel() + await publisher.close() + await sessions.close() + await db.close() + log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"]) + telemetry.shutdown() + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +@app.middleware("http") +async def guard(request: Request, call_next): + rid = uuid.uuid4().hex + request.state.rid = rid + # X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request. + telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID) + exempt = request.url.path in EXEMPT_PATHS + if state["draining"] and request.url.path != "/health/live": + count_rejection("draining") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"}) + if not exempt and not state["ready"]: + count_rejection("not_ready") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"}) + if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS: + count_rejection("inflight") + return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"}) + state["inflight"] += 1 + try: + fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else "" + if fault: + await inject_http_fault(fault) + response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT) + except asyncio.TimeoutError: + count_rejection("request_timeout") + log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path) + response = error_body( + rid, config.INSTANCE_ID, 503, "unavailable", + "Запрос не удалось обработать за отведённое время", {"Retry-After": "1"}, + ) + except Exception as exc: + telemetry.record_exception(exc) + log.exception("unhandled_error request_id=%s", rid) + response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера") + finally: + state["inflight"] -= 1 + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +async def inject_http_fault(fault: str) -> None: + """Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error` + raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose + trace survival can be counted (span attribute messenger.fault).""" + telemetry.set_attributes(messenger__fault=fault[:16]) + if fault == "slow": + await asyncio.sleep(0.6) + elif fault == "error": + raise RuntimeError("injected handler failure") + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + if exc.status in (413, 429, 503): + count_rejection(exc.code) + return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers) + + +def parse_uuid(value: object) -> Optional[str]: + if not isinstance(value, str): + return None + try: + return str(uuid.UUID(value)) + except ValueError: + return None + + +def non_blank(value: object, max_len: int) -> bool: + if not isinstance(value, str) or not 1 <= len(value) <= max_len: + return False + return value.translate(ZERO_WIDTH).strip() != "" + + +async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict: + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое") + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: + raise too_large + raw = bytearray() + async for chunk in request.stream(): + raw += chunk + if len(raw) > config.MAX_BODY_BYTES: + raise too_large + try: + body = json.loads(bytes(raw).decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Некорректный JSON") from None + if not isinstance(body, dict): + raise bad("Ожидается JSON-объект") + allowed = set(required) | set(optional) + if set(body) - allowed: + raise bad("Лишние поля в запросе") + if set(required) - set(body): + raise bad("Отсутствуют обязательные поля") + return body + + +def ensure_capacity() -> None: + if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB: + raise unavailable("Достигнут лимит памяти процесса, запись недоступна") + + +def bearer_token(request: Request) -> str: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token: + raise unauthorized() + return token + + +async def actor(request: Request) -> dict: + user = await sessions.resolve(bearer_token(request)) + if user is None: + raise unauthorized() + return user + + +def basic_credentials(request: Request) -> Tuple[str, str]: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized("Basic") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized("Basic") from None + username, separator, password = decoded.partition(":") + if not separator: + raise unauthorized("Basic") + return username, password + + +async def member_chat(user: dict, chat_id: str) -> dict: + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + async with db.connection() as conn: + chat = await repo.chat_for_member(conn, cid, user["id"]) + if chat is None: + raise not_found() + return chat + + +def page_params(request: Request) -> Tuple[int, Optional[str]]: + query = request.query_params + limit = 50 + if "limit" in query: + if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100: + raise bad("limit должен быть целым 1..100") + limit = int(query["limit"]) + cursor = query.get("cursor") + if cursor is not None and cursor == "": + raise bad("Некорректный cursor") + return limit, cursor + + +def keyset(scope: str, cursor: Optional[str]): + if cursor is None: + return None + try: + return decode_cursor(scope, cursor) + except BadCursor: + raise bad("Некорректный cursor") from None + + +def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse: + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +def detect_content_type(head: bytes) -> Optional[str]: + if head[:8] == PNG_MAGIC: + return "image/png" + if head[:3] == JPEG_MAGIC: + return "image/jpeg" + return None + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."): + raise bad("filename: 1..255 символов, не только пробелы") + if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename): + raise bad("filename содержит недопустимые символы") + + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def ready(): + if not await db.ready(): + raise unavailable("База данных недоступна") + if not state["s3_ok"]: + raise unavailable("Хранилище файлов недоступно") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + body = await read_json(request, ("username", "display_name", "password")) + if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + if not non_blank(body["display_name"], 100): + raise bad("display_name: 1..100 символов, не только пробелы") + password = body["password"] + if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX)) + ensure_capacity() + password_hash = await run_in_threadpool(passwords.hash_password, password) + async with db.connection() as conn: + user = await repo.create_user(conn, body["username"], body["display_name"], password_hash) + if user is None: + raise ApiError(409, "conflict", "username уже занят") + log.info("user_created id=%s", user["id"]) + return JSONResponse(user, status_code=201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + if await request.body(): + raise bad("POST /auth/sessions не принимает тело запроса") + username, password = basic_credentials(request) + row = None + if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + async with db.connection() as conn: + row = await repo.credentials(conn, username) + stored = row["password_hash"] if row else passwords.DUMMY_HASH + matches = await run_in_threadpool(passwords.verify_password, stored, password) + if row is None or not matches: + log.info("login_failed username_known=%s", row is not None) + raise unauthorized("Basic") + user = repo.public_user(row) + token, expires_at = await sessions.create(user) + log.info("session_created user=%s", user["id"]) + return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + token = bearer_token(request) + user = await sessions.resolve(token) + if user is None: + raise unauthorized() + await sessions.revoke(token) + log.info("session_revoked user=%s", user["id"]) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + return await actor(request) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await actor(request) + body = await read_json(request, ("title", "member_ids")) + if not non_blank(body["title"], 100): + raise bad("title: 1..100 символов, не только пробелы") + raw_ids = body["member_ids"] + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100: + raise bad("member_ids: массив из 1..100 UUID") + ids: List[str] = [] + for value in raw_ids: + parsed = parse_uuid(value) + if parsed is None: + raise bad("member_ids содержит некорректный UUID") + ids.append(parsed) + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user["id"] not in ids: + raise bad("Создатель должен входить в member_ids") + ensure_capacity() + + async def insert(conn): + return await repo.create_chat(conn, body["title"], ids) + + try: + chat = await db.in_transaction(insert) + except repo.UnknownMember: + raise bad("member_ids содержит неизвестного пользователя") from None + log.info("chat_created id=%s members=%d", chat["id"], len(ids)) + return JSONResponse(chat, status_code=201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await actor(request) + limit, cursor = page_params(request) + scope = "chats:" + user["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_chats(conn, user["id"], limit, after) + return page(scope, items, has_more) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + return await member_chat(await actor(request), chat_id) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user = await actor(request) + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + + idem_key = request.headers.get("idempotency-key", "") + if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key): + raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + + body = await read_json(request, ("text",), ("attachment_ids",)) + + text = body["text"] + attachment_ids_raw = body.get("attachment_ids", []) + + if not isinstance(attachment_ids_raw, list): + raise bad("attachment_ids должен быть массивом UUID") + if len(attachment_ids_raw) > 10: + raise bad("attachment_ids: максимум 10 вложений") + + attachment_ids: List[str] = [] + for value in attachment_ids_raw: + parsed = parse_uuid(value) + if parsed is None: + raise bad("attachment_ids содержит некорректный UUID") + attachment_ids.append(parsed) + + if len(set(attachment_ids)) != len(attachment_ids): + raise bad("attachment_ids содержит дубликаты") + + has_text = isinstance(text, str) and non_blank(text, 4000) + has_attachments = len(attachment_ids) > 0 + + if not has_text and not has_attachments: + raise bad("text: 1..4000 символов, не только пробелы") + + if isinstance(text, str) and len(text) > 4000: + raise bad("text: 1..4000 символов, не только пробелы") + + if not isinstance(text, str): + raise bad("text должен быть строкой") + + ensure_capacity() + + normalized = {"text": text} + if attachment_ids: + normalized["attachment_ids"] = attachment_ids + body_hash = hashlib.sha256( + json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode() + ).hexdigest() + + async def do(conn): + return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash) + + status, message = await db.in_transaction(do) + + if status == "conflict": + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса") + if status == "not_member": + raise not_found() + if status == "attachment_not_found": + raise not_found() + if status == "attachment_failed": + raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить") + return JSONResponse(message, status_code=201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + chat = await member_chat(await actor(request), chat_id) + limit, cursor = page_params(request) + scope = "messages:" + chat["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_messages(conn, chat["id"], limit, after) + return page(scope, items, has_more) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: str, request: Request): + user = await actor(request) + chat = await member_chat(user, chat_id) + + filename = request.query_params.get("filename", "") + validate_filename(filename) + + declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared_ct not in ALLOWED_CONTENT_TYPES: + raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg") + + declared_length = request.headers.get("content-length") + too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB") + if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES: + raise too_large + ensure_capacity() + # Backpressure: do not accept more work than the pipeline can hold; the client retries later. + if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS: + count_rejection("queue_full") + raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"}) + + sha = hashlib.sha256() + size = 0 + with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool: + head = b"" + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise too_large + if len(head) < 8: + head += chunk[: 8 - len(head)] + if len(head) == 8 and detect_content_type(head) is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + sha.update(chunk) + spool.write(chunk) + if size == 0: + raise bad("Пустой файл") + actual_ct = detect_content_type(head) + if actual_ct is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + if actual_ct != declared_ct: + raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла") + async with upload_slots: + await run_in_threadpool(inspect_image, spool, size, actual_ct) + + attachment_id = str(uuid.uuid4()) + staging_key = "staging/" + attachment_id + object_key = "attachments/%s/original" % attachment_id + async with db.connection() as conn: + await repo.begin_attachment( + conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), + staging_key, object_key, + ) + try: + async with upload_slots: + await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct) + except BaseException: + defer(compensate(attachment_id, staging_key)) + raise + + # Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox. + try: + att = await db.in_transaction(lambda conn: repo.queue_attachment( + conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS)) + except BaseException: + await compensate(attachment_id, staging_key) + raise + if att == repo.QUEUE_FULL: + defer(compensate(attachment_id, staging_key)) + count_rejection("queue_full") + raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"}) + if att is None: + raise unavailable("Загрузка отменена сверкой, повторите запрос") + telemetry.set_attributes(messenger__attachment_id=att["id"]) + if faults.active(filename, "api_crash_after_commit"): + faults.crash("API after COMMIT of job/outbox, before publish") + relay.notify() + log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size) + return JSONResponse(att, status_code=202) + + +def defer(coro) -> None: + task = asyncio.ensure_future(coro) + background.add(task) + task.add_done_callback(background.discard) + + +async def compensate(attachment_id: str, object_key: str) -> None: + """Best effort; whatever is left behind is collected by the reconciler after the grace period.""" + try: + async with db.connection() as conn: + await repo.abort_attachment(conn, attachment_id) + await run_in_threadpool(s3.delete, object_key) + except BaseException: + log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id) + + +async def accessible_attachment(request: Request, attachment_id: str) -> dict: + user = await actor(request) + aid = parse_uuid(attachment_id) + if aid is None: + raise bad("attachment_id должен быть UUID") + async with db.connection() as conn: + row = await repo.get_attachment(conn, aid) + if row is None: + raise not_found() + chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"]) + if chat is None: + raise not_found() + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variants = [] + if row["status"] == "ready": + async with db.connection() as conn: + variants = await repo.get_variants(conn, row["id"], row["pipeline_version"]) + return repo._attachment(row, variants) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variant = request.query_params.get("variant", "original") + if variant not in VARIANTS: + raise bad("variant: original, thumbnail, crop или optimized") + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing") + else "Обработка вложения завершилась ошибкой") + key, content_type = row["object_key"], row["content_type"] + if variant != "original": + async with db.connection() as conn: + found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant] + if not found: + raise not_found() + key, content_type = found[0]["object_key"], found[0]["content_type"] + opened = await run_in_threadpool(s3.open_stream, key) + if opened is None: + log.error("object_missing attachment=%s variant=%s", row["id"], variant) + raise unavailable("Файл временно недоступен") + chunks, length = opened + return StreamingResponse( + iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)}, + ) + + +@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def fallback(request: Request): + raise not_found() + + +# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel +# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics. +telemetry.setup("messenger-api", app) diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 0000000..6e7d854 --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,100 @@ +"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work). + +Rows are locked while they are published; `published_at` is set in the same transaction only after the +broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are +expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish +leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS. +""" +import asyncio +import logging + +from opentelemetry.trace import SpanKind + +from app import config, telemetry + +log = logging.getLogger("messenger.outbox") + + +class Relay: + def __init__(self, db, publisher) -> None: + self.db = db + self.publisher = publisher + self.wake = asyncio.Event() + self.broker_ok = False + self.published = 0 + + def notify(self) -> None: + self.wake.set() + + async def run_once(self) -> int: + sent, failure = 0, None + async with self.db.connection() as conn: + async with conn.transaction(): + cur = await conn.execute( + "SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()" + " ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED", + (config.OUTBOX_BATCH,), + ) + for row in await cur.fetchall(): + try: + await self.publish(row["payload"]) + except Exception as exc: + # Keep what was already confirmed: commit those rows, record the error, stop the batch. + await conn.execute( + "UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s", + (type(exc).__name__, row["id"]), + ) + failure = exc + break + await conn.execute( + "UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1" + " WHERE id = %s", + (row["id"],), + ) + sent += 1 + self.published += sent + if failure is not None: + raise failure + return sent + + async def publish(self, payload: dict) -> None: + """PRODUCER span, child of the context stored with the job (upload request or failed attempt); + the message then carries the producer's own traceparent, so the consumer becomes its child.""" + attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "publish", + "messaging.destination.name": config.BROKER_EXCHANGE, "messaging.message.id": payload["job_id"], + "messenger.job.attempt": payload["attempt"], "messenger.attachment_id": payload["attachment_id"]} + with telemetry.span(config.BROKER_QUEUE + " publish", kind=SpanKind.PRODUCER, + context=telemetry.extract(payload), attributes=attrs): + body = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")} + body.update(telemetry.inject()) + await self.publisher.publish(body) + + async def loop(self) -> None: + delay = config.OUTBOX_POLL_SECONDS + while True: + try: + await asyncio.wait_for(self.wake.wait(), delay) + except asyncio.TimeoutError: + pass + self.wake.clear() + try: + while await self.run_once() == config.OUTBOX_BATCH: + pass + if not self.broker_ok: + log.info("outbox_relay_resumed: broker reachable") + self.broker_ok = True + delay = config.OUTBOX_POLL_SECONDS + except Exception as exc: + if self.broker_ok: + log.warning("outbox_relay_paused: %s; jobs stay in outbox", type(exc).__name__) + self.broker_ok = False + await self.publisher.close() + delay = min(10.0, max(delay * 2, 1.0)) + + async def cleanup(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute( + "DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)", + (config.OUTBOX_RETENTION_SECONDS,), + ) + return cur.rowcount diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 0000000..b509e4a --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,16 @@ +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error, VerifyMismatchError + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash("there-is-no-such-user-password") + + +def hash_password(password: str) -> str: + return HASHER.hash(password) + + +def verify_password(stored: str, password: str) -> bool: + try: + return HASHER.verify(stored, password) + except (VerifyMismatchError, Argon2Error, ValueError): + return False diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 0000000..60a138f --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,155 @@ +"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout). + +Key layout (all keys are generated by the server): + staging/ written by the API before 202, read by the worker + attachments//original server-side copy made by the worker + attachments//v/.* variants of pipeline version N +Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row. + +Cases (all gated by a grace period so in-flight work is never touched): + stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both + orphan_object object without any row/variant row, older than grace -> delete object + staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete + staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete + (after that a replay is refused; the attachment stays failed and visible) + missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing +Queued/processing work is never touched: its staging object is the input of a live job. +""" +import logging +import time +from datetime import datetime, timedelta, timezone + +from starlette.concurrency import run_in_threadpool + +from app import config + +log = logging.getLogger("messenger.reconcile") + +LOCK_ID = 2026_0104 +PREFIXES = ("staging/", "attachments/") +BATCH = 500 + + +async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict: + started = time.monotonic() + result = { + "dry_run": dry_run, "grace_seconds": grace_seconds, + "stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0, + "deleted_objects": 0, "removed_rows": 0, "marked_failed": 0, + "objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0, + } + async with db.connection() as conn: + cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,)) + if not (await cur.fetchone())["got"]: + return {"skipped": "another reconciler is running"} + try: + now = datetime.now(timezone.utc) + cutoff = now - timedelta(seconds=grace_seconds) + await _stale_uploading(conn, s3, cutoff, dry_run, result) + await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result) + await _orphan_objects(conn, s3, cutoff, dry_run, result) + await _missing_objects(conn, s3, cutoff, dry_run, result) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + result["seconds"] = round(time.monotonic() - started, 3) + return result + + +async def _stale_uploading(conn, s3, cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, coalesce(staging_key, object_key) AS key FROM attachments" + " WHERE status = 'uploading' AND created_at < %s", + (cutoff,), + ) + for row in await cur.fetchall(): + modified = await run_in_threadpool(s3.head, row["key"]) + if modified is not None and modified >= cutoff: + result["protected_fresh_objects"] += 1 + continue + result["stale_uploading"] += 1 + if dry_run: + continue + if modified is not None: + await run_in_threadpool(s3.delete, row["key"]) + result["deleted_objects"] += 1 + done = await conn.execute( + "DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff), + ) + result["removed_rows"] += done.rowcount + + +async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL" + " AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))", + (retention_cutoff,), + ) + for row in await cur.fetchall(): + if await run_in_threadpool(s3.head, row["staging_key"]) is None: + continue + result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1 + if dry_run: + continue + again = await conn.execute( + "SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))", + (row["id"], retention_cutoff), + ) + if await again.fetchone() is not None: + await run_in_threadpool(s3.delete, row["staging_key"]) + result["deleted_objects"] += 1 + + +async def _known_keys(conn, keys): + cur = await conn.execute( + "SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)" + " UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)" + " UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)", + (keys, keys, keys), + ) + return {row["k"] for row in await cur.fetchall()} + + +async def _orphan_objects(conn, s3, cutoff, dry_run, result): + for prefix in PREFIXES: + pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix))) + while True: + page = await run_in_threadpool(next, pages, None) + if page is None: + break + result["objects_scanned"] += len(page) + keys = [o["Key"] for o in page] + if not keys: + continue + known = await _known_keys(conn, keys) + for obj in page: + if obj["Key"] in known or obj["LastModified"] >= cutoff: + continue + result["orphan_objects"] += 1 + if not dry_run and not await _known_keys(conn, [obj["Key"]]): + await run_in_threadpool(s3.delete, obj["Key"]) + result["deleted_objects"] += 1 + + +async def _missing_objects(conn, s3, cutoff, dry_run, result): + last = None + while True: + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s" + " AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s", + (cutoff, last, last, BATCH), + ) + rows = await cur.fetchall() + if not rows: + return + for row in rows: + result["rows_scanned"] += 1 + if await run_in_threadpool(s3.exists, row["object_key"]): + continue + result["missing_objects"] += 1 + if not dry_run: + done = await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'object_missing'," + " updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],), + ) + result["marked_failed"] += done.rowcount + last = str(rows[-1]["id"]) diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 0000000..8eb215d --- /dev/null +++ b/app/repo.py @@ -0,0 +1,338 @@ +import hashlib +import json +from typing import List, Optional, Tuple + +import psycopg + +from app import config, jobs +from app.cursor import fmt + +MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" +CHAT_QUERY = ( + "SELECT c.id, c.title, c.created_at," + " ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids" +) + + +class UnknownMember(Exception): + pass + + +def decoded(value) -> str: + return bytes(value).decode("utf-8") + + +def public_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": decoded(row["display_name"]), + "created_at": fmt(row["created_at"]), + } + + +def _chat(row) -> dict: + return { + "id": str(row["id"]), + "title": decoded(row["title"]), + "member_ids": [str(value) for value in row["member_ids"]], + "created_at": fmt(row["created_at"]), + } + + +def _message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": decoded(row["text"]), + "created_at": fmt(row["created_at"]), + } + + +def _message_with_attachments(row, attachment_ids: List[str]) -> dict: + msg = _message(row) + msg["attachment_ids"] = attachment_ids + return msg + + +VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2} + + +def _variant(row) -> dict: + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + + +def _attachment(row, variants=None) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [], + "error_code": row["error_code"], + "created_at": fmt(row["created_at"]), + "updated_at": fmt(row["updated_at"]), + } + + +async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]: + try: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())" + " RETURNING id, username, display_name, created_at", + (username, display_name.encode("utf-8"), password_hash), + ) + except psycopg.errors.UniqueViolation: + return None + return public_user(await cur.fetchone()) + + +async def credentials(conn, username: str): + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,), + ) + return await cur.fetchone() + + +async def create_chat(conn, title: str, member_ids: List[str]) -> dict: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_at)" + " VALUES (gen_random_uuid(), %s, clock_timestamp())" + " RETURNING id, title, created_at", + (title.encode("utf-8"),), + ) + chat = await cur.fetchone() + cur = await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])", + (chat["id"], member_ids), + ) + if cur.rowcount != len(member_ids): + raise UnknownMember() + chat["member_ids"] = list(member_ids) + return _chat(chat) + + +async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]: + cur = await conn.execute( + CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id), + ) + row = await cur.fetchone() + return _chat(row) if row else None + + +async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + CHAT_QUERY + + " FROM chats c JOIN chat_members m ON m.chat_id = c.id" + " WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))" + " ORDER BY c.created_at, c.id LIMIT %s", + (user_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + return [_chat(row) for row in rows[:limit]], len(rows) > limit + + +async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]: + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id), + ) + row = await cur.fetchone() + return _message(row) if row else None + + +async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]: + cur = await conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ) + rows = await cur.fetchall() + return [str(row["attachment_id"]) for row in rows] + + +async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None: + for pos, aid in enumerate(attachment_ids): + await conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message_id, aid, pos), + ) + + +async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + "SELECT " + MESSAGE_COLUMNS + " FROM messages" + " WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))" + " ORDER BY created_at, id LIMIT %s", + (chat_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + messages = [] + for row in rows[:limit]: + aids = await _fetch_message_attachment_ids(conn, row["id"]) + messages.append(_message_with_attachments(row, aids)) + return messages, len(rows) > limit + + +def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int: + raw = "%s:%s:%s" % (user_id, chat_id, key) + return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16) + + +async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str, + attachment_ids: List[str], idem_key: str, body_hash: str): + lock_id = _idem_lock_id(sender_id, chat_id, idem_key) + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ) + if await cur.fetchone() is None: + return "not_member", None + + cur = await conn.execute( + "SELECT request_body_hash, response_json FROM idempotency_keys" + " WHERE user_id = %s AND chat_id = %s AND key = %s", + (sender_id, chat_id, idem_key), + ) + existing = await cur.fetchone() + + if existing: + if existing["request_body_hash"] != body_hash: + return "conflict", None + return "replay", json.loads(existing["response_json"]) + + if attachment_ids: + cur = await conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'", + (attachment_ids,), + ) + att_rows = await cur.fetchall() + found_ids = {str(row["id"]) for row in att_rows} + for aid in attachment_ids: + if aid not in found_ids: + return "attachment_not_found", None + for row in att_rows: + if str(row["chat_id"]) != chat_id: + return "attachment_not_found", None + if row["status"] == "failed": + return "attachment_failed", None + + text_bytes = text.encode("utf-8") if text else b"" + + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text_bytes, chat_id, sender_id), + ) + row = await cur.fetchone() + if row is None: + return "not_member", None + + if attachment_ids: + await _link_attachments(conn, row["id"], chat_id, attachment_ids) + + message = _message_with_attachments(row, list(attachment_ids)) + + await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)" + " VALUES (%s, %s, %s, %s, %s, clock_timestamp())", + (sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)), + ) + + return "created", message + + +async def cleanup_idempotency(conn, ttl_seconds: int) -> int: + cur = await conn.execute( + "DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)", + (ttl_seconds,), + ) + return cur.rowcount + + +async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str, + content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None: + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key," + " staging_key, status, error_code, pipeline_version, created_at, updated_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())", + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, + config.PIPELINE_VERSION), + ) + + +ATTACHMENT_COLUMNS = ( + "id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status," + " error_code, pipeline_version, created_at, updated_at" +) + + +QUEUE_FULL = "queue_full" +ADMISSION_LOCK = 2026_0105 + + +async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0): + """uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff). + + With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock, + so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight. + """ + if max_pending: + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,)) + if await jobs.pending_count(conn) >= max_pending: + return QUEUE_FULL + cur = await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS, + (attachment_id,), + ) + row = await cur.fetchone() + if row is None: + return None + await jobs.enqueue(conn, attachment_id, actor) + return _attachment(row) + + +async def abort_attachment(conn, attachment_id: str) -> None: + await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,)) + + +async def get_attachment(conn, attachment_id: str): + cur = await conn.execute( + "SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'", + (attachment_id,), + ) + return await cur.fetchone() + + +async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]: + cur = await conn.execute( + "SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants" + " WHERE attachment_id = %s AND pipeline_version = %s", + (attachment_id, pipeline_version), + ) + rows = await cur.fetchall() + return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99)) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 0000000..642dd22 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,122 @@ +import logging +from typing import Iterator, Optional, Tuple + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.errors import unavailable + +log = logging.getLogger("messenger.s3") + +MISSING = ("NoSuchKey", "404", "NotFound") + + +class S3Storage: + def __init__(self) -> None: + self.client = None + self.probe_client = None + + def open(self) -> None: + self.probe_client = self._client(1.0, 1.5) + self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT) + + def _client(self, connect: float, read: float): + return boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect, + read_timeout=read, + retries={"max_attempts": 0}, + ), + ) + + def probe(self) -> bool: + try: + self.probe_client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False + + def put(self, key: str, fileobj, size: int, content_type: str) -> None: + try: + fileobj.seek(0) + self.client.put_object( + Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type, + ) + except (BotoCoreError, ClientError) as exc: + log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + + def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]: + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + except BotoCoreError as exc: + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + body = response["Body"] + return body.iter_chunks(65536), int(response["ContentLength"]) + + def head(self, key: str): + """LastModified of the object, or None when it does not exist.""" + try: + return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"] + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + raise + + def exists(self, key: str) -> bool: + return self.head(key) is not None + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=config.S3_BUCKET, Key=key) + + def list_pages(self, prefix: str): + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}): + yield page.get("Contents", []) + + # ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient) + + def put_file(self, key: str, path: str, content_type: str) -> None: + with open(path, "rb") as handle: + self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type) + + def download(self, key: str, path: str) -> bool: + """False when the object does not exist.""" + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return False + raise + with open(path, "wb") as handle: + for chunk in response["Body"].iter_chunks(1 << 20): + handle.write(chunk) + return True + + def copy(self, source_key: str, target_key: str) -> None: + """Server-side copy: the original bytes never pass through the worker again.""" + self.client.copy_object( + Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key}, + ) + + def ready(self) -> bool: + try: + self.client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 0000000..ffc0355 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,154 @@ +import hashlib +import json +import logging +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +import valkey.asyncio as valkey + +from app import config, repo +from app.cursor import fmt +from app.errors import unavailable + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionStore: + def __init__(self, db) -> None: + self.db = db + self.client = None + self.stats = {"hits": 0, "misses": 0, "errors": 0} + + async def open(self) -> None: + if not config.CACHE_URL: + log.info("session cache disabled: CACHE_URL is empty") + return + self.client = valkey.Valkey.from_url( + config.CACHE_URL, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ) + + async def close(self) -> None: + if self.client is not None: + await self.client.aclose() + self.client = None + + @staticmethod + def digest(token: str) -> bytes: + return hashlib.sha256(token.encode("utf-8")).digest() + + @staticmethod + def key(digest: bytes) -> str: + return "session:" + digest.hex() + + async def create(self, user: dict) -> tuple: + token = secrets.token_urlsafe(32) + digest = self.digest(token) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + async with self.db.connection() as conn: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at)" + " VALUES (%s, %s, clock_timestamp(), %s)", + (digest, user["id"], expires_at), + ) + await self._cache_put(digest, user, expires_at) + return token, fmt(expires_at) + + async def resolve(self, token: str) -> Optional[dict]: + digest = self.digest(token) + cached = await self._cache_get(digest) + if cached == REVOKED: + self.stats["hits"] += 1 + return None + if cached is not None: + self.stats["hits"] += 1 + return cached + self.stats["misses"] += 1 + async with self.db.connection() as conn: + cur = await conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token_hash = %s AND s.expires_at > now()", + (digest,), + ) + row = await cur.fetchone() + if row is None: + return None + user = repo.public_user(row) + await self._cache_put(digest, user, row["expires_at"]) + return user + + async def revoke(self, token: str) -> None: + digest = self.digest(token) + await self._cache_revoke(digest) + try: + async with self.db.connection() as conn: + await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,)) + except Exception: + await self._cache_release(digest) + raise + + async def sweep(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + return cur.rowcount + + def _cache_ttl(self, expires_at: datetime) -> int: + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + return min(config.SESSION_CACHE_TTL_SECONDS, remaining) + + async def _cache_get(self, digest: bytes) -> Optional[dict]: + if self.client is None: + return None + try: + raw = await self.client.get(self.key(digest)) + except Exception: + self._cache_failed("get") + return None + if not raw: + return None + if raw == REVOKED: + return REVOKED + try: + return json.loads(raw) + except ValueError: + return None + + async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None: + if self.client is None: + return + ttl = self._cache_ttl(expires_at) + if ttl <= 0: + return + payload = json.dumps(user) + try: + await self.client.set(self.key(digest), payload, ex=ttl, nx=True) + except Exception: + self._cache_failed("set") + + async def _cache_release(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.delete(self.key(digest)) + except Exception: + self._cache_failed("release") + + async def _cache_revoke(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS) + except Exception: + self._cache_failed("revoke") + raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None + + def _cache_failed(self, operation: str) -> None: + self.stats["errors"] += 1 + if self.stats["errors"] % 50 == 1: + log.warning("session_cache_failed %s, falling back to postgres (errors=%d)", operation, self.stats["errors"]) diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..6922c0d --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,393 @@ +"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector. + +Everything is configured from the standard OTEL_* environment variables plus a few of our own: + + OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline) + OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision) + OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs) + OTEL_METRIC_EXPORT_INTERVAL metric push period, ms + OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request + +Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed +URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job +type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels. +""" +import json +import logging +import os +import re +import resource +import sys +import time +from contextlib import contextmanager +from typing import Optional + +from opentelemetry import context as otel_context +from opentelemetry import metrics, propagate, trace +from opentelemetry.trace import SpanKind, Status, StatusCode + +from app import config + +ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true" +SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0") +TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") +QUEUE = config.BROKER_QUEUE +JOB_TYPE = "attachment.process.v1" + +# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md. +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30) +JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120) +READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600) + +tracer = trace.get_tracer("messenger") +meter = metrics.get_meter("messenger") +_state = {"service": "messenger", "providers": []} + + +# ---------------------------------------------------------------------------------------------- logging + +class EventFilter(logging.Filter): + """Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word.""" + + EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)") + + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "event", None): + match = self.EVENT_RE.match(str(record.msg)) + record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log" + return True + + +class JsonFormatter(logging.Formatter): + """One JSON object per line on stdout, with trace_id/span_id of the active span (container logs).""" + + def format(self, record: logging.LogRecord) -> str: + out = { + "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs, + "level": record.levelname, + "service": _state["service"], + "event": getattr(record, "event", ""), + "logger": record.name, + "msg": record.getMessage(), + } + ctx = trace.get_current_span().get_span_context() + if ctx.is_valid: + out["trace_id"] = format(ctx.trace_id, "032x") + out["span_id"] = format(ctx.span_id, "016x") + if record.exc_info: + out["exc"] = self.formatException(record.exc_info) + return json.dumps(out, ensure_ascii=False) + + +def _setup_logging(logger_provider=None) -> None: + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + root.setLevel(logging.INFO) + stdout = logging.StreamHandler(sys.stdout) + stdout.setFormatter(JsonFormatter()) + stdout.addFilter(EventFilter()) + root.addHandler(stdout) + # The exporter's own warnings (backend down) must not be fed back into the exporter. + sdk_log = logging.getLogger("opentelemetry") + sdk_log.propagate = False + sdk_log.addHandler(stdout) + for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"): + logging.getLogger(noisy).setLevel(logging.WARNING) + if logger_provider is not None: + from opentelemetry.sdk._logs import LoggingHandler + + otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otlp.addFilter(EventFilter()) + root.addHandler(otlp) + + +# ---------------------------------------------------------------------------------------------- sampling + +def _build_sampler(): + """ParentBased(root=KindFilter(TraceIdRatio)). + + Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling, + lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces. + Children always follow the parent's decision, so an upload and its worker job are sampled together. + """ + from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult, + TraceIdRatioBased) + + ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0")) + decisions = meter.create_counter( + "messenger.telemetry.sampler.decisions", unit="{span}", + description="Head sampler decisions for spans (sample/drop) and the rule that decided") + + class KindFilter(Sampler): + def __init__(self, delegate): + self.delegate = delegate + + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER): + decisions.add(1, {"decision": "drop", "rule": "root_kind"}) + return SamplingResult(Decision.DROP) + result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links, + trace_state) + decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"}) + return result + + def get_description(self): + return "KindFilter{%s}" % self.delegate.get_description() + + class CountingParentBased(ParentBased): + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state) + parent = trace.get_current_span(parent_context).get_span_context() + if parent.is_valid: + decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", + "rule": "parent"}) + return result + + return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio))) + + +class _CountingExporter: + """Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side).""" + + def __init__(self, delegate): + from opentelemetry.sdk.trace.export import SpanExportResult + + self.delegate = delegate + self.ok = SpanExportResult.SUCCESS + self.counter = meter.create_counter( + "messenger.telemetry.spans.exported", unit="{span}", + description="Spans handed to the OTLP exporter by outcome (success/failure)") + + def export(self, spans): + result = self.delegate.export(spans) + self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"}) + return result + + def shutdown(self): + return self.delegate.shutdown() + + def force_flush(self, timeout_millis: int = 30000): + return self.delegate.force_flush(timeout_millis) + + +# ---------------------------------------------------------------------------------------------- setup + +def setup(service_name: str, fastapi_app=None) -> None: + _state["service"] = service_name + if not ENABLED: + _setup_logging(None) + logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true") + return + + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab). + res = Resource.create({ + "service.name": service_name, + "service.namespace": "messenger", + "service.version": SERVICE_VERSION, + "service.instance.id": config.INSTANCE_ID, + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + }) + + views = [ + # Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc. + View(instrument_name="http.server.request.duration", + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS), + attribute_keys={"http.request.method", "http.route", "http.response.status_code"}), + View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + # No catch-all View("*"): an instrument matched by two views would be exported twice. + # Instruments without a matching view get the SDK default aggregation. + ] + meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())], + views=views) + metrics.set_meter_provider(meter_provider) + + tracer_provider = TracerProvider(resource=res, sampler=_build_sampler()) + tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter()))) + trace.set_tracer_provider(tracer_provider) + + logger_provider = LoggerProvider(resource=res) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + _setup_logging(logger_provider) + _state["providers"] = [tracer_provider, meter_provider, logger_provider] + + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + + # PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s + # placeholders), so values, message bodies and hashes never reach db.query.text. + BotocoreInstrumentor().instrument() + if fastapi_app is not None: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + # Health probes (LB every second) are excluded from spans and from the HTTP SLI. + FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready", + exclude_spans=["receive", "send"]) + _process_metrics() + logging.getLogger("messenger.telemetry").info( + "telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name, + os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0")) + + +def shutdown() -> None: + for provider in _state["providers"]: + try: + provider.shutdown() + except Exception: + pass + + +# ---------------------------------------------------------------------------------------------- context carrier + +def inject() -> dict: + """W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}.""" + carrier: dict = {} + if trace.get_current_span().get_span_context().is_valid: + propagate.inject(carrier) + return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")} + + +def extract(carrier: Optional[dict]): + """Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context.""" + if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))): + return otel_context.Context() + clean = {"traceparent": str(carrier["traceparent"])} + if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512: + clean["tracestate"] = carrier["tracestate"] + return propagate.extract(clean, context=otel_context.Context()) + + +def set_attributes(**attrs) -> None: + span = trace.get_current_span() + for key, value in attrs.items(): + if value is not None: + span.set_attribute(key.replace("__", "."), value) + + +@contextmanager +def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None): + with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {}, + record_exception=True, set_status_on_exception=True) as current: + yield current + + +def record_exception(exc: BaseException) -> None: + """Exception event on the active span (type + stack, never request data) for errors handled by our code.""" + current = trace.get_current_span() + current.record_exception(exc) + current.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + + +def mark_error(current, description: str) -> None: + current.set_status(Status(StatusCode.ERROR, description)) + + +# ---------------------------------------------------------------------------------------------- instruments + +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 + + +def _rss_bytes() -> int: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE + except OSError: + return 0 + + +def _process_metrics() -> None: + from opentelemetry.metrics import Observation + + def cpu(_options): + own = resource.getrusage(resource.RUSAGE_SELF) + kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit) + yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"}) + yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"}) + + meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)") + meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By", + description="Resident set size of the process") + + +def register_pool(db) -> None: + """DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats).""" + if not ENABLED: + return + from opentelemetry.metrics import Observation + + def count(_options): + stats = db.pool.get_stats() + size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0) + yield Observation(max(0, size - idle), {"db.client.connection.state": "used"}) + yield Observation(idle, {"db.client.connection.state": "idle"}) + + meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}") + meter.create_observable_up_down_counter( + "db.client.connection.pending_requests", + [lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}") + meter.create_observable_up_down_counter( + "db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}") + + +def register_queue(state: dict) -> None: + """Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second. + + Reported by every API replica (identical values): dashboards and alerts use max() across replicas. + """ + if not ENABLED: + return + from opentelemetry.metrics import Observation + + def depth(_options): + queue = state.get("queue") + if queue is None: + return [] + return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}), + Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}), + Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})] + + def oldest(_options): + queue = state.get("queue") + return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})] + + meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}", + description="Jobs accepted but not finished, by state") + meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s", + description="Age of the oldest queued/processing job since durable acceptance") + + +def register_worker(worker) -> None: + if not ENABLED: + return + from opentelemetry.metrics import Observation + + meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]], + unit="{job}", description="Jobs being processed right now") + meter.create_observable_gauge("messenger.worker.capacity", + [lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}", + description="Concurrent job slots of this worker") + + +jobs_processed = meter.create_counter( + "messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome") +job_duration = meter.create_histogram( + "messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome") +time_to_ready = meter.create_histogram( + "messenger.attachment.time_to_ready", unit="s", + description="From durable acceptance (job COMMIT, 202) to the ready COMMIT") diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..8d75c26 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,367 @@ +"""Attachment worker: a separate process consuming attachments.process (python -m app.worker). + +Per delivery: + 1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ + 2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op + 3. download staging object, verify size+sha256 against DB metadata + 4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants + 5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes) + 6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner + 7. delete staging (best effort, reconciler finishes it), then ack +Ack happens only after the durable effect (6) or after the failure/retry decision is committed. +A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases. +""" +import asyncio +import hashlib +import json +import logging +import os +import resource +import signal +import sys +import tempfile +import time +import uuid +from datetime import datetime, timezone + +from opentelemetry.trace import SpanKind + +from app import config, faults, jobs, telemetry +from app.broker import connect, declare +from app.db import Database +from app.s3 import S3Storage + +log = logging.getLogger("messenger.worker") + +PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"} + + +class Permanent(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail or code + + +class Transient(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail or code + + +class LeaseLost(Exception): + pass + + +def child_limits() -> None: + memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + cpu = config.JOB_TIMEOUT_SECONDS + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + os.nice(5) + + +def sha256_file(path: str): + digest, size = hashlib.sha256(), 0 + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +class Worker: + def __init__(self) -> None: + self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8]) + self.db = Database() + self.s3 = S3Storage() + self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY) + self.stopping = asyncio.Event() + self.inflight = 0 + self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0} + + # ------------------------------------------------------------------ lifecycle + + async def run(self) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self.stopping.set) + await self.db.open() + telemetry.register_pool(self.db) + telemetry.register_worker(self) + await asyncio.to_thread(self.s3.open) + connection = await connect() + channel = await connection.channel() + await channel.set_qos(prefetch_count=config.WORKER_PREFETCH) + _, queue = await declare(channel) + tag = await queue.consume(self.on_message, no_ack=False) + background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())] + log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB", + self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS, + config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB) + await self.stopping.wait() + log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight) + try: + await queue.cancel(tag) + except Exception: + pass + deadline = time.monotonic() + config.WORKER_GRACE_SECONDS + while self.inflight and time.monotonic() < deadline: + await asyncio.sleep(0.2) + for task in background: + task.cancel() + # Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed. + await connection.close() + await self.db.close() + log.info("worker_stopped stats=%s", self.stats) + + async def alive(self) -> None: + while True: + try: + with open(config.WORKER_HEARTBEAT_FILE, "w") as handle: + handle.write(str(time.time())) + except OSError: + pass + await asyncio.sleep(2) + + async def sweeper(self) -> None: + while True: + await asyncio.sleep(config.JOB_SWEEP_SECONDS) + try: + swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner)) + if swept: + log.warning("lease_expired recovered=%s", swept) + except Exception: + log.warning("lease_sweep_skipped: database unavailable") + + # ------------------------------------------------------------------ delivery + + async def on_message(self, message) -> None: + async with self.slots: + self.inflight += 1 + started = time.monotonic() + # CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier). + try: + carrier = json.loads(message.body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + carrier = None + if not isinstance(carrier, dict) or "traceparent" not in carrier: + carrier = dict(message.headers or {}) + attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process", + "messaging.destination.name": config.BROKER_QUEUE, + "messaging.message.id": str(message.message_id or ""), + "messaging.rabbitmq.message.redelivered": bool(message.redelivered)} + outcome = "requeued" + with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER, + context=telemetry.extract(carrier), attributes=attrs) as current: + try: + outcome = await self.handle(message) + except Exception: + log.exception("unexpected_error, message returned to queue") + await asyncio.sleep(1) + await self.settle(message.nack(requeue=True)) + finally: + self.inflight -= 1 + current.set_attribute("messenger.job.outcome", outcome) + if outcome in ("retry", "failed", "dlq", "requeued", "lost"): + telemetry.mark_error(current, outcome) + labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome} + telemetry.jobs_processed.add(1, labels) + telemetry.job_duration.record(time.monotonic() - started, labels) + + @staticmethod + async def settle(awaitable) -> None: + try: + await awaitable + except Exception: + log.warning("could not settle delivery (connection lost); broker will redeliver") + + async def handle(self, message) -> str: + """Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued.""" + msg = jobs.parse_message(message.body) + if msg is None: + self.stats["dlq"] += 1 + log.error("job_invalid_message -> DLQ message_id=%s", message.message_id) + await self.settle(message.reject(requeue=False)) + return "dlq" + telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"]) + try: + decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner)) + except Exception: + log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"]) + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return "requeued" + if decision == "done" and job["state"] == "succeeded": + await self.cleanup_staging(job["attachment_id"]) + if decision != "claimed": + self.stats["noop"] += 1 + telemetry.set_attributes(messenger__job__decision=decision) + log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision, + message.redelivered) + await self.settle(message.ack()) + return "noop" + started = time.monotonic() + log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"], + job["max_attempts"]) + lease_lost = asyncio.Event() + keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost)) + outcome = None + work = asyncio.create_task(self.process(job, message)) + lost = asyncio.create_task(lease_lost.wait()) + try: + done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2, + return_when=asyncio.FIRST_COMPLETED) + if work not in done: + work.cancel() + raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout") + outcome = "succeeded" if work.result() else "lost" + except LeaseLost: + outcome = "lost" + except Permanent as exc: + outcome = await self.record_failure(job, exc.code, exc.detail, transient=False) + except Transient as exc: + outcome = await self.record_failure(job, exc.code, exc.detail, transient=True) + except Exception as exc: # S3/DB/network errors + outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True) + finally: + keeper.cancel() + lost.cancel() + if outcome is None: + # Could not record the decision: keep the message; the lease expires and the sweeper takes over. + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return "requeued" + self.stats[outcome] = self.stats.get(outcome, 0) + 1 + log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome, + time.monotonic() - started) + await self.settle(message.ack()) + return outcome + + async def cleanup_staging(self, attachment_id) -> None: + """Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup.""" + try: + async with self.db.connection() as conn: + src = await jobs.attachment_source(conn, attachment_id) + if src and src["status"] == "ready" and src["staging_key"]: + await asyncio.to_thread(self.s3.delete, src["staging_key"]) + except Exception: + log.warning("staging_cleanup_deferred attachment=%s", attachment_id) + + async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None: + interval = max(1.0, config.JOB_LEASE_SECONDS / 3) + while True: + await asyncio.sleep(interval) + try: + alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner)) + except Exception: + continue # DB blip: keep trying until the lease really expires + if not alive: + log.warning("lease_lost job=%s: another worker owns it now", job_id) + lease_lost.set() + return + + async def record_failure(self, job, code: str, detail: str, transient: bool): + log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code, + transient) + try: + return await self.db.in_transaction( + lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient)) + except Exception: + log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"]) + return None + + # ------------------------------------------------------------------ the actual work + + async def process(self, job, message) -> bool: + async with self.db.connection() as conn: + src = await jobs.attachment_source(conn, job["attachment_id"]) + if src is None or not src["staging_key"]: + raise Permanent("staging_missing", "attachment has no staging object") + filename = src["filename"] + first_delivery = job["attempt"] == 1 and not message.redelivered + if faults.active(filename, "worker_hang") and job["attempt"] == 1: + log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5)) + time.sleep(config.JOB_LEASE_SECONDS * 2.5) + if faults.active(filename, "permanent"): + raise Permanent("invalid_image", "injected permanent failure") + if job["attempt"] <= faults.transient_until(filename): + raise Transient("dependency_unavailable", "injected transient failure") + + with tempfile.TemporaryDirectory(prefix="job-") as tmp: + source = os.path.join(tmp, "source") + if not await asyncio.to_thread(self.s3.download, src["staging_key"], source): + raise Permanent("staging_missing", "staging object not found") + digest, size = await asyncio.to_thread(sha256_file, source) + if digest != src["sha256"] or size != src["size_bytes"]: + raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload") + out_dir = os.path.join(tmp, "out") + os.mkdir(out_dir) + with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}): + produced = await self.run_child(source, out_dir) + + variants = [] + prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"]) + await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"]) + for item in produced: + digest, size = await asyncio.to_thread(sha256_file, item["file"]) + key = prefix + os.path.basename(item["file"]) + await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"]) + variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"], + "size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest}) + if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS): + raise Permanent("processing_failed", "not all variants were produced") + + if first_delivery and faults.active(filename, "worker_crash_after_put"): + faults.crash("after S3 put, before result COMMIT") + committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants)) + if not committed: + log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"]) + return False + telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()), + {"job.type": telemetry.JOB_TYPE}) + if first_delivery and faults.active(filename, "worker_crash_after_commit"): + faults.crash("after result COMMIT, before ack") + try: + await asyncio.to_thread(self.s3.delete, src["staging_key"]) + except Exception: + log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"]) + return True + + async def run_child(self, source: str, out_dir: str) -> list: + env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app", + "JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)} + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS), + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits, + ) + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None + except asyncio.CancelledError: + proc.kill() + raise + code = proc.returncode + if code == 0: + return json.loads(stdout) + if code in PERMANENT_EXIT: + raise Permanent(PERMANENT_EXIT[code]) + if code == -signal.SIGXCPU: + raise Permanent("image_too_complex", "CPU limit exceeded") + if code == -signal.SIGKILL: + raise Transient("worker_resource_exhausted", "child killed (OOM)") + raise Permanent("processing_failed", "child exit %s" % code) + + +def main() -> None: + telemetry.setup("messenger-worker") + try: + asyncio.run(Worker().run()) + finally: + telemetry.shutdown() + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..650aa62 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,460 @@ +name: mtusi-lab6 + +# OpenTelemetry SDK settings shared by the API replicas and the worker (track 1 overrides them from the shell). +x-otel-env: &otel-env + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_TIMEOUT: "5" + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_SEMCONV_STABILITY_OPT_IN: http + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-10000} + OTEL_BSP_MAX_QUEUE_SIZE: "2048" + OTEL_BSP_SCHEDULE_DELAY: "2000" + OTEL_BLRP_MAX_QUEUE_SIZE: "2048" + SERVICE_VERSION: "1.6.0" + +x-api: &api-defaults + build: + context: . + image: mtusi-messenger:lab6 + env_file: + - .env + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + rabbitmq: + condition: service_healthy + postgres: + condition: service_healthy + valkey: + condition: service_healthy + minio-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 15s + restart: unless-stopped + +services: + nginx: + image: nginx:1.27.3-alpine + command: ["/bin/sh", "/opt/nginx-health.sh"] + environment: + API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2} + HEALTH_INTERVAL: "1" + HEALTH_FALL: "2" + HEALTH_RISE: "2" + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx-health.sh:/opt/nginx-health.sh:ro + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + restart: unless-stopped + + api-1: + <<: *api-defaults + environment: + <<: *otel-env + INSTANCE_ID: api-1 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://minio:9000 + BROKER_URL_FILE: /run/secrets/broker_url + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + + api-2: + <<: *api-defaults + environment: + <<: *otel-env + INSTANCE_ID: api-2 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://minio:9000 + BROKER_URL_FILE: /run/secrets/broker_url + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + + worker: + build: + context: . + image: mtusi-messenger:lab6 + command: ["python", "-m", "app.worker"] + env_file: + - .env + environment: + <<: *otel-env + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://minio:9000 + CACHE_URL_FILE: "" + INSTANCE_ID: "" + DB_POOL_MAX: "6" + # Overridable from the shell for the track-1 experiment (shell env wins over .env interpolation). + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + rabbitmq: + condition: service_healthy + postgres: + condition: service_healthy + minio-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + stop_grace_period: 30s + deploy: + replicas: ${WORKER_REPLICAS:-2} + resources: + limits: + cpus: "${WORKER_CPUS:-2.00}" + memory: 1536M + healthcheck: + test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 15s + restart: unless-stopped + + rabbitmq: + image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32 + hostname: rabbitmq + entrypoint: ["/bin/sh", "/opt/entrypoint.sh"] + secrets: + - rabbitmq_password + volumes: + - rabbitmqdata:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + - ./infra/rabbitmq/entrypoint.sh:/opt/entrypoint.sh:ro + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 5s + timeout: 10s + retries: 20 + start_period: 20s + restart: unless-stopped + + postgres: + image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: --data-checksums + command: + - postgres + - -c + - synchronous_commit=on + - -c + - shared_buffers=128MB + - -c + - work_mem=8MB + - -c + - max_connections=50 + - -c + - track_io_timing=on + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: + - valkey-server + - --save + - "" + - --appendonly + - "no" + - --maxmemory + - 64mb + - --maxmemory-policy + - allkeys-lru + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + restart: unless-stopped + + minio: + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + command: ["server", "/data"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + secrets: + - minio_root_user + - minio_root_password + volumes: + - miniodata:/data + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "mc ready local || exit 1"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + restart: unless-stopped + + minio-init: + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + entrypoint: ["/bin/sh", "/opt/init.sh"] + environment: + S3_BUCKET: messenger-attachments + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/init.sh:/opt/init.sh:ro + - ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro + depends_on: + minio: + condition: service_healthy + restart: "no" + + migrate: + build: + context: . + image: mtusi-messenger:lab6 + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: "no" + + # ------------------------------------------------------------------ observability (lab6) + otel-collector: + image: otel/opentelemetry-collector-contrib:0.120.0@sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917 + command: + - --config=/etc/otelcol/collector.yaml + - --config=/etc/otelcol/profile-${COLLECTOR_PROFILE:-default}.yaml + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + - ./infra/otel/profile-default.yaml:/etc/otelcol/profile-default.yaml:ro + - ./infra/otel/profile-tail.yaml:/etc/otelcol/profile-tail.yaml:ro + # OTLP 4317/4318 and self-metrics 8888 stay inside the compose network. + deploy: + resources: + limits: + cpus: "1.00" + memory: 256M + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.2.1@sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --web.enable-otlp-receiver + - --web.enable-lifecycle + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - promdata:/prometheus + ports: + - "127.0.0.1:9090:9090" # query API for evidence scripts (localhost only) + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + loki: + image: grafana/loki:3.4.2@sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + ports: + - "127.0.0.1:3100:3100" + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.1@sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + ports: + - "127.0.0.1:3200:3200" + deploy: + resources: + limits: + cpus: "1.00" + memory: 1024M + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + restart: unless-stopped + + grafana: + image: grafana/grafana:11.5.2@sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + secrets: + - grafana_admin_password + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_started + tempo: + condition: service_started + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + restart: unless-stopped + +volumes: + pgdata: + miniodata: + rabbitmqdata: + promdata: + lokidata: + tempodata: + grafanadata: + +secrets: + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + broker_url: + file: ./secrets/broker_url + rabbitmq_password: + file: ./secrets/rabbitmq_password + grafana_admin_password: + file: ./secrets/grafana_admin_password diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 0000000..c02e4d8 --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,244 @@ +# Панели dashboard «Messenger — наблюдаемость (lab6)» + +Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. `$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval). + +## SLO (окно 30 мин) и алерты + +**Доступность (SLO 99%)** (stat, единица: доля (0..1)) +— 1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы. + +```promql +messenger:slo_availability:ratio30m +``` + +**Латентность ≤500 ms (SLO 95%)** (stat, единица: доля (0..1)) +— Доля запросов в bucket le=0.5 s за 30 мин. + +```promql +messenger:slo_latency:ratio30m +``` + +**Error ratio 5xx (2m)** (stat, единица: доля (0..1)) + +```promql +messenger:http_error_ratio:rate2m +``` + +**Старейшее задание** (stat, единица: секунды) + +```promql +max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) +``` + +**Алерты Prometheus (firing/pending)** (alertlist) + +Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`). + + +**RPS API** (stat, единица: запросы/с) + +```promql +messenger:http_requests:rate2m +``` + +**p95 API (5m)** (stat, единица: секунды) + +```promql +messenger:http_latency_p95:rate5m +``` + +**Загрузка worker** (stat, единица: доля (0..1)) + +```promql +messenger:worker_saturation:ratio +``` + +**Ожидают пул БД** (stat, единица: штуки) + +```promql +max(db_client_connection_pending_requests) +``` + +## API (messenger-api, все реплики) + +**RPS по маршруту и методу** (timeseries, единица: запросы/с) +— Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID. + +```promql +sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # {{http_request_method}} {{http_route}} +``` + +**RPS по классу статуса** (timeseries, единица: запросы/с) + +```promql +sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d)..")) # {{status_class}} +``` + +**Латентность API p50/p95/p99 (из histogram buckets)** (timeseries, единица: секунды) +— histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик. + +```promql +histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p50 +``` +```promql +histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p95 +``` +```promql +histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p99 +``` + +**Доля ошибок 5xx** (timeseries, единица: доля (0..1)) +— Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом. + +```promql +(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # 5xx / все +``` + +**p95 по маршруту** (timeseries, единица: секунды) + +```promql +histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # {{http_route}} +``` + +**Запросы в обработке** (timeseries, единица: штуки) + +```promql +sum by (instance) (http_server_active_requests{service_name="messenger-api"}) # {{instance}} +``` + +## Очередь и worker + +**Jobs по исходу (в сек.)** (timeseries, единица: операции/с) +— succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1 + +```promql +sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval])) # {{outcome}} +``` + +**Длительность job (succeeded) p50/p95/p99** (timeseries, единица: секунды) + +```promql +histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p50 +``` +```promql +histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p95 +``` +```promql +histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p99 +``` + +**Глубина очереди (из PostgreSQL)** (timeseries, единица: штуки) +— max() по репликам API: каждая публикует одно и то же значение из БД. + +```promql +max by (queue, state) (messenger_queue_depth) # {{queue}} {{state}} +``` + +**Возраст старейшей job** (timeseries, единица: секунды) +— От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency. + +```promql +max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) # oldest age +``` + +**Время до ready p50/p95/p99** (timeseries, единица: секунды) +— messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready. + +```promql +histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p50 +``` +```promql +histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p95 +``` +```promql +histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p99 +``` + +**Насыщение worker: занято / ёмкость** (timeseries, единица: штуки) + +```promql +sum(messenger_worker_busy) # busy +``` +```promql +sum(messenger_worker_capacity) # capacity +``` +```promql +messenger:worker_saturation:ratio # saturation +``` + +## Ресурсы: пул БД, CPU, RAM + +**Пул БД: used / idle / waiting** (timeseries, единица: штуки) + +```promql +sum by (service_name, db_client_connection_state) (db_client_connection_count) # {{service_name}} {{db_client_connection_state}} +``` +```promql +sum by (service_name) (db_client_connection_pending_requests) # {{service_name}} waiting +``` +```promql +sum by (service_name) (db_client_connection_max) # {{service_name}} max +``` + +**CPU процессов (ядра)** (timeseries, единица: доля (0..1)) +— worker: self + завершённые дочерние процессы обработки изображений. + +```promql +sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval])) # {{service_name}} {{instance}} +``` +```promql +rate(otelcol_process_cpu_seconds_total[$__rate_interval]) # otel-collector +``` + +**RSS процессов** (timeseries, единица: байты) + +```promql +process_memory_usage_bytes # {{service_name}} {{instance}} +``` +```promql +otelcol_process_memory_rss_bytes # otel-collector +``` + +**Telemetry pipeline: принято / отправлено / ошибки (items/s)** (timeseries, единица: штуки) + +```promql +sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval])) # accepted spans +``` +```promql +sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval])) # accepted log records +``` +```promql +sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval])) # accepted metric points +``` +```promql +sum by (exporter) (rate({__name__=~"otelcol_exporter_sent_(spans|log_records|metric_points)_total"}[$__rate_interval])) # sent {{exporter}} +``` +```promql +sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0) # send failed spans +``` +```promql +sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0) # dropped spans (queue full) +``` +```promql +sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0) # send failed logs +``` +```promql +sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0) # dropped logs (queue full) +``` +```promql +sum by (exporter) (otelcol_exporter_queue_size) # queue {{exporter}} +``` + +## Логи и трассы + +**Логи API и worker (warning/error)** (logs) + +``` +{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR" +``` + +**Трассы с ошибками или медленнее 500 ms (TraceQL)** (table) + +``` +{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)} +``` diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..d0fa9ec --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,90 @@ +# Runbooks алертов (lab6) + +Алерты вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`) каждые 15 с. Состояние смотрится в Grafana +(dashboard «Messenger — наблюдаемость», панель «Алерты Prometheus») или командой: + +```sh +curl -s http://127.0.0.1:9090/api/v1/alerts | python3 -m json.tool +``` + +Доставка во внешние каналы не настроена (по заданию достаточно локального firing/resolved). + +Общий порядок диагностики: **метрика → трасса → лог**. Метрика говорит «что и сколько», трасса — «где в +цепочке тратится время / где ошибка», лог — «какое событие и с какими техническими ID». +Все шаги ниже используют только сохранённую telemetry. + +--- + +## MessengerApiHighErrorRate + +**Условие:** `messenger:http_error_ratio:rate2m > 0.05` при `messenger:http_requests:rate2m > 0.2`, `for: 1m`. +Числитель — ответы 5xx, знаменатель — все бизнес-запросы messenger-api (health-пробы не инструментированы). +4xx (неверный пароль, чужой чат, 409) в числитель не входят. + +1. Dashboard → «RPS по классу статуса» и «Доля ошибок 5xx»: какой код (500 или 503) и с какого момента. +2. Prometheus: на каком маршруте ошибки + `sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))`. +3. Отличить БД от handler: + - пул: `max(db_client_connection_pending_requests)`, `db_client_connection_count{db_client_connection_state="used"}`. + Ожидание пула > 0 → соединения заняты долгими запросами к БД; + - Tempo (Explore → Tempo → TraceQL): + `{resource.service.name="messenger-api" && span.db.system="postgresql" && status=error}` — если в трассе + db-span (`INSERT`/`SELECT`) занимает почти всё время запроса и завершился исключением + `LockNotAvailable`/`QueryCanceled`/`OperationalError` — причина в PostgreSQL (блокировка, перегрузка, недоступность); + - если server span длинный, а db-spans короткие (единицы ms) — медленный/падающий handler + (`{span.messenger.fault="slow"}` в учебном стенде даёт именно такую картину). +4. Loki: `{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"` — события `unhandled_error` + (500, есть stack trace в атрибуте), `not_ready` (БД недоступна), `request_timeout`. + По `X-Request-Id`, который получил клиент: `{service_name="messenger-api"} |= ""` → поле `trace_id` + → кнопка «Open trace». +5. Если виновата БД: `docker compose exec postgres psql -U messenger -d messenger -c "SELECT pid, state, + wait_event_type, now()-xact_start AS age, left(query,80) FROM pg_stat_activity WHERE state <> 'idle'"` — + найти блокирующую транзакцию; согласовать и выполнить `SELECT pg_terminate_backend()`. +6. Восстановление: доля 5xx падает сразу, alert уходит в resolved, когда окно 2m очистится (~2–3 мин). + +## MessengerQueueLagGrowing + +**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60`, `for: 1m`. +Возраст считается от COMMIT задания (момент ответа 202) по данным PostgreSQL; публикуют все реплики API, +поэтому `max()`. + +1. Dashboard → «Глубина очереди», «Jobs по исходу», «Насыщение worker». + - `sum(rate(messenger_jobs_processed_total[1m])) == 0` при растущей глубине → worker не работает; + - `messenger:worker_saturation:ratio ≈ 1` и jobs/s > 0 → worker не успевает (нужно больше реплик/конкурентности); + - много `outcome="retry"` → ошибки зависимостей (S3/DB), см. логи `job_attempt_failed`. +2. Tempo: найти upload по attachment_id `{span.messenger.attachment_id=""}`. Если в трассе есть + PRODUCER `attachments.process publish`, но нет CONSUMER — сообщение в брокере, worker его не забрал. + Если нет и PRODUCER — outbox не опубликован: смотреть `max(messenger_queue_depth{queue="outbox"})` и лог + `outbox_relay_paused` (брокер недоступен). +3. Loki: `{service_name="messenger-worker"}` — последние события `worker_stopping`/`worker_stopped` или отсутствие + `job_*` событий; `{service_name="messenger-worker"} | event="job_attempt_failed"` — причины retry. +4. Действие: `docker compose ps worker`, `docker compose start worker` (или масштабирование + `docker compose up -d --scale worker=3`). Задания не теряются: они в PostgreSQL/outbox и в durable очереди. +5. Восстановление: backlog уходит за секунды, возраст падает до 0, alert resolved через ~15–30 с. + +## MessengerApiLatencyHigh + +**Условие:** p95 из buckets всех реплик за 5m > 1 s, `for: 2m`. Шаги — как в HighErrorRate, пункт 3: +по трассам `{resource.service.name="messenger-api" && duration > 1s}` определить, какой span занимает время +(db / S3 / handler). + +## MessengerDbPoolWaiting + +**Условие:** `max(db_client_connection_pending_requests) > 0`, `for: 1m`. Пул исчерпан: долгие запросы или +блокировки в БД (см. HighErrorRate, п. 3 и 5) либо пул мал для нагрузки (`DB_POOL_MAX`). + +## MessengerWorkerSaturated + +**Условие:** занято > 90% слотов всех worker 5 минут. Если очередь при этом растёт — масштабировать worker; +проверить `messenger_job_duration_seconds` p95 (не стали ли задания медленнее). + +## TelemetryExportFailing + +**Условие:** очередь exporter заполнена больше чем на 20% либо есть send_failed/enqueue_failed > 0, `for: 1m`. +Бизнес-API **не** затронут: SDK и collector имеют ограниченные очереди. + +1. `docker compose ps tempo loki prometheus` — какой backend недоступен. +2. Панель «Telemetry pipeline»: `otelcol_exporter_queue_size` по exporter (ёмкость 100 батчей) и + `otelcol_exporter_enqueue_failed_*` — сколько данных уже отброшено. +3. `docker compose start tempo loki`. Данные, отброшенные при переполнении очереди, не восстанавливаются — + их объём виден в счётчиках и фиксируется в отчёте инцидента. diff --git a/evidence/alerts_history.txt b/evidence/alerts_history.txt new file mode 100644 index 0000000..23f19d4 --- /dev/null +++ b/evidence/alerts_history.txt @@ -0,0 +1,6 @@ +ALERTS{alertstate=firing} во время финального прогона инцидентов A/B/C (источник: Prometheus TSDB), UTC +MessengerApiHighErrorRate page ['14:37:00..14:39:00'] +MessengerApiLatencyHigh ticket ['14:38:00..14:41:45'] +MessengerDbPoolWaiting ticket ['14:36:30..14:37:00'] +MessengerQueueLagGrowing page ['14:35:00..14:35:00'] +TelemetryExportFailing ticket ['14:41:00..14:42:45'] diff --git a/evidence/clean_clone.txt b/evidence/clean_clone.txt new file mode 100644 index 0000000..cf62b0f --- /dev/null +++ b/evidence/clean_clone.txt @@ -0,0 +1,9 @@ +# 2026-11-09T14:55:39Z clean copy of git files, steps of .github/workflows/stand.yml (-p lab6clean to keep the main stand volumes) +secrets ready in secrets/ (.env: present) +compose config OK + Container lab6clean-tempo-1 Healthy +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy) +grafana dashboards: ['Messenger — наблюдаемость (lab6)'] +prometheus rules: 13 diff --git a/evidence/dashboard_check.txt b/evidence/dashboard_check.txt new file mode 100644 index 0000000..ff54819 --- /dev/null +++ b/evidence/dashboard_check.txt @@ -0,0 +1,92 @@ +ok 1 series Доступность (SLO 99%) Доступность (SLO 99%) +ok 1 series Латентность ≤500 ms (SLO 95%) Латентность ≤500 ms (SLO 95%) +ok 1 series Error ratio 5xx (2m) Error ratio 5xx (2m) +ok 1 series Старейшее задание Старейшее задание +ok 1 series RPS API RPS API +ok 1 series p95 API (5m) p95 API (5m) +ok 1 series Загрузка worker Загрузка worker +ok 1 series Ожидают пул БД Ожидают пул БД +ok 6 series RPS по маршруту и методу {{http_request_method}} {{http_route}} +ok 2 series RPS по классу статуса {{status_class}} +ok 1 series Латентность API p50/p95/p99 (из histogram bucket p50 +ok 1 series Латентность API p50/p95/p99 (из histogram bucket p95 +ok 1 series Латентность API p50/p95/p99 (из histogram bucket p99 +ok 1 series Доля ошибок 5xx 5xx / все +ok 5 series p95 по маршруту {{http_route}} +ok 2 series Запросы в обработке {{instance}} +ok 1 series Jobs по исходу (в сек.) {{outcome}} +ok 1 series Длительность job (succeeded) p50/p95/p99 p50 +ok 1 series Длительность job (succeeded) p50/p95/p99 p95 +ok 1 series Длительность job (succeeded) p50/p95/p99 p99 +ok 3 series Глубина очереди (из PostgreSQL) {{queue}} {{state}} +ok 1 series Возраст старейшей job oldest age +ok 1 series Время до ready p50/p95/p99 p50 +ok 1 series Время до ready p50/p95/p99 p95 +ok 1 series Время до ready p50/p95/p99 p99 +ok 1 series Насыщение worker: занято / ёмкость busy +ok 1 series Насыщение worker: занято / ёмкость capacity +ok 1 series Насыщение worker: занято / ёмкость saturation +ok 4 series Пул БД: used / idle / waiting {{service_name}} {{db_client_connection_state}} +ok 2 series Пул БД: used / idle / waiting {{service_name}} waiting +ok 2 series Пул БД: used / idle / waiting {{service_name}} max +ok 4 series CPU процессов (ядра) {{service_name}} {{instance}} +ok 1 series CPU процессов (ядра) otel-collector +ok 4 series RSS процессов {{service_name}} {{instance}} +ok 1 series RSS процессов otel-collector +ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted spans +ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted log records +ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted metric points +ok 3 series Telemetry pipeline: принято / отправлено / ошибк sent {{exporter}} +ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed spans +ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped spans (queue full) +ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed logs +ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped logs (queue full) +ok 3 series Telemetry pipeline: принято / отправлено / ошибк queue {{exporter}} +panels: 32, prometheus queries without data: 0 +grafana prometheus ok 3 values Доступность (SLO 99%) +grafana prometheus ok 3 values Латентность ≤500 ms (SLO 95%) +grafana prometheus ok 3 values Error ratio 5xx (2m) +grafana prometheus ok 9 values Старейшее задание +grafana prometheus ok 3 values RPS API +grafana prometheus ok 3 values p95 API (5m) +grafana prometheus ok 9 values Загрузка worker +grafana prometheus ok 9 values Ожидают пул БД +grafana prometheus ok 18 values RPS по маршруту и методу +grafana prometheus ok 5 values RPS по классу статуса +grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket +grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket +grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket +grafana prometheus ok 3 values Доля ошибок 5xx +grafana prometheus ok 15 values p95 по маршруту +grafana prometheus ok 6 values Запросы в обработке +grafana prometheus ok 3 values Jobs по исходу (в сек.) +grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99 +grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99 +grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99 +grafana prometheus ok 27 values Глубина очереди (из PostgreSQL) +grafana prometheus ok 9 values Возраст старейшей job +grafana prometheus ok 3 values Время до ready p50/p95/p99 +grafana prometheus ok 3 values Время до ready p50/p95/p99 +grafana prometheus ok 3 values Время до ready p50/p95/p99 +grafana prometheus ok 9 values Насыщение worker: занято / ёмкость +grafana prometheus ok 9 values Насыщение worker: занято / ёмкость +grafana prometheus ok 9 values Насыщение worker: занято / ёмкость +grafana prometheus ok 36 values Пул БД: used / idle / waiting +grafana prometheus ok 18 values Пул БД: used / idle / waiting +grafana prometheus ok 18 values Пул БД: used / idle / waiting +grafana prometheus ok 34 values CPU процессов (ядра) +grafana prometheus ok 8 values CPU процессов (ядра) +grafana prometheus ok 36 values RSS процессов +grafana prometheus ok 9 values RSS процессов +grafana prometheus ok 2 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 8 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 8 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 18 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк +grafana prometheus ok 27 values Telemetry pipeline: принято / отправлено / ошибк +grafana loki ok 7 values Логи API и worker (warning/error) +grafana tempo ok 17 traces Трассы с ошибками или медленнее 500 ms (TraceQL) (datasource proxy) +via Grafana: queries without data or with errors: 0 diff --git a/evidence/exporter_queue_full.txt b/evidence/exporter_queue_full.txt new file mode 100644 index 0000000..9d2aa48 --- /dev/null +++ b/evidence/exporter_queue_full.txt @@ -0,0 +1,20 @@ +17:41:17 tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=100 batches +17:41:31 + 14 s queue=9/100 collector_rss=68.7MiB dropped=0 refused=0 +17:41:44 + 27 s queue=24/100 collector_rss=81.4MiB dropped=0 refused=0 +17:41:58 + 41 s queue=39/100 collector_rss=102MiB dropped=0 refused=0 +17:42:12 + 55 s queue=39/100 collector_rss=123.9MiB dropped=0 refused=0 +17:42:26 + 69 s queue=55/100 collector_rss=133.1MiB dropped=0 refused=0 +17:42:40 + 83 s queue=70/100 collector_rss=160.9MiB dropped=0 refused=0 +17:42:54 + 97 s queue=85/100 collector_rss=161.1MiB dropped=0 refused=0 +17:43:08 +111 s queue=100/100 collector_rss=164.1MiB dropped=1062 refused=0 +17:43:22 +125 s queue=100/100 collector_rss=173.4MiB dropped=15664 refused=0 +17:43:36 +139 s queue=99/100 collector_rss=177.4MiB dropped=16214 refused=4248 +17:43:50 +153 s queue=99/100 collector_rss=184MiB dropped=16214 refused=5310 +17:44:04 +167 s queue=99/100 collector_rss=185.5MiB dropped=16214 refused=6372 +17:44:18 +181 s queue=99/100 collector_rss=205.9MiB dropped=16214 refused=6372 +17:44:32 +195 s queue=100/100 collector_rss=204.4MiB dropped=24597 refused=12004 +17:44:46 +209 s queue=100/100 collector_rss=204.1MiB dropped=25147 refused=16252 +17:45:00 +223 s queue=100/100 collector_rss=203.9MiB dropped=25147 refused=16802 +17:45:01 result: peak queue=100/100, dropped by full queue=25147 spans, refused by memory_limiter=16802 spans, peak RSS=205.9 MiB +17:45:01 API: requests=24686 ok=24686 unexpected_errors=0 p99=55.79 ms +PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..34afe98 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,16 @@ +# 2026-11-09T13:40:58Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 1) +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 1.997} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab6-api-1-1 Up 47 hours (healthy) +$ docker compose start api-1 api-2 +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 2) +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 0.903} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab6-api-2-1 Up 41 seconds (healthy) +$ docker compose start api-1 api-2 +FAILOVER_DONE diff --git a/evidence/incidents.json b/evidence/incidents.json new file mode 100644 index 0000000..9b0cd69 --- /dev/null +++ b/evidence/incidents.json @@ -0,0 +1,195 @@ +{ + "A_worker_stopped": { + "alert_pending_s": 70.62756085395813, + "alert_firing_s": 131, + "age_at_firing_s": 126, + "drain_after_restart_s": 3.6, + "resolved_after_restart_s": 14, + "lost_jobs": 0 + }, + "B_db_lock": { + "alert_pending_s": 45.0726158618927, + "alert_firing_s": 105, + "error_ratio_at_firing": 0.707, + "p95_at_firing_s": 6.92, + "db_span_ms": 5001, + "request_ms": 5004, + "cause": "QueryCanceled", + "resolved_after_unlock_s": 135 + }, + "C1_backends_down": { + "outage_s": 192, + "api_requests": 1703, + "api_unexpected_errors": 0, + "collector_samples": [ + { + "t": 14, + "collector_mem": "190.5MiB", + "cpu": "0.05%", + "queue_tempo": 6.0, + "queue_loki": 0.0 + }, + { + "t": 26, + "collector_mem": "190.5MiB", + "cpu": "0.07%", + "queue_tempo": 6.0, + "queue_loki": 0.0 + }, + { + "t": 38, + "collector_mem": "190.5MiB", + "cpu": "0.04%", + "queue_tempo": 14.0, + "queue_loki": 0.0 + }, + { + "t": 50, + "collector_mem": "191.5MiB", + "cpu": "0.58%", + "queue_tempo": 21.0, + "queue_loki": 0.0 + }, + { + "t": 62, + "collector_mem": "190.8MiB", + "cpu": "0.31%", + "queue_tempo": 29.0, + "queue_loki": 0.0 + }, + { + "t": 74, + "collector_mem": "190.7MiB", + "cpu": "0.02%", + "queue_tempo": 36.0, + "queue_loki": 0.0 + }, + { + "t": 86, + "collector_mem": "190.3MiB", + "cpu": "0.13%", + "queue_tempo": 36.0, + "queue_loki": 0.0 + }, + { + "t": 98, + "collector_mem": "190.3MiB", + "cpu": "0.01%", + "queue_tempo": 44.0, + "queue_loki": 0.0 + }, + { + "t": 110, + "collector_mem": "191.7MiB", + "cpu": "0.02%", + "queue_tempo": 51.0, + "queue_loki": 0.0 + }, + { + "t": 122, + "collector_mem": "192.7MiB", + "cpu": "0.37%", + "queue_tempo": 57.0, + "queue_loki": 0.0 + }, + { + "t": 134, + "collector_mem": "191.9MiB", + "cpu": "0.10%", + "queue_tempo": 64.0, + "queue_loki": 0.0 + }, + { + "t": 146, + "collector_mem": "192.4MiB", + "cpu": "0.38%", + "queue_tempo": 64.0, + "queue_loki": 0.0 + }, + { + "t": 158, + "collector_mem": "192.4MiB", + "cpu": "0.22%", + "queue_tempo": 72.0, + "queue_loki": 0.0 + }, + { + "t": 170, + "collector_mem": "192MiB", + "cpu": "0.03%", + "queue_tempo": 79.0, + "queue_loki": 0.0 + } + ], + "alert": "firing", + "dropped_items": 0.0, + "send_failed_items": 387.0, + "traces_after_restart": 20, + "alert_resolved_after_restart_s": 35 + }, + "C2_collector_down": { + "outage_s": 60, + "api1_mem_before": "108.6MiB", + "api1_mem_during": [ + "108.8MiB", + "109.3MiB", + "109.3MiB", + "109.1MiB", + "109.3MiB", + "119.3MiB" + ], + "api_requests": 740, + "api_unexpected_errors": 0 + }, + "traffic": { + "profile": { + "rate_rps": 10, + "duration_s": 3600, + "upload_rate_rps": 0, + "slow_pct": 2, + "error_pct": 0, + "threads": 48, + "mix": "75% POST message / 25% GET history(limit=20); uploads separate" + }, + "started_at": 1794061933.7705631, + "finished_at": 1794062670.181464, + "generator_backlog_at_end": 0, + "requests": 7365, + "ok": 6335, + "successful_rps": 1.76, + "status_counts": { + "200": 1593, + "201": 4742, + "503": 816, + "502": 214 + }, + "server_errors_unexpected": 1030, + "latency_ms_plain_ok": { + "p50": 14.87, + "p95": 22.77, + "p99": 27.78, + "max": 8280.75 + }, + "faults_sent": { + "slow": 157, + "error": 0 + }, + "by_kind": { + "post_message": { + "n": 5510, + "p50_ms": 14.32, + "p99_ms": 25.09 + }, + "get_history": { + "n": 1855, + "p50_ms": 17.84, + "p99_ms": 43.91 + }, + "upload": { + "n": 0, + "p50_ms": null, + "p99_ms": null + } + } + } +} diff --git a/evidence/incidents.txt b/evidence/incidents.txt new file mode 100644 index 0000000..3789f1a --- /dev/null +++ b/evidence/incidents.txt @@ -0,0 +1,58 @@ +17:32:13 background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow) +17:32:43 === A. worker stopped (queue lag) +17:32:44 worker stopped at 17:32:44 +17:32:44 3 uploads accepted (202) while worker is down: ['62859e73', '99403faf', '4ad2b534'] +17:33:54 MessengerQueueLagGrowing pending (+71 s) +17:34:54 MessengerQueueLagGrowing firing (+131 s) +17:34:54 diagnosis from telemetry: +17:34:54 metrics: oldest job age=126 s, depth={'processing': 0.0, 'queued': 3.0}, worker jobs/s over 1m=0.000 +17:34:54 logs: worker job_* events after stop: 0; last worker events: ['worker_stopped', 'worker_stopped'] +17:34:54 trace of upload 62859e73: span kinds ['PRODUCER', 'SERVER'] -> published, no CONSUMER: the job waits in the queue +17:34:57 worker started (+131 s) +17:34:58 backlog drained: all 3 ready 3.6 s after worker start +17:35:08 MessengerQueueLagGrowing resolved (+14 s) +17:35:08 === B. DB lock on messages (slow/unavailable DB for writes and history reads) +17:35:08 ACCESS EXCLUSIVE lock taken on table messages at 17:35:08 +17:35:53 MessengerApiHighErrorRate pending (+45 s) +17:36:53 MessengerApiHighErrorRate firing (+105 s) +17:36:53 diagnosis from telemetry: +17:36:53 metrics: 5xx ratio=70.7%, p95=6.92 s, pool used(max replica)=10 waiting=19 +17:36:53 5xx/s /api/v1/chats 500 0.00 +17:36:53 5xx/s /api/v1/chats/{chat_id}/messages 503 9.59 +17:36:53 trace 23982edb8bd1246afc6bd6522faaa524 (failed request): + api SERVER POST /api/v1/chats/{chat_id}/messages 5004.2 ms ERROR {"messenger.request_id": "0f42d95e28624ecb9e443dc88c491177", "messenger.instance_id": "api-1", "http.response.status_code": "503"} + api CLIENT SELECT 0.1 ms UNSET {"db.system": "postgresql"} + api CLIENT SELECT chat_members 0.1 ms UNSET {"db.system": "postgresql"} + api CLIENT SELECT idempotency_keys 0.1 ms UNSET {"db.system": "postgresql"} + api CLIENT INSERT messages 5001.0 ms ERROR {"db.system": "postgresql"} +17:36:53 => DB span INSERT messages took 5001 ms of 5004 ms request, status ERROR, exception QueryCanceled: the time is spent in PostgreSQL +17:36:53 contrast: slow HANDLER trace 47ee6988aff06d8a0f5592a97fb9a967: request 615 ms, longest DB span 1.1 ms, all DB 8.3 ms +17:36:53 logs: 29 API warning/error records in 5 min, events=['not_ready', 'outbox_relay_paused'] +17:36:53 operator step (pg_stat_activity) finds the blocker: 4246|active|Timeout|BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT +17:36:53 lock released (+106 s) +17:39:09 MessengerApiHighErrorRate resolved (+135 s) +17:39:09 === C1. Tempo and Loki stopped (telemetry backends down) +17:39:22 + 14 s collector mem=190.5MiB cpu=0.05% queue tempo=6 loki=0 (capacity 100 batches) +17:39:35 + 26 s collector mem=190.5MiB cpu=0.07% queue tempo=6 loki=0 (capacity 100 batches) +17:39:47 + 38 s collector mem=190.5MiB cpu=0.04% queue tempo=14 loki=0 (capacity 100 batches) +17:39:59 + 50 s collector mem=191.5MiB cpu=0.58% queue tempo=21 loki=0 (capacity 100 batches) +17:40:10 + 62 s collector mem=190.8MiB cpu=0.31% queue tempo=29 loki=0 (capacity 100 batches) +17:40:23 + 74 s collector mem=190.7MiB cpu=0.02% queue tempo=36 loki=0 (capacity 100 batches) +17:40:35 + 86 s collector mem=190.3MiB cpu=0.13% queue tempo=36 loki=0 (capacity 100 batches) +17:40:47 + 98 s collector mem=190.3MiB cpu=0.01% queue tempo=44 loki=0 (capacity 100 batches) +17:40:59 +110 s collector mem=191.7MiB cpu=0.02% queue tempo=51 loki=0 (capacity 100 batches) +17:41:10 +122 s collector mem=192.7MiB cpu=0.37% queue tempo=57 loki=0 (capacity 100 batches) +17:41:23 +134 s collector mem=191.9MiB cpu=0.10% queue tempo=64 loki=0 (capacity 100 batches) +17:41:35 +146 s collector mem=192.4MiB cpu=0.38% queue tempo=64 loki=0 (capacity 100 batches) +17:41:47 +158 s collector mem=192.4MiB cpu=0.22% queue tempo=72 loki=0 (capacity 100 batches) +17:41:59 +170 s collector mem=192MiB cpu=0.03% queue tempo=79 loki=0 (capacity 100 batches) +17:41:59 alert TelemetryExportFailing during outage: firing +17:41:59 API during the outage: 1703 requests, unexpected errors=0; alert TelemetryExportFailing=firing +17:42:20 tempo/loki started and ready (+192 s) +17:42:30 new traces after restart visible: 20 (export resumed) +17:42:56 TelemetryExportFailing resolved (+35 s) +17:43:16 telemetry lost and counted: enqueue_failed(queue full)=0 items, send_failed(retries exhausted)=387 items +17:43:16 === C2. Collector stopped (SDK side buffering) +17:44:30 api-1 RSS before=108.6MiB during=['108.8MiB', '109.3MiB', '109.3MiB', '109.1MiB', '109.3MiB', '119.3MiB']; 740 requests, unexpected errors=0 +17:44:30 summary written to /Users/alenchha/mtusi2026lab6/evidence/incidents.json +PASS: incidents A,B,C diagnosed from telemetry, alerts fired and resolved diff --git a/evidence/lab5_failure_windows.txt b/evidence/lab5_failure_windows.txt new file mode 100644 index 0000000..6a34a38 --- /dev/null +++ b/evidence/lab5_failure_windows.txt @@ -0,0 +1,80 @@ +17:36:11 scenario A: scenario_a +17:36:11 client saw: missing X-Request-Id +17:36:11 committed attachment 8f23222e-3018-4692-a4ce-f67bfcf43871 survived the API crash +17:36:12 statuses: queued -> ready +17:36:12 audit attempt=1 enqueued api:api-2 +17:36:12 audit attempt=1 claimed 832caf57a286:a39a11c5 +17:36:12 audit attempt=1 succeeded 832caf57a286:a39a11c5 +17:36:13 scenario A PASS (2.4s) +17:36:13 scenario B: scenario_b +17:36:37 statuses: queued -> processing -> queued -> processing -> ready +17:36:37 audit attempt=1 enqueued api:api-1 +17:36:37 audit attempt=1 claimed 18bdb5d574fa:8873b063 +17:36:37 audit attempt=1 retry_scheduled sweeper:18bdb5d574fa:4f98f68 lease_expired delay=1.97s +17:36:37 audit attempt=2 claimed 832caf57a286:a39a11c5 +17:36:37 audit attempt=2 succeeded 832caf57a286:a39a11c5 +17:36:38 scenario B PASS (24.9s) +17:36:38 scenario C: scenario_c +17:36:39 statuses: queued -> ready +17:36:39 audit attempt=1 enqueued api:api-2 +17:36:39 audit attempt=1 claimed 18bdb5d574fa:4f98f687 +17:36:39 audit attempt=1 succeeded 18bdb5d574fa:4f98f687 +17:36:49 scenario C PASS (10.4s) +17:36:49 scenario D: scenario_d +17:36:58 statuses: processing -> queued -> ready +17:36:58 audit attempt=1 enqueued api:api-1 +17:36:58 audit attempt=1 claimed 832caf57a286:a39a11c5 +17:36:58 audit attempt=1 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=1.25s +17:36:58 audit attempt=2 claimed 18bdb5d574fa:79c71044 +17:36:58 audit attempt=2 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=2.58s +17:36:58 audit attempt=3 claimed 832caf57a286:a39a11c5 +17:36:58 audit attempt=3 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=4.3s +17:36:58 audit attempt=4 claimed 18bdb5d574fa:79c71044 +17:36:58 audit attempt=4 succeeded 18bdb5d574fa:79c71044 +17:36:59 backoff delays: [1.25, 2.58, 4.3] +17:36:59 scenario D PASS (10.4s) +17:36:59 scenario E: scenario_e +17:37:00 audit attempt=1 enqueued api:api-1 +17:37:00 audit attempt=1 claimed 832caf57a286:a39a11c5 +17:37:00 audit attempt=1 failed 832caf57a286:a39a11c5 invalid_image +17:37:00 scenario E PASS (0.7s) +17:37:00 scenario F: scenario_f +17:37:24 failed registry contains the job; replay +17:37:47 statuses: queued -> processing -> ready +17:37:47 audit attempt=1 enqueued api:api-2 +17:37:47 audit attempt=1 claimed 18bdb5d574fa:79c71044 +17:37:47 audit attempt=1 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=1.99s +17:37:47 audit attempt=2 claimed 832caf57a286:a39a11c5 +17:37:47 audit attempt=2 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=2.59s +17:37:47 audit attempt=3 claimed 18bdb5d574fa:79c71044 +17:37:47 audit attempt=3 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=4.09s +17:37:47 audit attempt=4 claimed 832caf57a286:a39a11c5 +17:37:47 audit attempt=4 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=11.17s +17:37:47 audit attempt=5 claimed 18bdb5d574fa:79c71044 +17:37:47 audit attempt=5 failed 18bdb5d574fa:79c71044 retry_exhausted +17:37:47 audit attempt=6 replayed operator:test-suite +17:37:47 audit attempt=6 claimed 832caf57a286:a39a11c5 +17:37:47 audit attempt=6 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=19.41s +17:37:47 audit attempt=7 claimed 18bdb5d574fa:79c71044 +17:37:47 audit attempt=7 succeeded 18bdb5d574fa:79c71044 +17:37:49 second replay refused: job is succeeded, attachment is ready +17:37:50 scenario F PASS (50.4s) +17:37:50 scenario G: scenario_g +17:38:17 statuses: processing -> queued -> processing -> ready +17:38:17 audit attempt=1 enqueued api:api-1 +17:38:17 audit attempt=1 claimed 832caf57a286:a39a11c5 +17:38:17 audit attempt=1 retry_scheduled sweeper:18bdb5d574fa:79c7104 lease_expired delay=1.77s +17:38:17 audit attempt=2 claimed 18bdb5d574fa:79c71044 +17:38:17 audit attempt=2 succeeded 18bdb5d574fa:79c71044 +17:38:49 scenario G PASS (59.1s) +17:38:49 scenario H: scenario_h +17:38:53 DLQ depth 0 -> 1 +17:38:53 scenario H PASS (3.8s) +17:38:53 scenario I: scenario_i +17:38:58 broker down: upload 202, outbox holds 1 unpublished job +17:39:03 statuses: queued -> ready +17:39:03 audit attempt=1 enqueued api:api-2 +17:39:03 audit attempt=1 claimed 832caf57a286:a39a11c5 +17:39:03 audit attempt=1 succeeded 832caf57a286:a39a11c5 +17:39:04 scenario I PASS (11.4s) +PASS: failure windows A B C D E F G H I diff --git a/evidence/lab5_worker_down.txt b/evidence/lab5_worker_down.txt new file mode 100644 index 0000000..74f7054 --- /dev/null +++ b/evidence/lab5_worker_down.txt @@ -0,0 +1,14 @@ +17:26:48 stop all workers +17:26:49 upload 202 queued id=bc973dcb-254c-4a56-8746-5accb5e85267 +17:26:49 message e459cc31-bf5f-431c-9c1d-e473e95a59a1 created with queued attachment +17:26:54 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0} +17:26:54 restart api-1 api-2 rabbitmq (volumes kept) +17:26:59 started before: mtusi-lab6-api-1-1 2026-11-09T14:18:47.635837802Z; mtusi-lab6-api-2-1 2026-11-09T14:18:47.722431011Z; mtusi-lab6-rabbitmq-1 2026-11-07T14:44:49.033246884Z +17:26:59 started after: mtusi-lab6-api-1-1 2026-11-09T14:26:55.137236792Z; mtusi-lab6-api-2-1 2026-11-09T14:26:55.0483775Z; mtusi-lab6-rabbitmq-1 2026-11-09T14:26:55.741984625Z +17:27:03 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0} +17:27:03 start workers +17:27:06 status sequence observed: queued -> ready +17:27:07 objects: ['attachments/bc973dcb-254c-4a56-8746-5accb5e85267/original', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/crop.jpg', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/optimized.webp', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/thumbnail.jpg'] +17:27:07 redelivery: republish the finished job through the outbox (duplicate message) +17:27:14 duplicate acked as no-op: variants=3 successes=1 +PASS: worker down / API+broker restart / redelivery no-op diff --git a/evidence/observability.txt b/evidence/observability.txt new file mode 100644 index 0000000..5b29e3f --- /dev/null +++ b/evidence/observability.txt @@ -0,0 +1,27 @@ +09:16:45 message sent request_id=07f7f48bf1c74c01b3f546bd98c19492 (text marker kept private) +09:16:47 upload 5791d45a-87cb-465a-b8af-47a02fcfe006: queued -> queued -> ready +09:16:51 Tempo: trace ef3b4c403f734bcb04ee9b5ce085aa68, 42 spans; tree (DB client spans omitted): + api SERVER POST /api/v1/chats/{chat_id}/attachments 10.2 ms UNSET {"http.response.status_code": "202", "messenger.request_id": "f33563a1565d46bea878a83207f42bef", "messenger.instance_id": "api-1", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"} + api CLIENT S3.PutObject 3.9 ms UNSET {"rpc.method": "PutObject"} + api PRODUCER attachments.process publish 1.5 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.job.attempt": "1", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"} + worker CONSUMER attachments.process process 4.9 ms ERROR {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006", "messenger.job.attempt": "1", "messenger.job.outcome": "retry"} + api PRODUCER attachments.process publish 4.6 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.job.attempt": "2", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"} + worker CONSUMER attachments.process process 169.0 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006", "messenger.job.attempt": "2", "messenger.job.outcome": "succeeded"} + worker CLIENT S3.GetObject 6.8 ms UNSET {"rpc.method": "GetObject"} + worker INTERNAL imageproc 137.9 ms UNSET {"messenger.pipeline_version": "1"} + worker CLIENT S3.CopyObject 3.8 ms UNSET {"rpc.method": "CopyObject"} + worker CLIENT S3.PutObject 2.2 ms UNSET {"rpc.method": "PutObject"} + worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"} + worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"} + worker CLIENT S3.DeleteObject 1.0 ms UNSET {"rpc.method": "DeleteObject"} +09:16:51 PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links +09:16:51 Loki: 6 records with trace_id=ef3b4c403f734bcb04ee9b5ce085aa68: [('api', 'attachment_queued'), ('worker', 'job_claimed'), ('worker', 'job_attempt_failed'), ('worker', 'job_finished'), ('worker', 'job_claimed'), ('worker', 'job_finished')] +09:16:51 Loki record fields: ['code_filepath', 'code_function', 'code_lineno', 'deployment_environment_name', 'detected_level', 'event', 'flags', 'observed_timestamp', 'scope_name', 'service_instance_id', 'service_name', 'service_namespace', 'service_version', 'severity_number', 'severity_text', 'span_id', 'telemetry_sdk_language', 'telemetry_sdk_name', 'telemetry_sdk_version', 'trace_id', 'ts'] +09:16:55 HTTP 500 X-Request-Id=9302b0182c0243df8acc92865370b939 -> Loki event=unhandled_error trace_id=6a254a70c77d5419df95d6f8d2e681ce +09:16:55 Tempo: GET /api/v1/chats status=ERROR request_id matches; exception event recorded=True +09:16:55 message POST trace 26d27e3a639403ae1576707024358b6e: ['POST /api/v1/chats/{chat_id}/messages', 'SELECT', 'SELECT chat_members', 'SELECT idempotency_keys', 'INSERT messages', 'INSERT idempotency_keys'] +09:16:55 PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from traces and logs +09:16:55 PASS: 994 series, label names: ['__name__', 'db_client_connection_state', 'decision', 'deployment_environment_name', 'error_type', 'http_request_method', 'http_response_status_code', 'http_route', 'instance', 'job', 'job_type', 'le', 'network_protocol_version', 'outcome', 'process_cpu_owner', 'queue', 'rule', 'service_instance_id', 'service_name', 'service_version', 'state', 'url_scheme'] +09:16:55 http_route values (templates only): ['/api/v1/attachments/{attachment_id}', '/api/v1/auth/sessions', '/api/v1/chats', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users'] +09:17:07 PASS: required metrics present: http_server_request_duration_seconds_bucket, messenger_jobs_processed_total, messenger_job_duration_seconds_bucket, messenger_attachment_time_to_ready_seconds_bucket, messenger_queue_depth, messenger_queue_oldest_age_seconds, messenger_worker_busy, messenger_worker_capacity, db_client_connection_count, db_client_connection_pending_requests, process_cpu_time_seconds_total, process_memory_usage_bytes, otelcol_receiver_accepted_spans_total, otelcol_exporter_sent_spans_total, otelcol_exporter_queue_size +PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy) diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..bfd0670 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,32 @@ +# 2026-11-11T06:17:14Z scripts/restart.sh restarts business and telemetry stores, volumes kept +$ docker compose ps (before) +mtusi-lab6-api-1-1 Up 3 minutes (healthy) +mtusi-lab6-api-2-1 Up 3 minutes (healthy) +mtusi-lab6-grafana-1 Up 3 minutes (healthy) +mtusi-lab6-loki-1 Up 3 minutes (healthy) +mtusi-lab6-minio-1 Up 3 minutes (healthy) +mtusi-lab6-otel-collector-1 Up 3 minutes +mtusi-lab6-postgres-1 Up 3 minutes (healthy) +mtusi-lab6-prometheus-1 Up 3 minutes (healthy) +mtusi-lab6-rabbitmq-1 Up 3 minutes (healthy) +mtusi-lab6-tempo-1 Up 3 minutes (healthy) +mtusi-lab6-valkey-1 Up 3 minutes (healthy) +mtusi-lab6-worker-1 Up 3 minutes (healthy) +mtusi-lab6-worker-2 Up 3 minutes (healthy) +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +$ docker compose ps (after) +mtusi-lab6-api-1-1 Up 10 seconds (healthy) +mtusi-lab6-api-2-1 Up 10 seconds (healthy) +mtusi-lab6-grafana-1 Up 21 seconds (healthy) +mtusi-lab6-loki-1 Up 20 seconds (healthy) +mtusi-lab6-minio-1 Up 23 seconds (healthy) +mtusi-lab6-otel-collector-1 Up 21 seconds +mtusi-lab6-postgres-1 Up 22 seconds (healthy) +mtusi-lab6-prometheus-1 Up 21 seconds (healthy) +mtusi-lab6-rabbitmq-1 Up 21 seconds (healthy) +mtusi-lab6-tempo-1 Up 11 seconds (health: starting) +mtusi-lab6-valkey-1 Up 22 seconds (healthy) +mtusi-lab6-worker-1 Up 11 seconds (healthy) +mtusi-lab6-worker-2 Up 11 seconds (healthy) diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..ca11cd7 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,7 @@ +# 2026-11-09T14:26:13Z +$ make check +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +$ make validate +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +$ make test +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/telemetry_persistence.txt b/evidence/telemetry_persistence.txt new file mode 100644 index 0000000..1444e81 --- /dev/null +++ b/evidence/telemetry_persistence.txt @@ -0,0 +1,16 @@ +# 2026-11-09T14:29:49Z restart of telemetry stores, volumes kept +mtusi-lab6-grafana-1 Up 2 days (healthy) +mtusi-lab6-loki-1 Up 2 days (healthy) +mtusi-lab6-prometheus-1 Up 2 days (healthy) +mtusi-lab6-tempo-1 Up 2 days (healthy) + Container mtusi-lab6-prometheus-1 Started + Container mtusi-lab6-grafana-1 Started + Container mtusi-lab6-tempo-1 Started + Container mtusi-lab6-loki-1 Started +mtusi-lab6-grafana-1 Up 27 seconds (healthy) +mtusi-lab6-loki-1 Up 25 seconds (healthy) +mtusi-lab6-prometheus-1 Up 27 seconds (healthy) +mtusi-lab6-tempo-1 Up 26 seconds (healthy) +Tempo: trace of 2026-11-07 incident B 23982edb8bd1246afc6bd6522faaa524 spans: 5 +Loki: API warn/error records for that window: 48 streams +Prometheus: firing samples of MessengerApiHighErrorRate in that window: 9 diff --git a/evidence/track1/results.jsonl b/evidence/track1/results.jsonl new file mode 100644 index 0000000..0b43d75 --- /dev/null +++ b/evidence/track1/results.jsonl @@ -0,0 +1,12 @@ +{"config": "off", "rep": 1, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794231942.6489458, "finished_at": 1794232032.6500669, "generator_backlog_at_end": 1, "requests": 5490, "ok": 5428, "successful_rps": 60.31, "status_counts": {"201": 4003, "202": 90, "200": 1335, "500": 62}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.43, "p95": 10.28, "p99": 31.67, "max": 77.52}, "faults_sent": {"slow": 56, "error": 62}, "by_kind": {"post_message": {"n": 4003, "p50_ms": 6.1, "p99_ms": 27.37}, "get_history": {"n": 1397, "p50_ms": 7.63, "p99_ms": 32.03}, "upload": {"n": 90, "p50_ms": 19.99, "p99_ms": 166.23}}}, "cpu_pct_avg": {"api-2": 9.01, "api-1": 9.09, "worker": 15.35, "otel-collector": 0.07, "tempo": 0.43, "nginx": 1.29, "grafana": 0.98, "postgres": 5.1, "rabbitmq": 6.69, "valkey": 0.55, "minio": 1.63, "loki": 0.57, "prometheus": 0.36}, "mem_mib_max": {"api-2": 102.9, "api-1": 93.6, "worker": 134.2, "otel-collector": 35.7, "tempo": 291.9, "nginx": 5.2, "grafana": 135.7, "postgres": 58.6, "rabbitmq": 153.1, "valkey": 12.4, "minio": 306.3, "loki": 82.9, "prometheus": 273.4}, "telemetry_delta": {"sdk_span_decisions_sample": -38.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -38.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 26288.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794231942.6489012, 1794232032.66341], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "on100", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232111.334157, "finished_at": 1794232201.333892, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5440, "successful_rps": 60.44, "status_counts": {"201": 4029, "202": 90, "200": 1321, "500": 50}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.63, "p95": 16.28, "p99": 54.35, "max": 484.0}, "faults_sent": {"slow": 60, "error": 50}, "by_kind": {"post_message": {"n": 4029, "p50_ms": 8.21, "p99_ms": 54.1}, "get_history": {"n": 1371, "p50_ms": 10.46, "p99_ms": 54.35}, "upload": {"n": 90, "p50_ms": 21.71, "p99_ms": 61.23}}}, "cpu_pct_avg": {"api-2": 11.5, "api-1": 11.33, "worker": 15.68, "otel-collector": 0.6, "tempo": 1.08, "nginx": 1.26, "grafana": 0.91, "postgres": 4.65, "rabbitmq": 6.94, "valkey": 0.49, "minio": 1.57, "loki": 0.55, "prometheus": 0.36}, "mem_mib_max": {"api-2": 111.2, "api-1": 115.5, "worker": 144.0, "otel-collector": 71.3, "tempo": 420.8, "nginx": 13.8, "grafana": 136.0, "postgres": 79.0, "rabbitmq": 141.6, "valkey": 12.0, "minio": 299.2, "loki": 92.9, "prometheus": 276.3}, "telemetry_delta": {"sdk_span_decisions_sample": 56361.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 56361.0, "sdk_spans_exported_failed": null, "col_spans_in": 57045.0, "col_logs_in": 340.0, "col_points_in": 1224.0, "col_spans_out": 57045.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21529255.0, "tempo_spans_in": 57045.0, "tempo_bytes_in": 21033821.0, "loki_bytes_in": 135765.0, "loki_lines_in": 340.0, "prom_samples_in": 35456.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 60, "error": 50, "pipeline_consumer": 90}, "window": [1794232111.3341079, 1794232201.347548], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "head10", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232280.328288, "finished_at": 1794232370.6738691, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5435, "successful_rps": 60.39, "status_counts": {"201": 4045, "202": 90, "200": 1300, "500": 55}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.29, "p95": 12.98, "p99": 39.25, "max": 71.67}, "faults_sent": {"slow": 68, "error": 55}, "by_kind": {"post_message": {"n": 4045, "p50_ms": 8.03, "p99_ms": 39.25}, "get_history": {"n": 1355, "p50_ms": 9.8, "p99_ms": 39.23}, "upload": {"n": 90, "p50_ms": 20.56, "p99_ms": 28.61}}}, "cpu_pct_avg": {"worker": 14.92, "api-2": 9.9, "api-1": 9.7, "otel-collector": 0.24, "tempo": 0.35, "nginx": 1.22, "grafana": 0.85, "postgres": 4.5, "rabbitmq": 6.47, "valkey": 0.51, "minio": 1.47, "loki": 0.5, "prometheus": 0.33}, "mem_mib_max": {"worker": 146.7, "api-2": 101.2, "api-1": 113.3, "otel-collector": 67.4, "tempo": 500.3, "nginx": 5.3, "grafana": 135.7, "postgres": 66.7, "rabbitmq": 91.2, "valkey": 11.9, "minio": 301.4, "loki": 87.8, "prometheus": 277.5}, "telemetry_delta": {"sdk_span_decisions_sample": 5553.0, "sdk_span_decisions_drop_ratio": 51184.0, "sdk_spans_exported_ok": 5553.0, "sdk_spans_exported_failed": null, "col_spans_in": 5553.0, "col_logs_in": 345.0, "col_points_in": 1290.0, "col_spans_out": 5553.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2546103.0, "tempo_spans_in": 5553.0, "tempo_bytes_in": 2070193.0, "loki_bytes_in": 137005.0, "loki_lines_in": 345.0, "prom_samples_in": 35514.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 530, "slow": 7, "error": 6, "pipeline_consumer": 6}, "window": [1794232280.3282502, 1794232370.689455], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "tail", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232449.187008, "finished_at": 1794232539.181068, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5454, "successful_rps": 60.6, "status_counts": {"201": 4070, "202": 90, "200": 1294, "500": 36}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.29, "p95": 14.45, "p99": 42.01, "max": 106.41}, "faults_sent": {"slow": 56, "error": 36}, "by_kind": {"post_message": {"n": 4070, "p50_ms": 7.89, "p99_ms": 42.6}, "get_history": {"n": 1330, "p50_ms": 9.98, "p99_ms": 41.92}, "upload": {"n": 90, "p50_ms": 19.19, "p99_ms": 77.61}}}, "cpu_pct_avg": {"api-1": 10.92, "api-2": 11.23, "worker": 15.35, "otel-collector": 0.54, "tempo": 0.5, "nginx": 1.18, "grafana": 0.83, "postgres": 4.47, "rabbitmq": 6.56, "valkey": 0.47, "minio": 1.45, "loki": 0.52, "prometheus": 0.34}, "mem_mib_max": {"api-1": 137.4, "api-2": 116.8, "worker": 143.7, "otel-collector": 82.1, "tempo": 504.9, "nginx": 5.2, "grafana": 135.9, "postgres": 70.0, "rabbitmq": 157.4, "valkey": 11.9, "minio": 302.9, "loki": 90.2, "prometheus": 277.9}, "telemetry_delta": {"sdk_span_decisions_sample": 54799.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 54799.0, "sdk_spans_exported_failed": null, "col_spans_in": 56699.0, "col_logs_in": 326.0, "col_points_in": 1224.0, "col_spans_out": 10097.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21397727.0, "tempo_spans_in": 9711.0, "tempo_bytes_in": 3949068.0, "loki_bytes_in": 126917.0, "loki_lines_in": 326.0, "prom_samples_in": 36752.0, "tail_policies": {"baseline-10pct/false": 5090.0, "baseline-10pct/true": 620.0, "keep-async-pipeline/false": 5619.0, "keep-async-pipeline/true": 91.0, "keep-errors/false": 5673.0, "keep-errors/true": 37.0, "keep-slow/false": 5649.0, "keep-slow/true": 61.0}}, "traces_in_tempo": {"server_api": 758, "slow": 56, "error": 36, "pipeline_consumer": 90}, "window": [1794232449.186968, 1794232539.195683], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "off", "rep": 2, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232618.428749, "finished_at": 1794232708.432197, "generator_backlog_at_end": 1, "requests": 5490, "ok": 5439, "successful_rps": 60.43, "status_counts": {"200": 1202, "202": 90, "201": 4147, "500": 51}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.33, "p95": 12.21, "p99": 32.69, "max": 68.21}, "faults_sent": {"slow": 56, "error": 51}, "by_kind": {"post_message": {"n": 4147, "p50_ms": 8.1, "p99_ms": 33.23}, "get_history": {"n": 1253, "p50_ms": 9.5, "p99_ms": 27.42}, "upload": {"n": 90, "p50_ms": 20.59, "p99_ms": 28.39}}}, "cpu_pct_avg": {"api-1": 8.49, "api-2": 8.47, "worker": 14.85, "otel-collector": 0.06, "tempo": 0.82, "nginx": 1.25, "grafana": 0.93, "postgres": 4.84, "rabbitmq": 6.54, "valkey": 0.49, "minio": 2.04, "loki": 0.54, "prometheus": 0.3}, "mem_mib_max": {"api-1": 93.7, "api-2": 93.3, "worker": 134.7, "otel-collector": 35.6, "tempo": 423.4, "nginx": 5.2, "grafana": 135.9, "postgres": 74.4, "rabbitmq": 154.2, "valkey": 11.9, "minio": 303.4, "loki": 89.5, "prometheus": 276.7}, "telemetry_delta": {"sdk_span_decisions_sample": -133.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -133.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 23378.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794232618.428705, 1794232708.44837], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "on100", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232786.969322, "finished_at": 1794232876.97309, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5439, "successful_rps": 60.43, "status_counts": {"202": 90, "201": 4040, "200": 1309, "500": 51}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.09, "p95": 14.67, "p99": 43.98, "max": 89.3}, "faults_sent": {"slow": 57, "error": 51}, "by_kind": {"post_message": {"n": 4040, "p50_ms": 7.49, "p99_ms": 43.73}, "get_history": {"n": 1360, "p50_ms": 10.06, "p99_ms": 45.13}, "upload": {"n": 90, "p50_ms": 20.2, "p99_ms": 53.57}}}, "cpu_pct_avg": {"worker": 15.89, "api-1": 11.4, "api-2": 11.46, "otel-collector": 0.56, "tempo": 1.03, "nginx": 1.22, "grafana": 0.93, "postgres": 4.6, "rabbitmq": 6.71, "valkey": 0.45, "minio": 1.71, "loki": 0.58, "prometheus": 0.33}, "mem_mib_max": {"worker": 155.8, "api-1": 121.5, "api-2": 107.1, "otel-collector": 74.9, "tempo": 715.3, "nginx": 5.5, "grafana": 136.0, "postgres": 84.5, "rabbitmq": 154.9, "valkey": 12.1, "minio": 304.7, "loki": 91.0, "prometheus": 281.9}, "telemetry_delta": {"sdk_span_decisions_sample": 55086.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 55086.0, "sdk_spans_exported_failed": null, "col_spans_in": 56986.0, "col_logs_in": 341.0, "col_points_in": 1224.0, "col_spans_out": 56986.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21513949.0, "tempo_spans_in": 56986.0, "tempo_bytes_in": 21018520.0, "loki_bytes_in": 136272.0, "loki_lines_in": 341.0, "prom_samples_in": 34616.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 57, "error": 51, "pipeline_consumer": 90}, "window": [1794232786.969286, 1794232876.987838], "stats_samples": 14, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "head10", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232959.123384, "finished_at": 1794233049.151171, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5442, "successful_rps": 60.47, "status_counts": {"200": 1281, "201": 4071, "202": 90, "500": 48}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.6, "p95": 10.27, "p99": 36.75, "max": 114.16}, "faults_sent": {"slow": 62, "error": 48}, "by_kind": {"post_message": {"n": 4071, "p50_ms": 6.31, "p99_ms": 33.63}, "get_history": {"n": 1329, "p50_ms": 8.13, "p99_ms": 38.58}, "upload": {"n": 90, "p50_ms": 16.04, "p99_ms": 74.32}}}, "cpu_pct_avg": {"api-1": 9.85, "api-2": 9.72, "worker": 15.02, "otel-collector": 0.23, "tempo": 0.44, "nginx": 1.23, "grafana": 0.91, "postgres": 4.53, "rabbitmq": 6.55, "valkey": 0.51, "minio": 1.56, "loki": 0.55, "prometheus": 0.34}, "mem_mib_max": {"api-1": 99.3, "api-2": 101.9, "worker": 156.9, "otel-collector": 75.6, "tempo": 408.7, "nginx": 5.2, "grafana": 136.0, "postgres": 90.7, "rabbitmq": 91.4, "valkey": 12.4, "minio": 305.8, "loki": 92.2, "prometheus": 282.7}, "telemetry_delta": {"sdk_span_decisions_sample": 5559.0, "sdk_span_decisions_drop_ratio": 50970.0, "sdk_spans_exported_ok": 5559.0, "sdk_spans_exported_failed": null, "col_spans_in": 5559.0, "col_logs_in": 338.0, "col_points_in": 1296.0, "col_spans_out": 5559.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2555586.0, "tempo_spans_in": 5559.0, "tempo_bytes_in": 2082660.0, "loki_bytes_in": 132915.0, "loki_lines_in": 338.0, "prom_samples_in": 35520.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 554, "slow": 3, "error": 4, "pipeline_consumer": 9}, "window": [1794232959.123342, 1794233049.1668348], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "tail", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233127.526659, "finished_at": 1794233217.5204532, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5424, "successful_rps": 60.27, "status_counts": {"201": 3988, "202": 90, "200": 1346, "500": 66}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.71, "p95": 11.88, "p99": 43.18, "max": 135.99}, "faults_sent": {"slow": 70, "error": 66}, "by_kind": {"post_message": {"n": 3988, "p50_ms": 6.4, "p99_ms": 42.94}, "get_history": {"n": 1412, "p50_ms": 8.66, "p99_ms": 59.68}, "upload": {"n": 90, "p50_ms": 17.45, "p99_ms": 108.82}}}, "cpu_pct_avg": {"api-1": 11.36, "api-2": 11.19, "worker": 15.2, "otel-collector": 0.53, "tempo": 0.63, "nginx": 1.22, "grafana": 0.84, "postgres": 4.41, "rabbitmq": 6.66, "valkey": 0.47, "minio": 1.52, "loki": 0.53, "prometheus": 0.32}, "mem_mib_max": {"api-1": 102.4, "api-2": 103.0, "worker": 148.8, "otel-collector": 77.9, "tempo": 486.8, "nginx": 5.4, "grafana": 135.6, "postgres": 96.5, "rabbitmq": 151.3, "valkey": 12.1, "minio": 307.6, "loki": 93.8, "prometheus": 282.2}, "telemetry_delta": {"sdk_span_decisions_sample": 55543.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 55543.0, "sdk_spans_exported_failed": null, "col_spans_in": 57443.0, "col_logs_in": 356.0, "col_points_in": 1326.0, "col_spans_out": 10531.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21716724.0, "tempo_spans_in": 9969.0, "tempo_bytes_in": 4062445.0, "loki_bytes_in": 145752.0, "loki_lines_in": 356.0, "prom_samples_in": 41766.0, "tail_policies": {"baseline-10pct/false": 5284.0, "baseline-10pct/true": 612.0, "keep-async-pipeline/false": 5804.0, "keep-async-pipeline/true": 92.0, "keep-errors/false": 5826.0, "keep-errors/true": 70.0, "keep-slow/false": 5824.0, "keep-slow/true": 72.0}}, "traces_in_tempo": {"server_api": 774, "slow": 70, "error": 66, "pipeline_consumer": 90}, "window": [1794233127.526627, 1794233217.535255], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "off", "rep": 3, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233296.514442, "finished_at": 1794233386.5104158, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5440, "successful_rps": 60.44, "status_counts": {"201": 4051, "202": 90, "200": 1299, "500": 50}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.56, "p95": 10.45, "p99": 33.86, "max": 79.09}, "faults_sent": {"slow": 52, "error": 50}, "by_kind": {"post_message": {"n": 4051, "p50_ms": 6.23, "p99_ms": 33.4}, "get_history": {"n": 1349, "p50_ms": 7.95, "p99_ms": 50.36}, "upload": {"n": 90, "p50_ms": 18.98, "p99_ms": 37.17}}}, "cpu_pct_avg": {"api-1": 9.13, "api-2": 9.18, "worker": 15.32, "otel-collector": 0.08, "tempo": 0.86, "nginx": 1.34, "grafana": 0.98, "postgres": 5.15, "rabbitmq": 6.66, "valkey": 0.57, "minio": 1.66, "loki": 0.6, "prometheus": 0.36}, "mem_mib_max": {"api-1": 105.8, "api-2": 124.5, "worker": 144.7, "otel-collector": 35.6, "tempo": 695.5, "nginx": 5.2, "grafana": 136.0, "postgres": 94.7, "rabbitmq": 157.3, "valkey": 12.6, "minio": 309.7, "loki": 95.0, "prometheus": 289.2}, "telemetry_delta": {"sdk_span_decisions_sample": -209.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -209.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 23378.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794233296.514399, 1794233386.524056], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "on100", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233465.4906292, "finished_at": 1794233555.486665, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5445, "successful_rps": 60.5, "status_counts": {"201": 4032, "202": 90, "200": 1323, "500": 45}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.63, "p95": 11.84, "p99": 43.42, "max": 76.29}, "faults_sent": {"slow": 50, "error": 45}, "by_kind": {"post_message": {"n": 4032, "p50_ms": 6.34, "p99_ms": 42.73}, "get_history": {"n": 1368, "p50_ms": 8.43, "p99_ms": 43.79}, "upload": {"n": 90, "p50_ms": 18.37, "p99_ms": 75.49}}}, "cpu_pct_avg": {"api-1": 11.12, "api-2": 11.21, "worker": 15.33, "otel-collector": 0.54, "tempo": 1.29, "nginx": 1.19, "grafana": 0.89, "postgres": 4.63, "rabbitmq": 6.56, "valkey": 0.49, "minio": 1.7, "loki": 0.57, "prometheus": 0.38}, "mem_mib_max": {"api-1": 102.6, "api-2": 107.7, "worker": 143.0, "otel-collector": 73.6, "tempo": 575.9, "nginx": 5.2, "grafana": 135.5, "postgres": 99.4, "rabbitmq": 92.2, "valkey": 12.3, "minio": 311.5, "loki": 94.7, "prometheus": 286.5}, "telemetry_delta": {"sdk_span_decisions_sample": 55247.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 55247.0, "sdk_spans_exported_failed": null, "col_spans_in": 57147.0, "col_logs_in": 335.0, "col_points_in": 1224.0, "col_spans_out": 57147.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21561062.0, "tempo_spans_in": 57147.0, "tempo_bytes_in": 21069123.0, "loki_bytes_in": 132480.0, "loki_lines_in": 335.0, "prom_samples_in": 35456.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 50, "error": 45, "pipeline_consumer": 90}, "window": [1794233465.490568, 1794233555.500811], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "head10", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233634.709635, "finished_at": 1794233725.1747258, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5427, "successful_rps": 60.3, "status_counts": {"201": 4006, "202": 90, "200": 1331, "500": 63}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.57, "p95": 10.44, "p99": 36.56, "max": 264.95}, "faults_sent": {"slow": 68, "error": 63}, "by_kind": {"post_message": {"n": 4006, "p50_ms": 6.25, "p99_ms": 29.64}, "get_history": {"n": 1394, "p50_ms": 8.14, "p99_ms": 44.81}, "upload": {"n": 90, "p50_ms": 18.47, "p99_ms": 63.61}}}, "cpu_pct_avg": {"api-1": 10.01, "api-2": 10.04, "worker": 15.08, "otel-collector": 0.23, "tempo": 0.43, "nginx": 1.25, "grafana": 0.88, "postgres": 4.63, "rabbitmq": 6.55, "valkey": 0.48, "minio": 1.74, "loki": 0.58, "prometheus": 0.34}, "mem_mib_max": {"api-1": 101.0, "api-2": 99.4, "worker": 148.2, "otel-collector": 69.7, "tempo": 359.2, "nginx": 5.1, "grafana": 136.2, "postgres": 108.8, "rabbitmq": 156.7, "valkey": 12.1, "minio": 314.6, "loki": 122.6, "prometheus": 284.1}, "telemetry_delta": {"sdk_span_decisions_sample": 5568.0, "sdk_span_decisions_drop_ratio": 51652.0, "sdk_spans_exported_ok": 5568.0, "sdk_spans_exported_failed": null, "col_spans_in": 5568.0, "col_logs_in": 353.0, "col_points_in": 1287.0, "col_spans_out": 5568.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2566444.0, "tempo_spans_in": 5568.0, "tempo_bytes_in": 2085852.0, "loki_bytes_in": 142091.0, "loki_lines_in": 353.0, "prom_samples_in": 39528.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 546, "slow": 8, "error": 5, "pipeline_consumer": 10}, "window": [1794233634.709598, 1794233725.183886], "stats_samples": 14, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} +{"config": "tail", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233805.530024, "finished_at": 1794233895.525802, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5428, "successful_rps": 60.31, "status_counts": {"201": 4009, "202": 90, "200": 1329, "500": 62}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.69, "p95": 14.69, "p99": 48.23, "max": 153.66}, "faults_sent": {"slow": 54, "error": 62}, "by_kind": {"post_message": {"n": 4009, "p50_ms": 8.35, "p99_ms": 45.87}, "get_history": {"n": 1391, "p50_ms": 10.45, "p99_ms": 51.17}, "upload": {"n": 90, "p50_ms": 20.58, "p99_ms": 84.12}}}, "cpu_pct_avg": {"worker": 15.24, "api-1": 10.99, "api-2": 11.11, "otel-collector": 0.52, "tempo": 0.8, "nginx": 1.16, "grafana": 0.88, "postgres": 4.49, "rabbitmq": 6.64, "valkey": 0.46, "minio": 1.69, "loki": 0.5, "prometheus": 0.32}, "mem_mib_max": {"worker": 143.4, "api-1": 106.7, "api-2": 102.9, "otel-collector": 83.0, "tempo": 487.7, "nginx": 5.1, "grafana": 135.8, "postgres": 114.2, "rabbitmq": 157.5, "valkey": 11.9, "minio": 316.2, "loki": 114.4, "prometheus": 284.0}, "telemetry_delta": {"sdk_span_decisions_sample": 55277.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 55277.0, "sdk_spans_exported_failed": null, "col_spans_in": 57177.0, "col_logs_in": 352.0, "col_points_in": 1224.0, "col_spans_out": 9512.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21589232.0, "tempo_spans_in": 9559.0, "tempo_bytes_in": 3899582.0, "loki_bytes_in": 143224.0, "loki_lines_in": 352.0, "prom_samples_in": 36752.0, "tail_policies": {"baseline-10pct/false": 4962.0, "baseline-10pct/true": 548.0, "keep-async-pipeline/false": 5420.0, "keep-async-pipeline/true": 90.0, "keep-errors/false": 5448.0, "keep-errors/true": 62.0, "keep-slow/false": 5456.0, "keep-slow/true": 54.0}}, "traces_in_tempo": {"server_api": 737, "slow": 54, "error": 62, "pipeline_consumer": 90}, "window": [1794233805.529983, 1794233895.540648], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}} diff --git a/evidence/track1/summary.md b/evidence/track1/summary.md new file mode 100644 index 0000000..6e0c3e7 --- /dev/null +++ b/evidence/track1/summary.md @@ -0,0 +1,31 @@ +Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются. + +| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % | +|---|---|---|---|---|---|---|---|---| +| off | 3 | 60.4 | 0 | 7.1 (6.4…8.3) | 11.0 (10.3…12.2) | 32.7 (31.7…33.9) | 17.8 (17.0…18.3) | 15.2 (14.8…15.3) | +| on100 | 3 | 60.5 | 0 | 7.8 (6.6…8.6) | 14.3 (11.8…16.3) | 47.2 (43.4…54.4) | 22.7 (22.3…22.9) | 15.6 (15.3…15.9) | +| head10 | 3 | 60.4 | 0 | 7.2 (6.6…8.3) | 11.2 (10.3…13.0) | 37.5 (36.6…39.2) | 19.7 (19.6…20.0) | 15.0 (14.9…15.1) | +| tail | 3 | 60.4 | 0 | 7.9 (6.7…8.7) | 13.7 (11.9…14.7) | 44.5 (42.0…48.2) | 22.3 (22.1…22.5) | 15.3 (15.2…15.3) | + +| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB | +|---|---|---|---|---|---|---|---| +| off | 0.07 (0.06…0.08) | 0.70 (0.43…0.86) | 0.57 (0.54…0.60) | 0.34 (0.30…0.36) | 98 | 36 | 470 | +| on100 | 0.57 (0.54…0.60) | 1.13 (1.03…1.29) | 0.57 (0.55…0.58) | 0.36 (0.33…0.38) | 113 | 73 | 571 | +| head10 | 0.23 (0.23…0.24) | 0.41 (0.35…0.44) | 0.54 (0.50…0.58) | 0.34 (0.33…0.34) | 105 | 71 | 423 | +| tail | 0.53 (0.52…0.54) | 0.64 (0.50…0.80) | 0.52 (0.50…0.53) | 0.33 (0.32…0.34) | 116 | 81 | 493 | + +| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB | +|---|---|---|---|---|---|---| +| off | — | 0 | 0 | — | 0 | 0 | +| on100 | 57059 | 57059 | 0 | 21030 | 20547 | 132 | +| head10 | 5560 | 5560 | 0 | 2496 | 2031 | 134 | +| tail | 57106 | 9746 | 0 | 21062 | 3877 | 135 | + +Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна. + +| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено | +|---|---|---|---|---|---| +| off | 0 / 16470 | 0 / 164 | 0 / 163 | 0 / 270 | — / 163 | +| on100 | 16530 / 16470 | 167 / 167 | 146 / 146 | 270 / 270 | 146 / 146 | +| head10 | 1630 / 16470 | 18 / 198 | 15 / 166 | 25 / 270 | 166 / 166 | +| tail | 2269 / 16470 | 180 / 180 | 164 / 164 | 270 / 270 | 164 / 164 | diff --git a/evidence/worker_saturation.json b/evidence/worker_saturation.json new file mode 100644 index 0000000..da17f24 --- /dev/null +++ b/evidence/worker_saturation.json @@ -0,0 +1,20 @@ +{ + "alert_pending_s": 60.01730489730835, + "alert_firing_s": 340, + "saturation_at_firing": 1.0, + "queued_at_firing": 194.0, + "oldest_age_s": 63, + "jobs_per_s": 3.16, + "job_p95_s": 0.98, + "alerts_at_firing": { + "MessengerApiHighErrorRate": "firing", + "MessengerQueueLagGrowing": "firing", + "MessengerWorkerSaturated": "firing" + }, + "resolved_after_restore_s": 17, + "drained_after_restore_s": 22, + "uploads": { + "202": 1255, + "503": 108 + } +} diff --git a/evidence/worker_saturation.txt b/evidence/worker_saturation.txt new file mode 100644 index 0000000..1731031 --- /dev/null +++ b/evidence/worker_saturation.txt @@ -0,0 +1,10 @@ +17:19:44 lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas +17:19:51 uploads: 4/s open-loop (sample.png) +17:20:51 fresh capacity=2 slots (stale series of recreated containers are excluded) +17:20:51 MessengerWorkerSaturated pending (+60 s) +17:25:31 MessengerWorkerSaturated firing (+340 s) +17:25:31 at firing: saturation=1.00, queued=194, oldest age=63 s, succeeded=3.16 jobs/s, job p95=0.98 s, alerts={'MessengerApiHighErrorRate': 'firing', 'MessengerQueueLagGrowing': 'firing', 'MessengerWorkerSaturated': 'firing'} +17:25:31 restore default worker configuration +17:25:49 MessengerWorkerSaturated resolved (+17 s) +17:25:54 backlog drained: True (+22 s after restore) +PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..2991e9d --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,1823 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — наблюдаемость (lab6)", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": false, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "graphTooltip": 1, + "panels": [ + { + "id": 1, + "type": "row", + "title": "SLO (окно 30 мин) и алерты", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Доступность (SLO 99%)", + "description": "1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.99 + }, + { + "color": "green", + "value": 0.995 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_availability:ratio30m", + "legendFormat": "Доступность (SLO 99%)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Латентность ≤500 ms (SLO 95%)", + "description": "Доля запросов в bucket le=0.5 s за 30 мин.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 4, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.95 + }, + { + "color": "green", + "value": 0.97 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:slo_latency:ratio30m", + "legendFormat": "Латентность ≤500 ms (SLO 95%)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Error ratio 5xx (2m)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_error_ratio:rate2m", + "legendFormat": "Error ratio 5xx (2m)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Старейшее задание", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 1, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachments.process\"})", + "legendFormat": "Старейшее задание", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 6, + "type": "alertlist", + "title": "Алерты Prometheus (firing/pending)", + "gridPos": { + "x": 16, + "y": 1, + "w": 8, + "h": 8 + }, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "firing": true, + "pending": true, + "noData": false, + "normal": false, + "error": true + }, + "viewMode": "list", + "datasource": "Prometheus" + } + }, + { + "id": 7, + "type": "stat", + "title": "RPS API", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_requests:rate2m", + "legendFormat": "RPS API", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 8, + "type": "stat", + "title": "p95 API (5m)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 4, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:http_latency_p95:rate5m", + "legendFormat": "p95 API (5m)", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 9, + "type": "stat", + "title": "Загрузка worker", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:worker_saturation:ratio", + "legendFormat": "Загрузка worker", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "Ожидают пул БД", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 4, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(db_client_connection_pending_requests)", + "legendFormat": "Ожидают пул БД", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 11, + "type": "row", + "title": "API (messenger-api, все реплики)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "RPS по маршруту и методу", + "description": "Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 10, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\"}[$__rate_interval]))", + "legendFormat": "{{http_request_method}} {{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "RPS по классу статуса", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 10, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "normal" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\"}[$__rate_interval]), \"status_class\", \"${1}xx\", \"http_response_status_code\", \"(\\\\d)..\")) ", + "legendFormat": "{{status_class}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Латентность API p50/p95/p99 (из histogram buckets)", + "description": "histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 18, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Доля ошибок 5xx", + "description": "Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 18, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\"}[$__rate_interval]))", + "legendFormat": "5xx / все", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "p95 по маршруту", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 26, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Запросы в обработке", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 26, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (instance) (http_server_active_requests{service_name=\"messenger-api\"})", + "legendFormat": "{{instance}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 18, + "type": "row", + "title": "Очередь и worker", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 34, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 19, + "type": "timeseries", + "title": "Jobs по исходу (в сек.)", + "description": "succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 35, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "normal" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Длительность job (succeeded) p50/p95/p99", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 35, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "Глубина очереди (из PostgreSQL)", + "description": "max() по репликам API: каждая публикует одно и то же значение из БД.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 43, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max by (queue, state) (messenger_queue_depth)", + "legendFormat": "{{queue}} {{state}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "Возраст старейшей job", + "description": "От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 43, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachments.process\"})", + "legendFormat": "oldest age", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 23, + "type": "timeseries", + "title": "Время до ready p50/p95/p99", + "description": "messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 51, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 24, + "type": "timeseries", + "title": "Насыщение worker: занято / ёмкость", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 51, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_busy)", + "legendFormat": "busy", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "capacity", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "messenger:worker_saturation:ratio", + "legendFormat": "saturation", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 25, + "type": "row", + "title": "Ресурсы: пул БД, CPU, RAM", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 59, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 26, + "type": "timeseries", + "title": "Пул БД: used / idle / waiting", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 60, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name, db_client_connection_state) (db_client_connection_count)", + "legendFormat": "{{service_name}} {{db_client_connection_state}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name) (db_client_connection_pending_requests)", + "legendFormat": "{{service_name}} waiting", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name) (db_client_connection_max)", + "legendFormat": "{{service_name}} max", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 27, + "type": "timeseries", + "title": "CPU процессов (ядра)", + "description": "worker: self + завершённые дочерние процессы обработки изображений.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 60, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "legendFormat": "{{service_name}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(otelcol_process_cpu_seconds_total[$__rate_interval])", + "legendFormat": "otel-collector", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 28, + "type": "timeseries", + "title": "RSS процессов", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 68, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_memory_usage_bytes", + "legendFormat": "{{service_name}} {{instance}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "otelcol_process_memory_rss_bytes", + "legendFormat": "otel-collector", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 29, + "type": "timeseries", + "title": "Telemetry pipeline: принято / отправлено / ошибки (items/s)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 68, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", + "legendFormat": "accepted spans", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", + "legendFormat": "accepted log records", + "refId": "E", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", + "legendFormat": "accepted metric points", + "refId": "F", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))", + "legendFormat": "sent {{exporter}}", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", + "legendFormat": "send failed spans", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", + "legendFormat": "dropped spans (queue full)", + "refId": "G", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", + "legendFormat": "send failed logs", + "refId": "H", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", + "legendFormat": "dropped logs (queue full)", + "refId": "I", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "queue {{exporter}}", + "refId": "D", + "range": true, + "instant": false + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Логи и трассы", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 76, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 31, + "type": "logs", + "title": "Логи API и worker (warning/error)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 77, + "w": 12, + "h": 10 + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "{service_name=~\"messenger-api|messenger-worker\"} | severity_text=~\"WARN.*|ERROR\"" + } + ] + }, + { + "id": 32, + "type": "table", + "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 12, + "y": 77, + "w": 12, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "refId": "A", + "queryType": "traceql", + "limit": 20, + "query": "{resource.service.name=~\"messenger-.*\" && (status=error || duration>500ms)}" + } + ] + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "templating": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..d6b09ae --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..562b315 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,44 @@ +# Grafana 11.5.2 datasources (fixed uids are referenced by the dashboard JSON). Grafana stores no signals itself. +# "$$" escapes "$" from Grafana's env-variable interpolation in provisioning files. +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s + prometheusType: Prometheus + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + maxLines: 1000 + derivedFields: + # trace_id is OTLP structured metadata -> link "Open trace" to Tempo. + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: Open trace + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"' + nodeGraph: + enabled: true diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..8436f30 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,39 @@ +# Loki 3.4.2 single binary: log store. Logs arrive by OTLP (http://loki:3100/otlp) from the Collector. +# service.name -> stream label service_name; log attributes (event, code.*) and trace_id/span_id -> structured metadata. +auth_enabled: false +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + volume_enabled: true +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/minio/app-policy.json b/infra/minio/app-policy.json new file mode 100644 index 0000000..9195e44 --- /dev/null +++ b/infra/minio/app-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/attachments/*", "arn:aws:s3:::messenger-attachments/staging/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/minio/init.sh b/infra/minio/init.sh new file mode 100755 index 0000000..2b69c79 --- /dev/null +++ b/infra/minio/init.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API. +set -eu +BUCKET="${S3_BUCKET:-messenger-attachments}" +ROOT_USER="$(cat /run/secrets/minio_root_user)" +ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +APP_USER="$(cat /run/secrets/s3_access_key_id)" +APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)" + +until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do + echo "waiting for minio"; sleep 2 +done +mc mb --ignore-existing "local/$BUCKET" +mc anonymous set none "local/$BUCKET" +mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \ + || mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true +mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true +mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true +echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*" diff --git a/infra/nginx-health.sh b/infra/nginx-health.sh new file mode 100644 index 0000000..692cab7 --- /dev/null +++ b/infra/nginx-health.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -u + +SERVERS="${API_UPSTREAMS:-api-1 api-2}" +INTERVAL="${HEALTH_INTERVAL:-1}" +FALL="${HEALTH_FALL:-2}" +RISE="${HEALTH_RISE:-2}" +OUT=/etc/nginx/upstream/servers.conf + +mkdir -p "$(dirname "$OUT")" + +var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; } + +render() { + any_up=0 + for s in $SERVERS; do + eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1 + done + for s in $SERVERS; do + mark="" + if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then + mark=" down" + fi + echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;" + done +} + +for s in $SERVERS; do + v=$(var "$s") + eval "up_$v=1 ok_$v=0 bad_$v=0" +done +render > "$OUT" + +watch() { + pending=0 + while sleep "$INTERVAL"; do + changed=0 + for s in $SERVERS; do + v=$(var "$s") + if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then + eval "ok_$v=\$((ok_$v + 1)) bad_$v=0" + if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then + eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream" + fi + else + eval "bad_$v=\$((bad_$v + 1)) ok_$v=0" + if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then + eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down" + fi + fi + done + if [ "$changed" = 1 ]; then + render > "$OUT" + pending=1 + fi + if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then + pending=0 + fi + done +} + +watch & +exec nginx -g 'daemon off;' diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 0000000..5d791ed --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,75 @@ +worker_processes 2; + +events { + worker_connections 256; +} + +http { + resolver 127.0.0.11 valid=5s ipv6=off; + + upstream api { + zone api 64k; + include /etc/nginx/upstream/servers.conf; + keepalive 32; + keepalive_timeout 4s; + } + + log_format main '$remote_addr - $upstream_addr [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$upstream_response_time" "$request_time"'; + + access_log /var/log/nginx/access.log main; + + server { + listen 8080; + client_max_body_size 64k; + error_page 413 @payload_too_large; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + location ~ ^/api/v1/chats/[^/]+/messages$ { + proxy_pass http://api; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + } + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + client_max_body_size 10m; + proxy_request_buffering off; + proxy_pass http://api; + proxy_next_upstream off; + } + + location @payload_too_large { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id lb always; + add_header Connection close always; + return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}'; + } + + location /health/ { + proxy_pass http://api; + proxy_next_upstream error timeout; + } + + location ~ ^/api/v1/attachments/ { + proxy_max_temp_file_size 0; + proxy_pass http://api; + proxy_next_upstream error timeout; + } + + location / { + proxy_pass http://api; + proxy_next_upstream error timeout http_502 http_503; + } + } +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..98af881 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,88 @@ +# OpenTelemetry Collector (contrib 0.120.0): receiver/processor/exporter only, it stores nothing long-term. +# API, worker --OTLP/HTTP:4318--> collector --> Prometheus (metrics, OTLP), Loki (logs, OTLP), Tempo (traces, OTLP/gRPC) +# Ports are reachable only inside the compose network; nothing here is published to the host. +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 4194304 # 4 MiB per OTLP request + +processors: + # First in every pipeline: refuse new data (receivers answer 503 -> SDK retries/drops) before the container OOMs. + # Container limit is 256 MiB; soft limit = limit_mib - spike_limit_mib = 140 MiB. + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + # Defense in depth: even if an instrumentation adds them, these attributes never leave the collector. + # url.query would contain ?filename=...; headers could contain Authorization. + attributes/scrub: + actions: + - {key: url.query, action: delete} + - {key: url.full, action: delete} + - {key: http.target, action: delete} + - {key: http.url, action: delete} + - {key: http.request.header.authorization, action: delete} + - {key: http.request.header.cookie, action: delete} + - {key: db.query.parameters, action: delete} + +exporters: + # Every exporter: bounded in-memory queue (batches) + bounded retry. Backend down => queue fills, then new + # batches are dropped and counted (otelcol_exporter_enqueue_failed_*); RAM stays bounded, apps are unaffected. + otlphttp/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp + compression: gzip + timeout: 5s + sending_queue: {enabled: true, num_consumers: 2, queue_size: 100} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s} + otlphttp/loki: + endpoint: http://loki:3100/otlp + compression: gzip + timeout: 5s + sending_queue: {enabled: true, num_consumers: 2, queue_size: 100} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s} + otlp/tempo: + endpoint: tempo:4317 + tls: {insecure: true} + compression: gzip + timeout: 5s + sending_queue: {enabled: true, num_consumers: 2, queue_size: 100} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s} + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlphttp/loki] diff --git a/infra/otel/profile-default.yaml b/infra/otel/profile-default.yaml new file mode 100644 index 0000000..a2fa84a --- /dev/null +++ b/infra/otel/profile-default.yaml @@ -0,0 +1,5 @@ +# Default profile: no extra processing. All spans that the SDK head-sampled are exported. +# Selected with COLLECTOR_PROFILE=default (see compose.yaml); merged over collector.yaml. +processors: + batch: + timeout: 2s diff --git a/infra/otel/profile-tail.yaml b/infra/otel/profile-tail.yaml new file mode 100644 index 0000000..1e38de5 --- /dev/null +++ b/infra/otel/profile-tail.yaml @@ -0,0 +1,31 @@ +# Track 1 "balanced" profile (COLLECTOR_PROFILE=tail): SDK keeps 100% head sampling, the collector decides per +# whole trace after decision_wait. Keeps: every trace with an error, every trace slower than 500 ms, every trace of +# the attachment pipeline (rare, async, valuable), and 10% of the remaining ordinary traffic. +# Metrics are computed in the SDK BEFORE any sampling, so RPS/error rate/percentiles are unaffected. +processors: + tail_sampling: + decision_wait: 10s + num_traces: 20000 + expected_new_traces_per_sec: 200 + decision_cache: + sampled_cache_size: 50000 + policies: + - name: keep-errors + type: status_code + status_code: {status_codes: [ERROR]} + - name: keep-slow + type: latency + latency: {threshold_ms: 500} + - name: keep-async-pipeline + type: string_attribute + string_attribute: {key: messaging.system, values: [rabbitmq]} + - name: baseline-10pct + type: probabilistic + probabilistic: {sampling_percentage: 10} + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, tail_sampling, batch] + exporters: [otlp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..10b00ca --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,31 @@ +# Prometheus v3.2.1: metrics store. Application metrics arrive by OTLP push from the Collector +# (--web.enable-otlp-receiver, /api/v1/otlp); infrastructure self-metrics are scraped. +global: + scrape_interval: 15s + evaluation_interval: 15s + +otlp: + # job=service.name, instance=service.instance.id are set by Prometheus; these become extra labels. + promote_resource_attributes: + - service.name + - service.instance.id + - service.version + - deployment.environment.name + +storage: + tsdb: + # OTLP pushes from several SDKs/batches may arrive slightly out of order. + out_of_order_time_window: 10m + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: otel-collector + static_configs: [{targets: ["otel-collector:8888"]}] + - job_name: tempo + static_configs: [{targets: ["tempo:3200"]}] + - job_name: loki + static_configs: [{targets: ["loki:3100"]}] + - job_name: prometheus + static_configs: [{targets: ["localhost:9090"]}] diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..aad2c6a --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,111 @@ +# Recording rules (SLI/SLO) and alert rules. Evaluated by Prometheus every 15 s; state (pending/firing/inactive) +# is visible in Prometheus /alerts, the API /api/v1/alerts and in Grafana (alert list panel). No external delivery. +# Runbooks: docs/runbooks.md (anchors in runbook_url). +# +# Label mapping (OTLP -> Prometheus 3): service.name -> service_name, service.instance.id -> instance, +# http.route -> http_route, http.response.status_code -> http_response_status_code, job.type -> job_type. +# HTTP SLI denominator: every business request of messenger-api (health probes are not instrumented). +# Numerator of errors: 5xx only. 4xx (wrong password, 404 of a foreign chat, 409) are client outcomes, not failures. +groups: + - name: messenger-sli + interval: 15s + rules: + - record: messenger:http_requests:rate2m + expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[2m])) + - record: messenger:http_errors_5xx:rate2m + expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[2m])) or vector(0) + - record: messenger:http_error_ratio:rate2m + expr: messenger:http_errors_5xx:rate2m / messenger:http_requests:rate2m + - record: messenger:http_latency_p95:rate5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[5m]))) + # SLO window: rolling 30 min (lab-sized; production would use 28-30 days with the same formulas). + # Availability SLO: 99% of business requests without 5xx. + - record: messenger:slo_availability:ratio30m + expr: | + 1 - ( + (sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[30m])) or vector(0)) + / sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m])) + ) + # Latency SLO: 95% of business requests complete in <= 500 ms (bucket le="0.5"). + - record: messenger:slo_latency:ratio30m + expr: | + sum(increase(http_server_request_duration_seconds_bucket{service_name="messenger-api",le="0.5"}[30m])) + / sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m])) + - record: messenger:worker_saturation:ratio + # OTLP-pushed series have no staleness markers: a recreated worker would stay in the denominator for the + # 5 min lookback. Only samples fresher than 30 s (3 export intervals) are counted. + expr: | + sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 30)) + / sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30)) + + - name: messenger-alerts + rules: + - alert: MessengerApiHighErrorRate + expr: messenger:http_error_ratio:rate2m > 0.05 and messenger:http_requests:rate2m > 0.2 + for: 1m + labels: + severity: page + annotations: + summary: "API: доля 5xx {{ $value | humanizePercentage }} за 2 мин (порог 5%)" + description: "Больше 5% бизнес-запросов messenger-api завершаются 5xx (окно 2m, держится 1m). Расходуется бюджет SLO доступности 99%." + runbook_url: "docs/runbooks.md#messengerapihigherrorrate" + + - alert: MessengerQueueLagGrowing + expr: max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60 + for: 1m + labels: + severity: page + annotations: + summary: "Очередь вложений: старейшее задание ждёт {{ $value | humanizeDuration }}" + description: "Принятое задание не завершено дольше 60 с (порог), условие держится 1m: worker остановлен, перегружен или не может писать в S3/DB." + runbook_url: "docs/runbooks.md#messengerqueuelaggrowing" + + - alert: MessengerApiLatencyHigh + expr: messenger:http_latency_p95:rate5m > 1 + for: 2m + labels: + severity: ticket + annotations: + summary: "API p95 = {{ $value | humanizeDuration }} (порог 1 s, окно 5m)" + description: "p95 из histogram buckets всех реплик выше 1 s." + runbook_url: "docs/runbooks.md#messengerapilatencyhigh" + + - alert: MessengerDbPoolWaiting + expr: max(db_client_connection_pending_requests) > 0 + for: 1m + labels: + severity: ticket + annotations: + summary: "Запросы ждут соединение из пула БД ({{ $value }} в очереди)" + description: "Пул psycopg исчерпан дольше минуты: БД медленная/блокирована или пул мал." + runbook_url: "docs/runbooks.md#messengerdbpoolwaiting" + + - alert: MessengerWorkerSaturated + expr: messenger:worker_saturation:ratio > 0.9 + for: 5m + labels: + severity: ticket + annotations: + summary: "Worker занят на {{ $value | humanizePercentage }} ёмкости 5 минут" + runbook_url: "docs/runbooks.md#messengerworkersaturated" + + - alert: TelemetryExportFailing + # Each term has "or vector(0)": the *_failed_* series do not exist until the first failure, and a sum with an + # empty operand is empty (the alert would never fire). Early signal: exporter queue filling up (backend + # down, retries in progress); late signal: data dropped (queue full) or retries exhausted. + expr: | + ( + (sum(rate(otelcol_exporter_send_failed_spans_total[2m])) or vector(0)) + + (sum(rate(otelcol_exporter_send_failed_log_records_total[2m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_spans_total[2m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_log_records_total[2m])) or vector(0)) + ) > 0 + or + max(max by (exporter) (otelcol_exporter_queue_size) / max by (exporter) (otelcol_exporter_queue_capacity)) > 0.2 + for: 1m + labels: + severity: ticket + annotations: + summary: "Collector не может доставить telemetry в backend" + description: "Очередь exporter заполнена >20% или данные отбрасываются: Tempo/Loki/Prometheus недоступен. Бизнес-API не затронут, RAM collector ограничена очередью и memory_limiter." + runbook_url: "docs/runbooks.md#telemetryexportfailing" diff --git a/infra/rabbitmq/entrypoint.sh b/infra/rabbitmq/entrypoint.sh new file mode 100644 index 0000000..2e99ea0 --- /dev/null +++ b/infra/rabbitmq/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Credentials come from a Docker secret; RabbitMQ's config cannot read files, so render a conf.d fragment at start. +set -eu +PASS="$(cat /run/secrets/rabbitmq_password)" +mkdir -p /etc/rabbitmq/conf.d +cat > /etc/rabbitmq/conf.d/20-credentials.conf < staging -> job/outbox -> broker -> worker -> final prefix + variants. +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status + CHECK (status IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +-- staging_key: private S3 prefix written by the API before 202; object_key: final original written by the worker. +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +CREATE UNIQUE INDEX attachments_staging_key_idx ON attachments (staging_key) WHERE staging_key IS NOT NULL; + +-- Effect key of the pipeline: (attachment_id, pipeline_version, name). A repeated job cannot add a second set. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CONSTRAINT variants_ct CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CONSTRAINT variants_size CHECK (size_bytes > 0), + width integer NOT NULL CONSTRAINT variants_width CHECK (width > 0), + height integer NOT NULL CONSTRAINT variants_height CHECK (height > 0), + sha256 text NOT NULL CONSTRAINT variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- One logical job per (attachment, pipeline_version); job_id never changes on redelivery or retry. +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + state text NOT NULL + CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed')), + attempt integer NOT NULL DEFAULT 1 CONSTRAINT jobs_attempt CHECK (attempt >= 1), + max_attempts integer NOT NULL CONSTRAINT jobs_max_attempts CHECK (max_attempts >= 1), + next_attempt_at timestamptz NOT NULL, + lease_owner text, + lease_until timestamptz, + error_code text, + last_error text, + created_at timestamptz NOT NULL, + first_started_at timestamptz, + started_at timestamptz, + finished_at timestamptz, + updated_at timestamptz NOT NULL, + UNIQUE (attachment_id, pipeline_version) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE state = 'processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE state = 'failed'; +CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (state) WHERE state IN ('queued', 'processing'); + +-- Transactional outbox: written in the same transaction as the job; the relay publishes with confirms. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz, + publish_attempts integer NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL +); +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_published_idx ON outbox (published_at) WHERE published_at IS NOT NULL; + +-- Operator-visible history of every state change of a job (including explicit replay). +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL, + action text NOT NULL, + actor text NOT NULL, + attempt integer NOT NULL, + detail jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL +); +CREATE INDEX job_audit_job_idx ON job_audit (job_id, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8228e2a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,60 @@ +aio-pika==9.5.5 +aiormq==6.8.1 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +asgiref==3.13.0 +boto3==1.35.81 +botocore==1.35.81 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +Deprecated==3.0.1 +exceptiongroup==1.2.2 +fastapi==0.115.6 +googleapis-common-protos==1.75.0 +h11==0.16.0 +httptools==0.9.0 +idna==3.20 +importlib_metadata==8.5.0 +jmespath==1.0.1 +multidict==6.1.0 +opentelemetry-api==1.30.0 +opentelemetry-exporter-otlp-proto-common==1.30.0 +opentelemetry-exporter-otlp-proto-http==1.30.0 +opentelemetry-instrumentation==0.51b0 +opentelemetry-instrumentation-asgi==0.51b0 +opentelemetry-instrumentation-botocore==0.51b0 +opentelemetry-instrumentation-fastapi==0.51b0 +opentelemetry-propagator-aws-xray==1.0.2 +opentelemetry-proto==1.30.0 +opentelemetry-sdk==1.30.0 +opentelemetry-semantic-conventions==0.51b0 +opentelemetry-util-http==0.51b0 +packaging==26.3 +pamqp==3.3.0 +pillow==11.0.0 +propcache==0.2.1 +protobuf==5.29.6 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.11 +pydantic==2.14.1 +pydantic_core==2.50.1 +python-dateutil==2.9.0.post0 +requests==2.34.2 +s3transfer==0.10.4 +six==1.17.0 +starlette==0.41.3 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.3.0 +uvicorn==0.34.0 +uvloop==0.23.0 +valkey==6.2.0 +wrapt==1.17.3 +yarl==1.18.3 +zipp==4.1.2 diff --git a/scripts/gen_dashboard.py b/scripts/gen_dashboard.py new file mode 100644 index 0000000..4c742dc --- /dev/null +++ b/scripts/gen_dashboard.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup). + +The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py +Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов"). +""" +import json +from pathlib import Path + +OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json" +PROM = {"type": "prometheus", "uid": "prometheus"} +LOKI = {"type": "loki", "uid": "loki"} +TEMPO = {"type": "tempo", "uid": "tempo"} +API = 'service_name="messenger-api"' +HTTP = "http_server_request_duration_seconds" +_ids = iter(range(1, 1000)) + + +def target(expr, legend="", ref="A", instant=False): + return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant, + "instant": instant} + + +def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None): + panel = { + "id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM, + "gridPos": {"x": x, "y": y, "w": w, "h": h}, + "fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, + "stacking": {"mode": "normal" if stack else "none"}, + "showPoints": "never"}}, + "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, + "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [target(*t) if isinstance(t, tuple) else t for t in targets], + } + if extra: + panel["fieldConfig"]["defaults"].update(extra) + return panel + + +def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None): + steps = thresholds or [{"color": "green", "value": None}] + panel = { + "id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM, + "gridPos": {"x": x, "y": y, "w": w, "h": h}, + "fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}}, + "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "colorMode": "value", "graphMode": "area", "textMode": "auto"}, + "targets": [target(expr, title)], + } + if decimals is not None: + panel["fieldConfig"]["defaults"]["decimals"] = decimals + return panel + + +def row(title, y): + return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, + "panels": []} + + +def q(p, expr_inner): + return "histogram_quantile(%s, %s)" % (p, expr_inner) + + +panels = [] +y = 0 +# ------------------------------------------------------------------ SLO and alerts +panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1 +red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad}, + {"color": "green", "value": warn}] +panels += [ + stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y, + desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.", + thresholds=red_yellow(0.99, 0.995), decimals=2), + stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y, + desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2), + stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2), + stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]), + {"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)", + "gridPos": {"x": 16, "y": y, "w": 8, "h": 8}, + "options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [], + "groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter": + {"firing": True, "pending": True, "noData": False, "normal": False, "error": True}, + "viewMode": "list", "datasource": "Prometheus"}}, +] +y += 4 +panels += [ + stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1), + stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]), + stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]), + stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]), +] +y += 4 +# ------------------------------------------------------------------ API +panels.append(row("API (messenger-api, все реплики)", y)); y += 1 +rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API) +panels += [ + ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count, + "{{http_request_method}} {{http_route}}")], "reqps", 0, y, + desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."), + ts("RPS по классу статуса", [( + 'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count, + "{{status_class}}")], "reqps", 12, y, stack=True), +] +y += 8 +bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API) +panels += [ + ts("Латентность API p50/p95/p99 (из histogram buckets)", [ + (q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y, + desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."), + ts("Доля ошибок 5xx", [ + ('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count), + "5xx / все", "A")], "percentunit", 12, y, + desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."), +] +y += 8 +panels += [ + ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)), + "{{http_route}}")], "s", 0, y), + ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], + "short", 12, y), +] +y += 8 +# ------------------------------------------------------------------ worker and queue +panels.append(row("Очередь и worker", y)); y += 1 +jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))" +rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))" +panels += [ + ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", + "{{outcome}}")], "ops", 0, y, stack=True, + desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"), + ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"), + (q(0.99, jb), "p99", "C")], "s", 12, y), +] +y += 8 +panels += [ + ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")], + "short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."), + ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")], + "s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.", + extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never", + "thresholdsStyle": {"mode": "line"}}, + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None}, + {"color": "red", "value": 60}]}}), +] +y += 8 +panels += [ + ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")], + "s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."), + ts("Насыщение worker: занято / ёмкость", [ + ("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"), + ("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y), +] +y += 8 +# ------------------------------------------------------------------ resources +panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1 +panels += [ + ts("Пул БД: used / idle / waiting", [ + ("sum by (service_name, db_client_connection_state) (db_client_connection_count)", + "{{service_name}} {{db_client_connection_state}}", "A"), + ("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"), + ("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y), + ts("CPU процессов (ядра)", [ + ("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", + "{{service_name}} {{instance}}", "A"), + ("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y, + desc="worker: self + завершённые дочерние процессы обработки изображений."), +] +y += 8 +panels += [ + ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"), + ("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y), + ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [ + ("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"), + ("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"), + ("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"), + ("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))", + "sent {{exporter}}", "B"), + # Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist. + ("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"), + ("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"), + ("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"), + ("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"), + ("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y), +] +y += 8 +# ------------------------------------------------------------------ logs and traces +panels.append(row("Логи и трассы", y)); y += 1 +panels += [ + {"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI, + "gridPos": {"x": 0, "y": y, "w": 12, "h": 10}, + "options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}, + "targets": [{"datasource": LOKI, "refId": "A", + "expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]}, + {"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO, + "gridPos": {"x": 12, "y": y, "w": 12, "h": 10}, + "targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, + "query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]}, +] +y += 10 + +dashboard = { + "uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"], + "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s", + "time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels, + "annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, + "enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]}, + "templating": {"list": []}, +} +OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels))) + +# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script). +UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки", + "bytes": "байты"} +DOC = OUT.parents[3] / "docs/dashboard.md" +lines = ["# Панели dashboard «%s»" % dashboard["title"], "", + "Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. " + "`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""] +for panel in panels: + if panel["type"] == "row": + lines += ["## " + panel["title"], ""] + continue + unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "") + lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else "")) + if panel.get("description"): + lines.append("— " + panel["description"]) + lines.append("") + if panel["type"] == "alertlist": + lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""] + for t in panel.get("targets", []): + query = t.get("expr") or t.get("query") + legend = t.get("legendFormat", "") + lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""), + query + (" # " + legend if legend and legend != panel["title"] else ""), "```"] + lines.append("") +DOC.write_text("\n".join(lines), encoding="utf-8") +print("wrote docs/dashboard.md") diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 0000000..2b08611 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Creates a local .env and random credentials in secrets/ (both are git-ignored). Prints no secret values. +# Safe to repeat: existing files are kept, so running it on a live stand does not change its passwords. +set -eu +cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env +umask 077 +mkdir -p secrets +cd secrets + +put() { # put + [ -f "$1" ] || printf '%s' "$2" > "$1" +} +random() { python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')"; } + +put postgres_password "$(random)" +put minio_root_user messenger_root +put minio_root_password "$(random)" +put s3_access_key_id messenger_app +put s3_secret_access_key "$(random)" +put rabbitmq_password "$(random)" +put grafana_admin_password "$(random)" +put cache_url valkey://valkey:6379/0 +put database_url "postgresql://messenger:$(cat postgres_password)@postgres:5432/messenger" +put broker_url "amqp://messenger:$(cat rabbitmq_password)@rabbitmq:5672/messenger" +# Containers run under other uids and read these through Docker secrets (file bind mounts in Compose). +chmod 644 ./* +echo "secrets ready in secrets/ (.env: $( [ -f ../.env ] && echo present ))" diff --git a/scripts/jobs.py b/scripts/jobs.py new file mode 100755 index 0000000..91edc79 --- /dev/null +++ b/scripts/jobs.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...). + + failed registry of failed jobs (the "DLQ" of business failures) + show job state + audit trail + replay --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing + stats counts by state and outbox backlog +Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md). +""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import jobs # noqa: E402 +from app.cursor import fmt # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +def out(value) -> None: + print(json.dumps(value, ensure_ascii=False, indent=2, default=str)) + + +async def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("failed") + sub.add_parser("stats") + show = sub.add_parser("show") + show.add_argument("attachment_id") + rp = sub.add_parser("replay") + rp.add_argument("attachment_id") + rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)") + args = parser.parse_args() + + db = Database() + await db.open() + try: + if args.cmd == "failed": + async with db.connection() as conn: + out(await jobs.failed_jobs(conn)) + elif args.cmd == "stats": + async with db.connection() as conn: + cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state") + states = {r["state"]: r["n"] for r in await cur.fetchall()} + cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL") + backlog = (await cur.fetchone())["n"] + out({"jobs": states, "outbox_unpublished": backlog}) + elif args.cmd == "show": + async with db.connection() as conn: + cur = await conn.execute( + "SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until," + " a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.attachment_id = %s", (args.attachment_id,)) + job = await cur.fetchone() + cur = await conn.execute( + "SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id", + (args.attachment_id,)) + trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()] + cur = await conn.execute( + "SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts" + " FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id", + (args.attachment_id,)) + outbox = await cur.fetchall() + out({"job": job, "audit": trail, "outbox": outbox}) + elif args.cmd == "replay": + s3 = S3Storage() + await run_in_threadpool(s3.open) + + async def exists(key): + return await run_in_threadpool(s3.exists, key) + + result = await db.in_transaction( + lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists)) + out(result) + return 0 if result["result"] == "queued" else 3 + finally: + await db.close() + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 0000000..5c19e22 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run.""" +import hashlib +import os +import pathlib +import sys + +import psycopg + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MIGRATIONS = ROOT / "migrations" +LOCK_ID = 2026_0004 + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return pathlib.Path(path).read_text(encoding="utf-8").strip() + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + sys.exit("DATABASE_URL or DATABASE_URL_FILE is required") + return url + + +def main() -> None: + files = sorted(MIGRATIONS.glob("*.sql")) + if not files: + sys.exit("no migration files found in %s" % MIGRATIONS) + with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall() + applied = {row[0]: row[1] for row in rows} + for path in files: + version = path.stem + body = path.read_text(encoding="utf-8") + checksum = hashlib.sha256(body.encode("utf-8")).hexdigest() + if version in applied: + if applied[version] != checksum: + sys.exit("migration %s changed after it was applied" % version) + print("skip %s" % version) + continue + with conn.transaction(): + conn.execute(body) + conn.execute( + "INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", + (version, checksum), + ) + print("apply %s" % version) + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + print("migrations complete: %d file(s)" % len(files)) + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 0000000..10cd7b6 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything.""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import config, reconcile # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports") + parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS) + args = parser.parse_args() + db, s3 = Database(), S3Storage() + await db.open() + await run_in_threadpool(s3.open) + try: + print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True)) + finally: + await db.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..5037c43 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Restart API, LB, worker and every store keeping all volumes: business stores (postgres, valkey, minio, rabbitmq) +# and, since lab6, the telemetry stores (prometheus, loki, tempo) plus collector and grafana. +set -eu +cd "$(dirname "$0")/.." +docker compose restart -t 10 postgres valkey minio rabbitmq +docker compose restart -t 10 prometheus loki tempo otel-collector grafana +docker compose restart -t 25 worker api-1 api-2 nginx diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..23c94b5 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +docker compose stop -t 3 "$TARGET_INSTANCE_ID" diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..9ec34af --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Every Prometheus query of the provisioned dashboard must parse and return data (after some traffic). + +python3 student-tests/check_dashboard.py +""" +import json +import sys +import urllib.error +from pathlib import Path + +import obs + +ROOT = Path(__file__).resolve().parents[1] +dash = json.loads((ROOT / "infra/grafana/dashboards/messenger.json").read_text(encoding="utf-8")) +bad = 0 +for panel in dash["panels"]: + for t in panel.get("targets", []): + if t.get("datasource", {}).get("type") != "prometheus": + continue + expr = t["expr"].replace("$__rate_interval", "2m") + try: + rows = obs.prom(expr) + status = "ok %3d series" % len(rows) if rows else "EMPTY" + except urllib.error.HTTPError as exc: + status, rows = "ERROR %s" % exc.read()[:120], None + if not rows: + bad += 1 + print("%-16s %-48s %s" % (status, panel["title"][:48], t.get("legendFormat", ""))) +print("panels: %d, prometheus queries without data: %d" % (len(dash["panels"]), bad)) + + +# ---------------------------------------------------------------- the same panels executed BY GRAFANA (/api/ds/query) +# Proves the provisioned datasources work from Grafana itself (Prometheus, Loki and Tempo), with Grafana macros. +import base64 # noqa: E402 +import urllib.request # noqa: E402 + +password = (ROOT / "secrets/grafana_admin_password").read_text().strip() +auth = "Basic " + base64.b64encode(("admin:" + password).encode()).decode() +via_bad = 0 +for panel in dash["panels"]: + for t in panel.get("targets", []): + q = dict(t) + q.setdefault("datasource", panel.get("datasource")) + if q["datasource"]["type"] == "tempo": + # Grafana 11 runs TraceQL search in the browser through the datasource proxy, not /api/ds/query + # ("backend TraceQL search queries are not supported"): check exactly that proxy path. + import time as _t + import urllib.parse as _p + url = ("http://127.0.0.1:3000/api/datasources/proxy/uid/tempo/api/search?" + + _p.urlencode({"q": q["query"], "limit": q.get("limit", 20), "start": int(_t.time()) - 1800, + "end": int(_t.time())})) + req = urllib.request.Request(url, headers={"Authorization": auth}) + try: + with urllib.request.urlopen(req, timeout=60) as response: + n = len(json.loads(response.read()).get("traces", [])) + status = "ok %5d traces" % n if n else "EMPTY" + except urllib.error.HTTPError as exc: + status = "ERROR HTTP %s" % exc.code + if not status.startswith("ok"): + via_bad += 1 + print("grafana %-8s %-18s %-48s (datasource proxy)" % ("tempo", status, panel["title"][:48])) + continue + body = json.dumps({"queries": [q], "from": "now-30m", "to": "now"}).encode() + req = urllib.request.Request("http://127.0.0.1:3000/api/ds/query", data=body, method="POST", + headers={"Content-Type": "application/json", "Authorization": auth}) + try: + with urllib.request.urlopen(req, timeout=60) as response: + res = json.loads(response.read())["results"][q["refId"]] + frames = res.get("frames", []) + rows = sum(len((f.get("data") or {}).get("values", [[]])[0] or []) for f in frames) + status = "ERROR %s" % res["error"][:100] if res.get("error") else ("ok %5d values" % rows if rows else "EMPTY") + except urllib.error.HTTPError as exc: + status = "ERROR HTTP %s %s" % (exc.code, exc.read()[:100]) + if not status.startswith("ok"): + via_bad += 1 + print("grafana %-8s %-18s %-48s" % (q["datasource"]["type"], status, panel["title"][:48])) +print("via Grafana: queries without data or with errors: %d" % via_bad) diff --git a/student-tests/obs.py b/student-tests/obs.py new file mode 100644 index 0000000..8e30a2f --- /dev/null +++ b/student-tests/obs.py @@ -0,0 +1,139 @@ +"""Query helpers for the telemetry backends (stdlib only). Backends listen on 127.0.0.1 (compose.yaml). + +Never prints tokens or passwords; callers print only ids, counts, span names and attributes. +""" +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request + +PROM = os.environ.get("PROM_URL", "http://127.0.0.1:9090") +LOKI = os.environ.get("LOKI_URL", "http://127.0.0.1:3100") +TEMPO = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200") + + +def _get(url, params=None, timeout=15): + if params: + url += "?" + urllib.parse.urlencode(params) + with urllib.request.urlopen(url, timeout=timeout) as response: + return json.loads(response.read()) + + +def prom(expr, at=None): + """Instant query -> list of (labels, float value).""" + params = {"query": expr} + if at: + params["time"] = at + data = _get(PROM + "/api/v1/query", params)["data"]["result"] + return [(r["metric"], float(r["value"][1])) for r in data] + + +def prom_scalar(expr, default=0.0, at=None): + rows = prom(expr, at) + return rows[0][1] if rows else default + + +def prom_range(expr, start, end, step=5): + data = _get(PROM + "/api/v1/query_range", {"query": expr, "start": start, "end": end, "step": step}) + return data["data"]["result"] + + +def alerts(): + """{alertname: state} for pending/firing alerts in Prometheus.""" + out = {} + for a in _get(PROM + "/api/v1/alerts")["data"]["alerts"]: + out[a["labels"]["alertname"]] = a["state"] + return out + + +def loki(query, minutes=15, limit=500): + end = time.time_ns() + data = _get(LOKI + "/loki/api/v1/query_range", {"query": query, "limit": limit, "direction": "forward", + "start": end - minutes * 60 * 10**9, "end": end}) + lines = [] + for stream in data["data"]["result"]: + for ts, line in stream["values"]: + lines.append({"ts": int(ts), "line": line, **stream["stream"]}) + return sorted(lines, key=lambda item: item["ts"]) + + +def tempo_search(traceql, minutes=30, limit=100, start=None, end=None): + end = int(end or time.time() + 60) + start = int(start or end - 60 - minutes * 60) + data = _get(TEMPO + "/api/search", {"q": traceql, "start": start, "end": end, "limit": limit, "spss": 50}, + timeout=60) + return data.get("traces", []) + + +def _hex(value): + return base64.b64decode(value).hex() if value else None + + +def _attr(value): + for key in ("stringValue", "intValue", "boolValue", "doubleValue"): + if key in value: + return value[key] + return None + + +def tempo_trace(trace_id): + """Flat list of spans: service, name, kind, span_id, parent_id, duration_ms, status, attrs (dict).""" + try: + data = _get(TEMPO + "/api/traces/" + trace_id) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return [] + raise + spans = [] + for batch in data.get("batches", []): + service = {a["key"]: _attr(a["value"]) for a in batch["resource"]["attributes"]}.get("service.name") + for scope in batch.get("scopeSpans", []): + for s in scope["spans"]: + spans.append({ + "service": service, "name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""), + "span_id": _hex(s["spanId"]), "parent_id": _hex(s.get("parentSpanId")), + "start": int(s["startTimeUnixNano"]), + "duration_ms": (int(s["endTimeUnixNano"]) - int(s["startTimeUnixNano"])) / 1e6, + "status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", "") or "UNSET", + "attrs": {a["key"]: _attr(a["value"]) for a in s.get("attributes", [])}, + }) + return sorted(spans, key=lambda item: item["start"]) + + +def tempo_raw(trace_id): + with urllib.request.urlopen(TEMPO + "/api/traces/" + trace_id, timeout=15) as response: + return response.read().decode("utf-8", "replace") + + +def print_tree(spans, out=print, skip_db=False): + children = {} + ids = {s["span_id"] for s in spans} + for s in spans: + children.setdefault(s["parent_id"] if s["parent_id"] in ids else None, []).append(s) + + def walk(parent, depth): + for s in children.get(parent, []): + if skip_db and s["kind"] == "CLIENT" and s["name"].split()[0] in ("SELECT", "UPDATE", "INSERT", "DELETE"): + continue + extra = {k: v for k, v in s["attrs"].items() if k.startswith(("messenger.", "messaging.message", "db.system", + "http.response.status_code", "rpc.method"))} + out("%s%-8s %-9s %-44s %8.1f ms %-5s %s" % (" " * depth, s["service"].replace("messenger-", ""), s["kind"], + s["name"][:44], s["duration_ms"], s["status"], + json.dumps(extra, ensure_ascii=False))) + walk(s["span_id"], depth + 1) + + walk(None, 0) + + +def wait_for(predicate, seconds=60, interval=2): + deadline = time.monotonic() + seconds + value = None + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(interval) + return value diff --git a/student-tests/openloop.py b/student-tests/openloop.py new file mode 100644 index 0000000..978e4ea --- /dev/null +++ b/student-tests/openloop.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Open-loop load generator (stdlib only): fixed arrival rate, latency measured from the SCHEDULED send time, +so a slow server cannot slow down the generator (no coordinated omission while threads are available). + +Mix per arrival: 75% POST message, 25% GET history (limit=20); plus an independent upload stream (sample.png). +Rare injected requests (FAULT_INJECTION=1 on the stand): `X-Fault: slow` (+600 ms) and `X-Fault: error` (500) +are added to GET history with the given percentages, to measure whether such traces survive sampling. + + python3 student-tests/openloop.py --rate 40 --duration 60 --upload-rate 1 --slow-pct 1 --error-pct 1 \ + --output evidence/x.json +Tokens stay in memory; the summary contains counts, percentiles and timestamps only. +""" +import argparse +import http.client +import json +import queue +import random +import sys +import threading +import time +import uuid +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def percentile(values, p): + if not values: + return None + values = sorted(values) + k = max(0, min(len(values) - 1, int(round(p / 100.0 * len(values) + 0.5)) - 1)) + return values[k] + + +class Load: + def __init__(self, base_url, users=4, threads=64, timeout=30.0): + self.base = urlsplit(base_url) + self.threads = threads + self.timeout = timeout + c = Client(base_url) + self.actors = [] + for i in range(users): + a, b = c.user("load%da" % i), c.user("load%db" % i) + chat = c.call("POST", "/api/v1/chats", 201, actor=a, + body={"title": "load %d" % i, "member_ids": [a["id"], b["id"]]}, schema="Chat")["json"] + self.actors.append((a["token"], chat["id"])) + self.lock = threading.Lock() + self.results = [] + self.local = threading.local() + + def conn(self): + c = getattr(self.local, "conn", None) + if c is None: + c = http.client.HTTPConnection(self.base.hostname, self.base.port or 80, timeout=self.timeout) + self.local.conn = c + return c + + def request(self, method, path, body=None, headers=None): + for attempt in (1, 2): # second attempt only for a dropped keep-alive connection BEFORE sending + conn = self.conn() + try: + conn.request(method, path, body=body, headers=headers or {}) + response = conn.getresponse() + response.read() + return response.status, response.getheader("X-Request-Id") + except (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError) as exc: + conn.close() + self.local.conn = None + if attempt == 2: + return "conn_error:" + type(exc).__name__, None + except (OSError, http.client.HTTPException) as exc: + conn.close() + self.local.conn = None + return "error:" + type(exc).__name__, None + return "error", None + + def one(self, kind, fault, scheduled): + token, chat = random.choice(self.actors) + headers = {"Authorization": "Bearer " + token} + if fault: + headers["X-Fault"] = fault + if kind == "post_message": + headers.update({"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}) + status, rid = self.request("POST", "/api/v1/chats/%s/messages" % chat, + json.dumps({"text": "нагрузка " + uuid.uuid4().hex[:8]}).encode(), headers) + elif kind == "get_history": + status, rid = self.request("GET", "/api/v1/chats/%s/messages?limit=20" % chat, None, headers) + else: + headers["Content-Type"] = "image/png" + status, rid = self.request("POST", "/api/v1/chats/%s/attachments?filename=load.png" % chat, SAMPLE, headers) + done = time.monotonic() + with self.lock: + self.results.append({"kind": kind, "fault": fault, "status": status, "latency": done - scheduled, + "at": time.time(), "rid": rid}) + + def run(self, rate, duration, upload_rate=0.0, slow_pct=0.0, error_pct=0.0, stop_event=None): + work = queue.Queue() + start = time.monotonic() + self.started_at = time.time() + + def worker(): + while True: + item = work.get() + if item is None: + return + self.one(*item) + + pool = [threading.Thread(target=worker, daemon=True) for _ in range(self.threads)] + for t in pool: + t.start() + + def schedule(r, picker): + if r <= 0: + return + n = 0 + while True: + due = start + n / r + if due - start >= duration or (stop_event and stop_event.is_set()): + return + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + work.put(picker() + (due,)) + n += 1 + + def pick_main(): + if random.random() < 0.75: + return ("post_message", None) + roll = random.random() * 100 + fault = "slow" if roll < slow_pct * 4 else "error" if roll < (slow_pct + error_pct) * 4 else None + return ("get_history", fault) # 25% share * 4 = requested share of ALL requests + + schedulers = [threading.Thread(target=schedule, args=(rate, pick_main), daemon=True), + threading.Thread(target=schedule, args=(upload_rate, lambda: ("upload", None)), daemon=True)] + for t in schedulers: + t.start() + for t in schedulers: + t.join() + backlog = work.qsize() + for _ in pool: + work.put(None) + for t in pool: + t.join(self.timeout + 5) + self.finished_at = time.time() + return self.summary(rate, duration, upload_rate, slow_pct, error_pct, backlog) + + def summary(self, rate, duration, upload_rate, slow_pct, error_pct, backlog): + res = self.results + ok = [r for r in res if isinstance(r["status"], int) and r["status"] < 400] + plain_ok = [r["latency"] for r in ok if r["fault"] is None and r["kind"] != "upload"] + out = { + "profile": {"rate_rps": rate, "duration_s": duration, "upload_rate_rps": upload_rate, + "slow_pct": slow_pct, "error_pct": error_pct, "threads": self.threads, + "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, + "started_at": self.started_at, "finished_at": self.finished_at, "generator_backlog_at_end": backlog, + "requests": len(res), "ok": len(ok), + "successful_rps": round(len(ok) / max(duration, 1e-9), 2), + "status_counts": {}, + "server_errors_unexpected": sum(1 for r in res if r["fault"] != "error" and ( + not isinstance(r["status"], int) or r["status"] >= 500)), + "latency_ms_plain_ok": {p: round(percentile(plain_ok, q) * 1000, 2) if plain_ok else None + for p, q in (("p50", 50), ("p95", 95), ("p99", 99), ("max", 100))}, + "faults_sent": {"slow": sum(1 for r in res if r["fault"] == "slow"), + "error": sum(1 for r in res if r["fault"] == "error")}, + "by_kind": {}, + } + for r in res: + key = str(r["status"]) + out["status_counts"][key] = out["status_counts"].get(key, 0) + 1 + for kind in ("post_message", "get_history", "upload"): + lat = [r["latency"] for r in ok if r["kind"] == kind and r["fault"] is None] + out["by_kind"][kind] = {"n": sum(1 for r in res if r["kind"] == kind), + "p50_ms": round(percentile(lat, 50) * 1000, 2) if lat else None, + "p99_ms": round(percentile(lat, 99) * 1000, 2) if lat else None} + return out + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--rate", type=float, default=40) + p.add_argument("--duration", type=float, default=60) + p.add_argument("--upload-rate", type=float, default=0) + p.add_argument("--slow-pct", type=float, default=0) + p.add_argument("--error-pct", type=float, default=0) + p.add_argument("--threads", type=int, default=64) + p.add_argument("--output") + a = p.parse_args() + load = Load(a.base_url, threads=a.threads) + summary = load.run(a.rate, a.duration, a.upload_rate, a.slow_pct, a.error_pct) + text = json.dumps(summary, ensure_ascii=False, indent=2) + print(text) + if a.output: + Path(a.output).write_text(text + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/queue_load.py b/student-tests/queue_load.py new file mode 100644 index 0000000..65f03b3 --- /dev/null +++ b/student-tests/queue_load.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB. + +Each run: + 1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH + 2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses) + 3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second + 4. waits until every accepted upload is ready/failed and computes from attachment_jobs: + queue lag = first_started_at - created_at (time a job waited for a worker) + time to ready = finished_at - attachment.created_at + throughput = finished jobs / (last finish - first arrival) + Rejections (503 queue_full from admission control) are counted, not retried. + + python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2 + python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30 +Results are appended as JSON lines to evidence/track1/results.jsonl. +""" +import argparse +import json +import os +import statistics +import subprocess +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready + +OUT = ROOT / "evidence/track1" +SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")} + +GEN = r''' +import io, random, sys +from PIL import Image, ImageChops, ImageDraw +w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3] +if fmt == "JPEG": # photo-like noise: realistic JPEG sizes + chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)] + grain = Image.effect_noise((w, h), 12) + img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan]) +else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling) + img = Image.linear_gradient("L").resize((w, h)).convert("RGB") + draw, rnd = ImageDraw.Draw(img), random.Random(1) + for _ in range(400): + x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300) + draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256))) +buf = io.BytesIO() +img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6})) +sys.stdout.buffer.write(buf.getvalue()) +''' + + +def image(size): + w, h, fmt = SIZES[size] + cache = OUT / ("fixture-%s.%s" % (size, fmt.lower())) + if not cache.exists(): + OUT.mkdir(parents=True, exist_ok=True) + data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt], + cwd=ROOT, capture_output=True, check=True).stdout + cache.write_bytes(data) + return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg" + + +def set_worker(replicas, concurrency, prefetch, cpus="2.00"): + env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency), + WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus) + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"], + cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) + + +def set_admission(max_pending): + env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending)) + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"], + cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) + + +def parse_mem(text): + value = text.split("/")[0].strip() + for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)): + if value.endswith(unit): + return float(value[: -len(unit)]) * mult + return 0.0 + + +class Sampler(threading.Thread): + def __init__(self, tag): + super().__init__(daemon=True) + self.tag = tag + self.stop = threading.Event() + self.rss, self.pending, self.cpu = [], [], [] + + def run(self): + while not self.stop.is_set(): + ids = compose("ps", "-q", "worker", check_rc=False).split() + if ids: + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids], + capture_output=True, text=True).stdout + rows = [line.split("|") for line in out.splitlines() if "|" in line] + self.rss.append(sum(parse_mem(r[0]) for r in rows)) + self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows)) + try: + self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" + % self.tag)[0][0])) + except Exception: + pass + self.stop.wait(1.0) + + +def pct(values, q): + if not values: + return None + values = sorted(values) + return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3) + + +def drive(c, actor, chat, body, ctype, rate, duration, tag): + """Open-loop: request i is sent at t0 + i/rate regardless of earlier responses.""" + total = int(rate * duration) + results = {"accepted": 0, "rejected_503": 0, "other_errors": 0} + lock = threading.Lock() + + def one(i): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i))) + try: + r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype}) + key = "accepted" if r["status"] == 202 else "rejected_503" + except Exception: # connection errors, timeouts: counted, never retried + key = "other_errors" + with lock: + results[key] += 1 + + t0 = time.monotonic() + with ThreadPoolExecutor(max_workers=64) as pool: + for i in range(total): + delay = t0 + i / rate - time.monotonic() + if delay > 0: + time.sleep(delay) + pool.submit(one, i) + results["send_seconds"] = round(time.monotonic() - t0, 2) + results["offered"] = total + return results + + +def wait_drained(tag, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0]) + if left == 0: + return True + time.sleep(1) + return False + + +def job_metrics(tag): + rows = sql( + "SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at)," + " j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%'" % tag) + done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"] + lag = [s - a for a, s, f in done] + ttr = [f - a for a, s, f in done] + span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0 + return { + "succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"), + "retries": sum(int(r[4]) - 1 for r in rows), + "throughput_jobs_s": round(len(done) / span, 3) if span else None, + "lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0), + "ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0), + } + + +def record(summary): + OUT.mkdir(parents=True, exist_ok=True) + with open(OUT / "results.jsonl", "a") as handle: + handle.write(json.dumps(summary, ensure_ascii=False) + "\n") + log(json.dumps(summary, ensure_ascii=False)) + + +def experiment(args, label, max_pending=None): + c = Client(timeout=60) + wait_http_ready(c) + alice = c.user("load") + chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]}, + schema="Chat")["json"] + body, ctype = image(args.size) + set_worker(args.replicas, args.concurrency, args.prefetch) + time.sleep(2) + tag = uuid.uuid4().hex[:8] + sampler = Sampler(tag) + sampler.start() + log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % ( + label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch)) + sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag) + load_end = time.monotonic() + drained = wait_drained(tag, args.drain_timeout) + drain_seconds = round(time.monotonic() - load_end, 2) + sampler.stop.set() + sampler.join() + summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate, + "duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency, + "prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained, + "drain_after_load_s": drain_seconds, **job_metrics(tag), + "pending_max": max(sampler.pending or [0]), + "worker_mem_peak_mib": round(max(sampler.rss or [0]), 1), + "worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None, + "worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1), + "at": time.strftime("%Y-%m-%dT%H:%M:%S")} + return summary, c, alice, chat + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=["run", "overload"]) + parser.add_argument("--size", choices=sorted(SIZES), default="medium") + parser.add_argument("--rate", type=float, default=4) + parser.add_argument("--duration", type=float, default=30) + parser.add_argument("--replicas", type=int, default=1) + parser.add_argument("--concurrency", type=int, default=2) + parser.add_argument("--prefetch", type=int, default=2) + parser.add_argument("--max-pending", type=int, default=30) + parser.add_argument("--drain-timeout", type=float, default=600) + parser.add_argument("--label", default="") + args = parser.parse_args() + if args.mode == "run": + summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch)) + record(summary) + return + # Overload: admission limit on, offered rate far above capacity, then recovery. + set_admission(args.max_pending) + try: + summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending) + # Recovery probe: after the backlog drained, a fresh upload must be ready quickly again. + body, ctype = image("small") + t = time.monotonic() + att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body, + headers={"Content-Type": ctype}, schema="Attachment")["json"] + while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready": + time.sleep(0.2) + summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2) + record(summary) + finally: + set_admission(200) + + +if __name__ == "__main__": + main() diff --git a/student-tests/stand.py b/student-tests/stand.py new file mode 100755 index 0000000..8ac5e7e --- /dev/null +++ b/student-tests/stand.py @@ -0,0 +1,132 @@ +"""Helpers for student resilience tests: the public HTTP client plus docker compose / psql / mc access. + +Standard library only (Python 3.9+). Images that need Pillow are generated inside the API container. +Never prints tokens or secrets. +""" +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402,F401 + +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def log(*parts): + print(time.strftime("%H:%M:%S"), *parts, flush=True) + + +def compose(*args, check_rc=True, capture=True, timeout=180): + done = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=capture, text=True, timeout=timeout) + if check_rc and done.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args[:2]), (done.stderr or "")[-300:])) + return done.stdout if capture else "" + + +def started_at(*services): + """Container name -> State.StartedAt, proof that a restart really happened.""" + ids = compose("ps", "-q", *services).split() + out = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids], + capture_output=True, text=True).stdout + return "; ".join(line.strip("/")[:60] for line in out.splitlines()) + + +def sql(query): + """Run a read query in postgres; returns rows as lists of strings (tab separated).""" + out = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t", "-c", query) + return [line.split("\t") for line in out.splitlines() if line] + + +def s3_keys(prefix): + """Object keys starting with prefix, listed with the root credentials inside a throwaway minio-init container.""" + wanted, prefix = prefix, prefix[:prefix.rfind("/") + 1] + script = ('mc alias set l http://minio:9000 "$(cat /run/secrets/minio_root_user)" ' + '"$(cat /run/secrets/minio_root_password)" >/dev/null && ' + 'mc ls --recursive --json l/messenger-attachments/%s || true' % prefix) + out = compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "minio-init", "-c", script) + keys = [] + for line in out.splitlines(): + try: + item = json.loads(line) + except ValueError: + continue + if item.get("key"): + keys.append(prefix + item["key"]) + return sorted(k for k in keys if k.startswith(wanted)) + + +def queue_depth(queue="attachments.process"): + out = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "--vhost", "messenger", "list_queues", + "name", "messages", "messages_ready", "messages_unacknowledged") + for line in out.splitlines(): + parts = line.split() + if parts and parts[0] == queue: + return {"messages": int(parts[1]), "ready": int(parts[2]), "unacked": int(parts[3])} + return None + + +def wait_queue(min_ready=1, seconds=30): + """Quorum queue counters are refreshed by the stats emitter (~5 s), so poll.""" + deadline = time.monotonic() + seconds + depth = None + while time.monotonic() < deadline: + depth = queue_depth() + if depth and depth["ready"] >= min_ready: + return depth + time.sleep(1) + return depth + + +def jobs_cli(*args): + done = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "scripts/jobs.py", *args], + cwd=ROOT, capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) if done.stdout.strip() else None + + +def wait_http_ready(c, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + return + except Failure: + time.sleep(1) + raise Failure("API not ready after %ds" % seconds) + + +def upload(c, actor, chat, body=SAMPLE, filename="sample.png", ctype="image/png", expected=202): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")) + schema = "Attachment" if expected == 202 else None + return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype}, schema=schema)["json"] + + +def get(c, actor, att_id): + return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"] + + +def wait_status(c, actor, att_id, statuses, seconds=120): + deadline = time.monotonic() + seconds + seen = [] + while time.monotonic() < deadline: + att = get(c, actor, att_id) + if not seen or seen[-1] != att["status"]: + seen.append(att["status"]) + if att["status"] in statuses: + return att, seen + time.sleep(0.5) + raise Failure("attachment %s did not reach %s in %ds (seen %s)" % (att_id, statuses, seconds, seen)) + + +def objects_of(att_id): + return s3_keys("attachments/%s/" % att_id) + s3_keys("staging/%s" % att_id) + + +def unique_name(fault=None): + tag = uuid.uuid4().hex[:8] + return ("fault-%s-%s.png" % (fault, tag)) if fault else ("plain-%s.png" % tag) diff --git a/student-tests/test_exporter_queue_full.py b/student-tests/test_exporter_queue_full.py new file mode 100644 index 0000000..75d59fc --- /dev/null +++ b/student-tests/test_exporter_queue_full.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Bounded buffering at the limit: Tempo stopped under 100 req/s until the exporter queue is FULL. +Expected: queue_size stops at capacity, extra spans are dropped and counted, collector RSS stays under the container +limit, API has no unexpected errors. python3 student-tests/test_exporter_queue_full.py (~5 min) +""" +import threading +import time + +import obs +from openloop import Load +from stand import check, compose, log, run +from test_incidents import docker_mem + + +def value(expr): + return obs.prom_scalar("sum(%s)" % expr) + + +def main(): + load = Load("http://localhost:8080", users=4, threads=96) + stop, summary = threading.Event(), {} + thread = threading.Thread(target=lambda: summary.update(load.run(100, 900, 1, stop_event=stop)), daemon=True) + thread.start() + time.sleep(20) + q = 'otelcol_exporter_queue_size{exporter="otlp/tempo"}' + capacity = value('otelcol_exporter_queue_capacity{exporter="otlp/tempo"}') + dropped0, refused0 = value("otelcol_exporter_enqueue_failed_spans_total"), value("otelcol_receiver_refused_spans_total") + compose("stop", "-t", "5", "tempo") + t0 = time.time() + log("tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=%.0f batches" % capacity) + peak_q, mems = 0, [] + try: + for _ in range(16): + time.sleep(12) + size, mem = value(q), docker_mem("otel-collector")[0] + peak_q, _ = max(peak_q, size), mems.append(mem) + log(" +%3d s queue=%.0f/%.0f collector_rss=%s dropped=%.0f refused=%.0f" % ( + time.time() - t0, size, capacity, mem, value("otelcol_exporter_enqueue_failed_spans_total") - dropped0, + value("otelcol_receiver_refused_spans_total") - refused0)) + dropped = value("otelcol_exporter_enqueue_failed_spans_total") - dropped0 + refused = value("otelcol_receiver_refused_spans_total") - refused0 + finally: + compose("start", "tempo") + stop.set() + thread.join(60) + log("result: peak queue=%.0f/%.0f, dropped by full queue=%.0f spans, refused by memory_limiter=%.0f spans, peak RSS=%s MiB" + % (peak_q, capacity, dropped, refused, max(float(m.replace("MiB", "")) for m in mems))) + log("API: requests=%s ok=%s unexpected_errors=%s p99=%s ms" % ( + summary["requests"], summary["ok"], summary["server_errors_unexpected"], summary["latency_ms_plain_ok"]["p99"])) + check(peak_q <= capacity, "queue must stay within capacity") + check(peak_q >= capacity * 0.95 or refused > 0, "the limit must really be reached") + check(dropped + refused > 0, "overflow must be counted") + check(summary["server_errors_unexpected"] == 0, "API must be unaffected") + print("PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_failure_windows.py b/student-tests/test_failure_windows.py new file mode 100755 index 0000000..cda0664 --- /dev/null +++ b/student-tests/test_failure_windows.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Grade 4 fault-injection suite (needs FAULT_INJECTION=1 on the stand, see REPORT.md). + +Each scenario uploads a file whose display name selects an injected fault (app/faults.py) and checks the +recovery through the public API plus DB/S3/broker state: + A api_crash_after_commit crash between COMMIT(job+outbox) and publish -> another replica's relay publishes + B worker_crash_after_put crash after S3 writes, before result COMMIT -> lease expiry -> same keys, one result + C worker_crash_after_commit crash after COMMIT, before ack -> redelivery is a no-op + D transient3 3 transient errors -> retries with growing jittered backoff -> ready + E permanent permanent error -> failed at attempt 1, no retry, content/attach 409 + F transient6 attempts exhausted -> failed/retry_exhausted -> registry -> explicit replay -> ready; + replay of a ready job is refused + G worker_hang hung worker (no heartbeats) -> other worker takes over, hung one is fenced + H poison message invalid broker message -> DLQ, not retried forever + I broker down RabbitMQ stopped: upload still 202 (job waits in outbox) -> broker back -> ready +usage: python3 student-tests/test_failure_windows.py [A B C ...] +""" +import sys +import time + +from stand import (Client, Failure, check, compose, get, jobs_cli, log, objects_of, queue_depth, run, sql, upload, + unique_name, wait_http_ready, wait_status) + + +def audit(att_id): + return sql("SELECT action, actor, attempt, coalesce(detail->>'error_code',''), coalesce(detail->>'delay_seconds','')" + " FROM job_audit WHERE attachment_id='%s' ORDER BY id" % att_id) + + +def show_audit(att_id): + for action, actor, attempt, code, delay in audit(att_id): + log(" audit attempt=%s %-16s %-28s %s %s" % (attempt, action, actor[:28], code, ("delay=%ss" % delay) if delay else "")) + + +def one_result(att_id): + variants = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att_id)[0][0] + succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att_id)[0][0] + keys = objects_of(att_id) + check(variants == "3", "exactly one set of 3 variant rows, got %s" % variants) + check(succeeded == "1", "exactly one committed success, got %s" % succeeded) + check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), + "original + 3 variants, staging cleaned: %s" % keys) + + +def ready_and_single(c, actor, att_id, seconds=120): + att, seen = wait_status(c, actor, att_id, ("ready", "failed"), seconds) + check(att["status"] == "ready", "expected ready, got %s/%s" % (att["status"], att["error_code"])) + log(" statuses:", " -> ".join(seen)) + show_audit(att_id) + one_result(att_id) + return att + + +def scenario_a(c, alice, bob, chat): + name = unique_name("api_crash_after_commit") + try: + upload(c, alice, chat, filename=name, expected=[202, 500, 502, 503]) + except Failure as exc: + log(" client saw:", exc) + wait_http_ready(c) + rows = sql("SELECT id FROM attachments WHERE filename='%s'" % name) + check(len(rows) == 1, "committed attachment must exist after API crash") + att_id = rows[0][0] + log(" committed attachment %s survived the API crash" % att_id) + ready_and_single(c, bob, att_id) + pub = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att_id)[0][0] + check(pub == "1", "outbox row published exactly once by the relay") + + +def scenario_b(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_crash_after_put")) + ready_and_single(c, bob, att["id"]) + actions = [r[0] for r in audit(att["id"])] + check("retry_scheduled" in actions, "lease expiry must schedule a new attempt") + + +def scenario_c(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_crash_after_commit")) + ready_and_single(c, bob, att["id"]) + time.sleep(8) + logs = compose("logs", "--since", "60s", "worker") + check("decision=done" in logs, "redelivered message after crash must be a no-op") + one_result(att["id"]) + + +def scenario_d(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("transient3")) + ready_and_single(c, bob, att["id"]) + delays = [float(r[4]) for r in audit(att["id"]) if r[0] == "retry_scheduled"] + check(len(delays) == 3, "three retries expected, got %s" % delays) + bounds = [(1, 2), (2, 4), (4, 8)] + check(all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "equal-jitter backoff bounds %s" % delays) + log(" backoff delays:", delays) + + +def scenario_e(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("permanent")) + failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", "permanent -> failed/invalid_image") + check(failed["variants"] == [], "failed attachment has no variants") + show_audit(att["id"]) + check([r[0] for r in audit(att["id"])].count("claimed") == 1, "permanent errors are not retried") + c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "k-" + att["id"]}) + + +def scenario_f(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("transient6")) + failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "exhausted -> retry_exhausted") + attempts = sql("SELECT attempt, max_attempts FROM attachment_jobs WHERE attachment_id='%s'" % att["id"])[0] + check(attempts == ["5", "5"], "bounded: stopped at attempt 5 of 5, got %s" % attempts) + rc, registry = jobs_cli("failed") + check(any(r["attachment_id"] == att["id"] for r in registry), "failed job listed in operator registry") + log(" failed registry contains the job; replay") + rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite") + check(rc == 0 and result["result"] == "queued", "replay must queue: %s" % result) + ready_and_single(c, bob, att["id"], 120) + rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite") + check(rc == 3 and result["result"] == "refused", "replay of a ready job must be refused: %s" % result) + log(" second replay refused:", result["reason"]) + one_result(att["id"]) + + +def scenario_g(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_hang")) + ready_and_single(c, bob, att["id"], 150) + rows = audit(att["id"]) + check(any(r[0] == "retry_scheduled" and r[3] == "lease_expired" for r in rows), "lease expiry detected") + time.sleep(30) # the hung worker wakes up and must be fenced + logs = compose("logs", "--since", "180s", "worker") + check("lease_lost" in logs or "job_result_not_committed" in logs, "hung worker must be fenced by the lease") + one_result(att["id"]) + + +POISON = ("import asyncio, aio_pika\n" + "from app import config, broker\n" + "async def m():\n" + " conn = await broker.connect(); ch = await conn.channel(publisher_confirms=True)\n" + " ex, _ = await broker.declare(ch)\n" + " await ex.publish(aio_pika.Message(b'{not json', delivery_mode=2), routing_key=broker.ROUTING_KEY)\n" + " await conn.close()\n" + "asyncio.run(m())\n") + + +def scenario_h(c, alice, bob, chat): + before = (queue_depth("attachments.dlq") or {}).get("messages", 0) + compose("exec", "-T", "api-1", "python", "-c", POISON) + deadline = time.monotonic() + 30 + after = before + while time.monotonic() < deadline and after <= before: + time.sleep(1) + after = (queue_depth("attachments.dlq") or {}).get("messages", 0) + check(after == before + 1, "poison message must land in attachments.dlq (%s -> %s)" % (before, after)) + log(" DLQ depth %s -> %s" % (before, after)) + + +def scenario_i(c, alice, bob, chat): + compose("stop", "-t", "10", "rabbitmq") + try: + att = upload(c, alice, chat, filename=unique_name()) + check(att["status"] == "queued", "upload accepted while broker is down") + c.message(alice, chat, "", attachments=[att["id"]]) + time.sleep(3) + pending = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0] + check(pending == "1", "job must wait in the outbox, got %s" % pending) + check(get(c, bob, att["id"])["status"] == "queued", "still queued while broker is down") + log(" broker down: upload 202, outbox holds 1 unpublished job") + finally: + compose("start", "rabbitmq") + ready_and_single(c, bob, att["id"], 120) + + +SCENARIOS = {"A": scenario_a, "B": scenario_b, "C": scenario_c, "D": scenario_d, "E": scenario_e, + "F": scenario_f, "G": scenario_g, "H": scenario_h, "I": scenario_i} + + +def main(): + c = Client(timeout=30) + alice, bob, eve, chat = c.bootstrap() + picked = sys.argv[1:] or list(SCENARIOS) + for key in picked: + log("scenario %s: %s" % (key, SCENARIOS[key].__name__)) + started = time.monotonic() + SCENARIOS[key](c, alice, bob, chat) + log("scenario %s PASS (%.1fs)" % (key, time.monotonic() - started)) + print("PASS: failure windows %s" % " ".join(picked)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_incidents.py b/student-tests/test_incidents.py new file mode 100644 index 0000000..6908652 --- /dev/null +++ b/student-tests/test_incidents.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Level 4 controlled incidents, diagnosed ONLY from stored telemetry (Prometheus / Loki / Tempo), with alert +firing -> resolved timings. A background open-loop traffic (messages + history, 2% slow handler requests) runs all +the time so that error ratios have a denominator. + + A. worker stopped -> MessengerQueueLagGrowing firing; start worker -> backlog drains -> resolved + B. DB table lock (slow DB) -> MessengerApiHighErrorRate firing; trace shows the INSERT span waiting 5 s on the lock + (vs. an X-Fault: slow request: long handler, short DB spans); unlock -> resolved + C. Tempo + Loki stopped -> API keeps working, collector queue bounded, drops counted, TelemetryExportFailing; + collector stopped -> API keeps working, SDK queues bounded (RSS flat) + +python3 student-tests/test_incidents.py [--only A,B,C] (stack running, FAULT_INJECTION=1; ~15 min) +""" +import argparse +import json +import subprocess +import threading +import time +from pathlib import Path + +import obs +from openloop import Load +from stand import Client, check, compose, log, run, upload, wait_status + +ROOT = Path(__file__).resolve().parents[1] +RESULTS = {} + + +def docker_mem(service): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}"], + capture_output=True, text=True).stdout + for line in out.splitlines(): + name, mem, cpu = line.split("\t") + if name.startswith("mtusi-lab6-%s-" % service): + return mem.split(" / ")[0], cpu + return "stopped", "-" + + +def wait_alert(name, want, seconds, t0): + """want='firing' waits for firing; want='resolved' waits until the alert is gone. Returns seconds since t0.""" + deadline = time.monotonic() + seconds + pending_at = None + while time.monotonic() < deadline: + state = obs.alerts().get(name) + if state == "pending" and pending_at is None: + pending_at = time.time() - t0 + log(" %s pending (+%.0f s)" % (name, pending_at)) + if (want == "firing" and state == "firing") or (want == "resolved" and state is None): + elapsed = time.time() - t0 + log(" %s %s (+%.0f s)" % (name, want, elapsed)) + return elapsed, pending_at + time.sleep(5) + raise AssertionError("%s did not become %s within %d s" % (name, want, seconds)) + + +def psql(query): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query) + + +# ------------------------------------------------------------------------------------------------ A +def incident_worker(c, alice, chat): + log("=== A. worker stopped (queue lag)") + t0 = time.time() + compose("stop", "-t", "20", "worker") + log("worker stopped at %s" % time.strftime("%H:%M:%S")) + atts = [upload(c, alice, chat, filename="incident-a.png") for _ in range(3)] + log("3 uploads accepted (202) while worker is down: %s" % [a["id"][:8] for a in atts]) + firing, pending = wait_alert("MessengerQueueLagGrowing", "firing", 300, t0) + + log("diagnosis from telemetry:") + age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})') + depth = {m["state"]: v for m, v in obs.prom('max by (state) (messenger_queue_depth{queue="attachments.process"})')} + processed = obs.prom_scalar('sum(rate(messenger_jobs_processed_total[1m]))') + log(" metrics: oldest job age=%.0f s, depth=%s, worker jobs/s over 1m=%.3f" % (age, depth, processed)) + since = int((time.time() - t0) / 60) + 1 + worker_lines = obs.loki('{service_name="messenger-worker"} | event=~"job_.*"', minutes=since) + worker_lines = [l for l in worker_lines if l["ts"] / 1e9 > t0 + 25] + stop_lines = obs.loki('{service_name="messenger-worker"} | event=~"worker_stop.*"', minutes=since + 1) + log(" logs: worker job_* events after stop: %d; last worker events: %s" % ( + len(worker_lines), [l.get("event") for l in stop_lines[-2:]])) + traces = obs.tempo_search('{span.messenger.attachment_id="%s"}' % atts[0]["id"], minutes=since + 1) + spans = obs.tempo_trace(traces[0]["traceID"]) if traces else [] + kinds = sorted({s["kind"] for s in spans if s["kind"] != "CLIENT"}) + log(" trace of upload %s: span kinds %s -> published, no CONSUMER: the job waits in the queue" % ( + atts[0]["id"][:8], kinds)) + check("PRODUCER" in kinds and "CONSUMER" not in kinds, "trace must stop at the producer while worker is down") + + t1 = time.time() + compose("start", "worker") + log("worker started (+%.0f s)" % (t1 - t0)) + for att in atts: + wait_status(c, alice, att["id"], ("ready",), 120) + drained = time.time() - t1 + log("backlog drained: all 3 ready %.1f s after worker start" % drained) + resolved, _ = wait_alert("MessengerQueueLagGrowing", "resolved", 300, t1) + RESULTS["A_worker_stopped"] = {"alert_pending_s": pending, "alert_firing_s": round(firing), "age_at_firing_s": + round(age), "drain_after_restart_s": round(drained, 1), + "resolved_after_restart_s": round(resolved), "lost_jobs": 0} + + +# ------------------------------------------------------------------------------------------------ B +def incident_db(c): + log("=== B. DB lock on messages (slow/unavailable DB for writes and history reads)") + t0 = time.time() + locker = subprocess.Popen( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", + "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;"], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + log("ACCESS EXCLUSIVE lock taken on table messages at %s" % time.strftime("%H:%M:%S")) + try: + firing, pending = wait_alert("MessengerApiHighErrorRate", "firing", 300, t0) + log("diagnosis from telemetry:") + ratio = obs.prom_scalar("messenger:http_error_ratio:rate2m") + p95 = obs.prom_scalar("messenger:http_latency_p95:rate5m") + by_route = obs.prom('sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count' + '{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))') + waiting = obs.prom_scalar("max(db_client_connection_pending_requests)") + used = obs.prom_scalar('max(db_client_connection_count{service_name="messenger-api",db_client_connection_state="used"})') + log(" metrics: 5xx ratio=%.1f%%, p95=%.2f s, pool used(max replica)=%.0f waiting=%.0f" % ( + ratio * 100, p95, used, waiting)) + for m, v in by_route: + log(" 5xx/s %-40s %s %.2f" % (m.get("http_route"), m.get("http_response_status_code"), v)) + db_err = obs.tempo_search('{resource.service.name="messenger-api" && span.db.operation.name="INSERT"' + ' && status=error}', minutes=5, limit=5) + check(db_err, "no trace with a failed DB span") + spans = obs.tempo_trace(db_err[0]["traceID"]) + log(" trace %s (failed request):" % db_err[0]["traceID"]) + obs.print_tree(spans, out=lambda line: print(" " + line)) + raw = obs.tempo_raw(db_err[0]["traceID"]) + cause = next((name for name in ("LockNotAvailable", "QueryCanceled", "PoolTimeout") if name in raw), "?") + db_span = max((s for s in spans if s["attrs"].get("db.system") == "postgresql"), key=lambda s: s["duration_ms"]) + server = [s for s in spans if s["kind"] == "SERVER"][0] + log(" => DB span %s took %.0f ms of %.0f ms request, status %s, exception %s: the time is spent in PostgreSQL" + % (db_span["name"], db_span["duration_ms"], server["duration_ms"], db_span["status"], cause)) + slow = obs.tempo_search('{span.messenger.fault="slow" && span.http.route="/api/v1/chats/{chat_id}/messages"}', + minutes=10, limit=50) + contrast = None + for item in slow: + sp = obs.tempo_trace(item["traceID"]) + srv = [s for s in sp if s["kind"] == "SERVER"] + dbs = [s["duration_ms"] for s in sp if s["attrs"].get("db.system") == "postgresql"] + if srv and dbs and srv[0]["attrs"].get("http.response.status_code") in ("200", 200) and max(dbs) < 100: + contrast = (item["traceID"], srv[0]["duration_ms"], max(dbs), sum(dbs)) + break + if contrast: + log(" contrast: slow HANDLER trace %s: request %.0f ms, longest DB span %.1f ms, all DB %.1f ms" + % contrast) + errors = obs.loki('{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"', minutes=5) + log(" logs: %d API warning/error records in 5 min, events=%s" % ( + len(errors), sorted({l.get("event") for l in errors}))) + blocker = psql("SELECT pid, state, wait_event_type, left(query, 60) FROM pg_stat_activity " + "WHERE query LIKE '%LOCK TABLE messages%' AND pid <> pg_backend_pid()").strip() + log(" operator step (pg_stat_activity) finds the blocker: %s" % blocker) + finally: + t1 = time.time() + psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%LOCK TABLE messages%'" + " AND pid <> pg_backend_pid()") + locker.wait(30) + log("lock released (+%.0f s)" % (t1 - t0)) + resolved, _ = wait_alert("MessengerApiHighErrorRate", "resolved", 400, t1) + RESULTS["B_db_lock"] = {"alert_pending_s": pending, "alert_firing_s": round(firing), + "error_ratio_at_firing": round(ratio, 3), "p95_at_firing_s": round(p95, 2), + "db_span_ms": round(db_span["duration_ms"]), "request_ms": round(server["duration_ms"]), + "cause": cause, "resolved_after_unlock_s": round(resolved)} + + +# ------------------------------------------------------------------------------------------------ C +def counter(expr): + return obs.prom_scalar("sum(%s)" % expr) + + +def incident_telemetry(load): + log("=== C1. Tempo and Loki stopped (telemetry backends down)") + before_res = len(load.results) + dropped0 = counter('otelcol_exporter_enqueue_failed_spans_total') + counter( + 'otelcol_exporter_enqueue_failed_log_records_total') + failed0 = counter('otelcol_exporter_send_failed_spans_total') + counter( + 'otelcol_exporter_send_failed_log_records_total') + t0 = time.time() + compose("stop", "-t", "5", "tempo", "loki") + samples = [] + for i in range(14): # ~140 s + time.sleep(10) + mem, cpu = docker_mem("otel-collector") + q_tempo = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlp/tempo"})') + q_loki = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})') + samples.append({"t": round(time.time() - t0), "collector_mem": mem, "cpu": cpu, "queue_tempo": q_tempo, + "queue_loki": q_loki}) + log(" +%3d s collector mem=%s cpu=%s queue tempo=%.0f loki=%.0f (capacity 100 batches)" % ( + samples[-1]["t"], mem, cpu, q_tempo, q_loki)) + alert_state = obs.alerts().get("TelemetryExportFailing") + log(" alert TelemetryExportFailing during outage: %s" % alert_state) + check(alert_state == "firing", "TelemetryExportFailing must fire while backends are down") + window = [r for r in load.results[before_res:]] + bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)] + log(" API during the outage: %d requests, unexpected errors=%d; alert TelemetryExportFailing=%s" % ( + len(window), len(bad), alert_state)) + check(not bad, "API must keep working while telemetry backends are down") + compose("start", "tempo", "loki") + obs.wait_for(lambda: subprocess.run(["curl", "-sf", obs.TEMPO + "/ready"], capture_output=True).returncode == 0 + and subprocess.run(["curl", "-sf", obs.LOKI + "/ready"], capture_output=True).returncode == 0, 120, 3) + t1 = time.time() + log(" tempo/loki started and ready (+%.0f s)" % (t1 - t0)) + fresh = obs.wait_for(lambda: obs.tempo_search('{resource.service.name="messenger-api"}', start=t1, + end=time.time() + 5, limit=20), 90, 5) + log(" new traces after restart visible: %d (export resumed)" % len(fresh or [])) + check(fresh, "traces must flow again after the backends are back") + resolved, _ = wait_alert("TelemetryExportFailing", "resolved", 300, t1) + time.sleep(20) + dropped = counter('otelcol_exporter_enqueue_failed_spans_total') + counter( + 'otelcol_exporter_enqueue_failed_log_records_total') - dropped0 + failed = counter('otelcol_exporter_send_failed_spans_total') + counter( + 'otelcol_exporter_send_failed_log_records_total') - failed0 + log(" telemetry lost and counted: enqueue_failed(queue full)=%.0f items, send_failed(retries exhausted)=%.0f items" + % (dropped, failed)) + RESULTS["C1_backends_down"] = {"outage_s": round(t1 - t0), "api_requests": len(window), "api_unexpected_errors": + len(bad), "collector_samples": samples, "alert": alert_state, + "dropped_items": dropped, "send_failed_items": failed, + "traces_after_restart": len(fresh or []), + "alert_resolved_after_restart_s": round(resolved)} + + log("=== C2. Collector stopped (SDK side buffering)") + before_res = len(load.results) + api_before = docker_mem("api-1")[0] + compose("stop", "-t", "5", "otel-collector") + mems = [] + for _ in range(6): + time.sleep(10) + mems.append(docker_mem("api-1")[0]) + window = load.results[before_res:] + bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)] + compose("start", "otel-collector") + log(" api-1 RSS before=%s during=%s; %d requests, unexpected errors=%d" % (api_before, mems, len(window), len(bad))) + check(not bad, "API must keep working while the collector is down") + RESULTS["C2_collector_down"] = {"outage_s": 60, "api1_mem_before": api_before, "api1_mem_during": mems, + "api_requests": len(window), "api_unexpected_errors": len(bad)} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--only", default="A,B,C") + p.add_argument("--output", default=str(ROOT / "evidence/incidents.json")) + args = p.parse_args() + c = Client() + alice, _bob, _eve, chat = c.bootstrap() + load = Load("http://localhost:8080", users=2, threads=48) + stop = threading.Event() + traffic = threading.Thread(target=lambda: RESULTS.setdefault("traffic", load.run( + 10, 3600, 0, slow_pct=2, error_pct=0, stop_event=stop)), daemon=True) + traffic.start() + log("background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow)") + time.sleep(30) + try: + if "A" in args.only: + incident_worker(c, alice, chat) + if "B" in args.only: + incident_db(c) + if "C" in args.only: + incident_telemetry(load) + finally: + stop.set() + traffic.join(60) + Path(args.output).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2, default=str) + "\n") + log("summary written to %s" % args.output) + print("PASS: incidents %s diagnosed from telemetry, alerts fired and resolved" % args.only) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_observability.py b/student-tests/test_observability.py new file mode 100644 index 0000000..3ff159a --- /dev/null +++ b/student-tests/test_observability.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Levels 3-4: one controlled upload and one failed request are found in traces, logs and metrics; privacy checks. + + 1. upload with fault-transient1 (attempt 1 fails transiently, attempt 2 succeeds) -> ONE trace: + API SERVER span -> PRODUCER -> CONSUMER (retry, ERROR) -> PRODUCER -> CONSUMER (succeeded) -> S3/DB spans + 2. the same trace_id in Loki for messenger-api and messenger-worker log records + 3. X-Request-Id of a 500 (X-Fault: error) -> Loki log line -> its trace_id -> Tempo server span with that request_id + 4. no password / token / message text / filename in logs or traces; no ids in metric labels + 5. all metric names used by the dashboard and alert rules exist and have samples + +python3 student-tests/test_observability.py (stack from REPORT.md must be running; FAULT_INJECTION=1) +""" +import re +import time +import uuid + +import obs +from stand import Client, check, log, run, unique_name, upload, wait_status + +UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url_full", "url_query", + "filename", "text", "http_target", "url_path"} +REQUIRED_METRICS = [ + "http_server_request_duration_seconds_bucket", "messenger_jobs_processed_total", + "messenger_job_duration_seconds_bucket", "messenger_attachment_time_to_ready_seconds_bucket", + "messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_worker_busy", + "messenger_worker_capacity", "db_client_connection_count", "db_client_connection_pending_requests", + "process_cpu_time_seconds_total", "process_memory_usage_bytes", "otelcol_receiver_accepted_spans_total", + "otelcol_exporter_sent_spans_total", "otelcol_exporter_queue_size", +] + + +def main(): + c = Client() + alice, bob, _eve, chat = c.bootstrap() + marker = "secret-text-" + uuid.uuid4().hex + sent = c.message(alice, chat, marker) + msg_rid = sent["headers"]["X-Request-Id"] + log("message sent request_id=%s (text marker kept private)" % msg_rid) + + # ---------------------------------------------------------------- 1. upload -> retry -> ready, one trace + filename = unique_name("transient1") + att = upload(c, alice, chat, filename=filename) + ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90) + check(ready["status"] == "ready", "upload must become ready after one retry") + log("upload %s: %s" % (att["id"], " -> ".join(["queued"] + seen))) + + def find(): + found = obs.tempo_search('{span.messenger.attachment_id="%s"}' % att["id"]) + if not found: + return None + spans = obs.tempo_trace(found[0]["traceID"]) + consumers = [s for s in spans if s["kind"] == "CONSUMER"] + producers = [s for s in spans if s["kind"] == "PRODUCER"] + # Spans of one trace arrive from 3 processes with different batch delays: wait until all are in. + return (found, spans) if len(consumers) >= 2 and len(producers) >= 2 and any( + s["attrs"].get("messenger.job.outcome") == "succeeded" for s in consumers) else None + + result = obs.wait_for(find, 60) + check(result, "trace of the upload with both attempts not found in Tempo") + found, spans = result + trace_ids = {t["traceID"] for t in found} + check(len(trace_ids) == 1, "upload and its jobs must be ONE trace, got %s" % trace_ids) + trace_id = found[0]["traceID"] + log("Tempo: trace %s, %d spans; tree (DB client spans omitted):" % (trace_id, len(spans))) + obs.print_tree(spans, out=lambda line: print(" " + line), skip_db=True) + + by_id = {s["span_id"]: s for s in spans} + server = [s for s in spans if s["kind"] == "SERVER" and s["service"] == "messenger-api"] + producers = [s for s in spans if s["kind"] == "PRODUCER"] + consumers = [s for s in spans if s["kind"] == "CONSUMER"] + check(len(server) == 1 and server[0]["name"] == "POST /api/v1/chats/{chat_id}/attachments", "API server span") + check(len(producers) >= 2 and len(consumers) >= 2, "two publish/consume pairs (attempt 1 + retry)") + for cons in consumers: + parent = by_id.get(cons["parent_id"]) + check(parent is not None and parent["kind"] == "PRODUCER", "consumer must be a child of a producer span") + check(by_id[producers[0]["parent_id"]]["kind"] == "SERVER", "first producer is a child of the upload request") + retry_parent = by_id[producers[1]["parent_id"]] + check(retry_parent["kind"] == "CONSUMER" and retry_parent["attrs"].get("messenger.job.outcome") == "retry", + "retry producer must be a child of the failed consumer") + check(consumers[0]["status"] == "ERROR" and consumers[-1]["attrs"].get("messenger.job.outcome") == "succeeded", + "attempt 1 ERROR/retry, last attempt succeeded") + worker_ops = {s["name"] for s in spans if s["service"] == "messenger-worker"} + for op in ("S3.GetObject", "S3.CopyObject", "S3.PutObject", "imageproc"): + check(op in worker_ops, "worker span %s missing" % op) + check(any(s["service"] == "messenger-worker" and s["attrs"].get("db.system") == "postgresql" for s in spans), + "worker DB spans missing") + log("PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links") + + # ---------------------------------------------------------------- 2. logs of both services with the trace_id + lines = obs.wait_for(lambda: [l for l in obs.loki('{service_name=~"messenger-api|messenger-worker"} | trace_id="%s"' + % trace_id) if True] or None, 30) + check(lines, "no Loki records with the trace_id") + services = {l["service_name"] for l in lines} + events = [(l["service_name"].replace("messenger-", ""), l.get("event")) for l in lines] + log("Loki: %d records with trace_id=%s: %s" % (len(lines), trace_id, events)) + check(services == {"messenger-api", "messenger-worker"}, "logs of both services must carry the trace_id") + check({"attachment_queued", "job_attempt_failed", "job_finished"} <= {e for _, e in events}, + "expected events attachment_queued, job_attempt_failed, job_finished") + sample = lines[0] + log("Loki record fields: %s" % sorted(k for k in sample if k not in ("line",))) + + # ---------------------------------------------------------------- 3. X-Request-Id of an HTTP error -> log -> trace + err = c.call("GET", "/api/v1/chats", 500, actor=alice, headers={"X-Fault": "error"}) + rid = err["headers"]["X-Request-Id"] + check(err["json"]["error"]["request_id"] == rid and "Traceback" not in err["bytes"].decode(), "500 body is clean") + err_lines = obs.wait_for(lambda: obs.loki('{service_name="messenger-api"} |= "%s"' % rid, minutes=5) or None, 30) + check(err_lines, "500 request_id not found in Loki") + err_trace = err_lines[0].get("trace_id") + log("HTTP 500 X-Request-Id=%s -> Loki event=%s trace_id=%s" % (rid, err_lines[0].get("event"), err_trace)) + err_spans = obs.wait_for(lambda: obs.tempo_trace(err_trace) or None, 30) + check(err_spans, "trace of the 500 not in Tempo") + srv = [s for s in err_spans if s["kind"] == "SERVER"][0] + check(srv["attrs"].get("messenger.request_id") == rid and srv["status"] == "ERROR" + and str(srv["attrs"].get("http.response.status_code")) == "500", "server span must carry request_id and ERROR") + log("Tempo: %s status=%s request_id matches; exception event recorded=%s" % ( + srv["name"], srv["status"], "exception" in obs.tempo_raw(err_trace))) + + msg_trace = obs.wait_for(lambda: obs.tempo_search('{span.messenger.request_id="%s"}' % msg_rid) or None, 30) + check(msg_trace, "message request trace not found by request_id") + msg_spans = obs.tempo_trace(msg_trace[0]["traceID"]) + log("message POST trace %s: %s" % (msg_trace[0]["traceID"], [s["name"] for s in msg_spans])) + + # ---------------------------------------------------------------- 4. privacy + secrets = {"password": alice["registration"]["password"], "token": alice["token"], "message text": marker, + "filename": filename} + raw = obs.tempo_raw(trace_id) + obs.tempo_raw(msg_trace[0]["traceID"]) + obs.tempo_raw(err_trace) + logs_raw = "\n".join(str(l) for l in obs.loki('{service_name=~"messenger-.*"}', minutes=10, limit=5000)) + for what, value in secrets.items(): + check(value not in raw, "%s leaked into traces" % what) + check(value not in logs_raw, "%s leaked into logs" % what) + # Image bytes: PNG signature/IHDR chunk as raw, escaped or base64 text (sample.png starts with iVBORw0KGgo). + for word in ("Authorization", "Bearer ", "filename=", "X-Amz-Signature", "iVBORw0KGgo", "IHDR", "\\x89PNG", + "\\u0089PNG"): + check(word not in raw and word not in logs_raw, "%r found in telemetry" % word) + log("PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from " + "traces and logs") + + series = obs.prom('{__name__=~"messenger_.*|http_server_.*|db_client_.*|process_.*"}') + labels = set() + for metric, _ in series: + labels |= set(metric) + for name, value in metric.items(): + check(not UUID_RE.search(value), "UUID-like value in metric label %s=%s" % (name, value)) + check(not labels & FORBIDDEN_LABELS, "forbidden metric labels: %s" % (labels & FORBIDDEN_LABELS)) + log("PASS: %d series, label names: %s" % (len(series), sorted(labels))) + routes = sorted({m.get("http_route") for m, _ in series if m.get("http_route")}) + log("http_route values (templates only): %s" % routes) + + # ---------------------------------------------------------------- 5. metrics referenced by dashboard/alerts + time.sleep(12) # one metric export interval + for name in REQUIRED_METRICS: + check(obs.prom(name), "metric %s has no samples" % name) + log("PASS: required metrics present: %s" % ", ".join(REQUIRED_METRICS)) + print("PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker_down.py b/student-tests/test_worker_down.py new file mode 100755 index 0000000..49fc71e --- /dev/null +++ b/student-tests/test_worker_down.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Grade 3 demo: worker stopped -> upload 202 -> message with attachment -> restart API + broker (volumes kept) +-> worker started -> ready + thumbnail. Then an explicit duplicate delivery of the finished job is a no-op. + +python3 student-tests/test_worker_down.py (stack from REPORT.md must be running) +""" +import hashlib +import time + +from stand import (Client, check, compose, started_at, get, log, objects_of, wait_queue, run, sql, upload, wait_http_ready, + wait_status, unique_name, SAMPLE) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + log("stop all workers") + compose("stop", "-t", "20", "worker") + try: + att = upload(c, alice, chat, filename=unique_name()) + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload must be 202 queued") + log("upload 202 queued id=%s" % att["id"]) + msg = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + log("message %s created with queued attachment" % msg["id"]) + c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob) + c.message(bob, chat, "другие сообщения работают") + depth = wait_queue() + log("broker queue while worker is down:", depth) + check(depth and depth["ready"] >= 1, "job must be waiting in the broker") + unpublished = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0] + check(unpublished == "0", "outbox row must be published with confirm") + + log("restart api-1 api-2 rabbitmq (volumes kept)") + before = started_at("api-1", "api-2", "rabbitmq") + compose("restart", "-t", "10", "rabbitmq", "api-1", "api-2") + wait_http_ready(c) + after = started_at("api-1", "api-2", "rabbitmq") + log("started before:", before) + log("started after: ", after) + check(before != after, "containers must really restart") + check(get(c, bob, att["id"])["status"] == "queued", "still queued after restart, nothing lost") + depth = wait_queue() + log("broker queue after broker restart:", depth) + check(depth and depth["ready"] >= 1, "durable message must survive broker restart") + finally: + log("start workers") + compose("start", "worker") + ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90) + check(ready["status"] == "ready", "expected ready, got %s" % ready["status"]) + log("status sequence observed:", " -> ".join(seen)) + names = sorted(v["name"] for v in ready["variants"]) + check(names == ["crop", "optimized", "thumbnail"], "expected thumbnail+crop+optimized, got %s" % names) + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail 256x160") + body = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"] + check(hashlib.sha256(body).hexdigest() == thumb["sha256"], "thumbnail checksum") + orig = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"] + check(orig == SAMPLE, "original unchanged byte for byte") + keys = objects_of(att["id"]) + log("objects:", keys) + check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), "original + 3 variants, staging removed") + + log("redelivery: republish the finished job through the outbox (duplicate message)") + sql("INSERT INTO outbox (job_id, payload, available_at, created_at) SELECT o.job_id, o.payload, now(), now()" + " FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='%s' ORDER BY o.id LIMIT 1" + % att["id"]) + time.sleep(5) + again = get(c, bob, att["id"]) + check(again["status"] == "ready" and again["variants"] == ready["variants"], "duplicate must not change the result") + rows = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att["id"])[0][0] + succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att["id"])[0][0] + check(rows == "3" and succeeded == "1", "one set of variants and one success (rows=%s succeeded=%s)" % (rows, succeeded)) + check(len(objects_of(att["id"])) == 4, "no new objects after duplicate") + noop = compose("logs", "--since", "30s", "worker") + check("decision=done" in noop, "worker must log the duplicate as no-op") + log("duplicate acked as no-op: variants=%s successes=%s" % (rows, succeeded)) + print("PASS: worker down / API+broker restart / redelivery no-op") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker_saturation.py b/student-tests/test_worker_saturation.py new file mode 100644 index 0000000..c3b6e2d --- /dev/null +++ b/student-tests/test_worker_saturation.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""MessengerWorkerSaturated: capacity is lowered (2 workers x 1 slot, 0.25 CPU each), uploads exceed it for longer than +`for: 5m`; the alert must fire, and resolve after the normal configuration is restored and the backlog drains. + +python3 student-tests/test_worker_saturation.py (stack running; ~10 min). Restores WORKER_* defaults at the end. +""" +import json +import os +import subprocess +import threading +import time +from pathlib import Path + +import obs +from openloop import Load +from stand import check, log, run +from test_incidents import wait_alert + +ROOT = Path(__file__).resolve().parents[1] +ALERT = "MessengerWorkerSaturated" + + +def worker(env): + done = subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--no-deps", "--force-recreate", + "worker"], cwd=ROOT, env={**os.environ, **env}, capture_output=True, text=True, timeout=240) + check(done.returncode == 0, "worker recreate failed: %s" % done.stderr[-300:]) + + +def main(): + result = {} + log("lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas") + worker({"WORKER_CONCURRENCY": "1", "WORKER_PREFETCH": "1", "WORKER_CPUS": "0.25"}) + load = Load("http://localhost:8080", users=2, threads=32) + stop = threading.Event() + summary = {} + thread = threading.Thread(target=lambda: summary.update(load.run(0, 900, upload_rate=4, stop_event=stop)), daemon=True) + t0 = time.time() + thread.start() + log("uploads: 4/s open-loop (sample.png)") + try: + time.sleep(60) + capacity = obs.prom_scalar("sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))") + log("fresh capacity=%.0f slots (stale series of recreated containers are excluded)" % capacity) + check(capacity == 2, "expected 2 slots, got %s" % capacity) + firing, pending = wait_alert(ALERT, "firing", 480, t0) + ratio = obs.prom_scalar("messenger:worker_saturation:ratio") + depth = obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})') + age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})') + rate = obs.prom_scalar('sum(rate(messenger_jobs_processed_total{outcome="succeeded"}[2m]))') + p95 = obs.prom_scalar('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[5m])))') + others = obs.alerts() + log("at firing: saturation=%.2f, queued=%.0f, oldest age=%.0f s, succeeded=%.2f jobs/s, job p95=%.2f s, alerts=%s" + % (ratio, depth, age, rate, p95, others)) + check(ratio > 0.9, "saturation must be above the threshold") + finally: + stop.set() + thread.join(60) + log("restore default worker configuration") + t1 = time.time() + worker({}) + resolved, _ = wait_alert(ALERT, "resolved", 240, t1) + drained = obs.wait_for(lambda: obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})', + default=-1) == 0, 240, 5) + log("backlog drained: %s (+%.0f s after restore)" % (bool(drained), time.time() - t1)) + check(drained, "backlog must drain") + result.update({"alert_pending_s": pending, "alert_firing_s": round(firing), "saturation_at_firing": round(ratio, 3), + "queued_at_firing": depth, "oldest_age_s": round(age), "jobs_per_s": round(rate, 2), + "job_p95_s": round(p95, 2), "alerts_at_firing": others, "resolved_after_restore_s": round(resolved), + "drained_after_restore_s": round(time.time() - t1), "uploads": summary.get("status_counts")}) + (ROOT / "evidence/worker_saturation.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print("PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track1.py b/student-tests/track1.py new file mode 100644 index 0000000..71ae407 --- /dev/null +++ b/student-tests/track1.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Track 5.1 "Цена наблюдаемости": the same open-loop load against several telemetry configurations. + +Configurations (applied by recreating api-1, api-2, worker and otel-collector with shell env overrides): + off OTEL_SDK_DISABLED=true no SDK, no instrumentation (baseline) + on100 sampler ratio 1.0, collector profile default every span exported + head10 sampler ratio 0.1, collector profile default head sampling of root spans (children follow) + tail sampler ratio 1.0, collector profile tail balanced: errors + >500 ms + async pipeline + 10% rest + +Per run: recreate -> ready -> warm-up 20 s -> idle 20 s -> counters A -> load (rate, duration) -> idle 30 s -> +counters B. Delta(B-A) = telemetry produced by the measured load. CPU is the exact cgroup delta over the load window +(Docker API); RSS is the maximum of `docker stats` samples taken every 5 s. Afterwards Tempo is searched in the load window for traces of injected slow/error requests and +upload pipelines, to count what survives sampling. + + python3 student-tests/track1.py run [--configs off,on100,head10,tail] [--repeats 2] [--rate 60] [--duration 90] + python3 student-tests/track1.py summary # evidence/track1/results.jsonl -> evidence/track1/summary.md +""" +import argparse +import json +import os +import re +import statistics +import subprocess +import threading +import time +from pathlib import Path + +import obs +from openloop import Load +from stand import Client, log + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence/track1" +CONFIGS = { + "off": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, + "on100": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, + "head10": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, + "tail": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, +} +SERVICES = ("api-1", "api-2", "worker", "otel-collector", "tempo", "loki", "prometheus", "postgres", "nginx") +COUNTERS = { + # SDK side (sum over all instances; absent when the SDK is off) + "sdk_span_decisions_sample": 'sum(messenger_telemetry_sampler_decisions_total{decision="sample"})', + "sdk_span_decisions_drop_ratio": 'sum(messenger_telemetry_sampler_decisions_total{decision="drop",rule=~"ratio|parent"})', + "sdk_spans_exported_ok": 'sum(messenger_telemetry_spans_exported_total{outcome="success"})', + "sdk_spans_exported_failed": 'sum(messenger_telemetry_spans_exported_total{outcome="failure"})', + # Collector + "col_spans_in": "sum(otelcol_receiver_accepted_spans_total)", + "col_logs_in": "sum(otelcol_receiver_accepted_log_records_total)", + "col_points_in": "sum(otelcol_receiver_accepted_metric_points_total)", + "col_spans_out": 'sum(otelcol_exporter_sent_spans_total{exporter="otlp/tempo"})', + "col_spans_refused": "sum(otelcol_receiver_refused_spans_total)", + "col_spans_dropped": 'sum(otelcol_exporter_enqueue_failed_spans_total) or vector(0)', + "col_otlp_bytes_in": 'sum({__name__="http.server.request.size_bytes_total"})', + # Backends + "tempo_spans_in": "sum(tempo_distributor_spans_received_total)", + "tempo_bytes_in": "sum(tempo_distributor_bytes_received_total)", + "loki_bytes_in": "sum(loki_distributor_bytes_received_total)", + "loki_lines_in": "sum(loki_distributor_lines_received_total)", + "prom_samples_in": "sum(prometheus_tsdb_head_samples_appended_total)", +} + + +def compose(*args, env=None, timeout=300): + done = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, + text=True, timeout=timeout, env={**os.environ, **(env or {})}) + if done.returncode != 0: + raise RuntimeError("compose %s: %s" % (args[:2], done.stderr[-400:])) + return done.stdout + + +def apply(name): + env = CONFIGS[name] + compose("up", "-d", "--wait", "--force-recreate", "--no-deps", "otel-collector", "api-1", "api-2", "worker", + env=env) + compose("restart", "nginx") # re-resolve recreated replicas + c = Client() + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + break + except Exception: + time.sleep(1) + out = compose("exec", "-T", "api-1", "printenv", "OTEL_SDK_DISABLED", "OTEL_TRACES_SAMPLER_ARG") + log("config %s applied: api-1 env %s" % (name, out.split())) + + +def counters(): + values = {} + for key, expr in COUNTERS.items(): + try: + rows = obs.prom(expr) + values[key] = rows[0][1] if rows else None + except Exception: + values[key] = None + tail = {} + for metric, value in obs.prom('{__name__=~"otelcol_processor_tail_sampling_count_traces_sampled.*"}'): + tail["%s/%s" % (metric.get("policy"), metric.get("sampled"))] = value + values["tail_policies"] = tail + return values + + +def delta(a, b): + out = {} + for key in COUNTERS: + if a.get(key) is not None and b.get(key) is not None: + out[key] = b[key] - a[key] + elif b.get(key) is not None: + out[key] = b[key] # series appeared during the window (counter started at 0) + else: + out[key] = None + out["tail_policies"] = {k: v - a["tail_policies"].get(k, 0) for k, v in b["tail_policies"].items()} + return out + + +def parse_mem(text): + """'123.4MiB' -> 123.4 (MiB).""" + match = re.match(r"\s*([\d.]+)\s*([A-Za-z]*)", text) + if not match: + return 0.0 + scale = {"B": 1 / 1048576, "kB": 1 / 1024, "KiB": 1 / 1024, "MiB": 1, "MB": 1, "GiB": 1024, "GB": 1024} + return float(match.group(1)) * scale.get(match.group(2), 1) + + +def sample_stats(stop, samples): + while not stop.is_set(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], + capture_output=True, text=True).stdout + row = {} + for line in out.splitlines(): + name, cpu, mem = line.split("\t") + if not name.startswith("mtusi-lab6-"): + continue + svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0] + cur = row.setdefault(svc, [0.0, 0.0]) + cur[0] += float(cpu.rstrip("%") or 0) + cur[1] += parse_mem(mem.split(" / ")[0]) + samples.append(row) + stop.wait(5) + + +def docker_socket(): + host = subprocess.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + capture_output=True, text=True).stdout.strip() + return host.replace("unix://", "") + + +def cpu_seconds(): + """Cumulative container CPU seconds per service from the Docker API (exact over a window, unlike docker stats).""" + sock = docker_socket() + ids = subprocess.run(["docker", "ps", "--filter", "label=com.docker.compose.project=mtusi-lab6", "--format", + "{{.ID}} {{.Names}}"], capture_output=True, text=True).stdout.splitlines() + out = {} + for line in ids: + cid, name = line.split() + svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0] + raw = subprocess.run(["curl", "-s", "--unix-socket", sock, + "http://localhost/containers/%s/stats?stream=false&one-shot=true" % cid], + capture_output=True, text=True).stdout + out[svc] = out.get(svc, 0.0) + json.loads(raw)["cpu_stats"]["cpu_usage"]["total_usage"] / 1e9 + return out + + +def count_traces(query, start, end): + try: + return len(obs.tempo_search(query, limit=10000, start=start, end=end)) + except Exception as exc: + return "error:%s" % type(exc).__name__ + + +def one_run(name, rep, rate, duration): + apply(name) + load = Load("http://localhost:8080", users=4, threads=96) + load.run(rate, 20, 0.5, 1, 1) # warm-up, same mix + time.sleep(20) + a = counters() + load = Load("http://localhost:8080", users=4, threads=96) + stop, samples = threading.Event(), [] + sampler = threading.Thread(target=sample_stats, args=(stop, samples), daemon=True) + sampler.start() + cpu_a = cpu_seconds() + t_start = time.time() + summary = load.run(rate, duration, 1, 1, 1) + t_end = time.time() + cpu_b = cpu_seconds() + stop.set() + sampler.join(15) + time.sleep(30) # SDK batch (2 s) + collector batch (2 s) + tail decision_wait (10 s) + scrape (15 s) + b = counters() + d = delta(a, b) + # % of one core averaged over the load window (cumulative cgroup CPU; several replicas of a service are summed) + cpu = {svc: round((cpu_b[svc] - cpu_a[svc]) / (t_end - t_start) * 100, 2) for svc in cpu_b if svc in cpu_a} + mem = {svc: round(max(s[svc][1] for s in samples if svc in s), 1) for svc in samples[0]} if samples else {} + w0, w1 = int(t_start) - 2, int(t_end) + 15 + traces = { + "server_api": count_traces('{resource.service.name="messenger-api" && kind=server}', w0, w1), + "slow": count_traces('{span.messenger.fault="slow"}', w0, w1), + "error": count_traces('{span.messenger.fault="error"}', w0, w1), + "pipeline_consumer": count_traces('{kind=consumer && span.messenger.job.outcome="succeeded"}', w0, w1), + } + result = {"config": name, "rep": rep, "env": CONFIGS[name], "load": summary, "cpu_pct_avg": cpu, + "mem_mib_max": mem, "telemetry_delta": d, "traces_in_tempo": traces, + "window": [t_start, t_end], "stats_samples": len(samples)} + log("%s#%d: ok_rps=%s p50/p95/p99=%s/%s/%s ms api_cpu=%s+%s%% spans_in=%s tempo_spans=%s slow=%s/%d err=%s/%d" % ( + name, rep, summary["successful_rps"], summary["latency_ms_plain_ok"]["p50"], + summary["latency_ms_plain_ok"]["p95"], summary["latency_ms_plain_ok"]["p99"], cpu.get("api-1"), + cpu.get("api-2"), d.get("col_spans_in"), d.get("tempo_spans_in"), traces["slow"], + summary["faults_sent"]["slow"], traces["error"], summary["faults_sent"]["error"])) + return result + + +def cmd_run(args): + OUT.mkdir(parents=True, exist_ok=True) + path = OUT / "results.jsonl" + names = args.configs.split(",") + try: + for rep in range(1, args.repeats + 1): + for name in names: + result = one_run(name, rep, args.rate, args.duration) + result["hardware"] = HARDWARE + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(result, ensure_ascii=False) + "\n") + finally: + log("restore default configuration (on100)") + apply("on100") + + +def fmt(v, digits=0): + if v is None: + return "—" + if isinstance(v, str): + return v + return ("%%.%df" % digits) % v + + +def metric_500s(run_result): + """Injected 500s as seen by METRICS for this run: metrics are computed in the SDK, before any trace sampling.""" + start, end = run_result["window"] + if run_result["config"] == "off": + return None + # Exact counter difference at the window borders (increase() extrapolates and overstates by ~10% here). + expr = 'sum(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code="500"})' + try: + return obs.prom_scalar(expr, at=end + 20) - obs.prom_scalar(expr, at=start - 5) + except Exception: + return None + + +def cmd_summary(_args): + rows = [json.loads(line) for line in (OUT / "results.jsonl").read_text(encoding="utf-8").splitlines() if line] + by = {} + for r in rows: + by.setdefault(r["config"], []).append(r) + + def values(items, fn): + return [v for v in (fn(x) for x in items) if isinstance(v, (int, float))] + + def avg(items, fn): + vals = values(items, fn) + return statistics.mean(vals) if vals else None + + def spread(items, fn, digits=1): + vals = values(items, fn) + if not vals: + return "—" + return "%s (%s…%s)" % (fmt(statistics.mean(vals), digits), fmt(min(vals), digits), fmt(max(vals), digits)) + + lat = lambda q: (lambda x: x["load"]["latency_ms_plain_ok"][q]) # noqa: E731 + cpu = lambda *svc: (lambda x: sum(x["cpu_pct_avg"].get(s, 0) for s in svc)) # noqa: E731 + delta_of = lambda key: (lambda x: x["telemetry_delta"].get(key)) # noqa: E731 + traces = lambda key: (lambda x: x["traces_in_tempo"].get(key)) # noqa: E731 + faults = lambda key: (lambda x: x["load"]["faults_sent"][key]) # noqa: E731 + names = [n for n in CONFIGS if n in by] + + lines = ["Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.", "", + "| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |", + "|---|---|---|---|---|---|---|---|---|"] + for n in names: + i = by[n] + lines.append("| %s | %d | %s | %s | %s | %s | %s | %s | %s |" % ( + n, len(i), fmt(avg(i, lambda x: x["load"]["successful_rps"]), 1), + fmt(avg(i, lambda x: x["load"]["server_errors_unexpected"])), spread(i, lat("p50")), spread(i, lat("p95")), + spread(i, lat("p99")), spread(i, cpu("api-1", "api-2")), spread(i, cpu("worker")))) + lines += ["", "| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |", + "|---|---|---|---|---|---|---|---|"] + for n in names: + i = by[n] + mem = lambda s: (lambda x: x["mem_mib_max"].get(s)) # noqa: E731 + lines.append("| %s | %s | %s | %s | %s | %s | %s | %s |" % ( + n, spread(i, cpu("otel-collector"), 2), spread(i, cpu("tempo"), 2), spread(i, cpu("loki"), 2), + spread(i, cpu("prometheus"), 2), fmt(avg(i, mem("api-1"))), fmt(avg(i, mem("otel-collector"))), + fmt(avg(i, mem("tempo"))))) + lines += ["", "| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |", + "|---|---|---|---|---|---|---|"] + for n in names: + i = by[n] + kib = lambda key: (avg(i, delta_of(key)) / 1024 if avg(i, delta_of(key)) is not None else None) # noqa: E731 + lost = avg(i, lambda x: (x["telemetry_delta"].get("col_spans_refused") or 0) + (x["telemetry_delta"].get("col_spans_dropped") or 0)) + lines.append("| %s | %s | %s | %s | %s | %s | %s |" % ( + n, fmt(avg(i, delta_of("col_spans_in"))), fmt(avg(i, delta_of("tempo_spans_in"))), fmt(lost), + fmt(kib("col_otlp_bytes_in")), fmt(kib("tempo_bytes_in")), fmt(kib("loki_bytes_in")))) + lines += ["", "Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.", "", + "| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |", + "|---|---|---|---|---|---|"] + for n in names: + i = by[n] + total = lambda fn: sum(values(i, fn)) # noqa: E731 + m500 = [metric_500s(x) for x in i] + lines.append("| %s | %d / %d | %d / %d | %d / %d | %d / %d | %s / %d |" % ( + n, total(traces("server_api")), total(lambda x: x["load"]["requests"]), + total(traces("slow")), total(faults("slow")), total(traces("error")), total(faults("error")), + total(traces("pipeline_consumer")), total(lambda x: x["load"]["by_kind"]["upload"]["n"]), + "—" if None in m500 else "%.0f" % sum(m500), total(faults("error")))) + text = "\n".join(lines) + "\n" + (OUT / "summary.md").write_text(text, encoding="utf-8") + print(text) + + +HARDWARE = {} + + +def main(): + p = argparse.ArgumentParser() + sub = p.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("run") + r.add_argument("--configs", default="off,on100,head10,tail") + r.add_argument("--repeats", type=int, default=2) + r.add_argument("--rate", type=float, default=60) + r.add_argument("--duration", type=float, default=90) + sub.add_parser("summary") + args = p.parse_args() + if args.cmd == "run": + info = subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} {{.MemTotal}}"], + capture_output=True, text=True).stdout.split() + HARDWARE.update({"docker": info[0], "vm_cpus": int(info[1]), "vm_mem_gib": round(int(info[2]) / 2**30, 1)}) + cmd_run(args) + else: + cmd_summary(args) + + +if __name__ == "__main__": + main() diff --git a/student-tests/variants_check.py b/student-tests/variants_check.py new file mode 100644 index 0000000..6ab763c --- /dev/null +++ b/student-tests/variants_check.py @@ -0,0 +1,126 @@ +"""Variant rules and worker resource limits. Runs INSIDE the worker container (Pillow + app code): + +docker compose exec -T --index 1 worker python - < student-tests/variants_check.py +""" +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +import time + +from PIL import Image + +sys.path.insert(0, "/app") +from app import config # noqa: E402 + +FAILS = [] + + +def check(cond, msg): + print(("ok " if cond else "FAIL ") + msg, flush=True) + if not cond: + FAILS.append(msg) + + +def run_child(path, mem_mb=config.WORKER_CHILD_MEMORY_MB, cpu=config.JOB_TIMEOUT_SECONDS, max_pixels=config.MAX_IMAGE_PIXELS): + out = tempfile.mkdtemp() + + def limits(): + resource.setrlimit(resource.RLIMIT_AS, (mem_mb << 20, mem_mb << 20)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + + start = time.monotonic() + done = subprocess.run([sys.executable, "-m", "app.imageproc", path, out, str(max_pixels)], capture_output=True, + preexec_fn=limits, timeout=120, env={"PATH": os.environ["PATH"], "MALLOC_ARENA_MAX": "2"}) + usage = resource.getrusage(resource.RUSAGE_CHILDREN) + produced = {v["name"]: v for v in json.loads(done.stdout)} if done.returncode == 0 else {} + return done.returncode, produced, time.monotonic() - start, usage.ru_maxrss / 1024 + + +def save(img, fmt, **kw): + path = tempfile.mktemp(suffix="." + fmt.lower()) + img.save(path, fmt, **kw) + return path + + +def dims(v): + return v["width"], v["height"] + + +def real(v): + with Image.open(v["file"]) as im: + return im.format, im.size, "exif" in im.info, im + + +# 1. fixture 320x200 -> 256x160, crop 128, optimized not upscaled +rc, p, _, _ = run_child("/app/tests/fixtures/sample.png" if os.path.exists("/app/tests/fixtures/sample.png") else + save(Image.new("RGB", (320, 200), "red"), "PNG")) +check(rc == 0 and dims(p["thumbnail"]) == (256, 160), "320x200 -> thumbnail 256x160") +check(dims(p["crop"]) == (128, 128), "crop 128x128") +check(dims(p["optimized"]) == (320, 200) and real(p["optimized"])[0] == "WEBP", "optimized WebP 320x200, no upscale") +check(real(p["thumbnail"])[0] == "JPEG" and real(p["crop"])[0] == "JPEG", "thumbnail/crop are JPEG") + +# 2. small source: no upscale for thumbnail, upscale allowed for crop +rc, p, _, _ = run_child(save(Image.new("RGB", (40, 10), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (40, 10) and dims(p["crop"]) == (128, 128), "40x10: thumbnail 40x10, crop 128x128") + +# 3. extreme aspect ratio: floor rounding, minimum 1 px +rc, p, _, _ = run_child(save(Image.new("RGB", (3000, 2), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (256, 1) and dims(p["optimized"]) == (1280, 1), "3000x2 -> 256x1 / 1280x1 (min 1px)") +rc, p, _, _ = run_child(save(Image.new("RGB", (1001, 999), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (256, 255), "1001x999 -> 256x255 (floor)") + +# 4. EXIF orientation 6 (rotate 90): stored 400x200 -> upright 200x400 -> thumbnail 128x256; EXIF stripped +exif = Image.Exif() +exif[0x0112] = 6 +img = Image.new("RGB", (400, 200), "white") +img.paste((255, 0, 0), (0, 0, 200, 200)) # left half red +rc, p, _, _ = run_child(save(img, "JPEG", exif=exif.tobytes(), quality=95)) +fmt, size, has_exif, im = real(p["thumbnail"]) +check(size == (128, 256), "EXIF orientation applied: 400x200 (o=6) -> 128x256") +with Image.open(p["thumbnail"]["file"]) as t: + top = t.convert("RGB").getpixel((64, 20)) +check(top[0] > 200 and top[1] < 80, "rotated content: red half on top after orientation 6") +check(not has_exif and "exif" not in Image.open(p["optimized"]["file"]).info, "EXIF removed from thumbnail/optimized") + +# 5. alpha composited on white for JPEG, kept for WebP +rgba = Image.new("RGBA", (100, 100), (0, 0, 0, 0)) +rc, p, _, _ = run_child(save(rgba, "PNG")) +with Image.open(p["thumbnail"]["file"]) as t: + px = t.convert("RGB").getpixel((50, 50)) +check(min(px) >= 250, "transparent PNG -> white JPEG thumbnail") +check(Image.open(p["optimized"]["file"]).mode == "RGBA", "WebP optimized keeps alpha") + +# 6. 16-bit grayscale PNG, CMYK JPEG, palette PNG +rc16, _, _, _ = run_child(save(Image.new("I;16", (300, 300), 40000), "PNG")) +rcc, _, _, _ = run_child(save(Image.new("CMYK", (300, 300), (0, 255, 0, 0)), "JPEG")) +rcp, _, _, _ = run_child(save(Image.new("P", (300, 300), 3), "PNG", transparency=3)) +check(rc16 == 0 and rcc == 0 and rcp == 0, "16-bit PNG, CMYK JPEG, palette+transparency PNG are processed") + +# 7. limits: pixels (exit 4), not an image (exit 3), memory (exit 5), CPU (SIGXCPU) +rc, _, _, _ = run_child(save(Image.new("RGB", (100, 100)), "PNG"), max_pixels=5000) +check(rc == 4, "pixel limit enforced before decode -> image_too_large (exit 4)") +junk = tempfile.mktemp() +open(junk, "wb").write(b"\x89PNG\r\n\x1a\n" + os.urandom(2000)) +rc, _, _, _ = run_child(junk) +check(rc == 3, "corrupted PNG -> invalid_image (exit 3)") +big = save(Image.new("RGB", (6000, 6000), "green"), "PNG") +rc, _, _, _ = run_child(big, mem_mb=160) +check(rc == 5, "36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got %s" % rc) + +# 8. largest allowed inputs under the real limits: time and child peak RSS +for name, w, h, fmt in (("png", 7900, 5000, "PNG"), ("jpeg", 7900, 5000, "JPEG")): + noise = Image.effect_noise((w // 10, h // 10), 60).convert("RGB").resize((w, h)) + path = save(noise, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 1})) + rc, p, secs, rss = run_child(path) + check(rc == 0 and dims(p["thumbnail"]) == (256, 162), "%.1f Mpx %s (%d MB file) ok in %.1fs, peak child RSS %.0f MiB" + % (w * h / 1e6, name.upper(), os.path.getsize(path) >> 20, secs, rss)) + if fmt == "PNG": + rc, _, _, _ = run_child(path, cpu=0) + check(rc == -24, "CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got %s" % rc) + +print("RESULT:", "PASS" if not FAILS else "FAIL %s" % FAILS) +sys.exit(1 if FAILS else 0) -- 2.54.0