From baee48072c1c1c92f5ffd97322cf20fe787881ca Mon Sep 17 00:00:00 2001 From: David Date: Tue, 17 Nov 2026 15:23:33 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=BA=D0=BB=D1=8E=D1=87?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20OpenTelemetry,=20Collector=20=D0=B8=20Graf?= =?UTF-8?q?ana=20=D0=B4=D0=BB=D1=8F=20=D0=BE=D1=86=D0=B5=D0=BD=D0=BA=D0=B8?= =?UTF-8?q?=205.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit API и worker отдают метрики, логи и трассы. Правила SLO записаны, прогон инцидентов ещё впереди. --- .gitattributes | 2 + .github/pull_request_template.md | 10 +- REPORT.md | 63 ++- app/broker.py | 40 +- app/config.py | 5 + app/jobs.py | 68 ++- app/main.py | 12 +- app/middleware.py | 72 ++- app/telemetry.py | 430 ++++++++++++++++++ app/worker.py | 75 +-- compose.yaml | 84 ++++ infra/grafana/dashboards/messenger.json | 156 +++++++ .../provisioning/dashboards/dashboards.yml | 11 + .../provisioning/datasources/datasources.yml | 38 ++ infra/grafana/runbooks/high-error-rate.md | 9 + infra/grafana/runbooks/queue-lag.md | 8 + infra/grafana/runbooks/slo-burn.md | 13 + infra/loki/loki.yml | 30 ++ infra/otel/README.md | 12 +- infra/otel/collector.yaml | 89 ++++ infra/prometheus/alerts.yml | 112 +++++ infra/prometheus/prometheus.yml | 6 + infra/slo.md | 12 + infra/tempo/tempo.yml | 28 ++ migrations/007_job_trace.sql | 3 + requirements.txt | 4 + 26 files changed, 1279 insertions(+), 113 deletions(-) create mode 100644 app/telemetry.py create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yml create mode 100644 infra/grafana/provisioning/datasources/datasources.yml create mode 100644 infra/grafana/runbooks/high-error-rate.md create mode 100644 infra/grafana/runbooks/queue-lag.md create mode 100644 infra/grafana/runbooks/slo-burn.md create mode 100644 infra/loki/loki.yml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/alerts.yml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/slo.md create mode 100644 infra/tempo/tempo.yml create mode 100644 migrations/007_job_trace.sql diff --git a/.gitattributes b/.gitattributes index 67613ca..c9b9506 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,4 @@ *.sh text eol=lf infra/s3/* text eol=lf +*.yml text eol=lf +*.yaml text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..2321919 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,11 +1,11 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Булацев, БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 2, SLO и обнаружение инцидентов -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO +Что реализовано и что перенесено: база lab5; OTLP у API и worker, Collector, Prometheus, Loki, Tempo, Grafana provisioning. Инциденты и измерения burn rate ещё не сняты. +Команды запуска из чистого clone: см. REPORT.md (`cp .env.example .env`, `docker compose --env-file .env up -d --build --scale api=2`). Результаты проверок и ссылка на REPORT.md: TODO Ограничения / известные ошибки: TODO Источники и использование ИИ: TODO diff --git a/REPORT.md b/REPORT.md index b97ae42..0e734f0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,70 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Булацев, БПИ2301 +- Целевая оценка; выбранный трек 5: 5, трек 2 «SLO и обнаружение инцидентов». Эксперименты со всплеском и длинной деградацией ещё не прогнаны. +- Commit/PR: локальные коммиты, PR не открыт +- Использованные источники, библиотеки и помощь ИИ: контракт lab6, OpenTelemetry Python SDK 1.29.0, Collector Contrib 0.128.0, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.7.1, Grafana 11.4.0. Реализация перенесена из lab5 и дополнена с помощью Cursor. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose v2 и Python 3.9+ на хосте для `make check` / `make test`. Образы закреплены: `python:3.12.8-slim-bookworm`, `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`, `chrislusf/seaweedfs:3.85`, `rabbitmq:3.13.7-alpine`, `otel/opentelemetry-collector-contrib:0.128.0`, `prom/prometheus:v2.55.1`, `grafana/loki:3.3.2`, `grafana/tempo:2.7.1`, `grafana/grafana:11.4.0`. + +```sh +cp .env.example .env +# заменить синтетические пароли в локальном .env; файл не коммитить +docker compose --env-file .env up -d --build --scale api=2 +``` + +`migrate` применяет SQL, включая `007_job_trace.sql`. Снаружи: HAProxy `127.0.0.1:8080`, Grafana `127.0.0.1:3000` (анонимный Admin только на localhost), Prometheus `127.0.0.1:9090`, S3 `127.0.0.1:9000` для presigned PUT. OTLP наружу не публикуется. + +Готовность API: `GET http://127.0.0.1:8080/health/ready`. Дашборд «Messenger observability» поднимается provisioning-ом, руками его импортировать не нужно. Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных volumes: `docker compose --env-file .env down -v`. + +Проверки после готовности: + +```sh +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab5 перенесены две реплики API за HAProxy, Postgres, Redis, SeaweedFS, RabbitMQ и отдельный worker. Бизнес-контракт не менялся: upload по-прежнему `202`, обработка в worker. + +Новое в lab6: API и worker шлют OTLP/HTTP в Collector. Collector раскладывает сигналы в Prometheus (метрики), Loki (логи) и Tempo (трассы). Grafana только читает эти хранилища. + +`traceparent` запроса, который создал job, пишется в `attachment_jobs` и при публикации становится родителем producer span. Worker читает заголовок AMQP и открывает consumer span. Внутри обработки есть spans `s3.get`, `s3.put`, `image.render`, `db.complete`. Повторная публикация использует тот же сохранённый `traceparent`, поэтому retry остаётся в той же трассе. + +Метрики не используют `user_id`, `chat_id`, `message_id`, `attachment_id` как labels. Маршрут — шаблон. Доля 5xx считается только по бизнес-запросам: health, docs и openapi в неё не входят. Процентили на дашборде — `histogram_quantile` по bucket. Глубина очереди и возраст старейшей queued-job читаются из Postgres раз в 5 секунд; на дашборде берётся `max`, чтобы две реплики API не удваивали gauge. + +SLO и burn-rate: `infra/slo.md`, правила `infra/prometheus/alerts.yml`. Состояние firing/resolved смотрит Prometheus (`ALERTS`) и нижняя панель дашборда. Доставка наружу не настроена. + +Если Collector или хранилище телеметрии недоступны, SDK держит очередь не длиннее 2048 и отбрасывает лишнее. Запросы и обработка job от этого не останавливаются. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| База lab5 перенесена | код и compose | API, worker, broker, S3, Postgres на месте | compose.yaml | +| Сигналы и дашборд | provisioning Grafana | конфиги в Git, прогон стенда ещё впереди | infra/grafana/dashboards/messenger.json | +| Трасса upload → worker | traceparent в job и AMQP | код связи есть, снимок трассы ещё не снят | migrations/007_job_trace.sql | +| SLO / burn rate | правила Prometheus | пороги заданы, firing на стенде ещё не снят | infra/prometheus/alerts.yml | +| Инциденты трека 2 | короткий всплеск и длинная деградация | не выполнено в этом коммите | — | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза трека 2: пара окон 5m+1h ловит короткий всплеск 5xx быстрее, чем пара 30m+6h, а длинная деградация остаётся видна после того, как короткое окно уже чистое. Ложные срабатывания на одном шумном интервале режет требование совпадения обоих окон. + +Прогон, время обнаружения и время восстановления ещё не измерены. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка worker | растёт oldest job age, alert QueueLagHigh | ещё не прогнано | — | +| Медленная или недоступная БД | 5xx и pool waiting, это не медленный handler | ещё не прогнано | — | +| Недоступный backend телеметрии | API жив, очередь SDK ограничена | ещё не прогнано | — | +| Короткий всплеск и длинная деградация | fast burn и slow burn расходятся по времени | ещё не прогнано | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Алерты вычисляет Prometheus; отдельной доставки нет. Анонимный Admin Grafana слушает только `127.0.0.1`. Возраст очереди — возраст строки `queued` в Postgres, а не lag consumer RabbitMQ. Две реплики публикуют один и тот же gauge глубины, поэтому на графике `max`. + +Дальше: поднять стенд, снять одну трассу и связанный лог, остановить worker и замедлить БД, затем прогнать оба сценария burn rate и записать времена. \ No newline at end of file diff --git a/app/broker.py b/app/broker.py index 812e2dc..cb24e76 100644 --- a/app/broker.py +++ b/app/broker.py @@ -8,7 +8,7 @@ import logging import aio_pika -from app import config +from app import config, telemetry logger = logging.getLogger("messenger.broker") @@ -49,19 +49,33 @@ async def _ensure() -> tuple[aio_pika.RobustChannel, aio_pika.Exchange]: return _channel, _exchange -async def publish_job(payload: dict) -> None: +async def publish_job(payload: dict, traceparent: str | None = None, tracestate: str | None = None) -> None: channel, exchange = await _ensure() - body = json.dumps(payload, separators=(",", ":")).encode("utf-8") - await exchange.publish( - aio_pika.Message( - body=body, - content_type="application/json", - delivery_mode=aio_pika.DeliveryMode.PERSISTENT, - message_id=payload["job_id"], - ), - routing_key=ROUTING_KEY, - mandatory=True, - ) + parent = telemetry.remote_context(traceparent, tracestate) + with telemetry.span( + "attachment.job.publish", + context=parent, + kind="producer", + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "messaging.operation": "publish", + "job.id": payload.get("job_id"), + }, + ): + headers = telemetry.current_carrier() or None + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + await exchange.publish( + aio_pika.Message( + body=body, + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=payload["job_id"], + headers=headers, + ), + routing_key=ROUTING_KEY, + mandatory=True, + ) async def publish_delay(payload: dict, delay_seconds: float) -> None: diff --git a/app/config.py b/app/config.py index b682a0d..6c19b08 100644 --- a/app/config.py +++ b/app/config.py @@ -56,6 +56,11 @@ JOB_BACKOFF_BASE_SECONDS = float(_env("JOB_BACKOFF_BASE_SECONDS", "0.5")) JOB_BACKOFF_CAP_SECONDS = float(_env("JOB_BACKOFF_CAP_SECONDS", "30")) STAGING_RETENTION_SECONDS = int(_env("STAGING_RETENTION_SECONDS", "86400")) WORKER_CRASH_POINT = _env("WORKER_CRASH_POINT", "") +TELEMETRY_ENABLED = _env("TELEMETRY_ENABLED", "1") == "1" +OTEL_SERVICE_NAME = _env("OTEL_SERVICE_NAME", "messenger-api") +OTEL_EXPORTER_OTLP_ENDPOINT = _env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318") +TRACE_SAMPLE_RATIO = float(_env("TRACE_SAMPLE_RATIO", "1")) +SERVICE_VERSION = _env("SERVICE_VERSION", "lab6") def lab_mode() -> bool: diff --git a/app/jobs.py b/app/jobs.py index a9dc730..844941e 100644 --- a/app/jobs.py +++ b/app/jobs.py @@ -7,10 +7,11 @@ import hashlib import logging import os import random +import time from datetime import datetime, timezone from uuid import UUID, uuid4 -from app import broker, config, db, pipeline, s3 +from app import broker, config, db, pipeline, s3, telemetry from app.errors import AppError from app.pipeline import PermanentJobError, TransientJobError from app.store import format_ts @@ -37,19 +38,27 @@ def backoff_seconds(attempt: int) -> float: return base * random.uniform(0.5, 1.5) +def _trace_columns() -> tuple[str | None, str | None]: + carrier = telemetry.current_carrier() + return carrier.get("traceparent") or None, carrier.get("tracestate") or None + + async def insert_job(conn, attachment_id: UUID, created: datetime) -> UUID: job_id = uuid4() + traceparent, tracestate = _trace_columns() await conn.execute( """ INSERT INTO attachment_jobs ( job_id, attachment_id, pipeline_version, attempt, status, - next_attempt_at, created_at, updated_at + next_attempt_at, created_at, updated_at, traceparent, tracestate ) - VALUES ($1, $2, 1, 1, 'queued', $3, $3, $3) + VALUES ($1, $2, 1, 1, 'queued', $3, $3, $3, $4, $5) """, job_id, attachment_id, created, + traceparent, + tracestate, ) await conn.execute( """ @@ -78,7 +87,8 @@ async def publish_pending(limit: int = 20) -> int: async with conn.transaction(): rows = await conn.fetch( """ - SELECT job_id, attachment_id, pipeline_version, attempt, created_at + SELECT job_id, attachment_id, pipeline_version, attempt, created_at, + traceparent, tracestate FROM attachment_jobs WHERE published_at IS NULL AND status = 'queued' @@ -90,7 +100,11 @@ async def publish_pending(limit: int = 20) -> int: limit, ) for row in rows: - await broker.publish_job(message_of(row)) + await broker.publish_job( + message_of(row), + traceparent=row["traceparent"], + tracestate=row["tracestate"], + ) await conn.execute( "UPDATE attachment_jobs SET published_at = now() WHERE job_id = $1", row["job_id"], @@ -445,6 +459,7 @@ async def load_attachment(attachment_id: UUID): async def process(job_id: UUID, pipeline_version: int) -> str: + started = time.perf_counter() state, row = await claim(job_id, pipeline_version) if state != "claimed": return state @@ -452,15 +467,23 @@ async def process(job_id: UUID, pipeline_version: int) -> str: lease = row["lease_token"] try: await asyncio.wait_for(_effect(row, lease), config.JOB_TIMEOUT_SECONDS) + outcome = "ready" except PermanentJobError as exc: await fail_job(job_id, lease, exc.code) - return "failed" + outcome = "failed" except (TransientJobError, asyncio.TimeoutError, AppError): - return await schedule_retry(job_id, lease, "processing_failed") + outcome = await schedule_retry(job_id, lease, "processing_failed") except Exception: logger.warning("job processing failed job_id=%s", job_id) - return await schedule_retry(job_id, lease, "processing_failed") - return "ready" + outcome = await schedule_retry(job_id, lease, "processing_failed") + elapsed = time.perf_counter() - started + if outcome in ("ready", "failed", "retry", "lost"): + telemetry.record_job(outcome, elapsed) + if outcome == "ready": + created = _aware(row["created_at"]) + if created is not None: + telemetry.record_time_to_ready((datetime.now(timezone.utc) - created).total_seconds()) + return outcome async def _effect(row, lease: UUID) -> None: @@ -479,20 +502,24 @@ async def _effect(row, lease: UUID) -> None: os._exit(1) if fault == "fail_once": raise PermanentJobError("processing_failed") - data = await s3.get_bytes(attachment["object_key"]) + with telemetry.span("s3.get", kind="client", attributes={"s3.operation": "get"}): + data = await s3.get_bytes(attachment["object_key"]) if len(data) != attachment["size_bytes"] or hashlib.sha256(data).hexdigest() != attachment["sha256"]: raise PermanentJobError("invalid_image") final_key = f"originals/{attachment['id']}" if attachment["object_key"] != final_key: await _intent(final_key) - await s3.put_bytes(final_key, data, attachment["content_type"]) - rendered = await asyncio.to_thread(pipeline.render, data, attachment["content_type"], version) + with telemetry.span("s3.put", kind="client", attributes={"s3.operation": "put", "variant": "original"}): + await s3.put_bytes(final_key, data, attachment["content_type"]) + with telemetry.span("image.render"): + rendered = await asyncio.to_thread(pipeline.render, data, attachment["content_type"], version) stored = [] for name, (payload, content_type, width, height) in rendered.items(): suffix = "webp" if content_type == "image/webp" else "jpg" key = f"variants/{attachment['id']}/v{version}/{name}.{suffix}" await _intent(key) - await s3.put_bytes(key, payload, content_type) + with telemetry.span("s3.put", kind="client", attributes={"s3.operation": "put", "variant": name}): + await s3.put_bytes(key, payload, content_type) stored.append( { "name": name, @@ -511,7 +538,8 @@ async def _effect(row, lease: UUID) -> None: if fresh is not None and int(fresh["applied_pipeline_version"]) > version: stored = [] final_key = "" - committed = await complete_job(row["job_id"], lease, attachment["id"], final_key, stored, version) + with telemetry.span("db.complete", kind="client"): + committed = await complete_job(row["job_id"], lease, attachment["id"], final_key, stored, version) if not committed: raise TransientJobError("processing_failed") if stored and attachment["object_key"] != final_key and final_key: @@ -571,18 +599,21 @@ async def bump_pipeline(attachment_id: UUID) -> dict: return {"result": "missing"} version = int(current) + 1 job_id = uuid4() + traceparent, tracestate = _trace_columns() await conn.execute( """ INSERT INTO attachment_jobs ( job_id, attachment_id, pipeline_version, attempt, status, - next_attempt_at, created_at, updated_at + next_attempt_at, created_at, updated_at, traceparent, tracestate ) - VALUES ($1, $2, $3, 1, 'queued', $4, $4, $4) + VALUES ($1, $2, $3, 1, 'queued', $4, $4, $4, $5, $6) """, job_id, attachment_id, version, created, + traceparent, + tracestate, ) await conn.execute( """ @@ -600,7 +631,8 @@ async def redeliver(job_id: UUID) -> dict: async with db.require().acquire() as conn: row = await conn.fetchrow( """ - SELECT job_id, attachment_id, pipeline_version, attempt, created_at, status + SELECT job_id, attachment_id, pipeline_version, attempt, created_at, status, + traceparent, tracestate FROM attachment_jobs WHERE job_id = $1 """, @@ -608,7 +640,7 @@ async def redeliver(job_id: UUID) -> dict: ) if row is None: return {"result": "missing"} - await broker.publish_job(message_of(row)) + await broker.publish_job(message_of(row), traceparent=row["traceparent"], tracestate=row["tracestate"]) return { "result": "published", "job_id": str(row["job_id"]), diff --git a/app/main.py b/app/main.py index c06bdf5..d2c658b 100644 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,6 @@ import asyncio import logging import signal -import sys import uuid from pathlib import Path @@ -12,7 +11,7 @@ from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route -from app import attachments, config, db, jobs, rediscli +from app import attachments, config, db, jobs, rediscli, telemetry from app.docs import openapi_json, swagger_ui from app.errors import AppError, error_response, from_app_error from app.middleware import RequestContextMiddleware @@ -76,13 +75,16 @@ async def startup() -> None: await rediscli.connect() attachments.start_sweeper() jobs.start_outbox() + telemetry.start_probes() async def shutdown() -> None: + await telemetry.stop_probes() await attachments.stop_sweeper() await jobs.stop_outbox() await rediscli.close() await db.close() + telemetry.shutdown() def build_app() -> Starlette: @@ -125,11 +127,7 @@ def build_app() -> Starlette: def configure_logging() -> None: - logging.basicConfig( - level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), - format="%(asctime)s %(levelname)s %(name)s %(message)s", - stream=sys.stdout, - ) + telemetry.configure_logging() def main() -> None: diff --git a/app/middleware.py b/app/middleware.py index f79e98e..c0d1ed3 100644 --- a/app/middleware.py +++ b/app/middleware.py @@ -1,11 +1,13 @@ import logging import time import uuid +from contextlib import nullcontext from starlette.datastructures import MutableHeaders from starlette.responses import Response from starlette.types import ASGIApp, Message, Receive, Scope, Send +from app import telemetry from app.config import INSTANCE_ID from app.errors import AppError, error_response, from_app_error @@ -30,34 +32,62 @@ class RequestContextMiddleware: scope["state"] = raw_state raw_state["request_id"] = request_id path = scope.get("path", "") + method = scope.get("method", "") started = time.perf_counter() + status_code = 500 + record = telemetry.observe_http(path) + route = telemetry.route_template(path) async def send_wrapper(message: Message) -> None: + nonlocal status_code if message["type"] == "http.response.start": + status_code = int(message["status"]) response_headers = MutableHeaders(scope=message) response_headers["X-Request-Id"] = request_id response_headers["X-Instance-Id"] = INSTANCE_ID await send(message) - app = scope.get("app") - shutting_down = bool(app and getattr(app.state, "shutting_down", False)) - if shutting_down and path not in ("/health/live", "/health/ready"): - response: Response = error_response(503, "service is shutting down", request_id) - await response(scope, receive, send_wrapper) - return - try: - await self.app(scope, receive, send_wrapper) - except AppError as exc: - await from_app_error(exc, request_id)(scope, receive, send_wrapper) - except Exception: - logger.exception("request_id=%s unhandled error path=%s", request_id, path) - await error_response(500, "internal error", request_id)(scope, receive, send_wrapper) - return - elapsed_ms = (time.perf_counter() - started) * 1000 - logger.info( - "request_id=%s method=%s path=%s duration_ms=%.2f", - request_id, - scope.get("method", ""), - path, - elapsed_ms, + parent = telemetry.remote_context(headers.get("traceparent"), headers.get("tracestate")) + span_cm = ( + telemetry.span( + "%s %s" % (method, route), + context=parent, + kind="server", + attributes={ + "http.request.method": method, + "http.route": route, + "http.request_id": request_id, + }, + ) + if record + else nullcontext(None) ) + with span_cm as current: + app = scope.get("app") + shutting_down = bool(app and getattr(app.state, "shutting_down", False)) + try: + if shutting_down and path not in ("/health/live", "/health/ready"): + response: Response = error_response(503, "service is shutting down", request_id) + await response(scope, receive, send_wrapper) + else: + await self.app(scope, receive, send_wrapper) + except AppError as exc: + status_code = exc.status + await from_app_error(exc, request_id)(scope, receive, send_wrapper) + except Exception: + status_code = 500 + logger.exception("request failed request_id=%s route=%s", request_id, route) + await error_response(500, "internal error", request_id)(scope, receive, send_wrapper) + finally: + if record: + telemetry.finish_http_span(current, status_code) + telemetry.record_http(method, route, status_code, time.perf_counter() - started) + if record: + logger.info( + "request completed request_id=%s method=%s route=%s status=%s duration_ms=%.2f", + request_id, + method, + route, + status_code, + (time.perf_counter() - started) * 1000, + ) diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..438300b --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,430 @@ +"""OTLP telemetry for the API and the worker. + +Metric labels stay low-cardinality: method, route template, status class, +job type, outcome, queue name. User, chat, message and attachment ids are +never metric labels. Trace and log records may carry a job id so one upload +can be found; passwords, tokens, message text and image bytes are not exported. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +import sys +import time +from contextlib import contextmanager + +import psutil +from opentelemetry import metrics, propagate, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased +from opentelemetry.trace import SpanKind, Status, StatusCode + +from app import config + +logger = logging.getLogger("messenger.telemetry") + +_HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10) +_JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60) +_READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300) +_JOB_TYPE = "attachment.process.v1" +_QUEUE = "attachment.jobs" + +_ROUTES = ( + (re.compile(r"^/api/v1/chats/[^/]+/messages$"), "/api/v1/chats/{chat_id}/messages"), + (re.compile(r"^/api/v1/chats/[^/]+/attachments$"), "/api/v1/chats/{chat_id}/attachments"), + (re.compile(r"^/api/v1/chats/[^/]+/uploads$"), "/api/v1/chats/{chat_id}/uploads"), + (re.compile(r"^/api/v1/chats/[^/]+$"), "/api/v1/chats/{chat_id}"), + (re.compile(r"^/api/v1/uploads/[^/]+/complete$"), "/api/v1/uploads/{upload_id}/complete"), + (re.compile(r"^/api/v1/attachments/[^/]+/content$"), "/api/v1/attachments/{attachment_id}/content"), + (re.compile(r"^/api/v1/attachments/[^/]+$"), "/api/v1/attachments/{attachment_id}"), + (re.compile(r"^/api/v1/_lab/attachments/[^/]+/jobs$"), "/api/v1/_lab/attachments/{attachment_id}/jobs"), + (re.compile(r"^/api/v1/_lab/attachments/[^/]+/pipelines$"), "/api/v1/_lab/attachments/{attachment_id}/pipelines"), + (re.compile(r"^/api/v1/_lab/attachments/[^/]+/replay$"), "/api/v1/_lab/attachments/{attachment_id}/replay"), + (re.compile(r"^/api/v1/_lab/jobs/[^/]+/redeliver$"), "/api/v1/_lab/jobs/{job_id}/redeliver"), +) +_SECRET = re.compile(r"(?i)(authorization|password|token|secret|session)([\"']?\s*[:=]\s*[\"']?)(\S+)") + +_ready = False +_tracer = None +_http_requests = None +_http_duration = None +_jobs = None +_job_duration = None +_time_to_ready = None +_log_handler = None +_providers = [] +_process = psutil.Process() +_process.cpu_percent(interval=None) +_inflight = 0 +_queue_depth = 0 +_queue_oldest = 0.0 +_probe_task: asyncio.Task | None = None + +_KIND = { + "internal": SpanKind.INTERNAL, + "server": SpanKind.SERVER, + "client": SpanKind.CLIENT, + "producer": SpanKind.PRODUCER, + "consumer": SpanKind.CONSUMER, +} + + +class JsonFormatter(logging.Formatter): + def __init__(self, service_name: str) -> None: + super().__init__() + self.service_name = service_name + + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() + payload = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime(record.created)), + "severity": record.levelname, + "service.name": self.service_name, + "event": _SECRET.sub(r"\1\2[redacted]", record.getMessage()), + "logger": record.name, + } + if ctx.is_valid: + payload["trace_id"] = format(ctx.trace_id, "032x") + payload["span_id"] = format(ctx.span_id, "016x") + if record.exc_info: + payload["exception"] = _SECRET.sub(r"\1\2[redacted]", self.formatException(record.exc_info)) + return json.dumps(payload, ensure_ascii=False) + + +def route_template(path: str) -> str: + for pattern, template in _ROUTES: + if pattern.match(path): + return template + return path + + +def observe_http(path: str) -> bool: + return path not in ("/health/live", "/health/ready", "/docs", "/openapi.json") + + +def status_class(code: int) -> str: + return "%sxx" % (int(code) // 100) + + +def configure_logging() -> None: + setup() + root = logging.getLogger() + root.handlers.clear() + root.setLevel(getattr(logging, config.LOG_LEVEL.upper(), logging.INFO)) + stream = logging.StreamHandler(sys.stdout) + stream.setFormatter(JsonFormatter(config.OTEL_SERVICE_NAME)) + root.addHandler(stream) + if _log_handler is not None: + root.addHandler(_log_handler) + logging.getLogger("opentelemetry").setLevel(logging.ERROR) + + +def setup() -> None: + global _ready, _tracer, _http_requests, _http_duration, _jobs, _job_duration, _time_to_ready, _log_handler + if _ready: + return + _ready = True + if not config.TELEMETRY_ENABLED: + logger.info("telemetry disabled") + return + resource = Resource.create( + { + "service.name": config.OTEL_SERVICE_NAME, + "service.version": config.SERVICE_VERSION, + "service.instance.id": config.INSTANCE_ID, + "deployment.environment.name": "lab", + } + ) + ratio = min(1.0, max(0.0, config.TRACE_SAMPLE_RATIO)) + tracer_provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(ratio))) + tracer_provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=_signal_endpoint("traces"), timeout=5), + max_queue_size=2048, + schedule_delay_millis=5000, + max_export_batch_size=512, + export_timeout_millis=5000, + ) + ) + trace.set_tracer_provider(tracer_provider) + _tracer = trace.get_tracer("messenger", config.SERVICE_VERSION) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=_signal_endpoint("metrics"), timeout=5), + export_interval_millis=10000, + export_timeout_millis=5000, + ) + meter_provider = MeterProvider( + resource=resource, + metric_readers=[reader], + views=[ + View( + instrument_name="messenger.http.request.duration", + aggregation=ExplicitBucketHistogramAggregation(_HTTP_BUCKETS), + ), + View( + instrument_name="messenger.job.duration", + aggregation=ExplicitBucketHistogramAggregation(_JOB_BUCKETS), + ), + View( + instrument_name="messenger.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(_READY_BUCKETS), + ), + ], + ) + metrics.set_meter_provider(meter_provider) + meter = metrics.get_meter("messenger", config.SERVICE_VERSION) + _http_requests = meter.create_counter("messenger.http.requests", unit="1") + _http_duration = meter.create_histogram("messenger.http.request.duration", unit="s") + _jobs = meter.create_counter("messenger.jobs", unit="1") + _job_duration = meter.create_histogram("messenger.job.duration", unit="s") + _time_to_ready = meter.create_histogram("messenger.time_to_ready", unit="s") + meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used]) + meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting]) + meter.create_observable_gauge("process.cpu.percent", callbacks=[_cpu]) + meter.create_observable_gauge("process.memory.rss", unit="By", callbacks=[_rss]) + if config.OTEL_SERVICE_NAME == "messenger-api": + meter.create_observable_gauge("messenger.queue.depth", callbacks=[_depth]) + meter.create_observable_gauge("messenger.queue.oldest_age", unit="s", callbacks=[_oldest]) + if config.OTEL_SERVICE_NAME == "messenger-worker": + meter.create_observable_gauge("messenger.worker.busy", callbacks=[_busy]) + meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_capacity]) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=_signal_endpoint("logs"), timeout=5), + max_queue_size=2048, + schedule_delay_millis=5000, + max_export_batch_size=512, + export_timeout_millis=5000, + ) + ) + set_logger_provider(logger_provider) + _log_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + _providers.extend([tracer_provider, meter_provider, logger_provider]) + + +def shutdown() -> None: + for provider in _providers: + try: + provider.shutdown() + except Exception: + logger.warning("telemetry shutdown failed") + _providers.clear() + + +def _signal_endpoint(signal: str) -> str: + base = config.OTEL_EXPORTER_OTLP_ENDPOINT.rstrip("/") + return "%s/v1/%s" % (base, signal) + + +def current_carrier() -> dict: + if _tracer is None: + return {} + carrier: dict = {} + propagate.inject(carrier) + return carrier + + +def remote_context(traceparent: str | None, tracestate: str | None = None): + carrier = {} + if traceparent: + carrier["traceparent"] = traceparent + if tracestate: + carrier["tracestate"] = tracestate + if not carrier: + return None + return propagate.extract(carrier) + + +def carrier_from_headers(headers) -> dict: + carrier = {} + if not headers: + return carrier + for key in ("traceparent", "tracestate"): + value = headers.get(key) + if value is None: + continue + if isinstance(value, bytes): + value = value.decode("latin1") + carrier[key] = str(value) + return carrier + + +@contextmanager +def span(name: str, context=None, kind: str = "internal", attributes: dict | None = None): + if _tracer is None: + yield None + return + kwargs = {"kind": _KIND.get(kind, SpanKind.INTERNAL)} + if context is not None: + kwargs["context"] = context + with _tracer.start_as_current_span(name, **kwargs) as current: + if attributes: + for key, value in attributes.items(): + if value is not None: + current.set_attribute(key, value) + yield current + + +def finish_http_span(current, status_code: int) -> None: + if current is None: + return + current.set_attribute("http.response.status_code", int(status_code)) + if status_code >= 500: + current.set_status(Status(StatusCode.ERROR)) + else: + current.set_status(Status(StatusCode.OK)) + + +def record_http(method: str, route: str, status_code: int, duration_seconds: float) -> None: + if _http_requests is None: + return + attributes = { + "method": method, + "route": route, + "status_class": status_class(status_code), + } + _http_requests.add(1, attributes) + _http_duration.record(max(0.0, duration_seconds), attributes) + + +def record_job(outcome: str, duration_seconds: float) -> None: + if _jobs is None: + return + attributes = {"job_type": _JOB_TYPE, "outcome": outcome} + _jobs.add(1, attributes) + _job_duration.record(max(0.0, duration_seconds), attributes) + + +def record_time_to_ready(duration_seconds: float) -> None: + if _time_to_ready is None: + return + _time_to_ready.record(max(0.0, duration_seconds), {"job_type": _JOB_TYPE}) + + +def worker_enter() -> None: + global _inflight + _inflight += 1 + + +def worker_leave() -> None: + global _inflight + _inflight = max(0, _inflight - 1) + + +def start_probes() -> None: + global _probe_task + if _tracer is None or config.OTEL_SERVICE_NAME != "messenger-api": + return + if _probe_task is None or _probe_task.done(): + _probe_task = asyncio.create_task(_probe_loop()) + + +async def stop_probes() -> None: + global _probe_task + task = _probe_task + _probe_task = None + if task is None: + return + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +async def _probe_loop() -> None: + from app import db + + global _queue_depth, _queue_oldest + while True: + try: + async with db.require().acquire() as conn: + row = await conn.fetchrow( + """ + SELECT + count(*) FILTER (WHERE status IN ('queued', 'processing')) AS depth, + COALESCE( + EXTRACT(EPOCH FROM (now() - min(created_at) FILTER (WHERE status = 'queued'))), + 0 + ) AS oldest + FROM attachment_jobs + """ + ) + _queue_depth = int(row["depth"] or 0) + _queue_oldest = float(row["oldest"] or 0) + except asyncio.CancelledError: + raise + except Exception: + logger.warning("queue probe failed") + await asyncio.sleep(5) + + +def _observation(value, attributes=None): + from opentelemetry.metrics import Observation + + return [Observation(value, attributes or {})] + + +def _pool_used(_options): + from app import db + + pool = db.pool + if pool is None: + return _observation(0) + return _observation(max(0, pool.get_size() - pool.get_idle_size())) + + +def _pool_waiting(_options): + from app import db + + pool = db.pool + if pool is None: + return _observation(0) + waiters = getattr(pool, "_waiters", None) + if waiters is not None: + return _observation(len(waiters)) + queue = getattr(pool, "_queue", None) + getters = getattr(queue, "_getters", None) + if getters is not None: + return _observation(len(getters)) + return _observation(0) + + +def _depth(_options): + return _observation(_queue_depth, {"queue": _QUEUE}) + + +def _oldest(_options): + return _observation(_queue_oldest, {"queue": _QUEUE}) + + +def _busy(_options): + return _observation(_inflight) + + +def _capacity(_options): + return _observation(config.WORKER_CONCURRENCY) + + +def _cpu(_options): + return _observation(_process.cpu_percent(interval=None)) + + +def _rss(_options): + return _observation(_process.memory_info().rss) diff --git a/app/worker.py b/app/worker.py index 1be412e..21ff051 100644 --- a/app/worker.py +++ b/app/worker.py @@ -5,50 +5,64 @@ from __future__ import annotations import asyncio import logging import signal -import sys from uuid import UUID import aio_pika -from app import broker, config, db, jobs +from app import broker, config, db, jobs, telemetry from app.broker import payload_from logger = logging.getLogger("messenger.worker") def configure_logging() -> None: - logging.basicConfig( - level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), - format="%(asctime)s %(levelname)s %(name)s %(message)s", - stream=sys.stdout, - ) + telemetry.configure_logging() async def _settle(message: aio_pika.IncomingMessage) -> None: - try: - payload = payload_from(message.body) - job_id = UUID(payload["job_id"]) - pipeline_version = int(payload["pipeline_version"]) - except (ValueError, KeyError, TypeError): - logger.warning("dropping a job message that does not match the schema") - await message.ack() - return - try: - waits = 0 - while True: - state = await jobs.process(job_id, pipeline_version) - if state == "busy": - waits += 1 - if waits > config.JOB_LEASE_SECONDS + 5: - await message.reject(requeue=True) + carrier = telemetry.carrier_from_headers(message.headers) + parent = telemetry.remote_context(carrier.get("traceparent"), carrier.get("tracestate")) + with telemetry.span( + "attachment.job.consume", + context=parent, + kind="consumer", + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": broker.QUEUE, + "messaging.operation": "consume", + }, + ) as current: + telemetry.worker_enter() + try: + try: + payload = payload_from(message.body) + job_id = UUID(payload["job_id"]) + pipeline_version = int(payload["pipeline_version"]) + except (ValueError, KeyError, TypeError): + logger.warning("dropping a job message that does not match the schema") + await message.ack() + return + if current is not None: + current.set_attribute("job.id", str(job_id)) + try: + waits = 0 + while True: + state = await jobs.process(job_id, pipeline_version) + if state == "busy": + waits += 1 + if waits > config.JOB_LEASE_SECONDS + 5: + await message.reject(requeue=True) + return + await asyncio.sleep(1) + continue + logger.info("job settled job_id=%s state=%s", job_id, state) + await message.ack() return - await asyncio.sleep(1) - continue - await message.ack() - return - except Exception: - logger.warning("job settle failed job_id=%s", payload.get("job_id")) - await message.reject(requeue=True) + except Exception: + logger.warning("job settle failed job_id=%s", payload.get("job_id")) + await message.reject(requeue=True) + finally: + telemetry.worker_leave() async def _reap_loop(stop: asyncio.Event) -> None: @@ -86,6 +100,7 @@ async def serve() -> None: pass await broker.close() await db.close() + telemetry.shutdown() def main() -> None: diff --git a/compose.yaml b/compose.yaml index 6fa38c2..c9104bc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,7 @@ # Client -> lb:8080 -> api x2 -> Postgres + Redis + private SeaweedFS S3 + RabbitMQ. # Worker is a separate process and is the only place that resizes images. +# API and worker export OTLP to the collector. Prometheus, Loki, Tempo and Grafana +# keep metrics, logs and traces. The collector is not required for the API to start. # API replicas do not publish host ports. MinIO console is not published. # API replicas do not publish host ports. Migrations run once in `migrate`. # Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead @@ -37,6 +39,11 @@ services: APP_PORT: "8080" APP_ENV: lab LOG_LEVEL: info + TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1} + TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-1} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + SERVICE_VERSION: lab6 SHUTDOWN_GRACE_SECONDS: "3" LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} @@ -196,6 +203,11 @@ services: environment: APP_ENV: lab LOG_LEVEL: info + TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1} + TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-1} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + SERVICE_VERSION: lab6 DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger S3_ENDPOINT: http://s3:9000 S3_REGION: ${S3_REGION:-us-east-1} @@ -250,8 +262,80 @@ services: mem_limit: 128m cpus: "0.50" + otel-collector: + image: otel/opentelemetry-collector-contrib:0.128.0 + command: ["--config=/etc/otel/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro + restart: unless-stopped + mem_limit: 320m + cpus: "0.50" + + prometheus: + image: prom/prometheus:v2.55.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-remote-write-receiver + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + ports: + - "127.0.0.1:9090:9090" + restart: unless-stopped + mem_limit: 512m + cpus: "0.50" + + loki: + image: grafana/loki:3.3.2 + command: ["-config.file=/etc/loki/loki.yml"] + volumes: + - ./infra/loki/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + restart: unless-stopped + mem_limit: 512m + cpus: "0.50" + + tempo: + image: grafana/tempo:2.7.1 + command: ["-config.file=/etc/tempo/tempo.yml"] + volumes: + - ./infra/tempo/tempo.yml:/etc/tempo/tempo.yml:ro + - tempo_data:/var/tempo + restart: unless-stopped + mem_limit: 512m + cpus: "0.50" + + grafana: + image: grafana/grafana:11.4.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-lab6-dev-only-change-me} + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_USERS_DEFAULT_THEME: light + ports: + - "127.0.0.1:3000:3000" + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + - loki + - tempo + restart: unless-stopped + mem_limit: 256m + cpus: "0.50" + volumes: pg_data: redis_data: s3_data: rabbitmq_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..7779ace --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,156 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger observability", + "tags": ["messenger", "lab6"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": {"from": "now-1h", "to": "now"}, + "description": "RPS and errors use messenger_http_requests_total. Durations are histogram_quantile over buckets, in seconds. Queue age is seconds. API error ratio excludes health.", + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "API RPS", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line"}}, "overrides": []}, + "targets": [ + { + "refId": "A", + "expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "API error ratio (5xx / business)", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 1e-9)", + "legendFormat": "5xx ratio" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "API duration p50 / p95 / p99", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))", "legendFormat": "p99"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Jobs by outcome", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "legendFormat": "{{outcome}}" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Worker duration p50 / p95 / p99", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", "legendFormat": "p99"} + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Time to ready p50 / p95 / p99", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[1m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[1m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[1m])))", "legendFormat": "p99"} + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth and oldest job age", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24}, + "fieldConfig": {"defaults": {}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max(messenger_queue_depth)", "legendFormat": "depth (jobs)"}, + {"refId": "B", "expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest age (seconds)"} + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "DB pool used and waiting", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24}, + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max by (service_name, service_instance_id) (messenger_db_pool_used)", "legendFormat": "used {{service_name}} {{service_instance_id}}"}, + {"refId": "B", "expr": "max by (service_name, service_instance_id) (messenger_db_pool_waiting)", "legendFormat": "waiting {{service_name}} {{service_instance_id}}"} + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Worker busy / capacity", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max(messenger_worker_busy)", "legendFormat": "busy"}, + {"refId": "B", "expr": "max(messenger_worker_capacity)", "legendFormat": "capacity"} + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Process CPU and RSS", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, + "fieldConfig": {"defaults": {}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max by (service_name, service_instance_id) (process_cpu_percent)", "legendFormat": "cpu % {{service_name}} {{service_instance_id}}"}, + {"refId": "B", "expr": "max by (service_name, service_instance_id) (process_memory_rss_bytes)", "legendFormat": "rss {{service_name}} {{service_instance_id}}"} + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Firing alerts", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 40}, + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "ALERTS{alertstate=\"firing\"}", "legendFormat": "{{alertname}}"} + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..60c50f5 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yml b/infra/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..4675dfa --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,38 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 15s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([0-9a-f]+)"' + name: trace_id + url: "$${__value.raw}" + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true diff --git a/infra/grafana/runbooks/high-error-rate.md b/infra/grafana/runbooks/high-error-rate.md new file mode 100644 index 0000000..fa1d398 --- /dev/null +++ b/infra/grafana/runbooks/high-error-rate.md @@ -0,0 +1,9 @@ +# High API error rate + +Signal: `messenger_http_requests_total`. Numerator is `status_class="5xx"`. Denominator is every instrumented business request. `/health/*`, `/docs` and `/openapi.json` are not in this ratio, so a bad password (401) is not an incident. + +1. Open the Messenger dashboard and compare the 5xx ratio with p95 duration. +2. In Tempo, open a server span with `http.response.status_code >= 500`. The span attribute `http.route` is the template, not a user id. +3. In Loki, query `{service_name="messenger-api"}` and match `trace_id` from that span. The log line has `event`, `severity`, `trace_id` and `span_id`. +4. If the span stops inside the database client or the pool-used gauge is at `DB_POOL_MAX` with waiters, the cause is Postgres, not the HTTP handler itself. +5. After the dependency recovers, the 5m ratio falls and `HighApiErrorRate` leaves `firing`. diff --git a/infra/grafana/runbooks/queue-lag.md b/infra/grafana/runbooks/queue-lag.md new file mode 100644 index 0000000..34f33ff --- /dev/null +++ b/infra/grafana/runbooks/queue-lag.md @@ -0,0 +1,8 @@ +# Queue lag + +Signal: `messenger_queue_oldest_age_seconds` is `now() - min(created_at)` for rows still `queued`. `messenger_queue_depth` counts `queued` and `processing`. Age is not the same number as worker duration. + +1. If age grows and `messenger_worker_busy` is 0, the worker process is stopped or not consuming. Messages stay in RabbitMQ and in `attachment_jobs`. +2. If busy equals capacity and age still grows, the worker is saturated. Look at `messenger_job_duration_seconds` and the consumer span. +3. A consumer span's parent is the producer span, whose parent is the API server span that accepted the upload (`traceparent` stored on the job row). +4. Start the worker again. Depth and age return toward zero, then `QueueLagHigh` resolves. Do not delete the RabbitMQ volume. diff --git a/infra/grafana/runbooks/slo-burn.md b/infra/grafana/runbooks/slo-burn.md new file mode 100644 index 0000000..ec5487b --- /dev/null +++ b/infra/grafana/runbooks/slo-burn.md @@ -0,0 +1,13 @@ +# SLO burn + +Two SLOs, each with a 1% error budget over 30 days. + +- API availability: business requests that are not 5xx. Fast burn pages when both the 5 minute and 1 hour windows exceed 14.4% errors (14.4x budget). Slow burn warns when both 30 minutes and 6 hours exceed 6% (6x budget). +- Time to ready: jobs whose ready time is at most 30 seconds, read from the `le="30"` histogram bucket, not from an average. The same 14.4x and 6x windows apply. + +A short spike can trip only the fast pair. A long degradation keeps the slow pair firing after the spike is gone. Both windows must agree, so one noisy minute does not page by itself. + +1. Read which alert is firing and which SLO label it has. +2. For `api-availability`, follow `high-error-rate.md`. Separate 4xx from 5xx before calling it an incident. +3. For `time-to-ready`, follow `queue-lag.md`, then open the worker trace and see whether time is in `s3.get`, `image.render`, `s3.put` or `db.complete`. +4. Record the first firing timestamp and the first resolved timestamp. That pair is time-to-detect and time-to-recover for the experiment. diff --git a/infra/loki/loki.yml b/infra/loki/loki.yml new file mode 100644 index 0000000..477b68a --- /dev/null +++ b/infra/loki/loki.yml @@ -0,0 +1,30 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..3278ba3 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,11 @@ -# Конфигурация наблюдаемости — реализовать +# Collector -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +`collector.yaml` — OTel Collector Contrib 0.128.0. Принимает OTLP/HTTP `:4318` и OTLP/gRPC `:4317` только внутри сети Compose. -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +- traces → Tempo `http://tempo:4318/v1/traces` +- metrics → Prometheus remote write `http://prometheus:9090/api/v1/write` +- logs → Loki OTLP `http://loki:3100/otlp/v1/logs` -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +`memory_limiter` (192 MiB) и очереди exporter ограничены. Если backend недоступен, collector сбрасывает телеметрию после повторных попыток и не хранит её вечно. API от этого не зависит: SDK тоже пишет в ограниченную очередь и продолжает отвечать на запросы. + +Grafana provisioning и dashboard лежат в `infra/grafana`. Правила Prometheus — в `infra/prometheus/alerts.yml`. SLO — в `infra/slo.md`. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..b79deb1 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,89 @@ +# Collector 0.128.0. It forwards signals and is not a database. +# memory_limiter refuses data before the process grows without bound. +# Exporter queues are capped; a dead backend drops telemetry after retry, the API keeps serving. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + timeout: 5s + send_batch_size: 256 + send_batch_max_size: 512 + +exporters: + otlphttp/tempo: + traces_endpoint: http://tempo:4318/v1/traces + tls: + insecure: true + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 1000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true + remote_write_queue: + enabled: true + queue_size: 1000 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + otlphttp/loki: + logs_endpoint: http://loki:3100/otlp/v1/logs + tls: + insecure: true + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 1000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + telemetry: + logs: + level: info + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/prometheus/alerts.yml b/infra/prometheus/alerts.yml new file mode 100644 index 0000000..e431113 --- /dev/null +++ b/infra/prometheus/alerts.yml @@ -0,0 +1,112 @@ +groups: + - name: messenger + rules: + - alert: HighApiErrorRate + expr: | + ( + sum(rate(messenger_http_requests_total{status_class="5xx"}[5m])) + / + clamp_min(sum(rate(messenger_http_requests_total[5m])), 1e-9) + ) > 0.05 + for: 1m + labels: + severity: warning + slo: api-availability + annotations: + summary: API 5xx ratio is above 5% over 5 minutes + description: Numerator is status_class=5xx. Denominator is instrumented business requests. Health, docs and openapi are excluded. + runbook: infra/grafana/runbooks/high-error-rate.md + - alert: QueueLagHigh + expr: max(messenger_queue_oldest_age_seconds) > 30 + for: 30s + labels: + severity: warning + slo: time-to-ready + annotations: + summary: Oldest queued job is older than 30 seconds + description: Age is now minus created_at of jobs still in status queued. Depth is a separate gauge. + runbook: infra/grafana/runbooks/queue-lag.md + - alert: ApiAvailabilityFastBurn + expr: | + ( + sum(rate(messenger_http_requests_total{status_class="5xx"}[5m])) + / clamp_min(sum(rate(messenger_http_requests_total[5m])), 1e-9) + ) > 0.144 + and + ( + sum(rate(messenger_http_requests_total{status_class="5xx"}[1h])) + / clamp_min(sum(rate(messenger_http_requests_total[1h])), 1e-9) + ) > 0.144 + for: 2m + labels: + severity: critical + slo: api-availability + window: fast + annotations: + summary: API error budget burn rate is above 14.4x on both 5m and 1h + runbook: infra/grafana/runbooks/slo-burn.md + - alert: ApiAvailabilitySlowBurn + expr: | + ( + sum(rate(messenger_http_requests_total{status_class="5xx"}[30m])) + / clamp_min(sum(rate(messenger_http_requests_total[30m])), 1e-9) + ) > 0.06 + and + ( + sum(rate(messenger_http_requests_total{status_class="5xx"}[6h])) + / clamp_min(sum(rate(messenger_http_requests_total[6h])), 1e-9) + ) > 0.06 + for: 15m + labels: + severity: warning + slo: api-availability + window: slow + annotations: + summary: API error budget burn rate is above 6x on both 30m and 6h + runbook: infra/grafana/runbooks/slo-burn.md + - alert: TimeToReadyFastBurn + expr: | + ( + 1 - ( + sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[5m])) + / clamp_min(sum(rate(messenger_time_to_ready_seconds_count[5m])), 1e-9) + ) + ) > 0.144 + and + ( + 1 - ( + sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[1h])) + / clamp_min(sum(rate(messenger_time_to_ready_seconds_count[1h])), 1e-9) + ) + ) > 0.144 + for: 2m + labels: + severity: critical + slo: time-to-ready + window: fast + annotations: + summary: Share of jobs slower than 30s is burning the ready-time budget (5m and 1h) + runbook: infra/grafana/runbooks/slo-burn.md + - alert: TimeToReadySlowBurn + expr: | + ( + 1 - ( + sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[30m])) + / clamp_min(sum(rate(messenger_time_to_ready_seconds_count[30m])), 1e-9) + ) + ) > 0.06 + and + ( + 1 - ( + sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[6h])) + / clamp_min(sum(rate(messenger_time_to_ready_seconds_count[6h])), 1e-9) + ) + ) > 0.06 + for: 15m + labels: + severity: warning + slo: time-to-ready + window: slow + annotations: + summary: Share of jobs slower than 30s is burning the ready-time budget (30m and 6h) + runbook: infra/grafana/runbooks/slo-burn.md diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..2808750 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,6 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml diff --git a/infra/slo.md b/infra/slo.md new file mode 100644 index 0000000..a6517ae --- /dev/null +++ b/infra/slo.md @@ -0,0 +1,12 @@ +# SLI и SLO + +Окно цели: 30 суток. Бюджет ошибок: 1% у каждого SLO. Пороги burn rate — 14.4x (быстрое выгорание, окна 5 минут и 1 час) и 6x (медленное, окна 30 минут и 6 часов). Оба окна пары должны выполниться одновременно. + +| SLI | Хорошее событие | SLO | Единица | +| --- | --- | --- | --- | +| Доступность API | бизнес-запрос со `status_class` не `5xx` | 99% | доля запросов | +| Время до ready | job, у которой `messenger_time_to_ready_seconds` попала в bucket `le=30` | 99% готовы не дольше 30 секунд | секунды от durable acceptance до `ready` | + +Health, `/docs` и `/openapi.json` в SLI API не входят. 401 и 404 остаются в знаменателе, но не в числителе ошибок: это не отказ сервиса. Процентили дашборда считаются `histogram_quantile` по bucket, не средним значений и не средним p99 реплик. + +Правила лежат в `infra/prometheus/alerts.yml`. Runbook: `infra/grafana/runbooks/`. diff --git a/infra/tempo/tempo.yml b/infra/tempo/tempo.yml new file mode 100644 index 0000000..d013768 --- /dev/null +++ b/infra/tempo/tempo.yml @@ -0,0 +1,28 @@ +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/migrations/007_job_trace.sql b/migrations/007_job_trace.sql new file mode 100644 index 0000000..2f0780d --- /dev/null +++ b/migrations/007_job_trace.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachment_jobs + ADD COLUMN traceparent TEXT, + ADD COLUMN tracestate TEXT; diff --git a/requirements.txt b/requirements.txt index f8e8853..d374c53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,7 @@ argon2-cffi==23.1.0 boto3==1.35.99 pillow==11.1.0 aio-pika==9.5.5 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 +psutil==6.1.1