From 307b658bfb5049bcb29c30a770097fa397a74475 Mon Sep 17 00:00:00 2001 From: aegor Date: Fri, 20 Nov 2026 12:55:53 +0300 Subject: [PATCH] feat: instrument api, relay and worker with opentelemetry --- compose.yaml | 11 + infra/grafana/dashboards/messenger.json | 235 ++++++ package-lock.json | 820 +++++++++++++++++++- package.json | 15 + scripts/dashboard-check.py | 57 ++ scripts/latency-compare.py | 52 ++ scripts/trace-check.py | 134 ++++ src/common/filters/http-exception.filter.ts | 12 +- src/health/shutdown.ts | 15 +- src/jobs/job-message.ts | 33 +- src/jobs/processor.ts | 23 +- src/main.ts | 5 + src/relay.ts | 45 +- src/telemetry/http.ts | 66 ++ src/telemetry/json-logger.ts | 99 +++ src/telemetry/register.ts | 3 + src/telemetry/sdk.ts | 157 ++++ src/worker.ts | 110 ++- 18 files changed, 1843 insertions(+), 49 deletions(-) create mode 100644 infra/grafana/dashboards/messenger.json create mode 100755 scripts/dashboard-check.py create mode 100755 scripts/latency-compare.py create mode 100755 scripts/trace-check.py create mode 100644 src/telemetry/http.ts create mode 100644 src/telemetry/json-logger.ts create mode 100644 src/telemetry/register.ts create mode 100644 src/telemetry/sdk.ts diff --git a/compose.yaml b/compose.yaml index 7e95960..c5087a0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,10 @@ name: messenger-lab6 +x-otel: &otel + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + services: secrets-init: image: alpine:3.20.3 @@ -168,6 +173,8 @@ services: MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + <<: *otel + OTEL_SERVICE_NAME: messenger-api volumes: - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } } deploy: @@ -244,6 +251,8 @@ services: BROKER_URL_FILE: /run/secrets/broker_url JOB_STALE_SECONDS: ${JOB_STALE_SECONDS:-300} RELAY_FAULT: ${RELAY_FAULT:-} + <<: *otel + OTEL_SERVICE_NAME: messenger-relay volumes: - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: relay } } deploy: @@ -279,6 +288,8 @@ services: JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} WORKER_FAULT: ${WORKER_FAULT:-} + <<: *otel + OTEL_SERVICE_NAME: messenger-worker volumes: - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: worker } } deploy: diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..d935127 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,235 @@ +{ + "uid": "messenger-overview", + "title": "Messenger overview", + "tags": ["messenger", "lab6"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": false, + "graphTooltip": 1, + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "templating": { "list": [] }, + "annotations": { "list": [] }, + "panels": [ + { + "type": "row", + "title": "API", + "id": 100, + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "type": "timeseries", + "title": "RPS по маршрутам (без health)", + "id": 1, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum by (http_request_method, http_route) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))", + "legendFormat": "{{http_request_method}} {{http_route}}" + } + ] + }, + { + "type": "timeseries", + "title": "Ответы по классам статуса (без health)", + "id": 2, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum by (http_response_status_class) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))", + "legendFormat": "{{http_response_status_class}}" + } + ] + }, + { + "type": "timeseries", + "title": "Доля 5xx среди бизнес-запросов (окно 5m)", + "id": 3, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.05 }] }, + "custom": { "thresholdsStyle": { "mode": "line" } } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "(sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\", http_response_status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[5m]))", + "legendFormat": "5xx ratio" + } + ] + }, + { + "type": "timeseries", + "title": "Длительность API p50 / p95 / p99 (все реплики, без health)", + "id": 4, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 }, + "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ] + }, + { + "type": "timeseries", + "title": "p95 по маршрутам", + "id": 5, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 }, + "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum by (le, http_request_method, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))", + "legendFormat": "{{http_request_method}} {{http_route}}" + } + ] + }, + { + "type": "row", + "title": "Worker", + "id": 101, + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 } + }, + { + "type": "timeseries", + "title": "Задания по исходу (в секунду)", + "id": 6, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum by (job_type, outcome) (rate(jobs_processed_total{job=\"messenger-worker\"}[$__rate_interval]))", + "legendFormat": "{{job_type}} {{outcome}}" + } + ] + }, + { + "type": "timeseries", + "title": "Длительность обработки задания p50 / p95 / p99", + "id": 7, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))", + "legendFormat": "p50 success" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))", + "legendFormat": "p95 success" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))", + "legendFormat": "p99 success" + }, + { + "refId": "D", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome!=\"success\"}[$__rate_interval])))", + "legendFormat": "p95 retry/failed" + } + ] + }, + { + "type": "row", + "title": "Логи и трассы", + "id": 102, + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 } + }, + { + "type": "logs", + "title": "Логи API и worker (WARN и выше)", + "id": 8, + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 27 }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\"" + } + ] + }, + { + "type": "logs", + "title": "Все логи (http.request, job.handled, …)", + "id": 9, + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 27 }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=~\"messenger-.+\"}" + } + ] + }, + { + "type": "table", + "title": "Последние трассы (HTTP server и consumer spans)", + "id": 10, + "datasource": { "type": "tempo", "uid": "tempo" }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "tempo", "uid": "tempo" }, + "queryType": "traceql", + "query": "{resource.service.name=~\"messenger-.+\" && (kind=server || kind=consumer)}", + "limit": 20, + "tableType": "traces" + } + ] + } + ] +} diff --git a/package-lock.json b/package-lock.json index d306d52..d7834ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,22 @@ "@nestjs/core": "^10.4.15", "@nestjs/platform-fastify": "^10.4.15", "@nestjs/swagger": "7.4.2", + "@opentelemetry/api": "1.9.0", + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/exporter-logs-otlp-proto": "0.57.2", + "@opentelemetry/exporter-metrics-otlp-proto": "0.57.2", + "@opentelemetry/exporter-trace-otlp-proto": "0.57.2", + "@opentelemetry/instrumentation-aws-sdk": "0.49.1", + "@opentelemetry/instrumentation-http": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-logs": "0.57.2", + "@opentelemetry/sdk-metrics": "1.30.1", + "@opentelemetry/sdk-node": "0.57.2", + "@opentelemetry/sdk-trace-base": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0", "@prisma/client": "6.1.0", + "@prisma/instrumentation": "6.1.0", "amqplib": "0.10.5", "argon2": "0.41.1", "class-transformer": "^0.5.1", @@ -1961,6 +1976,37 @@ "glob": "^10.3.4" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.7.tgz", + "integrity": "sha512-W4W/fcBMGRVtG6dW2c7cw2mnpS3hvbBd+oQXjh2iYFTZDk1JJmcNqkM5KSaPubQpamlUUurrlb7G1s9NJeERmQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", @@ -3016,6 +3062,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@ljharb/through": { "version": "2.3.14", "resolved": "https://registry.npmjs.org/@ljharb/through/-/through-2.3.14.tgz", @@ -3484,6 +3540,556 @@ "npm": ">=5.0.0" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.57.2.tgz", + "integrity": "sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.30.1.tgz", + "integrity": "sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.57.2.tgz", + "integrity": "sha512-eovEy10n3umjKJl2Ey6TLzikPE+W4cUQ4gCwgGP1RqzTGtgDra0WjIqdy29ohiUKfvmbiL3MndZww58xfIvyFw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-grpc-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/sdk-logs": "0.57.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.57.2.tgz", + "integrity": "sha512-0rygmvLcehBRp56NQVLSleJ5ITTduq/QfU7obOkyWgPpFHulwpw2LYTqNIz5TczKZuy5YY+5D3SDnXZL1tXImg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/sdk-logs": "0.57.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.57.2.tgz", + "integrity": "sha512-ta0ithCin0F8lu9eOf4lEz9YAScecezCHkMMyDkvd9S7AnZNX5ikUmC5EQOQADU+oCcgo/qkQIaKcZvQ0TYKDw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-logs": "0.57.2", + "@opentelemetry/sdk-trace-base": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.57.2.tgz", + "integrity": "sha512-r70B8yKR41F0EC443b5CGB4rUaOMm99I5N75QQt6sHKxYDzSEc6gm48Diz1CI1biwa5tDPznpylTrywO/pT7qw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.57.2", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-grpc-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-metrics": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.57.2.tgz", + "integrity": "sha512-ttb9+4iKw04IMubjm3t0EZsYRNWr3kg44uUuzfo9CaccYlOh8cDooe4QObDUkvx9d5qQUrbEckhrWKfJnKhemA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-metrics": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.57.2.tgz", + "integrity": "sha512-HX068Q2eNs38uf7RIkNN9Hl4Ynl+3lP0++KELkXMCpsCbFO03+0XNNZ1SkwxPlP9jrhQahsMPMkzNXpq3fKsnw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.57.2", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-metrics": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.57.2.tgz", + "integrity": "sha512-VqIqXnuxWMWE/1NatAGtB1PvsQipwxDcdG4RwA/umdBcW3/iOHp0uejvFHTRN2O78ZPged87ErJajyUBPUhlDQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-metrics": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.57.2.tgz", + "integrity": "sha512-gHU1vA3JnHbNxEXg5iysqCWxN9j83d7/epTYBZflqQnTyCC4N7yZXn/dMM+bEmyhQPGjhCkNZLx4vZuChH1PYw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-grpc-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.57.2.tgz", + "integrity": "sha512-sB/gkSYFu+0w2dVQ0PWY9fAMl172PKMZ/JrHkkW8dmjCL0CYkmXeE+ssqIL/yBUTPOvpLIpenX5T9RwXRBW/3g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.57.2.tgz", + "integrity": "sha512-awDdNRMIwDvUtoRYxRhja5QYH6+McBLtoz1q9BeEsskhZcrGmH/V1fWpGx8n+Rc+542e8pJA6y+aullbIzQmlw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.30.1.tgz", + "integrity": "sha512-6S2QIMJahIquvFaaxmcwpvQQRD/YFaMTNoIxrfPIPOeITN+a8lfEcPDxNxn8JDAaxkg+4EnXhz8upVDYenoQjA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.57.2.tgz", + "integrity": "sha512-BdBGhQBh8IjZ2oIIX6F2/Q3LKm/FDDKi6ccYKcBTeilh6SNdNKveDOLk73BkSJjQLJk6qe4Yh+hHw1UPhCDdrg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@types/shimmer": "^1.2.0", + "import-in-the-middle": "^1.8.1", + "require-in-the-middle": "^7.1.1", + "semver": "^7.5.2", + "shimmer": "^1.2.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-aws-sdk": { + "version": "0.49.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-aws-sdk/-/instrumentation-aws-sdk-0.49.1.tgz", + "integrity": "sha512-Vbj4BYeV/1K4Pbbfk+gQ8gwYL0w+tBeUwG88cOxnF7CLPO1XnskGV8Q3Gzut2Ah/6Dg17dBtlzEqL3UiFP2Z6A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^1.8.0", + "@opentelemetry/instrumentation": "^0.57.1", + "@opentelemetry/propagation-utils": "^0.30.16", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.57.2.tgz", + "integrity": "sha512-1Uz5iJ9ZAlFOiPuwYg29Bf7bJJc/GeoeJIFKJYQf67nTVKFe8RHbEtxgkOmK4UGZNHKXcpW4P8cWBYzBn1USpg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/instrumentation": "0.57.2", + "@opentelemetry/semantic-conventions": "1.28.0", + "forwarded-parse": "2.1.2", + "semver": "^7.5.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.57.2.tgz", + "integrity": "sha512-XdxEzL23Urhidyebg5E6jZoaiW5ygP/mRjxLHixogbqwDy2Faduzb5N0o/Oi+XTIJu+iyxXdVORjXax+Qgfxag==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-transformer": "0.57.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.57.2.tgz", + "integrity": "sha512-USn173KTWy0saqqRB5yU9xUZ2xdgb1Rdu5IosJnm9aV4hMTuFFRTUsQxbgc24QxpCHeoKzzCSnS/JzdV0oM2iQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/otlp-exporter-base": "0.57.2", + "@opentelemetry/otlp-transformer": "0.57.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.57.2.tgz", + "integrity": "sha512-48IIRj49gbQVK52jYsw70+Jv+JbahT8BqT2Th7C4H7RCM9d0gZ5sgNPoMpWldmfjvIsSgiGJtjfk9MeZvjhoig==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-logs": "0.57.2", + "@opentelemetry/sdk-metrics": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "protobufjs": "^7.3.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/propagation-utils": { + "version": "0.30.16", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagation-utils/-/propagation-utils-0.30.16.tgz", + "integrity": "sha512-ZVQ3Z/PQ+2GQlrBfbMMMT0U7MzvYZLCPP800+ooyaBqm4hMvuQHfP028gB9/db0mwkmyEAMad9houukUVxhwcw==", + "deprecated": "The use of process spans has been removed from Messaging Semantic Conventions. It is now recommended to connect pub/sub spans via Span Links. See https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ for details.", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/propagator-b3": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.30.1.tgz", + "integrity": "sha512-oATwWWDIJzybAZ4pO76ATN5N6FFbOA1otibAVlS8v90B4S1wClnhRUk7K+2CHAwN1JKYuj4jh/lpCEG5BAqFuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.30.1.tgz", + "integrity": "sha512-Pj/BfnYEKIOImirH76M4hDaBSx6HyZ2CXUqk+Kj02m6BB80c/yo4BdWkn/1gDFfU+YPY+bPR2U0DKBfdxCKwmg==", + "deprecated": "Package no longer supported. See https://github.com/open-telemetry/opentelemetry-js/pull/7077", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz", + "integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.57.2.tgz", + "integrity": "sha512-TXFHJ5c+BKggWbdEQ/inpgIzEmS2BGQowLE9UhsMd7YYlUfBQJ4uax0VF/B5NYigdM/75OoJGhAV3upEhK+3gg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.30.1.tgz", + "integrity": "sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node": { + "version": "0.57.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.57.2.tgz", + "integrity": "sha512-8BaeqZyN5sTuPBtAoY+UtKwXBdqyuRKmekN5bFzAO40CgbGzAxfTpiL3PBerT7rhZ7p2nBdq7FaMv/tBQgHE4A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/exporter-logs-otlp-grpc": "0.57.2", + "@opentelemetry/exporter-logs-otlp-http": "0.57.2", + "@opentelemetry/exporter-logs-otlp-proto": "0.57.2", + "@opentelemetry/exporter-metrics-otlp-grpc": "0.57.2", + "@opentelemetry/exporter-metrics-otlp-http": "0.57.2", + "@opentelemetry/exporter-metrics-otlp-proto": "0.57.2", + "@opentelemetry/exporter-prometheus": "0.57.2", + "@opentelemetry/exporter-trace-otlp-grpc": "0.57.2", + "@opentelemetry/exporter-trace-otlp-http": "0.57.2", + "@opentelemetry/exporter-trace-otlp-proto": "0.57.2", + "@opentelemetry/exporter-zipkin": "1.30.1", + "@opentelemetry/instrumentation": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-logs": "0.57.2", + "@opentelemetry/sdk-metrics": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "@opentelemetry/sdk-trace-node": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.30.1.tgz", + "integrity": "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.30.1.tgz", + "integrity": "sha512-cBjYOINt1JxXdpw1e5MlHmFRc5fgj4GW/86vsKFxJCJ8AL4PdVtYH41gWwl4qd4uQjqEL1oJVrXkSy5cnduAnQ==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "1.30.1", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/propagator-b3": "1.30.1", + "@opentelemetry/propagator-jaeger": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "semver": "^7.5.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@paralleldrive/cuid2": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz", @@ -3587,6 +4193,107 @@ "@prisma/debug": "6.1.0" } }, + "node_modules/@prisma/instrumentation": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.1.0.tgz", + "integrity": "sha512-mNdBeKOvKeXWFmowPNVEciGzpWjd7o3hkTVZmbjuMhA1ZHQksS3Am4HposiAmvqpuYnCJ2h79sJsv2kW0CqlDg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.52.0 || ^0.53.0 || ^0.54.0 || ^0.55.0 || ^0.56.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.8" + } + }, + "node_modules/@prisma/instrumentation/node_modules/@opentelemetry/api-logs": { + "version": "0.56.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.56.0.tgz", + "integrity": "sha512-Wr39+94UNNG3Ei9nv3pHd4AJ63gq5nSemMRpCd8fPwDL9rN3vK26lzxfH27mw16XzOSO+TpyQwBAMaLxaPWG0g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@prisma/instrumentation/node_modules/@opentelemetry/instrumentation": { + "version": "0.56.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.56.0.tgz", + "integrity": "sha512-2KkGBKE+FPXU1F0zKww+stnlUxUTlBvLCiWdP63Z9sqXYeNI/ziNzsxAp4LAdUcTQmXjw1IWgvm5CAb/BHy99w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.56.0", + "@types/shimmer": "^1.2.0", + "import-in-the-middle": "^1.8.1", + "require-in-the-middle": "^7.1.1", + "semver": "^7.5.2", + "shimmer": "^1.2.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, "node_modules/@sinclair/typebox": { "version": "0.27.12", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.12.tgz", @@ -4513,12 +5220,17 @@ "version": "20.19.43", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, + "node_modules/@types/shimmer": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.2.0.tgz", + "integrity": "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg==", + "license": "MIT" + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -4998,7 +5710,6 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", - "dev": true, "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -5007,6 +5718,15 @@ "node": ">=0.4.0" } }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -5749,7 +6469,6 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", - "dev": true, "license": "MIT" }, "node_modules/class-transformer": { @@ -5825,7 +6544,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -5840,7 +6558,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -6377,7 +7094,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6424,7 +7140,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -7227,6 +7942,12 @@ "node": ">= 0.6" } }, + "node_modules/forwarded-parse": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", + "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", + "license": "MIT" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -7275,7 +7996,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7295,7 +8015,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -7574,7 +8293,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -7676,6 +8394,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/import-in-the-middle": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.15.0.tgz", + "integrity": "sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.14.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^1.2.2", + "module-details-from-path": "^1.0.3" + } + }, "node_modules/import-local": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", @@ -7808,7 +8538,6 @@ "version": "2.17.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz", "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==", - "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.4" @@ -8985,6 +9714,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -9028,6 +9763,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -9424,6 +10165,12 @@ "obliterator": "^2.0.1" } }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -9767,7 +10514,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, "license": "MIT" }, "node_modules/path-scurry": { @@ -10048,6 +10794,29 @@ "node": ">= 6" } }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", @@ -10239,7 +11008,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -10254,6 +11022,20 @@ "node": ">=0.10.0" } }, + "node_modules/require-in-the-middle": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.5.2.tgz", + "integrity": "sha512-gAZ+kLqBdHarXB64XpAe2VCjB7rIRv+mU8tfRWziHRJ5umKsIHN2tLLv6EtMw7WCdP19S0ERVMldNvxYCHnhSQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3", + "resolve": "^1.22.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, "node_modules/requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", @@ -10264,7 +11046,6 @@ "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -10690,6 +11471,12 @@ "node": ">=8" } }, + "node_modules/shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==", + "license": "BSD-2-Clause" + }, "node_modules/side-channel": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", @@ -11095,7 +11882,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -11817,7 +12603,6 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, "node_modules/universalify": { @@ -12227,7 +13012,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -12244,7 +13028,6 @@ "version": "17.7.3", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -12263,7 +13046,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" diff --git a/package.json b/package.json index b392238..dc58a83 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,22 @@ "@nestjs/core": "^10.4.15", "@nestjs/platform-fastify": "^10.4.15", "@nestjs/swagger": "7.4.2", + "@opentelemetry/api": "1.9.0", + "@opentelemetry/api-logs": "0.57.2", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/exporter-logs-otlp-proto": "0.57.2", + "@opentelemetry/exporter-metrics-otlp-proto": "0.57.2", + "@opentelemetry/exporter-trace-otlp-proto": "0.57.2", + "@opentelemetry/instrumentation-aws-sdk": "0.49.1", + "@opentelemetry/instrumentation-http": "0.57.2", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-logs": "0.57.2", + "@opentelemetry/sdk-metrics": "1.30.1", + "@opentelemetry/sdk-node": "0.57.2", + "@opentelemetry/sdk-trace-base": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0", "@prisma/client": "6.1.0", + "@prisma/instrumentation": "6.1.0", "amqplib": "0.10.5", "argon2": "0.41.1", "class-transformer": "^0.5.1", diff --git a/scripts/dashboard-check.py b/scripts/dashboard-check.py new file mode 100755 index 0000000..c5c1f5e --- /dev/null +++ b/scripts/dashboard-check.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels. + +Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py +""" +import json +import os +import sys +import time +import urllib.parse +import urllib.request + +GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') +DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json') + + +def get(path, **params): + url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '') + with urllib.request.urlopen(url, timeout=15) as response: + return json.loads(response.read()) + + +def run(target): + uid = target['datasource']['uid'] + now = int(time.time()) + if uid == 'prometheus': + expr = target['expr'].replace('$__rate_interval', '1m') + body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30) + series = body['data']['result'] + last = [s['values'][-1][1] for s in series if s['values']] + return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v) + for s, v in zip(series[:4], last[:4])) + if uid == 'loki': + body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m') + return sum(len(s['values']) for s in body['data']['result']), '' + if uid == 'tempo': + body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now) + return len(body.get('traces', [])), '' + return 0, 'unknown datasource' + + +def main(): + grafana = get('/api/dashboards/uid/messenger-overview') + print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle')) + dashboard = json.load(open(DASHBOARD, encoding='utf-8')) + empty = 0 + for panel in dashboard['panels']: + for target in panel.get('targets', []): + count, sample = run(target) + empty += count == 0 + print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70])) + print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty)) + sys.exit(0 if empty == 0 else 1) + + +if __name__ == '__main__': + main() diff --git a/scripts/latency-compare.py b/scripts/latency-compare.py new file mode 100755 index 0000000..edf7914 --- /dev/null +++ b/scripts/latency-compare.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus. + +Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30] +""" +import json +import os +import subprocess +import sys +import tempfile +import time +import urllib.parse +import urllib.request + +GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') +BASE = os.environ.get('BASE_URL', 'http://lb:8080') +SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}' + + +def prom(expr, at): + params = urllib.parse.urlencode({'query': expr, 'time': at}) + with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r: + result = json.loads(r.read())['data']['result'] + return float(result[0]['value'][1]) if result else None + + +def main(): + duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30' + output = os.path.join(tempfile.mkdtemp(), 'load.json') + started = time.time() + subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output], + check=True, stdout=subprocess.DEVNULL) + report = json.load(open(output)) + print('waiting 25 s for the metric export interval') + time.sleep(25) + at = time.time() + window = int(at - started) + 5 + print('client (tests/load.py): %d requests, %.1f rps, errors %d' % ( + report['successful_requests'], report['successful_rps'], report['failed_requests'])) + count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at) + print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window)) + print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms')) + for p in ('p50', 'p95', 'p99'): + q = int(p[1:]) / 100 + server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at) + print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000)) + print('server-side histogram excludes network/client overhead; values are interpolated inside buckets ' + '(…, 5, 10, 25, 50, 100 ms, …).') + + +if __name__ == '__main__': + main() diff --git a/scripts/trace-check.py b/scripts/trace-check.py new file mode 100755 index 0000000..68f6a97 --- /dev/null +++ b/scripts/trace-check.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki. + +Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py +""" +import base64 +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +BASE = os.environ.get('BASE_URL', 'http://localhost:8080') +GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') +SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read() + + +def call(method, url, body=None, headers=None, raw=None): + data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) + all_headers = dict(headers or {}) + if body is not None: + all_headers['Content-Type'] = 'application/json' + request = urllib.request.Request(url, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read() or b'null') + except urllib.error.HTTPError as error: + return error.code, None + + +def session(): + username = 'trace_' + uuid.uuid4().hex[:8] + password = 'Trace-check-' + uuid.uuid4().hex + _, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password}) + credential = base64.b64encode((username + ':' + password).encode()).decode() + _, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential}) + auth = {'Authorization': 'Bearer ' + login['token']} + _, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth) + return auth, chat['id'] + + +def wait_trace(trace_id, services, timeout=60): + deadline = time.monotonic() + timeout + spans = [] + while time.monotonic() < deadline: + code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id) + if code == 200 and body: + spans = [] + for batch in body.get('batches', []): + attrs = {a['key']: a['value'] for a in batch['resource']['attributes']} + service = attrs['service.name']['stringValue'] + for scope in batch.get('scopeSpans', []): + for span in scope.get('spans', []): + spans.append((service, span)) + if services <= {service for service, _ in spans}: + return spans + time.sleep(3) + return spans + + +def b64hex(value): + return base64.b64decode(value).hex() if value else '' + + +def main(): + trace_id = os.urandom(16).hex() + client_span = os.urandom(8).hex() + traceparent = '00-%s-%s-01' % (trace_id, client_span) + auth, chat = session() + started = time.monotonic() + code, attachment = call( + 'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat, + headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE, + ) + print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent) + status = None + while time.monotonic() - started < 60: + _, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth) + status = body['status'] + if status in ('ready', 'failed'): + break + time.sleep(0.5) + print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started)) + + spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'}) + by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans} + print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans))) + for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])): + if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'): + continue + parent = b64hex(span.get('parentSpanId')) + parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name'] + duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6 + print(' %-17s %-22s %-48s %8.2f ms parent=%s' % ( + service, span.get('kind', ''), span['name'][:48], duration, parent_name)) + + query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id + params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'}) + streams = [] + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + _, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params) + streams = (logs or {}).get('data', {}).get('result', []) + if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}: + break + time.sleep(3) + print('\nLoki %s' % query) + lines = 0 + for stream in streams: + labels = stream['stream'] + for _, line in stream['values']: + lines += 1 + print(' %-17s %-8s %-13s span_id=%s %s' % ( + labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''), + labels.get('span_id', ''), line[:60])) + + kinds = {(service, span.get('kind')) for service, span in spans} + ok = ( + status == 'ready' + and ('messenger-api', 'SPAN_KIND_SERVER') in kinds + and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds + and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds + and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans) + and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams} + ) + print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL')) + sys.exit(0 if ok else 1) + + +if __name__ == '__main__': + main() diff --git a/src/common/filters/http-exception.filter.ts b/src/common/filters/http-exception.filter.ts index 0a79a1b..911e3aa 100644 --- a/src/common/filters/http-exception.filter.ts +++ b/src/common/filters/http-exception.filter.ts @@ -8,6 +8,7 @@ import { } from '@nestjs/common'; import { FastifyReply, FastifyRequest } from 'fastify'; import { isDatabaseUnavailable } from '../../db/prisma.service'; +import { routeTemplate } from '../../telemetry/http'; import { ApiError, ErrorCode } from './api-error'; const CODE_BY_STATUS: Partial> = { @@ -39,9 +40,14 @@ export class HttpExceptionFilter implements ExceptionFilter { const { status, code, message } = this.resolve(exception); if (status >= 500) { - this.logger.error( - `${request.method} ${request.url} -> ${status}: ${message}`, - ); + this.logger.error({ + event: 'http.error', + message, + method: request.method, + route: routeTemplate(request), + status, + request_id: requestId, + }); } if (exception instanceof ApiError) { diff --git a/src/health/shutdown.ts b/src/health/shutdown.ts index 329f27d..90c28c1 100644 --- a/src/health/shutdown.ts +++ b/src/health/shutdown.ts @@ -1,4 +1,5 @@ import { INestApplication, Logger } from '@nestjs/common'; +import { shutdownTelemetry } from '../telemetry/sdk'; let draining = false; @@ -15,10 +16,16 @@ export function enableGracefulShutdown(app: INestApplication): void { draining = true; Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown'); setTimeout(() => { - app.close().then( - () => process.exit(0), - () => process.exit(1), - ); + app + .close() + .then( + () => 0, + () => 1, + ) + .then(async (code) => { + await shutdownTelemetry(); + process.exit(code); + }); }, delayMs); }; process.once('SIGTERM', () => shutdown('SIGTERM')); diff --git a/src/jobs/job-message.ts b/src/jobs/job-message.ts index 3fbeeb4..93ea6d7 100644 --- a/src/jobs/job-message.ts +++ b/src/jobs/job-message.ts @@ -1,9 +1,18 @@ +import { context, propagation } from '@opentelemetry/api'; import { validate as isUuid } from '../common/uuid'; export const PIPELINE_VERSION = 1; export const JOB_TYPE = 'attachment.process.v1'; -export interface JobMessage { +const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/; +const TRACESTATE_MAX = 512; + +export interface TraceCarrier { + traceparent?: string; + tracestate?: string; +} + +export interface JobMessage extends TraceCarrier { schema_version: 1; job_id: string; type: typeof JOB_TYPE; @@ -21,6 +30,8 @@ const FIELDS = [ 'pipeline_version', 'attempt', 'created_at', + 'traceparent', + 'tracestate', ]; export function jobMessage( @@ -35,9 +46,23 @@ export function jobMessage( pipeline_version: job.pipelineVersion, attempt: job.attempt, created_at: createdAt.toISOString(), + ...currentTraceCarrier(), }; } +export function currentTraceCarrier(): TraceCarrier { + const carrier: Record = {}; + propagation.inject(context.active(), carrier); + const result: TraceCarrier = {}; + if (TRACEPARENT.test(carrier.traceparent ?? '')) { + result.traceparent = carrier.traceparent; + if (carrier.tracestate && carrier.tracestate.length <= TRACESTATE_MAX) { + result.tracestate = carrier.tracestate; + } + } + return result; +} + export function parseJobMessage(raw: Buffer): JobMessage | null { let value: any; try { @@ -61,7 +86,11 @@ export function parseJobMessage(raw: Buffer): JobMessage | null { !Number.isInteger(value.attempt) || value.attempt < 1 || typeof value.created_at !== 'string' || - Number.isNaN(Date.parse(value.created_at)) + Number.isNaN(Date.parse(value.created_at)) || + (value.traceparent !== undefined && + (typeof value.traceparent !== 'string' || !TRACEPARENT.test(value.traceparent))) || + (value.tracestate !== undefined && + (typeof value.tracestate !== 'string' || value.tracestate.length > TRACESTATE_MAX)) ) { return null; } diff --git a/src/jobs/processor.ts b/src/jobs/processor.ts index a1f1a1b..9491ee5 100644 --- a/src/jobs/processor.ts +++ b/src/jobs/processor.ts @@ -1,4 +1,5 @@ import { Logger } from '@nestjs/common'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client'; import { createHash, randomUUID } from 'crypto'; import { @@ -50,6 +51,8 @@ class PermanentError extends Error { class TimeoutError extends Error {} +const tracer = trace.getTracer('messenger-worker'); + export class JobProcessor { private readonly logger = new Logger('JobProcessor'); @@ -141,11 +144,21 @@ export class JobProcessor { } let variants: RenderedVariant[]; try { - variants = await renderVariants( - original, - this.options.maxPixels, - Math.ceil(this.options.timeoutMs / 1000), - ); + variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => { + span.setAttribute('messenger.image.bytes', original.length); + try { + return await renderVariants( + original, + this.options.maxPixels, + Math.ceil(this.options.timeoutMs / 1000), + ); + } catch (error) { + span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' }); + throw error; + } finally { + span.end(); + } + }); } catch (error) { if (error instanceof InvalidImageError) { throw new PermanentError('invalid_image', 'image cannot be decoded'); diff --git a/src/main.ts b/src/main.ts index 4e6043c..68982df 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,3 +1,4 @@ +import './telemetry/register'; import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { @@ -8,6 +9,8 @@ import { AppModule } from './app.module'; import { configureApp } from './configure-app'; import { setupSwagger } from './docs/swagger'; import { enableGracefulShutdown } from './health/shutdown'; +import { registerHttpTelemetry } from './telemetry/http'; +import { JsonLogger } from './telemetry/json-logger'; async function bootstrap(): Promise { const app = await NestFactory.create( @@ -15,10 +18,12 @@ async function bootstrap(): Promise { new FastifyAdapter(), { bufferLogs: true, + logger: new JsonLogger(), }, ); configureApp(app); + registerHttpTelemetry(app.getHttpAdapter().getInstance()); setupSwagger(app); enableGracefulShutdown(app); diff --git a/src/relay.ts b/src/relay.ts index 5f54987..bcc43d6 100644 --- a/src/relay.ts +++ b/src/relay.ts @@ -1,9 +1,15 @@ +import './telemetry/register'; import { Logger } from '@nestjs/common'; +import { propagation, ROOT_CONTEXT, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; import { ConfirmChannel } from 'amqplib'; import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker'; import { PrismaService } from './db/prisma.service'; import { JobMessage } from './jobs/job-message'; import { recoverExpiredLeases, republishStale } from './jobs/job-store'; +import { JsonLogger } from './telemetry/json-logger'; +import { shutdownTelemetry } from './telemetry/sdk'; + +Logger.overrideLogger(new JsonLogger()); const BATCH_SIZE = 100; const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500); @@ -13,8 +19,29 @@ const FAULT = process.env.RELAY_FAULT; const logger = new Logger('OutboxRelay'); const prisma = new PrismaService(); +const tracer = trace.getTracer('messenger-relay'); let stopping = false; +function startPublishSpan(payload: JobMessage, headers: Record): Span { + const parent = propagation.extract(ROOT_CONTEXT, payload); + const span = tracer.startSpan( + `${JOB_QUEUE} publish`, + { + kind: SpanKind.PRODUCER, + attributes: { + 'messaging.system': 'rabbitmq', + 'messaging.operation.type': 'publish', + 'messaging.destination.name': JOB_QUEUE, + 'messaging.message.id': payload.job_id, + 'messaging.job.attempt': payload.attempt, + }, + }, + parent, + ); + propagation.inject(trace.setSpan(parent, span), headers); + return span; +} + async function publishBatch(channel: ConfirmChannel): Promise { return prisma.$transaction( async (tx) => { @@ -27,15 +54,30 @@ async function publishBatch(channel: ConfirmChannel): Promise { if (rows.length === 0) { return 0; } + const spans: Span[] = []; for (const { payload } of rows) { + const headers: Record = {}; + spans.push(startPublishSpan(payload, headers)); channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), { persistent: true, contentType: 'application/json', messageId: payload.job_id, type: payload.type, + headers, }); } - await channel.waitForConfirms(); + try { + await channel.waitForConfirms(); + } catch (error) { + for (const span of spans) { + span.setStatus({ code: SpanStatusCode.ERROR, message: 'broker did not confirm' }); + span.end(); + } + throw error; + } + for (const span of spans) { + span.end(); + } logger.log(`published ${rows.length} job message(s), broker confirmed`); if (FAULT === 'exit-after-confirm') { logger.warn('fault injection: exit after broker confirm, before outbox is marked'); @@ -121,6 +163,7 @@ run() .then(async () => { await prisma.$disconnect(); logger.log('stopped'); + await shutdownTelemetry(); process.exit(0); }) .catch((error) => { diff --git a/src/telemetry/http.ts b/src/telemetry/http.ts new file mode 100644 index 0000000..9f8e406 --- /dev/null +++ b/src/telemetry/http.ts @@ -0,0 +1,66 @@ +import { Logger } from '@nestjs/common'; +import { metrics, trace } from '@opentelemetry/api'; +import { FastifyInstance, FastifyRequest } from 'fastify'; + +export const LATENCY_BUCKETS = [ + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 5, 10, +]; + +const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']); + +const meter = metrics.getMeter('messenger-api'); +const requests = meter.createCounter('http.server.requests', { + description: 'HTTP requests by method, route template and status class', +}); +const duration = meter.createHistogram('http.server.request.duration', { + description: 'HTTP request duration', + unit: 's', + advice: { explicitBucketBoundaries: LATENCY_BUCKETS }, +}); +const logger = new Logger('Http'); + +export function routeTemplate(request: FastifyRequest): string { + const url = request.routeOptions?.url; + return url + ? url.replace(/:([A-Za-z0-9_]+)/g, (_, name: string) => `{${name.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)}}`) + : 'unmatched'; +} + +export function registerHttpTelemetry(instance: FastifyInstance): void { + instance.addHook('onRequest', (request, _reply, done) => { + const span = trace.getActiveSpan(); + if (span) { + const route = routeTemplate(request); + span.updateName(`${methodOf(request)} ${route}`); + span.setAttribute('http.route', route); + span.setAttribute('messenger.request_id', request.requestId); + } + done(); + }); + instance.addHook('onResponse', (request, reply, done) => { + const route = routeTemplate(request); + const method = methodOf(request); + const attributes = { + 'http.request.method': method, + 'http.route': route, + 'http.response.status_class': `${Math.floor(reply.statusCode / 100)}xx`, + }; + requests.add(1, attributes); + duration.record(reply.elapsedTime / 1000, attributes); + if (!route.startsWith('/health')) { + logger.log({ + event: 'http.request', + method, + route, + status: reply.statusCode, + duration_ms: Math.round(reply.elapsedTime * 100) / 100, + request_id: request.requestId, + }); + } + done(); + }); +} + +function methodOf(request: FastifyRequest): string { + return METHODS.has(request.method) ? request.method : '_OTHER'; +} diff --git a/src/telemetry/json-logger.ts b/src/telemetry/json-logger.ts new file mode 100644 index 0000000..181492e --- /dev/null +++ b/src/telemetry/json-logger.ts @@ -0,0 +1,99 @@ +import { LoggerService } from '@nestjs/common'; +import { trace, TraceFlags } from '@opentelemetry/api'; +import { AnyValueMap, logs, SeverityNumber } from '@opentelemetry/api-logs'; + +type Level = 'fatal' | 'error' | 'warn' | 'log' | 'debug' | 'verbose'; + +const SEVERITY: Record = { + fatal: [SeverityNumber.FATAL, 'FATAL'], + error: [SeverityNumber.ERROR, 'ERROR'], + warn: [SeverityNumber.WARN, 'WARN'], + log: [SeverityNumber.INFO, 'INFO'], + debug: [SeverityNumber.DEBUG, 'DEBUG'], + verbose: [SeverityNumber.TRACE, 'TRACE'], +}; + +type Field = string | number | boolean; + +export class JsonLogger implements LoggerService { + private readonly otel = logs.getLogger('messenger'); + + constructor(private readonly service = process.env.OTEL_SERVICE_NAME ?? 'messenger') {} + + fatal(message: unknown, ...params: unknown[]): void { + this.write('fatal', message, params); + } + + error(message: unknown, ...params: unknown[]): void { + this.write('error', message, params); + } + + warn(message: unknown, ...params: unknown[]): void { + this.write('warn', message, params); + } + + log(message: unknown, ...params: unknown[]): void { + this.write('log', message, params); + } + + debug(message: unknown, ...params: unknown[]): void { + this.write('debug', message, params); + } + + verbose(message: unknown, ...params: unknown[]): void { + this.write('verbose', message, params); + } + + private write(level: Level, message: unknown, params: unknown[]): void { + const last = params[params.length - 1]; + const logger = typeof last === 'string' ? last : 'app'; + const fields = toFields(message); + const event = typeof fields.event === 'string' ? fields.event : eventName(logger); + delete fields.event; + const [severityNumber, severityText] = SEVERITY[level]; + const active = trace.getActiveSpan()?.spanContext(); + const span = active && active.traceFlags & TraceFlags.SAMPLED ? active : undefined; + const line = JSON.stringify({ + timestamp: new Date().toISOString(), + severity: severityText, + 'service.name': this.service, + event, + logger, + ...fields, + trace_id: span?.traceId, + span_id: span?.spanId, + }); + process.stdout.write(line + '\n'); + const attributes: AnyValueMap = { event, logger }; + for (const [key, value] of Object.entries(fields)) { + if (key !== 'message') { + attributes[key] = value; + } + } + this.otel.emit({ + severityNumber, + severityText, + body: line, + attributes, + }); + } +} + +function toFields(message: unknown): Record { + if (typeof message === 'object' && message !== null && !(message instanceof Error)) { + const fields: Record = {}; + for (const [key, value] of Object.entries(message)) { + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + fields[key] = value; + } + } + return fields; + } + return { message: message instanceof Error ? message.message : String(message) }; +} + +function eventName(logger: string): string { + return logger + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .toLowerCase(); +} diff --git a/src/telemetry/register.ts b/src/telemetry/register.ts new file mode 100644 index 0000000..571273b --- /dev/null +++ b/src/telemetry/register.ts @@ -0,0 +1,3 @@ +import { startTelemetry } from './sdk'; + +startTelemetry(); diff --git a/src/telemetry/sdk.ts b/src/telemetry/sdk.ts new file mode 100644 index 0000000..96cebbc --- /dev/null +++ b/src/telemetry/sdk.ts @@ -0,0 +1,157 @@ +import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; +import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk'; +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { envDetectorSync, Resource } from '@opentelemetry/resources'; +import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'; +import { + DropAggregation, + PeriodicExportingMetricReader, + View, +} from '@opentelemetry/sdk-metrics'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { + BatchSpanProcessor, + ParentBasedSampler, + Sampler, + SamplingDecision, + SamplingResult, +} from '@opentelemetry/sdk-trace-base'; +import { PrismaInstrumentation } from '@prisma/instrumentation'; +import { readFileSync } from 'fs'; +import { IncomingMessage } from 'http'; +import { hostname } from 'os'; +import { join } from 'path'; + +const EXPORT_TIMEOUT_MS = 5000; + +const NOISY_LEAF_SPANS = new Set([ + 'prisma:client:serialize', + 'prisma:engine:serialize', + 'prisma:engine:response_json_serialization', + 'prisma:engine:connection', +]); + +const NOT_RECORDED: SamplingResult = { decision: SamplingDecision.NOT_RECORD }; + +class SkipRootDatabaseSpans implements Sampler { + shouldSample( + _context: Context, + _traceId: string, + name: string, + ): SamplingResult { + return name.startsWith('prisma:') + ? NOT_RECORDED + : { decision: SamplingDecision.RECORD_AND_SAMPLED }; + } + + toString(): string { + return 'SkipRootDatabaseSpans'; + } +} + +class MessengerSampler implements Sampler { + private readonly parentBased = new ParentBasedSampler({ root: new SkipRootDatabaseSpans() }); + + shouldSample( + context: Context, + traceId: string, + name: string, + kind: SpanKind, + attributes: Attributes, + links: Link[], + ): SamplingResult { + if (NOISY_LEAF_SPANS.has(name)) { + return NOT_RECORDED; + } + return this.parentBased.shouldSample(context, traceId, name, kind, attributes, links); + } + + toString(): string { + return 'MessengerSampler'; + } +} + +let sdk: NodeSDK | null = null; + +export function telemetryEnabled(): boolean { + return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT) && process.env.OTEL_SDK_DISABLED !== 'true'; +} + +export function startTelemetry(): void { + if (!telemetryEnabled() || sdk) { + return; + } + sdk = new NodeSDK({ + resource: new Resource({ + 'service.version': serviceVersion(), + 'service.instance.id': process.env.INSTANCE_ID ?? hostname(), + }), + resourceDetectors: [envDetectorSync], + sampler: new MessengerSampler(), + spanProcessors: [ + new BatchSpanProcessor(new OTLPTraceExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), { + maxQueueSize: 2048, + maxExportBatchSize: 512, + exportTimeoutMillis: EXPORT_TIMEOUT_MS, + }), + ], + logRecordProcessors: [ + new BatchLogRecordProcessor(new OTLPLogExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), { + maxQueueSize: 2048, + maxExportBatchSize: 512, + exportTimeoutMillis: EXPORT_TIMEOUT_MS, + }), + ], + metricReader: new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), + exportIntervalMillis: 10000, + exportTimeoutMillis: EXPORT_TIMEOUT_MS, + }), + views: [ + new View({ + meterName: '@opentelemetry/instrumentation-http', + aggregation: new DropAggregation(), + }), + ], + instrumentations: [ + new HttpInstrumentation({ + ignoreIncomingRequestHook: (request) => (request.url ?? '').startsWith('/health'), + ignoreOutgoingRequestHook: () => true, + requestHook: (span, request) => { + if (request instanceof IncomingMessage) { + const path = (request.url ?? '/').split('?')[0]; + span.setAttribute('http.target', path); + span.setAttribute('http.url', path); + } + }, + }), + new AwsInstrumentation({ suppressInternalInstrumentation: true }), + new PrismaInstrumentation(), + ], + }); + sdk.start(); +} + +export async function shutdownTelemetry(): Promise { + if (!sdk) { + return; + } + const current = sdk; + sdk = null; + await Promise.race([ + current.shutdown().catch(() => undefined), + new Promise((resolve) => setTimeout(resolve, EXPORT_TIMEOUT_MS)), + ]); +} + +function serviceVersion(): string { + try { + const raw = readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8'); + return String(JSON.parse(raw).version); + } catch { + return 'unknown'; + } +} diff --git a/src/worker.ts b/src/worker.ts index cec9146..9dab179 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,10 +1,25 @@ +import './telemetry/register'; import { Logger } from '@nestjs/common'; +import { + context, + Context, + metrics, + propagation, + ROOT_CONTEXT, + SpanKind, + SpanStatusCode, + trace, +} from '@opentelemetry/api'; import { Channel, ConsumeMessage } from 'amqplib'; import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker'; import { PrismaService } from './db/prisma.service'; -import { parseJobMessage } from './jobs/job-message'; +import { JOB_TYPE, JobMessage, parseJobMessage } from './jobs/job-message'; import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor'; import { S3Service } from './storage/s3.service'; +import { JsonLogger } from './telemetry/json-logger'; +import { shutdownTelemetry } from './telemetry/sdk'; + +Logger.overrideLogger(new JsonLogger()); const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2); const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000); @@ -18,6 +33,27 @@ const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks()) let stopping = false; let inFlight = 0; +const JOB_BUCKETS = [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 5, 10, 30, 60, 120]; + +const tracer = trace.getTracer('messenger-worker'); +const meter = metrics.getMeter('messenger-worker'); +const jobsProcessed = meter.createCounter('jobs.processed', { + description: 'Job deliveries handled by the worker, by job type and outcome', +}); +const jobDuration = meter.createHistogram('jobs.duration', { + description: 'Job delivery handling duration', + unit: 's', + advice: { explicitBucketBoundaries: JOB_BUCKETS }, +}); + +function parentContext(delivery: ConsumeMessage, message: JobMessage | null): Context { + const headers = delivery.properties.headers ?? {}; + if (typeof headers.traceparent === 'string') { + return propagation.extract(ROOT_CONTEXT, headers); + } + return message ? propagation.extract(ROOT_CONTEXT, message) : ROOT_CONTEXT; +} + function faultHooks(): ProcessorHooks { const crash = (point: string) => async () => { logger.warn(`fault injection: worker killed ${point}`); @@ -34,31 +70,74 @@ function faultHooks(): ProcessorHooks { async function handle(channel: Channel, delivery: ConsumeMessage): Promise { inFlight++; + const started = performance.now(); + const message = parseJobMessage(delivery.content); + const span = tracer.startSpan( + `${JOB_QUEUE} process`, + { + kind: SpanKind.CONSUMER, + attributes: { + 'messaging.system': 'rabbitmq', + 'messaging.operation.type': 'process', + 'messaging.destination.name': JOB_QUEUE, + 'messaging.message.id': message?.job_id ?? 'malformed', + 'messaging.rabbitmq.redelivered': delivery.fields.redelivered, + 'messaging.job.attempt': message?.attempt ?? 0, + }, + }, + parentContext(delivery, message), + ); + const active = trace.setSpan(context.active(), span); + let outcome = 'interrupted'; + let requeue = false; try { - const message = parseJobMessage(delivery.content); - if (!message) { - logger.warn('malformed job message moved to dead letter queue'); - channel.nack(delivery, false, false); - return; + await context.with(active, async () => { + if (!message) { + outcome = 'malformed'; + logger.warn('malformed job message moved to dead letter queue'); + channel.nack(delivery, false, false); + return; + } + const result = await processor.handle(message); + outcome = result === 'ready' ? 'success' : result; + logger.log({ + event: 'job.handled', + job_id: message.job_id, + attachment_id: message.attachment_id, + attempt: message.attempt, + redelivered: delivery.fields.redelivered, + outcome, + duration_ms: Math.round(performance.now() - started), + }); + channel.ack(delivery); + }); + span.setAttribute('messenger.job.outcome', outcome); + if (outcome === 'failed' || outcome === 'malformed') { + span.setStatus({ code: SpanStatusCode.ERROR, message: outcome }); } - const outcome = await processor.handle(message); - logger.log( - `job ${message.job_id} attempt ${message.attempt}` + - `${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`, - ); - channel.ack(delivery); } catch (error) { const reason = error instanceof Error ? error.name : 'unknown error'; - logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`); + span.setAttribute('messenger.job.outcome', outcome); + span.setStatus({ code: SpanStatusCode.ERROR, message: reason }); + context.with(active, () => + logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`), + ); + requeue = true; + } finally { + span.end(); + const attributes = { 'job.type': JOB_TYPE, outcome }; + jobsProcessed.add(1, attributes); + jobDuration.record((performance.now() - started) / 1000, attributes); + } + if (requeue) { await sleep(INFRA_RETRY_MS); try { channel.nack(delivery, false, true); } catch { logger.warn('channel closed, broker will redeliver unacked message'); } - } finally { - inFlight--; } + inFlight--; } async function run(): Promise { @@ -129,6 +208,7 @@ run() await prisma.$disconnect(); s3.onModuleDestroy(); logger.log('stopped'); + await shutdownTelemetry(); process.exit(0); }) .catch((error) => {