forked from mtusi2026/mtusi2026lab6
feat: instrument api, relay and worker with opentelemetry
This commit is contained in:
@@ -1,5 +1,10 @@
|
||||
name: messenger-lab6
|
||||
|
||||
x-otel: &otel
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
|
||||
|
||||
services:
|
||||
secrets-init:
|
||||
image: alpine:3.20.3
|
||||
@@ -168,6 +173,8 @@ services:
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
<<: *otel
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } }
|
||||
deploy:
|
||||
@@ -244,6 +251,8 @@ services:
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_STALE_SECONDS: ${JOB_STALE_SECONDS:-300}
|
||||
RELAY_FAULT: ${RELAY_FAULT:-}
|
||||
<<: *otel
|
||||
OTEL_SERVICE_NAME: messenger-relay
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: relay } }
|
||||
deploy:
|
||||
@@ -279,6 +288,8 @@ services:
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
WORKER_FAULT: ${WORKER_FAULT:-}
|
||||
<<: *otel
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: worker } }
|
||||
deploy:
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
{
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger overview",
|
||||
"tags": ["messenger", "lab6"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": false,
|
||||
"graphTooltip": 1,
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-30m", "to": "now" },
|
||||
"templating": { "list": [] },
|
||||
"annotations": { "list": [] },
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "API",
|
||||
"id": 100,
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "RPS по маршрутам (без health)",
|
||||
"id": 1,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] } },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (http_request_method, http_route) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Ответы по классам статуса (без health)",
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (http_response_status_class) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_response_status_class}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Доля 5xx среди бизнес-запросов (окно 5m)",
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.05 }] },
|
||||
"custom": { "thresholdsStyle": { "mode": "line" } }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "(sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\", http_response_status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[5m]))",
|
||||
"legendFormat": "5xx ratio"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Длительность API p50 / p95 / p99 (все реплики, без health)",
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "p95 по маршрутам",
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le, http_request_method, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Worker",
|
||||
"id": 101,
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Задания по исходу (в секунду)",
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (job_type, outcome) (rate(jobs_processed_total{job=\"messenger-worker\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{job_type}} {{outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Длительность обработки задания p50 / p95 / p99",
|
||||
"id": 7,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p50 success"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95 success"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p99 success"
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome!=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95 retry/failed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "row",
|
||||
"title": "Логи и трассы",
|
||||
"id": 102,
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }
|
||||
},
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Логи API и worker (WARN и выше)",
|
||||
"id": 8,
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 27 },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Все логи (http.request, job.handled, …)",
|
||||
"id": 9,
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 27 },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=~\"messenger-.+\"}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Последние трассы (HTTP server и consumer spans)",
|
||||
"id": 10,
|
||||
"datasource": { "type": "tempo", "uid": "tempo" },
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "tempo", "uid": "tempo" },
|
||||
"queryType": "traceql",
|
||||
"query": "{resource.service.name=~\"messenger-.+\" && (kind=server || kind=consumer)}",
|
||||
"limit": 20,
|
||||
"tableType": "traces"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Generated
+801
-19
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,22 @@
|
||||
"@nestjs/core": "^10.4.15",
|
||||
"@nestjs/platform-fastify": "^10.4.15",
|
||||
"@nestjs/swagger": "7.4.2",
|
||||
"@opentelemetry/api": "1.9.0",
|
||||
"@opentelemetry/api-logs": "0.57.2",
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/exporter-logs-otlp-proto": "0.57.2",
|
||||
"@opentelemetry/exporter-metrics-otlp-proto": "0.57.2",
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "0.57.2",
|
||||
"@opentelemetry/instrumentation-aws-sdk": "0.49.1",
|
||||
"@opentelemetry/instrumentation-http": "0.57.2",
|
||||
"@opentelemetry/resources": "1.30.1",
|
||||
"@opentelemetry/sdk-logs": "0.57.2",
|
||||
"@opentelemetry/sdk-metrics": "1.30.1",
|
||||
"@opentelemetry/sdk-node": "0.57.2",
|
||||
"@opentelemetry/sdk-trace-base": "1.30.1",
|
||||
"@opentelemetry/semantic-conventions": "1.28.0",
|
||||
"@prisma/client": "6.1.0",
|
||||
"@prisma/instrumentation": "6.1.0",
|
||||
"amqplib": "0.10.5",
|
||||
"argon2": "0.41.1",
|
||||
"class-transformer": "^0.5.1",
|
||||
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
|
||||
|
||||
|
||||
def get(path, **params):
|
||||
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def run(target):
|
||||
uid = target['datasource']['uid']
|
||||
now = int(time.time())
|
||||
if uid == 'prometheus':
|
||||
expr = target['expr'].replace('$__rate_interval', '1m')
|
||||
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
|
||||
series = body['data']['result']
|
||||
last = [s['values'][-1][1] for s in series if s['values']]
|
||||
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
|
||||
for s, v in zip(series[:4], last[:4]))
|
||||
if uid == 'loki':
|
||||
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
|
||||
return sum(len(s['values']) for s in body['data']['result']), ''
|
||||
if uid == 'tempo':
|
||||
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
|
||||
return len(body.get('traces', [])), ''
|
||||
return 0, 'unknown datasource'
|
||||
|
||||
|
||||
def main():
|
||||
grafana = get('/api/dashboards/uid/messenger-overview')
|
||||
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
|
||||
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
|
||||
empty = 0
|
||||
for panel in dashboard['panels']:
|
||||
for target in panel.get('targets', []):
|
||||
count, sample = run(target)
|
||||
empty += count == 0
|
||||
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
|
||||
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
|
||||
sys.exit(0 if empty == 0 else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30]
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}'
|
||||
|
||||
|
||||
def prom(expr, at):
|
||||
params = urllib.parse.urlencode({'query': expr, 'time': at})
|
||||
with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r:
|
||||
result = json.loads(r.read())['data']['result']
|
||||
return float(result[0]['value'][1]) if result else None
|
||||
|
||||
|
||||
def main():
|
||||
duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30'
|
||||
output = os.path.join(tempfile.mkdtemp(), 'load.json')
|
||||
started = time.time()
|
||||
subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
report = json.load(open(output))
|
||||
print('waiting 25 s for the metric export interval')
|
||||
time.sleep(25)
|
||||
at = time.time()
|
||||
window = int(at - started) + 5
|
||||
print('client (tests/load.py): %d requests, %.1f rps, errors %d' % (
|
||||
report['successful_requests'], report['successful_rps'], report['failed_requests']))
|
||||
count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at)
|
||||
print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window))
|
||||
print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms'))
|
||||
for p in ('p50', 'p95', 'p99'):
|
||||
q = int(p[1:]) / 100
|
||||
server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at)
|
||||
print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000))
|
||||
print('server-side histogram excludes network/client overhead; values are interpolated inside buckets '
|
||||
'(…, 5, 10, 25, 50, 100 ms, …).')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
|
||||
|
||||
def call(method, url, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
all_headers = dict(headers or {})
|
||||
if body is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status, json.loads(response.read() or b'null')
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
|
||||
def session():
|
||||
username = 'trace_' + uuid.uuid4().hex[:8]
|
||||
password = 'Trace-check-' + uuid.uuid4().hex
|
||||
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
auth = {'Authorization': 'Bearer ' + login['token']}
|
||||
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
|
||||
return auth, chat['id']
|
||||
|
||||
|
||||
def wait_trace(trace_id, services, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
spans = []
|
||||
while time.monotonic() < deadline:
|
||||
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
||||
if code == 200 and body:
|
||||
spans = []
|
||||
for batch in body.get('batches', []):
|
||||
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
|
||||
service = attrs['service.name']['stringValue']
|
||||
for scope in batch.get('scopeSpans', []):
|
||||
for span in scope.get('spans', []):
|
||||
spans.append((service, span))
|
||||
if services <= {service for service, _ in spans}:
|
||||
return spans
|
||||
time.sleep(3)
|
||||
return spans
|
||||
|
||||
|
||||
def b64hex(value):
|
||||
return base64.b64decode(value).hex() if value else ''
|
||||
|
||||
|
||||
def main():
|
||||
trace_id = os.urandom(16).hex()
|
||||
client_span = os.urandom(8).hex()
|
||||
traceparent = '00-%s-%s-01' % (trace_id, client_span)
|
||||
auth, chat = session()
|
||||
started = time.monotonic()
|
||||
code, attachment = call(
|
||||
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
|
||||
)
|
||||
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
|
||||
status = None
|
||||
while time.monotonic() - started < 60:
|
||||
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
|
||||
status = body['status']
|
||||
if status in ('ready', 'failed'):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
|
||||
|
||||
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
|
||||
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
|
||||
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
|
||||
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
|
||||
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
|
||||
continue
|
||||
parent = b64hex(span.get('parentSpanId'))
|
||||
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
|
||||
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
||||
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
|
||||
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
|
||||
|
||||
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
|
||||
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
|
||||
streams = []
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
|
||||
streams = (logs or {}).get('data', {}).get('result', [])
|
||||
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
|
||||
break
|
||||
time.sleep(3)
|
||||
print('\nLoki %s' % query)
|
||||
lines = 0
|
||||
for stream in streams:
|
||||
labels = stream['stream']
|
||||
for _, line in stream['values']:
|
||||
lines += 1
|
||||
print(' %-17s %-8s %-13s span_id=%s %s' % (
|
||||
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
|
||||
labels.get('span_id', ''), line[:60]))
|
||||
|
||||
kinds = {(service, span.get('kind')) for service, span in spans}
|
||||
ok = (
|
||||
status == 'ready'
|
||||
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
|
||||
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
|
||||
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
|
||||
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
|
||||
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
|
||||
)
|
||||
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { isDatabaseUnavailable } from '../../db/prisma.service';
|
||||
import { routeTemplate } from '../../telemetry/http';
|
||||
import { ApiError, ErrorCode } from './api-error';
|
||||
|
||||
const CODE_BY_STATUS: Partial<Record<number, ErrorCode>> = {
|
||||
@@ -39,9 +40,14 @@ export class HttpExceptionFilter implements ExceptionFilter {
|
||||
|
||||
const { status, code, message } = this.resolve(exception);
|
||||
if (status >= 500) {
|
||||
this.logger.error(
|
||||
`${request.method} ${request.url} -> ${status}: ${message}`,
|
||||
);
|
||||
this.logger.error({
|
||||
event: 'http.error',
|
||||
message,
|
||||
method: request.method,
|
||||
route: routeTemplate(request),
|
||||
status,
|
||||
request_id: requestId,
|
||||
});
|
||||
}
|
||||
|
||||
if (exception instanceof ApiError) {
|
||||
|
||||
+11
-4
@@ -1,4 +1,5 @@
|
||||
import { INestApplication, Logger } from '@nestjs/common';
|
||||
import { shutdownTelemetry } from '../telemetry/sdk';
|
||||
|
||||
let draining = false;
|
||||
|
||||
@@ -15,10 +16,16 @@ export function enableGracefulShutdown(app: INestApplication): void {
|
||||
draining = true;
|
||||
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
|
||||
setTimeout(() => {
|
||||
app.close().then(
|
||||
() => process.exit(0),
|
||||
() => process.exit(1),
|
||||
);
|
||||
app
|
||||
.close()
|
||||
.then(
|
||||
() => 0,
|
||||
() => 1,
|
||||
)
|
||||
.then(async (code) => {
|
||||
await shutdownTelemetry();
|
||||
process.exit(code);
|
||||
});
|
||||
}, delayMs);
|
||||
};
|
||||
process.once('SIGTERM', () => shutdown('SIGTERM'));
|
||||
|
||||
+31
-2
@@ -1,9 +1,18 @@
|
||||
import { context, propagation } from '@opentelemetry/api';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
|
||||
export const PIPELINE_VERSION = 1;
|
||||
export const JOB_TYPE = 'attachment.process.v1';
|
||||
|
||||
export interface JobMessage {
|
||||
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
|
||||
const TRACESTATE_MAX = 512;
|
||||
|
||||
export interface TraceCarrier {
|
||||
traceparent?: string;
|
||||
tracestate?: string;
|
||||
}
|
||||
|
||||
export interface JobMessage extends TraceCarrier {
|
||||
schema_version: 1;
|
||||
job_id: string;
|
||||
type: typeof JOB_TYPE;
|
||||
@@ -21,6 +30,8 @@ const FIELDS = [
|
||||
'pipeline_version',
|
||||
'attempt',
|
||||
'created_at',
|
||||
'traceparent',
|
||||
'tracestate',
|
||||
];
|
||||
|
||||
export function jobMessage(
|
||||
@@ -35,9 +46,23 @@ export function jobMessage(
|
||||
pipeline_version: job.pipelineVersion,
|
||||
attempt: job.attempt,
|
||||
created_at: createdAt.toISOString(),
|
||||
...currentTraceCarrier(),
|
||||
};
|
||||
}
|
||||
|
||||
export function currentTraceCarrier(): TraceCarrier {
|
||||
const carrier: Record<string, string> = {};
|
||||
propagation.inject(context.active(), carrier);
|
||||
const result: TraceCarrier = {};
|
||||
if (TRACEPARENT.test(carrier.traceparent ?? '')) {
|
||||
result.traceparent = carrier.traceparent;
|
||||
if (carrier.tracestate && carrier.tracestate.length <= TRACESTATE_MAX) {
|
||||
result.tracestate = carrier.tracestate;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parseJobMessage(raw: Buffer): JobMessage | null {
|
||||
let value: any;
|
||||
try {
|
||||
@@ -61,7 +86,11 @@ export function parseJobMessage(raw: Buffer): JobMessage | null {
|
||||
!Number.isInteger(value.attempt) ||
|
||||
value.attempt < 1 ||
|
||||
typeof value.created_at !== 'string' ||
|
||||
Number.isNaN(Date.parse(value.created_at))
|
||||
Number.isNaN(Date.parse(value.created_at)) ||
|
||||
(value.traceparent !== undefined &&
|
||||
(typeof value.traceparent !== 'string' || !TRACEPARENT.test(value.traceparent))) ||
|
||||
(value.tracestate !== undefined &&
|
||||
(typeof value.tracestate !== 'string' || value.tracestate.length > TRACESTATE_MAX))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
+18
-5
@@ -1,4 +1,5 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import {
|
||||
@@ -50,6 +51,8 @@ class PermanentError extends Error {
|
||||
|
||||
class TimeoutError extends Error {}
|
||||
|
||||
const tracer = trace.getTracer('messenger-worker');
|
||||
|
||||
export class JobProcessor {
|
||||
private readonly logger = new Logger('JobProcessor');
|
||||
|
||||
@@ -141,11 +144,21 @@ export class JobProcessor {
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await renderVariants(
|
||||
original,
|
||||
this.options.maxPixels,
|
||||
Math.ceil(this.options.timeoutMs / 1000),
|
||||
);
|
||||
variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => {
|
||||
span.setAttribute('messenger.image.bytes', original.length);
|
||||
try {
|
||||
return await renderVariants(
|
||||
original,
|
||||
this.options.maxPixels,
|
||||
Math.ceil(this.options.timeoutMs / 1000),
|
||||
);
|
||||
} catch (error) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' });
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw new PermanentError('invalid_image', 'image cannot be decoded');
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import './telemetry/register';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import {
|
||||
@@ -8,6 +9,8 @@ import { AppModule } from './app.module';
|
||||
import { configureApp } from './configure-app';
|
||||
import { setupSwagger } from './docs/swagger';
|
||||
import { enableGracefulShutdown } from './health/shutdown';
|
||||
import { registerHttpTelemetry } from './telemetry/http';
|
||||
import { JsonLogger } from './telemetry/json-logger';
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
@@ -15,10 +18,12 @@ async function bootstrap(): Promise<void> {
|
||||
new FastifyAdapter(),
|
||||
{
|
||||
bufferLogs: true,
|
||||
logger: new JsonLogger(),
|
||||
},
|
||||
);
|
||||
|
||||
configureApp(app);
|
||||
registerHttpTelemetry(app.getHttpAdapter().getInstance());
|
||||
setupSwagger(app);
|
||||
enableGracefulShutdown(app);
|
||||
|
||||
|
||||
+44
-1
@@ -1,9 +1,15 @@
|
||||
import './telemetry/register';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { propagation, ROOT_CONTEXT, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import { ConfirmChannel } from 'amqplib';
|
||||
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { JobMessage } from './jobs/job-message';
|
||||
import { recoverExpiredLeases, republishStale } from './jobs/job-store';
|
||||
import { JsonLogger } from './telemetry/json-logger';
|
||||
import { shutdownTelemetry } from './telemetry/sdk';
|
||||
|
||||
Logger.overrideLogger(new JsonLogger());
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500);
|
||||
@@ -13,8 +19,29 @@ const FAULT = process.env.RELAY_FAULT;
|
||||
|
||||
const logger = new Logger('OutboxRelay');
|
||||
const prisma = new PrismaService();
|
||||
const tracer = trace.getTracer('messenger-relay');
|
||||
let stopping = false;
|
||||
|
||||
function startPublishSpan(payload: JobMessage, headers: Record<string, string>): Span {
|
||||
const parent = propagation.extract(ROOT_CONTEXT, payload);
|
||||
const span = tracer.startSpan(
|
||||
`${JOB_QUEUE} publish`,
|
||||
{
|
||||
kind: SpanKind.PRODUCER,
|
||||
attributes: {
|
||||
'messaging.system': 'rabbitmq',
|
||||
'messaging.operation.type': 'publish',
|
||||
'messaging.destination.name': JOB_QUEUE,
|
||||
'messaging.message.id': payload.job_id,
|
||||
'messaging.job.attempt': payload.attempt,
|
||||
},
|
||||
},
|
||||
parent,
|
||||
);
|
||||
propagation.inject(trace.setSpan(parent, span), headers);
|
||||
return span;
|
||||
}
|
||||
|
||||
async function publishBatch(channel: ConfirmChannel): Promise<number> {
|
||||
return prisma.$transaction(
|
||||
async (tx) => {
|
||||
@@ -27,15 +54,30 @@ async function publishBatch(channel: ConfirmChannel): Promise<number> {
|
||||
if (rows.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const spans: Span[] = [];
|
||||
for (const { payload } of rows) {
|
||||
const headers: Record<string, string> = {};
|
||||
spans.push(startPublishSpan(payload, headers));
|
||||
channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), {
|
||||
persistent: true,
|
||||
contentType: 'application/json',
|
||||
messageId: payload.job_id,
|
||||
type: payload.type,
|
||||
headers,
|
||||
});
|
||||
}
|
||||
await channel.waitForConfirms();
|
||||
try {
|
||||
await channel.waitForConfirms();
|
||||
} catch (error) {
|
||||
for (const span of spans) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: 'broker did not confirm' });
|
||||
span.end();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
for (const span of spans) {
|
||||
span.end();
|
||||
}
|
||||
logger.log(`published ${rows.length} job message(s), broker confirmed`);
|
||||
if (FAULT === 'exit-after-confirm') {
|
||||
logger.warn('fault injection: exit after broker confirm, before outbox is marked');
|
||||
@@ -121,6 +163,7 @@ run()
|
||||
.then(async () => {
|
||||
await prisma.$disconnect();
|
||||
logger.log('stopped');
|
||||
await shutdownTelemetry();
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { metrics, trace } from '@opentelemetry/api';
|
||||
import { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
|
||||
export const LATENCY_BUCKETS = [
|
||||
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 5, 10,
|
||||
];
|
||||
|
||||
const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']);
|
||||
|
||||
const meter = metrics.getMeter('messenger-api');
|
||||
const requests = meter.createCounter('http.server.requests', {
|
||||
description: 'HTTP requests by method, route template and status class',
|
||||
});
|
||||
const duration = meter.createHistogram('http.server.request.duration', {
|
||||
description: 'HTTP request duration',
|
||||
unit: 's',
|
||||
advice: { explicitBucketBoundaries: LATENCY_BUCKETS },
|
||||
});
|
||||
const logger = new Logger('Http');
|
||||
|
||||
export function routeTemplate(request: FastifyRequest): string {
|
||||
const url = request.routeOptions?.url;
|
||||
return url
|
||||
? url.replace(/:([A-Za-z0-9_]+)/g, (_, name: string) => `{${name.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)}}`)
|
||||
: 'unmatched';
|
||||
}
|
||||
|
||||
export function registerHttpTelemetry(instance: FastifyInstance): void {
|
||||
instance.addHook('onRequest', (request, _reply, done) => {
|
||||
const span = trace.getActiveSpan();
|
||||
if (span) {
|
||||
const route = routeTemplate(request);
|
||||
span.updateName(`${methodOf(request)} ${route}`);
|
||||
span.setAttribute('http.route', route);
|
||||
span.setAttribute('messenger.request_id', request.requestId);
|
||||
}
|
||||
done();
|
||||
});
|
||||
instance.addHook('onResponse', (request, reply, done) => {
|
||||
const route = routeTemplate(request);
|
||||
const method = methodOf(request);
|
||||
const attributes = {
|
||||
'http.request.method': method,
|
||||
'http.route': route,
|
||||
'http.response.status_class': `${Math.floor(reply.statusCode / 100)}xx`,
|
||||
};
|
||||
requests.add(1, attributes);
|
||||
duration.record(reply.elapsedTime / 1000, attributes);
|
||||
if (!route.startsWith('/health')) {
|
||||
logger.log({
|
||||
event: 'http.request',
|
||||
method,
|
||||
route,
|
||||
status: reply.statusCode,
|
||||
duration_ms: Math.round(reply.elapsedTime * 100) / 100,
|
||||
request_id: request.requestId,
|
||||
});
|
||||
}
|
||||
done();
|
||||
});
|
||||
}
|
||||
|
||||
function methodOf(request: FastifyRequest): string {
|
||||
return METHODS.has(request.method) ? request.method : '_OTHER';
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { LoggerService } from '@nestjs/common';
|
||||
import { trace, TraceFlags } from '@opentelemetry/api';
|
||||
import { AnyValueMap, logs, SeverityNumber } from '@opentelemetry/api-logs';
|
||||
|
||||
type Level = 'fatal' | 'error' | 'warn' | 'log' | 'debug' | 'verbose';
|
||||
|
||||
const SEVERITY: Record<Level, [SeverityNumber, string]> = {
|
||||
fatal: [SeverityNumber.FATAL, 'FATAL'],
|
||||
error: [SeverityNumber.ERROR, 'ERROR'],
|
||||
warn: [SeverityNumber.WARN, 'WARN'],
|
||||
log: [SeverityNumber.INFO, 'INFO'],
|
||||
debug: [SeverityNumber.DEBUG, 'DEBUG'],
|
||||
verbose: [SeverityNumber.TRACE, 'TRACE'],
|
||||
};
|
||||
|
||||
type Field = string | number | boolean;
|
||||
|
||||
export class JsonLogger implements LoggerService {
|
||||
private readonly otel = logs.getLogger('messenger');
|
||||
|
||||
constructor(private readonly service = process.env.OTEL_SERVICE_NAME ?? 'messenger') {}
|
||||
|
||||
fatal(message: unknown, ...params: unknown[]): void {
|
||||
this.write('fatal', message, params);
|
||||
}
|
||||
|
||||
error(message: unknown, ...params: unknown[]): void {
|
||||
this.write('error', message, params);
|
||||
}
|
||||
|
||||
warn(message: unknown, ...params: unknown[]): void {
|
||||
this.write('warn', message, params);
|
||||
}
|
||||
|
||||
log(message: unknown, ...params: unknown[]): void {
|
||||
this.write('log', message, params);
|
||||
}
|
||||
|
||||
debug(message: unknown, ...params: unknown[]): void {
|
||||
this.write('debug', message, params);
|
||||
}
|
||||
|
||||
verbose(message: unknown, ...params: unknown[]): void {
|
||||
this.write('verbose', message, params);
|
||||
}
|
||||
|
||||
private write(level: Level, message: unknown, params: unknown[]): void {
|
||||
const last = params[params.length - 1];
|
||||
const logger = typeof last === 'string' ? last : 'app';
|
||||
const fields = toFields(message);
|
||||
const event = typeof fields.event === 'string' ? fields.event : eventName(logger);
|
||||
delete fields.event;
|
||||
const [severityNumber, severityText] = SEVERITY[level];
|
||||
const active = trace.getActiveSpan()?.spanContext();
|
||||
const span = active && active.traceFlags & TraceFlags.SAMPLED ? active : undefined;
|
||||
const line = JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
severity: severityText,
|
||||
'service.name': this.service,
|
||||
event,
|
||||
logger,
|
||||
...fields,
|
||||
trace_id: span?.traceId,
|
||||
span_id: span?.spanId,
|
||||
});
|
||||
process.stdout.write(line + '\n');
|
||||
const attributes: AnyValueMap = { event, logger };
|
||||
for (const [key, value] of Object.entries(fields)) {
|
||||
if (key !== 'message') {
|
||||
attributes[key] = value;
|
||||
}
|
||||
}
|
||||
this.otel.emit({
|
||||
severityNumber,
|
||||
severityText,
|
||||
body: line,
|
||||
attributes,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toFields(message: unknown): Record<string, Field> {
|
||||
if (typeof message === 'object' && message !== null && !(message instanceof Error)) {
|
||||
const fields: Record<string, Field> = {};
|
||||
for (const [key, value] of Object.entries(message)) {
|
||||
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
||||
fields[key] = value;
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
return { message: message instanceof Error ? message.message : String(message) };
|
||||
}
|
||||
|
||||
function eventName(logger: string): string {
|
||||
return logger
|
||||
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
|
||||
.toLowerCase();
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { startTelemetry } from './sdk';
|
||||
|
||||
startTelemetry();
|
||||
@@ -0,0 +1,157 @@
|
||||
import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
|
||||
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
|
||||
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk';
|
||||
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
|
||||
import { envDetectorSync, Resource } from '@opentelemetry/resources';
|
||||
import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
|
||||
import {
|
||||
DropAggregation,
|
||||
PeriodicExportingMetricReader,
|
||||
View,
|
||||
} from '@opentelemetry/sdk-metrics';
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import {
|
||||
BatchSpanProcessor,
|
||||
ParentBasedSampler,
|
||||
Sampler,
|
||||
SamplingDecision,
|
||||
SamplingResult,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import { PrismaInstrumentation } from '@prisma/instrumentation';
|
||||
import { readFileSync } from 'fs';
|
||||
import { IncomingMessage } from 'http';
|
||||
import { hostname } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
const EXPORT_TIMEOUT_MS = 5000;
|
||||
|
||||
const NOISY_LEAF_SPANS = new Set([
|
||||
'prisma:client:serialize',
|
||||
'prisma:engine:serialize',
|
||||
'prisma:engine:response_json_serialization',
|
||||
'prisma:engine:connection',
|
||||
]);
|
||||
|
||||
const NOT_RECORDED: SamplingResult = { decision: SamplingDecision.NOT_RECORD };
|
||||
|
||||
class SkipRootDatabaseSpans implements Sampler {
|
||||
shouldSample(
|
||||
_context: Context,
|
||||
_traceId: string,
|
||||
name: string,
|
||||
): SamplingResult {
|
||||
return name.startsWith('prisma:')
|
||||
? NOT_RECORDED
|
||||
: { decision: SamplingDecision.RECORD_AND_SAMPLED };
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return 'SkipRootDatabaseSpans';
|
||||
}
|
||||
}
|
||||
|
||||
class MessengerSampler implements Sampler {
|
||||
private readonly parentBased = new ParentBasedSampler({ root: new SkipRootDatabaseSpans() });
|
||||
|
||||
shouldSample(
|
||||
context: Context,
|
||||
traceId: string,
|
||||
name: string,
|
||||
kind: SpanKind,
|
||||
attributes: Attributes,
|
||||
links: Link[],
|
||||
): SamplingResult {
|
||||
if (NOISY_LEAF_SPANS.has(name)) {
|
||||
return NOT_RECORDED;
|
||||
}
|
||||
return this.parentBased.shouldSample(context, traceId, name, kind, attributes, links);
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return 'MessengerSampler';
|
||||
}
|
||||
}
|
||||
|
||||
let sdk: NodeSDK | null = null;
|
||||
|
||||
export function telemetryEnabled(): boolean {
|
||||
return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT) && process.env.OTEL_SDK_DISABLED !== 'true';
|
||||
}
|
||||
|
||||
export function startTelemetry(): void {
|
||||
if (!telemetryEnabled() || sdk) {
|
||||
return;
|
||||
}
|
||||
sdk = new NodeSDK({
|
||||
resource: new Resource({
|
||||
'service.version': serviceVersion(),
|
||||
'service.instance.id': process.env.INSTANCE_ID ?? hostname(),
|
||||
}),
|
||||
resourceDetectors: [envDetectorSync],
|
||||
sampler: new MessengerSampler(),
|
||||
spanProcessors: [
|
||||
new BatchSpanProcessor(new OTLPTraceExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), {
|
||||
maxQueueSize: 2048,
|
||||
maxExportBatchSize: 512,
|
||||
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
|
||||
}),
|
||||
],
|
||||
logRecordProcessors: [
|
||||
new BatchLogRecordProcessor(new OTLPLogExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), {
|
||||
maxQueueSize: 2048,
|
||||
maxExportBatchSize: 512,
|
||||
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
|
||||
}),
|
||||
],
|
||||
metricReader: new PeriodicExportingMetricReader({
|
||||
exporter: new OTLPMetricExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }),
|
||||
exportIntervalMillis: 10000,
|
||||
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
|
||||
}),
|
||||
views: [
|
||||
new View({
|
||||
meterName: '@opentelemetry/instrumentation-http',
|
||||
aggregation: new DropAggregation(),
|
||||
}),
|
||||
],
|
||||
instrumentations: [
|
||||
new HttpInstrumentation({
|
||||
ignoreIncomingRequestHook: (request) => (request.url ?? '').startsWith('/health'),
|
||||
ignoreOutgoingRequestHook: () => true,
|
||||
requestHook: (span, request) => {
|
||||
if (request instanceof IncomingMessage) {
|
||||
const path = (request.url ?? '/').split('?')[0];
|
||||
span.setAttribute('http.target', path);
|
||||
span.setAttribute('http.url', path);
|
||||
}
|
||||
},
|
||||
}),
|
||||
new AwsInstrumentation({ suppressInternalInstrumentation: true }),
|
||||
new PrismaInstrumentation(),
|
||||
],
|
||||
});
|
||||
sdk.start();
|
||||
}
|
||||
|
||||
export async function shutdownTelemetry(): Promise<void> {
|
||||
if (!sdk) {
|
||||
return;
|
||||
}
|
||||
const current = sdk;
|
||||
sdk = null;
|
||||
await Promise.race([
|
||||
current.shutdown().catch(() => undefined),
|
||||
new Promise((resolve) => setTimeout(resolve, EXPORT_TIMEOUT_MS)),
|
||||
]);
|
||||
}
|
||||
|
||||
function serviceVersion(): string {
|
||||
try {
|
||||
const raw = readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8');
|
||||
return String(JSON.parse(raw).version);
|
||||
} catch {
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
+95
-15
@@ -1,10 +1,25 @@
|
||||
import './telemetry/register';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import {
|
||||
context,
|
||||
Context,
|
||||
metrics,
|
||||
propagation,
|
||||
ROOT_CONTEXT,
|
||||
SpanKind,
|
||||
SpanStatusCode,
|
||||
trace,
|
||||
} from '@opentelemetry/api';
|
||||
import { Channel, ConsumeMessage } from 'amqplib';
|
||||
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
import { parseJobMessage } from './jobs/job-message';
|
||||
import { JOB_TYPE, JobMessage, parseJobMessage } from './jobs/job-message';
|
||||
import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor';
|
||||
import { S3Service } from './storage/s3.service';
|
||||
import { JsonLogger } from './telemetry/json-logger';
|
||||
import { shutdownTelemetry } from './telemetry/sdk';
|
||||
|
||||
Logger.overrideLogger(new JsonLogger());
|
||||
|
||||
const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2);
|
||||
const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000);
|
||||
@@ -18,6 +33,27 @@ const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks())
|
||||
let stopping = false;
|
||||
let inFlight = 0;
|
||||
|
||||
const JOB_BUCKETS = [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 5, 10, 30, 60, 120];
|
||||
|
||||
const tracer = trace.getTracer('messenger-worker');
|
||||
const meter = metrics.getMeter('messenger-worker');
|
||||
const jobsProcessed = meter.createCounter('jobs.processed', {
|
||||
description: 'Job deliveries handled by the worker, by job type and outcome',
|
||||
});
|
||||
const jobDuration = meter.createHistogram('jobs.duration', {
|
||||
description: 'Job delivery handling duration',
|
||||
unit: 's',
|
||||
advice: { explicitBucketBoundaries: JOB_BUCKETS },
|
||||
});
|
||||
|
||||
function parentContext(delivery: ConsumeMessage, message: JobMessage | null): Context {
|
||||
const headers = delivery.properties.headers ?? {};
|
||||
if (typeof headers.traceparent === 'string') {
|
||||
return propagation.extract(ROOT_CONTEXT, headers);
|
||||
}
|
||||
return message ? propagation.extract(ROOT_CONTEXT, message) : ROOT_CONTEXT;
|
||||
}
|
||||
|
||||
function faultHooks(): ProcessorHooks {
|
||||
const crash = (point: string) => async () => {
|
||||
logger.warn(`fault injection: worker killed ${point}`);
|
||||
@@ -34,31 +70,74 @@ function faultHooks(): ProcessorHooks {
|
||||
|
||||
async function handle(channel: Channel, delivery: ConsumeMessage): Promise<void> {
|
||||
inFlight++;
|
||||
const started = performance.now();
|
||||
const message = parseJobMessage(delivery.content);
|
||||
const span = tracer.startSpan(
|
||||
`${JOB_QUEUE} process`,
|
||||
{
|
||||
kind: SpanKind.CONSUMER,
|
||||
attributes: {
|
||||
'messaging.system': 'rabbitmq',
|
||||
'messaging.operation.type': 'process',
|
||||
'messaging.destination.name': JOB_QUEUE,
|
||||
'messaging.message.id': message?.job_id ?? 'malformed',
|
||||
'messaging.rabbitmq.redelivered': delivery.fields.redelivered,
|
||||
'messaging.job.attempt': message?.attempt ?? 0,
|
||||
},
|
||||
},
|
||||
parentContext(delivery, message),
|
||||
);
|
||||
const active = trace.setSpan(context.active(), span);
|
||||
let outcome = 'interrupted';
|
||||
let requeue = false;
|
||||
try {
|
||||
const message = parseJobMessage(delivery.content);
|
||||
if (!message) {
|
||||
logger.warn('malformed job message moved to dead letter queue');
|
||||
channel.nack(delivery, false, false);
|
||||
return;
|
||||
await context.with(active, async () => {
|
||||
if (!message) {
|
||||
outcome = 'malformed';
|
||||
logger.warn('malformed job message moved to dead letter queue');
|
||||
channel.nack(delivery, false, false);
|
||||
return;
|
||||
}
|
||||
const result = await processor.handle(message);
|
||||
outcome = result === 'ready' ? 'success' : result;
|
||||
logger.log({
|
||||
event: 'job.handled',
|
||||
job_id: message.job_id,
|
||||
attachment_id: message.attachment_id,
|
||||
attempt: message.attempt,
|
||||
redelivered: delivery.fields.redelivered,
|
||||
outcome,
|
||||
duration_ms: Math.round(performance.now() - started),
|
||||
});
|
||||
channel.ack(delivery);
|
||||
});
|
||||
span.setAttribute('messenger.job.outcome', outcome);
|
||||
if (outcome === 'failed' || outcome === 'malformed') {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: outcome });
|
||||
}
|
||||
const outcome = await processor.handle(message);
|
||||
logger.log(
|
||||
`job ${message.job_id} attempt ${message.attempt}` +
|
||||
`${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`,
|
||||
);
|
||||
channel.ack(delivery);
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`);
|
||||
span.setAttribute('messenger.job.outcome', outcome);
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: reason });
|
||||
context.with(active, () =>
|
||||
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`),
|
||||
);
|
||||
requeue = true;
|
||||
} finally {
|
||||
span.end();
|
||||
const attributes = { 'job.type': JOB_TYPE, outcome };
|
||||
jobsProcessed.add(1, attributes);
|
||||
jobDuration.record((performance.now() - started) / 1000, attributes);
|
||||
}
|
||||
if (requeue) {
|
||||
await sleep(INFRA_RETRY_MS);
|
||||
try {
|
||||
channel.nack(delivery, false, true);
|
||||
} catch {
|
||||
logger.warn('channel closed, broker will redeliver unacked message');
|
||||
}
|
||||
} finally {
|
||||
inFlight--;
|
||||
}
|
||||
inFlight--;
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
@@ -129,6 +208,7 @@ run()
|
||||
await prisma.$disconnect();
|
||||
s3.onModuleDestroy();
|
||||
logger.log('stopped');
|
||||
await shutdownTelemetry();
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((error) => {
|
||||
|
||||
Reference in New Issue
Block a user