feat: instrument api, relay and worker with opentelemetry

This commit is contained in:
2026-11-20 12:55:53 +03:00
parent 9ed32e2d15
commit 307b658bfb
18 changed files with 1843 additions and 49 deletions
+11
View File
@@ -1,5 +1,10 @@
name: messenger-lab6
x-otel: &otel
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
services:
secrets-init:
image: alpine:3.20.3
@@ -168,6 +173,8 @@ services:
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
<<: *otel
OTEL_SERVICE_NAME: messenger-api
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } }
deploy:
@@ -244,6 +251,8 @@ services:
BROKER_URL_FILE: /run/secrets/broker_url
JOB_STALE_SECONDS: ${JOB_STALE_SECONDS:-300}
RELAY_FAULT: ${RELAY_FAULT:-}
<<: *otel
OTEL_SERVICE_NAME: messenger-relay
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: relay } }
deploy:
@@ -279,6 +288,8 @@ services:
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
WORKER_FAULT: ${WORKER_FAULT:-}
<<: *otel
OTEL_SERVICE_NAME: messenger-worker
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: worker } }
deploy:
+235
View File
@@ -0,0 +1,235 @@
{
"uid": "messenger-overview",
"title": "Messenger overview",
"tags": ["messenger", "lab6"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"editable": false,
"graphTooltip": 1,
"refresh": "10s",
"time": { "from": "now-30m", "to": "now" },
"templating": { "list": [] },
"annotations": { "list": [] },
"panels": [
{
"type": "row",
"title": "API",
"id": 100,
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
},
{
"type": "timeseries",
"title": "RPS по маршрутам (без health)",
"id": 1,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (http_request_method, http_route) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
]
},
{
"type": "timeseries",
"title": "Ответы по классам статуса (без health)",
"id": 2,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (http_response_status_class) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
"legendFormat": "{{http_response_status_class}}"
}
]
},
{
"type": "timeseries",
"title": "Доля 5xx среди бизнес-запросов (окно 5m)",
"id": 3,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 },
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.05 }] },
"custom": { "thresholdsStyle": { "mode": "line" } }
},
"overrides": []
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "(sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\", http_response_status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[5m]))",
"legendFormat": "5xx ratio"
}
]
},
{
"type": "timeseries",
"title": "Длительность API p50 / p95 / p99 (все реплики, без health)",
"id": 4,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "p95"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "p99"
}
]
},
{
"type": "timeseries",
"title": "p95 по маршрутам",
"id": 5,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le, http_request_method, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
]
},
{
"type": "row",
"title": "Worker",
"id": 101,
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }
},
{
"type": "timeseries",
"title": "Задания по исходу (в секунду)",
"id": 6,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (job_type, outcome) (rate(jobs_processed_total{job=\"messenger-worker\"}[$__rate_interval]))",
"legendFormat": "{{job_type}} {{outcome}}"
}
]
},
{
"type": "timeseries",
"title": "Длительность обработки задания p50 / p95 / p99",
"id": 7,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
"legendFormat": "p50 success"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
"legendFormat": "p95 success"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
"legendFormat": "p99 success"
},
{
"refId": "D",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome!=\"success\"}[$__rate_interval])))",
"legendFormat": "p95 retry/failed"
}
]
},
{
"type": "row",
"title": "Логи и трассы",
"id": 102,
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }
},
{
"type": "logs",
"title": "Логи API и worker (WARN и выше)",
"id": 8,
"datasource": { "type": "loki", "uid": "loki" },
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 27 },
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "loki" },
"expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\""
}
]
},
{
"type": "logs",
"title": "Все логи (http.request, job.handled, …)",
"id": 9,
"datasource": { "type": "loki", "uid": "loki" },
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 27 },
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "loki" },
"expr": "{service_name=~\"messenger-.+\"}"
}
]
},
{
"type": "table",
"title": "Последние трассы (HTTP server и consumer spans)",
"id": 10,
"datasource": { "type": "tempo", "uid": "tempo" },
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 },
"targets": [
{
"refId": "A",
"datasource": { "type": "tempo", "uid": "tempo" },
"queryType": "traceql",
"query": "{resource.service.name=~\"messenger-.+\" && (kind=server || kind=consumer)}",
"limit": 20,
"tableType": "traces"
}
]
}
]
}
+801 -19
View File
File diff suppressed because it is too large Load Diff
+15
View File
@@ -19,7 +19,22 @@
"@nestjs/core": "^10.4.15",
"@nestjs/platform-fastify": "^10.4.15",
"@nestjs/swagger": "7.4.2",
"@opentelemetry/api": "1.9.0",
"@opentelemetry/api-logs": "0.57.2",
"@opentelemetry/core": "1.30.1",
"@opentelemetry/exporter-logs-otlp-proto": "0.57.2",
"@opentelemetry/exporter-metrics-otlp-proto": "0.57.2",
"@opentelemetry/exporter-trace-otlp-proto": "0.57.2",
"@opentelemetry/instrumentation-aws-sdk": "0.49.1",
"@opentelemetry/instrumentation-http": "0.57.2",
"@opentelemetry/resources": "1.30.1",
"@opentelemetry/sdk-logs": "0.57.2",
"@opentelemetry/sdk-metrics": "1.30.1",
"@opentelemetry/sdk-node": "0.57.2",
"@opentelemetry/sdk-trace-base": "1.30.1",
"@opentelemetry/semantic-conventions": "1.28.0",
"@prisma/client": "6.1.0",
"@prisma/instrumentation": "6.1.0",
"amqplib": "0.10.5",
"argon2": "0.41.1",
"class-transformer": "^0.5.1",
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
"""
import json
import os
import sys
import time
import urllib.parse
import urllib.request
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
def get(path, **params):
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
with urllib.request.urlopen(url, timeout=15) as response:
return json.loads(response.read())
def run(target):
uid = target['datasource']['uid']
now = int(time.time())
if uid == 'prometheus':
expr = target['expr'].replace('$__rate_interval', '1m')
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
series = body['data']['result']
last = [s['values'][-1][1] for s in series if s['values']]
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
for s, v in zip(series[:4], last[:4]))
if uid == 'loki':
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
return sum(len(s['values']) for s in body['data']['result']), ''
if uid == 'tempo':
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
return len(body.get('traces', [])), ''
return 0, 'unknown datasource'
def main():
grafana = get('/api/dashboards/uid/messenger-overview')
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
empty = 0
for panel in dashboard['panels']:
for target in panel.get('targets', []):
count, sample = run(target)
empty += count == 0
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
sys.exit(0 if empty == 0 else 1)
if __name__ == '__main__':
main()
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus.
Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30]
"""
import json
import os
import subprocess
import sys
import tempfile
import time
import urllib.parse
import urllib.request
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}'
def prom(expr, at):
params = urllib.parse.urlencode({'query': expr, 'time': at})
with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r:
result = json.loads(r.read())['data']['result']
return float(result[0]['value'][1]) if result else None
def main():
duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30'
output = os.path.join(tempfile.mkdtemp(), 'load.json')
started = time.time()
subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output],
check=True, stdout=subprocess.DEVNULL)
report = json.load(open(output))
print('waiting 25 s for the metric export interval')
time.sleep(25)
at = time.time()
window = int(at - started) + 5
print('client (tests/load.py): %d requests, %.1f rps, errors %d' % (
report['successful_requests'], report['successful_rps'], report['failed_requests']))
count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at)
print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window))
print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms'))
for p in ('p50', 'p95', 'p99'):
q = int(p[1:]) / 100
server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at)
print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000))
print('server-side histogram excludes network/client overhead; values are interpolated inside buckets '
'(…, 5, 10, 25, 50, 100 ms, …).')
if __name__ == '__main__':
main()
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
"""
import base64
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
def call(method, url, body=None, headers=None, raw=None):
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
all_headers = dict(headers or {})
if body is not None:
all_headers['Content-Type'] = 'application/json'
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status, json.loads(response.read() or b'null')
except urllib.error.HTTPError as error:
return error.code, None
def session():
username = 'trace_' + uuid.uuid4().hex[:8]
password = 'Trace-check-' + uuid.uuid4().hex
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
credential = base64.b64encode((username + ':' + password).encode()).decode()
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
auth = {'Authorization': 'Bearer ' + login['token']}
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
return auth, chat['id']
def wait_trace(trace_id, services, timeout=60):
deadline = time.monotonic() + timeout
spans = []
while time.monotonic() < deadline:
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
if code == 200 and body:
spans = []
for batch in body.get('batches', []):
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
service = attrs['service.name']['stringValue']
for scope in batch.get('scopeSpans', []):
for span in scope.get('spans', []):
spans.append((service, span))
if services <= {service for service, _ in spans}:
return spans
time.sleep(3)
return spans
def b64hex(value):
return base64.b64decode(value).hex() if value else ''
def main():
trace_id = os.urandom(16).hex()
client_span = os.urandom(8).hex()
traceparent = '00-%s-%s-01' % (trace_id, client_span)
auth, chat = session()
started = time.monotonic()
code, attachment = call(
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
)
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
status = None
while time.monotonic() - started < 60:
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
status = body['status']
if status in ('ready', 'failed'):
break
time.sleep(0.5)
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
continue
parent = b64hex(span.get('parentSpanId'))
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
streams = []
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
streams = (logs or {}).get('data', {}).get('result', [])
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
break
time.sleep(3)
print('\nLoki %s' % query)
lines = 0
for stream in streams:
labels = stream['stream']
for _, line in stream['values']:
lines += 1
print(' %-17s %-8s %-13s span_id=%s %s' % (
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
labels.get('span_id', ''), line[:60]))
kinds = {(service, span.get('kind')) for service, span in spans}
ok = (
status == 'ready'
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
)
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
sys.exit(0 if ok else 1)
if __name__ == '__main__':
main()
+9 -3
View File
@@ -8,6 +8,7 @@ import {
} from '@nestjs/common';
import { FastifyReply, FastifyRequest } from 'fastify';
import { isDatabaseUnavailable } from '../../db/prisma.service';
import { routeTemplate } from '../../telemetry/http';
import { ApiError, ErrorCode } from './api-error';
const CODE_BY_STATUS: Partial<Record<number, ErrorCode>> = {
@@ -39,9 +40,14 @@ export class HttpExceptionFilter implements ExceptionFilter {
const { status, code, message } = this.resolve(exception);
if (status >= 500) {
this.logger.error(
`${request.method} ${request.url} -> ${status}: ${message}`,
);
this.logger.error({
event: 'http.error',
message,
method: request.method,
route: routeTemplate(request),
status,
request_id: requestId,
});
}
if (exception instanceof ApiError) {
+11 -4
View File
@@ -1,4 +1,5 @@
import { INestApplication, Logger } from '@nestjs/common';
import { shutdownTelemetry } from '../telemetry/sdk';
let draining = false;
@@ -15,10 +16,16 @@ export function enableGracefulShutdown(app: INestApplication): void {
draining = true;
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
setTimeout(() => {
app.close().then(
() => process.exit(0),
() => process.exit(1),
);
app
.close()
.then(
() => 0,
() => 1,
)
.then(async (code) => {
await shutdownTelemetry();
process.exit(code);
});
}, delayMs);
};
process.once('SIGTERM', () => shutdown('SIGTERM'));
+31 -2
View File
@@ -1,9 +1,18 @@
import { context, propagation } from '@opentelemetry/api';
import { validate as isUuid } from '../common/uuid';
export const PIPELINE_VERSION = 1;
export const JOB_TYPE = 'attachment.process.v1';
export interface JobMessage {
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
const TRACESTATE_MAX = 512;
export interface TraceCarrier {
traceparent?: string;
tracestate?: string;
}
export interface JobMessage extends TraceCarrier {
schema_version: 1;
job_id: string;
type: typeof JOB_TYPE;
@@ -21,6 +30,8 @@ const FIELDS = [
'pipeline_version',
'attempt',
'created_at',
'traceparent',
'tracestate',
];
export function jobMessage(
@@ -35,9 +46,23 @@ export function jobMessage(
pipeline_version: job.pipelineVersion,
attempt: job.attempt,
created_at: createdAt.toISOString(),
...currentTraceCarrier(),
};
}
export function currentTraceCarrier(): TraceCarrier {
const carrier: Record<string, string> = {};
propagation.inject(context.active(), carrier);
const result: TraceCarrier = {};
if (TRACEPARENT.test(carrier.traceparent ?? '')) {
result.traceparent = carrier.traceparent;
if (carrier.tracestate && carrier.tracestate.length <= TRACESTATE_MAX) {
result.tracestate = carrier.tracestate;
}
}
return result;
}
export function parseJobMessage(raw: Buffer): JobMessage | null {
let value: any;
try {
@@ -61,7 +86,11 @@ export function parseJobMessage(raw: Buffer): JobMessage | null {
!Number.isInteger(value.attempt) ||
value.attempt < 1 ||
typeof value.created_at !== 'string' ||
Number.isNaN(Date.parse(value.created_at))
Number.isNaN(Date.parse(value.created_at)) ||
(value.traceparent !== undefined &&
(typeof value.traceparent !== 'string' || !TRACEPARENT.test(value.traceparent))) ||
(value.tracestate !== undefined &&
(typeof value.tracestate !== 'string' || value.tracestate.length > TRACESTATE_MAX))
) {
return null;
}
+18 -5
View File
@@ -1,4 +1,5 @@
import { Logger } from '@nestjs/common';
import { SpanStatusCode, trace } from '@opentelemetry/api';
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import {
@@ -50,6 +51,8 @@ class PermanentError extends Error {
class TimeoutError extends Error {}
const tracer = trace.getTracer('messenger-worker');
export class JobProcessor {
private readonly logger = new Logger('JobProcessor');
@@ -141,11 +144,21 @@ export class JobProcessor {
}
let variants: RenderedVariant[];
try {
variants = await renderVariants(
original,
this.options.maxPixels,
Math.ceil(this.options.timeoutMs / 1000),
);
variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => {
span.setAttribute('messenger.image.bytes', original.length);
try {
return await renderVariants(
original,
this.options.maxPixels,
Math.ceil(this.options.timeoutMs / 1000),
);
} catch (error) {
span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' });
throw error;
} finally {
span.end();
}
});
} catch (error) {
if (error instanceof InvalidImageError) {
throw new PermanentError('invalid_image', 'image cannot be decoded');
+5
View File
@@ -1,3 +1,4 @@
import './telemetry/register';
import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import {
@@ -8,6 +9,8 @@ import { AppModule } from './app.module';
import { configureApp } from './configure-app';
import { setupSwagger } from './docs/swagger';
import { enableGracefulShutdown } from './health/shutdown';
import { registerHttpTelemetry } from './telemetry/http';
import { JsonLogger } from './telemetry/json-logger';
async function bootstrap(): Promise<void> {
const app = await NestFactory.create<NestFastifyApplication>(
@@ -15,10 +18,12 @@ async function bootstrap(): Promise<void> {
new FastifyAdapter(),
{
bufferLogs: true,
logger: new JsonLogger(),
},
);
configureApp(app);
registerHttpTelemetry(app.getHttpAdapter().getInstance());
setupSwagger(app);
enableGracefulShutdown(app);
+44 -1
View File
@@ -1,9 +1,15 @@
import './telemetry/register';
import { Logger } from '@nestjs/common';
import { propagation, ROOT_CONTEXT, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
import { ConfirmChannel } from 'amqplib';
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
import { PrismaService } from './db/prisma.service';
import { JobMessage } from './jobs/job-message';
import { recoverExpiredLeases, republishStale } from './jobs/job-store';
import { JsonLogger } from './telemetry/json-logger';
import { shutdownTelemetry } from './telemetry/sdk';
Logger.overrideLogger(new JsonLogger());
const BATCH_SIZE = 100;
const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500);
@@ -13,8 +19,29 @@ const FAULT = process.env.RELAY_FAULT;
const logger = new Logger('OutboxRelay');
const prisma = new PrismaService();
const tracer = trace.getTracer('messenger-relay');
let stopping = false;
function startPublishSpan(payload: JobMessage, headers: Record<string, string>): Span {
const parent = propagation.extract(ROOT_CONTEXT, payload);
const span = tracer.startSpan(
`${JOB_QUEUE} publish`,
{
kind: SpanKind.PRODUCER,
attributes: {
'messaging.system': 'rabbitmq',
'messaging.operation.type': 'publish',
'messaging.destination.name': JOB_QUEUE,
'messaging.message.id': payload.job_id,
'messaging.job.attempt': payload.attempt,
},
},
parent,
);
propagation.inject(trace.setSpan(parent, span), headers);
return span;
}
async function publishBatch(channel: ConfirmChannel): Promise<number> {
return prisma.$transaction(
async (tx) => {
@@ -27,15 +54,30 @@ async function publishBatch(channel: ConfirmChannel): Promise<number> {
if (rows.length === 0) {
return 0;
}
const spans: Span[] = [];
for (const { payload } of rows) {
const headers: Record<string, string> = {};
spans.push(startPublishSpan(payload, headers));
channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), {
persistent: true,
contentType: 'application/json',
messageId: payload.job_id,
type: payload.type,
headers,
});
}
await channel.waitForConfirms();
try {
await channel.waitForConfirms();
} catch (error) {
for (const span of spans) {
span.setStatus({ code: SpanStatusCode.ERROR, message: 'broker did not confirm' });
span.end();
}
throw error;
}
for (const span of spans) {
span.end();
}
logger.log(`published ${rows.length} job message(s), broker confirmed`);
if (FAULT === 'exit-after-confirm') {
logger.warn('fault injection: exit after broker confirm, before outbox is marked');
@@ -121,6 +163,7 @@ run()
.then(async () => {
await prisma.$disconnect();
logger.log('stopped');
await shutdownTelemetry();
process.exit(0);
})
.catch((error) => {
+66
View File
@@ -0,0 +1,66 @@
import { Logger } from '@nestjs/common';
import { metrics, trace } from '@opentelemetry/api';
import { FastifyInstance, FastifyRequest } from 'fastify';
export const LATENCY_BUCKETS = [
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 5, 10,
];
const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']);
const meter = metrics.getMeter('messenger-api');
const requests = meter.createCounter('http.server.requests', {
description: 'HTTP requests by method, route template and status class',
});
const duration = meter.createHistogram('http.server.request.duration', {
description: 'HTTP request duration',
unit: 's',
advice: { explicitBucketBoundaries: LATENCY_BUCKETS },
});
const logger = new Logger('Http');
export function routeTemplate(request: FastifyRequest): string {
const url = request.routeOptions?.url;
return url
? url.replace(/:([A-Za-z0-9_]+)/g, (_, name: string) => `{${name.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)}}`)
: 'unmatched';
}
export function registerHttpTelemetry(instance: FastifyInstance): void {
instance.addHook('onRequest', (request, _reply, done) => {
const span = trace.getActiveSpan();
if (span) {
const route = routeTemplate(request);
span.updateName(`${methodOf(request)} ${route}`);
span.setAttribute('http.route', route);
span.setAttribute('messenger.request_id', request.requestId);
}
done();
});
instance.addHook('onResponse', (request, reply, done) => {
const route = routeTemplate(request);
const method = methodOf(request);
const attributes = {
'http.request.method': method,
'http.route': route,
'http.response.status_class': `${Math.floor(reply.statusCode / 100)}xx`,
};
requests.add(1, attributes);
duration.record(reply.elapsedTime / 1000, attributes);
if (!route.startsWith('/health')) {
logger.log({
event: 'http.request',
method,
route,
status: reply.statusCode,
duration_ms: Math.round(reply.elapsedTime * 100) / 100,
request_id: request.requestId,
});
}
done();
});
}
function methodOf(request: FastifyRequest): string {
return METHODS.has(request.method) ? request.method : '_OTHER';
}
+99
View File
@@ -0,0 +1,99 @@
import { LoggerService } from '@nestjs/common';
import { trace, TraceFlags } from '@opentelemetry/api';
import { AnyValueMap, logs, SeverityNumber } from '@opentelemetry/api-logs';
type Level = 'fatal' | 'error' | 'warn' | 'log' | 'debug' | 'verbose';
const SEVERITY: Record<Level, [SeverityNumber, string]> = {
fatal: [SeverityNumber.FATAL, 'FATAL'],
error: [SeverityNumber.ERROR, 'ERROR'],
warn: [SeverityNumber.WARN, 'WARN'],
log: [SeverityNumber.INFO, 'INFO'],
debug: [SeverityNumber.DEBUG, 'DEBUG'],
verbose: [SeverityNumber.TRACE, 'TRACE'],
};
type Field = string | number | boolean;
export class JsonLogger implements LoggerService {
private readonly otel = logs.getLogger('messenger');
constructor(private readonly service = process.env.OTEL_SERVICE_NAME ?? 'messenger') {}
fatal(message: unknown, ...params: unknown[]): void {
this.write('fatal', message, params);
}
error(message: unknown, ...params: unknown[]): void {
this.write('error', message, params);
}
warn(message: unknown, ...params: unknown[]): void {
this.write('warn', message, params);
}
log(message: unknown, ...params: unknown[]): void {
this.write('log', message, params);
}
debug(message: unknown, ...params: unknown[]): void {
this.write('debug', message, params);
}
verbose(message: unknown, ...params: unknown[]): void {
this.write('verbose', message, params);
}
private write(level: Level, message: unknown, params: unknown[]): void {
const last = params[params.length - 1];
const logger = typeof last === 'string' ? last : 'app';
const fields = toFields(message);
const event = typeof fields.event === 'string' ? fields.event : eventName(logger);
delete fields.event;
const [severityNumber, severityText] = SEVERITY[level];
const active = trace.getActiveSpan()?.spanContext();
const span = active && active.traceFlags & TraceFlags.SAMPLED ? active : undefined;
const line = JSON.stringify({
timestamp: new Date().toISOString(),
severity: severityText,
'service.name': this.service,
event,
logger,
...fields,
trace_id: span?.traceId,
span_id: span?.spanId,
});
process.stdout.write(line + '\n');
const attributes: AnyValueMap = { event, logger };
for (const [key, value] of Object.entries(fields)) {
if (key !== 'message') {
attributes[key] = value;
}
}
this.otel.emit({
severityNumber,
severityText,
body: line,
attributes,
});
}
}
function toFields(message: unknown): Record<string, Field> {
if (typeof message === 'object' && message !== null && !(message instanceof Error)) {
const fields: Record<string, Field> = {};
for (const [key, value] of Object.entries(message)) {
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
fields[key] = value;
}
}
return fields;
}
return { message: message instanceof Error ? message.message : String(message) };
}
function eventName(logger: string): string {
return logger
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
.toLowerCase();
}
+3
View File
@@ -0,0 +1,3 @@
import { startTelemetry } from './sdk';
startTelemetry();
+157
View File
@@ -0,0 +1,157 @@
import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk';
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
import { envDetectorSync, Resource } from '@opentelemetry/resources';
import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
import {
DropAggregation,
PeriodicExportingMetricReader,
View,
} from '@opentelemetry/sdk-metrics';
import { NodeSDK } from '@opentelemetry/sdk-node';
import {
BatchSpanProcessor,
ParentBasedSampler,
Sampler,
SamplingDecision,
SamplingResult,
} from '@opentelemetry/sdk-trace-base';
import { PrismaInstrumentation } from '@prisma/instrumentation';
import { readFileSync } from 'fs';
import { IncomingMessage } from 'http';
import { hostname } from 'os';
import { join } from 'path';
const EXPORT_TIMEOUT_MS = 5000;
const NOISY_LEAF_SPANS = new Set([
'prisma:client:serialize',
'prisma:engine:serialize',
'prisma:engine:response_json_serialization',
'prisma:engine:connection',
]);
const NOT_RECORDED: SamplingResult = { decision: SamplingDecision.NOT_RECORD };
class SkipRootDatabaseSpans implements Sampler {
shouldSample(
_context: Context,
_traceId: string,
name: string,
): SamplingResult {
return name.startsWith('prisma:')
? NOT_RECORDED
: { decision: SamplingDecision.RECORD_AND_SAMPLED };
}
toString(): string {
return 'SkipRootDatabaseSpans';
}
}
class MessengerSampler implements Sampler {
private readonly parentBased = new ParentBasedSampler({ root: new SkipRootDatabaseSpans() });
shouldSample(
context: Context,
traceId: string,
name: string,
kind: SpanKind,
attributes: Attributes,
links: Link[],
): SamplingResult {
if (NOISY_LEAF_SPANS.has(name)) {
return NOT_RECORDED;
}
return this.parentBased.shouldSample(context, traceId, name, kind, attributes, links);
}
toString(): string {
return 'MessengerSampler';
}
}
let sdk: NodeSDK | null = null;
export function telemetryEnabled(): boolean {
return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT) && process.env.OTEL_SDK_DISABLED !== 'true';
}
export function startTelemetry(): void {
if (!telemetryEnabled() || sdk) {
return;
}
sdk = new NodeSDK({
resource: new Resource({
'service.version': serviceVersion(),
'service.instance.id': process.env.INSTANCE_ID ?? hostname(),
}),
resourceDetectors: [envDetectorSync],
sampler: new MessengerSampler(),
spanProcessors: [
new BatchSpanProcessor(new OTLPTraceExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), {
maxQueueSize: 2048,
maxExportBatchSize: 512,
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
}),
],
logRecordProcessors: [
new BatchLogRecordProcessor(new OTLPLogExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }), {
maxQueueSize: 2048,
maxExportBatchSize: 512,
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
}),
],
metricReader: new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({ timeoutMillis: EXPORT_TIMEOUT_MS }),
exportIntervalMillis: 10000,
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
}),
views: [
new View({
meterName: '@opentelemetry/instrumentation-http',
aggregation: new DropAggregation(),
}),
],
instrumentations: [
new HttpInstrumentation({
ignoreIncomingRequestHook: (request) => (request.url ?? '').startsWith('/health'),
ignoreOutgoingRequestHook: () => true,
requestHook: (span, request) => {
if (request instanceof IncomingMessage) {
const path = (request.url ?? '/').split('?')[0];
span.setAttribute('http.target', path);
span.setAttribute('http.url', path);
}
},
}),
new AwsInstrumentation({ suppressInternalInstrumentation: true }),
new PrismaInstrumentation(),
],
});
sdk.start();
}
export async function shutdownTelemetry(): Promise<void> {
if (!sdk) {
return;
}
const current = sdk;
sdk = null;
await Promise.race([
current.shutdown().catch(() => undefined),
new Promise((resolve) => setTimeout(resolve, EXPORT_TIMEOUT_MS)),
]);
}
function serviceVersion(): string {
try {
const raw = readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8');
return String(JSON.parse(raw).version);
} catch {
return 'unknown';
}
}
+95 -15
View File
@@ -1,10 +1,25 @@
import './telemetry/register';
import { Logger } from '@nestjs/common';
import {
context,
Context,
metrics,
propagation,
ROOT_CONTEXT,
SpanKind,
SpanStatusCode,
trace,
} from '@opentelemetry/api';
import { Channel, ConsumeMessage } from 'amqplib';
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
import { PrismaService } from './db/prisma.service';
import { parseJobMessage } from './jobs/job-message';
import { JOB_TYPE, JobMessage, parseJobMessage } from './jobs/job-message';
import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor';
import { S3Service } from './storage/s3.service';
import { JsonLogger } from './telemetry/json-logger';
import { shutdownTelemetry } from './telemetry/sdk';
Logger.overrideLogger(new JsonLogger());
const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2);
const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000);
@@ -18,6 +33,27 @@ const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks())
let stopping = false;
let inFlight = 0;
const JOB_BUCKETS = [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 2.5, 5, 10, 30, 60, 120];
const tracer = trace.getTracer('messenger-worker');
const meter = metrics.getMeter('messenger-worker');
const jobsProcessed = meter.createCounter('jobs.processed', {
description: 'Job deliveries handled by the worker, by job type and outcome',
});
const jobDuration = meter.createHistogram('jobs.duration', {
description: 'Job delivery handling duration',
unit: 's',
advice: { explicitBucketBoundaries: JOB_BUCKETS },
});
function parentContext(delivery: ConsumeMessage, message: JobMessage | null): Context {
const headers = delivery.properties.headers ?? {};
if (typeof headers.traceparent === 'string') {
return propagation.extract(ROOT_CONTEXT, headers);
}
return message ? propagation.extract(ROOT_CONTEXT, message) : ROOT_CONTEXT;
}
function faultHooks(): ProcessorHooks {
const crash = (point: string) => async () => {
logger.warn(`fault injection: worker killed ${point}`);
@@ -34,31 +70,74 @@ function faultHooks(): ProcessorHooks {
async function handle(channel: Channel, delivery: ConsumeMessage): Promise<void> {
inFlight++;
const started = performance.now();
const message = parseJobMessage(delivery.content);
const span = tracer.startSpan(
`${JOB_QUEUE} process`,
{
kind: SpanKind.CONSUMER,
attributes: {
'messaging.system': 'rabbitmq',
'messaging.operation.type': 'process',
'messaging.destination.name': JOB_QUEUE,
'messaging.message.id': message?.job_id ?? 'malformed',
'messaging.rabbitmq.redelivered': delivery.fields.redelivered,
'messaging.job.attempt': message?.attempt ?? 0,
},
},
parentContext(delivery, message),
);
const active = trace.setSpan(context.active(), span);
let outcome = 'interrupted';
let requeue = false;
try {
const message = parseJobMessage(delivery.content);
if (!message) {
logger.warn('malformed job message moved to dead letter queue');
channel.nack(delivery, false, false);
return;
await context.with(active, async () => {
if (!message) {
outcome = 'malformed';
logger.warn('malformed job message moved to dead letter queue');
channel.nack(delivery, false, false);
return;
}
const result = await processor.handle(message);
outcome = result === 'ready' ? 'success' : result;
logger.log({
event: 'job.handled',
job_id: message.job_id,
attachment_id: message.attachment_id,
attempt: message.attempt,
redelivered: delivery.fields.redelivered,
outcome,
duration_ms: Math.round(performance.now() - started),
});
channel.ack(delivery);
});
span.setAttribute('messenger.job.outcome', outcome);
if (outcome === 'failed' || outcome === 'malformed') {
span.setStatus({ code: SpanStatusCode.ERROR, message: outcome });
}
const outcome = await processor.handle(message);
logger.log(
`job ${message.job_id} attempt ${message.attempt}` +
`${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`,
);
channel.ack(delivery);
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`);
span.setAttribute('messenger.job.outcome', outcome);
span.setStatus({ code: SpanStatusCode.ERROR, message: reason });
context.with(active, () =>
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`),
);
requeue = true;
} finally {
span.end();
const attributes = { 'job.type': JOB_TYPE, outcome };
jobsProcessed.add(1, attributes);
jobDuration.record((performance.now() - started) / 1000, attributes);
}
if (requeue) {
await sleep(INFRA_RETRY_MS);
try {
channel.nack(delivery, false, true);
} catch {
logger.warn('channel closed, broker will redeliver unacked message');
}
} finally {
inFlight--;
}
inFlight--;
}
async function run(): Promise<void> {
@@ -129,6 +208,7 @@ run()
await prisma.$disconnect();
s3.onModuleDestroy();
logger.log('stopped');
await shutdownTelemetry();
process.exit(0);
})
.catch((error) => {