lab6: каркас OpenTelemetry — SDK, инструменты метрик, трассировка, структурный логгер, HTTP-хуки, метрики очереди

This commit is contained in:
Ilya
2026-11-09 20:20:26 +03:00
parent 6889e77ab4
commit cf87f4dd43
17 changed files with 1428 additions and 0 deletions
+11
View File
@@ -35,6 +35,17 @@
"@nestjs/core": "^12.0.1",
"@nestjs/platform-express": "^12.0.1",
"@nestjs/platform-fastify": "^12.0.3",
"@opentelemetry/api": "1.9.0",
"@opentelemetry/api-logs": "0.203.0",
"@opentelemetry/core": "2.0.1",
"@opentelemetry/exporter-logs-otlp-proto": "0.203.0",
"@opentelemetry/exporter-metrics-otlp-proto": "0.203.0",
"@opentelemetry/exporter-trace-otlp-proto": "0.203.0",
"@opentelemetry/resources": "2.0.1",
"@opentelemetry/sdk-logs": "0.203.0",
"@opentelemetry/sdk-metrics": "2.0.1",
"@opentelemetry/sdk-trace-base": "2.0.1",
"@opentelemetry/sdk-trace-node": "2.0.1",
"@smithy/node-http-handler": "^4.12.1",
"amqplib": "^2.2.1",
"argon2": "^0.45.1",
+8
View File
@@ -170,6 +170,14 @@ export const appConfigSchema = z
86400,
),
WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081),
// lab6: как часто API пересчитывает глубину и возраст очереди задач для
// метрик messenger.jobs.queue.* (один дешёвый агрегат по частичным индексам).
QUEUE_STATS_INTERVAL_MS: numeric(
'QUEUE_STATS_INTERVAL_MS',
1000,
600000,
10000,
),
FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0),
QUEUED_RECONCILE_GRACE_SECONDS: numeric(
'QUEUED_RECONCILE_GRACE_SECONDS',
+5
View File
@@ -0,0 +1,5 @@
// Импортируется первым в main.ts: ESM выполняет модули в порядке импорта, и
// глобальные провайдеры OTel должны быть установлены до первого запроса.
import { startTelemetry } from './sdk.js';
startTelemetry('messenger-api');
+36
View File
@@ -0,0 +1,36 @@
import { ExportResultCode, type ExportResult } from '@opentelemetry/core';
import { instruments } from './instruments.js';
type BatchExporter<T> = {
export(items: T[], resultCallback: (result: ExportResult) => void): void;
shutdown(): Promise<void>;
forceFlush?(): Promise<void>;
};
/**
* Обёртка над OTLP-экспортёром: считает, сколько span и log records ушло и
* сколько потеряно. Счётчик кумулятивный, поэтому потери во время простоя
* Collector становятся видны в Prometheus сразу после его возвращения.
* Переполнение очереди BatchProcessor (drop до экспорта) SDK пишет в diag-лог.
*/
export function countingExporter<T>(
signal: 'traces' | 'logs',
inner: BatchExporter<T>,
): BatchExporter<T> {
return {
export(items: T[], resultCallback: (result: ExportResult) => void) {
inner.export(items, (result: ExportResult) => {
instruments().telemetryExport.add(items.length, {
signal,
outcome:
result.code === ExportResultCode.SUCCESS
? 'success'
: 'failed',
});
resultCallback(result);
});
},
shutdown: () => inner.shutdown(),
forceFlush: () => inner.forceFlush?.() ?? Promise.resolve(),
};
}
+35
View File
@@ -0,0 +1,35 @@
import { describe, expect, it } from 'vitest';
import { isTracedRoute, normalizeMethod, routeTemplate } from './http.js';
describe('routeTemplate', () => {
it('turns Fastify params into low-cardinality templates', () => {
expect(routeTemplate('/api/v1/chats/:chatId/messages')).toBe(
'/api/v1/chats/{chat_id}/messages',
);
expect(routeTemplate('/api/v1/attachments/:attachmentId/content')).toBe(
'/api/v1/attachments/{attachment_id}/content',
);
expect(routeTemplate('/api/v1/users/me')).toBe('/api/v1/users/me');
});
it('groups requests that matched no route', () => {
expect(routeTemplate(undefined)).toBe('unmatched');
});
});
describe('normalizeMethod', () => {
it('keeps standard methods and folds the rest', () => {
expect(normalizeMethod('get')).toBe('GET');
expect(normalizeMethod('DELETE')).toBe('DELETE');
expect(normalizeMethod('PROPFIND')).toBe('_OTHER');
});
});
describe('isTracedRoute', () => {
it('skips probes, internal endpoints and unmatched requests', () => {
expect(isTracedRoute('/health/ready')).toBe(false);
expect(isTracedRoute('/internal/metrics')).toBe(false);
expect(isTracedRoute('unmatched')).toBe(false);
expect(isTracedRoute('/api/v1/chats/{chat_id}/messages')).toBe(true);
});
});
+201
View File
@@ -0,0 +1,201 @@
import {
context,
propagation,
ROOT_CONTEXT,
SpanKind,
SpanStatusCode,
trace,
type Context,
type Span,
} from '@opentelemetry/api';
import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { instruments } from './instruments.js';
import { appLogger } from './logger.js';
import { tracer } from './tracing.js';
const STATE = Symbol('messenger.telemetry.http');
const KNOWN_METHODS = new Set([
'GET',
'HEAD',
'POST',
'PUT',
'PATCH',
'DELETE',
'OPTIONS',
]);
// Пробы HAProxy (каждые 500 мс на реплику) и внутренние endpoint не трассируются
// и не пишут access-log: они дали бы больше половины объёма Tempo и Loki без
// диагностической ценности. В метриках они остаются.
const UNTRACED_PREFIXES = ['/health', '/internal/'];
type HttpState = {
readonly ctx: Context;
readonly span: Span | undefined;
readonly method: string;
readonly route: string;
readonly startedAt: bigint;
finished: boolean;
};
type InstrumentedRequest = FastifyRequest & { [STATE]?: HttpState };
type RouteHandler = (
this: unknown,
request: FastifyRequest,
reply: FastifyReply,
) => unknown;
/** `/api/v1/chats/:chatId/messages` → `/api/v1/chats/{chat_id}/messages`. */
export function routeTemplate(url: string | undefined): string {
if (!url) return 'unmatched';
return url.replace(
/:([A-Za-z0-9_]+)/gu,
(_match, name: string) =>
`{${name.replace(/([a-z0-9])([A-Z])/gu, '$1_$2').toLowerCase()}}`,
);
}
export function normalizeMethod(method: string): string {
const upper = method.toUpperCase();
return KNOWN_METHODS.has(upper) ? upper : '_OTHER';
}
export function isTracedRoute(route: string): boolean {
return (
route !== 'unmatched' &&
!UNTRACED_PREFIXES.some((prefix) => route.startsWith(prefix))
);
}
function headerValue(value: unknown): string | undefined {
if (typeof value === 'string') return value;
if (typeof value === 'number') return String(value);
if (Array.isArray(value) && typeof value[0] === 'string') return value[0];
return undefined;
}
/**
* HTTP-инструментация на хуках Fastify:
* - onRequest: SERVER span с родителем из входящего `traceparent`;
* - onRoute: обработчик маршрута выполняется внутри контекста span, поэтому
* span PostgreSQL/S3/RabbitMQ и логи обработчика становятся его детьми;
* - onResponse / onRequestAbort: длительность в гистограмму, access-log, end span.
* В метрики попадают только метод, route template и код статуса.
* Регистрируется до app.listen(): маршруты Nest добавляются при init.
*/
export function installHttpTelemetry(fastify: FastifyInstance): void {
fastify.addHook('onRoute', (routeOptions) => {
const original = routeOptions.handler as unknown as RouteHandler | undefined;
if (typeof original !== 'function') return;
const wrapped: RouteHandler = function (
this: unknown,
request: FastifyRequest,
reply: FastifyReply,
) {
const state = (request as InstrumentedRequest)[STATE];
if (!state?.span) return original.call(this, request, reply);
return context.with(state.ctx, () =>
original.call(this, request, reply),
);
};
(routeOptions as unknown as { handler: RouteHandler }).handler = wrapped;
});
fastify.addHook(
'onRequest',
(request: FastifyRequest, _reply: FastifyReply, done: () => void) => {
const method = normalizeMethod(request.method);
const route = routeTemplate(request.routeOptions?.url);
let ctx = ROOT_CONTEXT;
let span: Span | undefined;
if (isTracedRoute(route)) {
const parent = propagation.extract(ROOT_CONTEXT, request.headers);
span = tracer().startSpan(
`${method} ${route}`,
{
kind: SpanKind.SERVER,
attributes: {
'http.request.method': method,
'http.route': route,
'url.scheme': 'http',
'network.protocol.version': request.raw.httpVersion,
},
},
parent,
);
ctx = trace.setSpan(parent, span);
}
(request as InstrumentedRequest)[STATE] = {
ctx,
span,
method,
route,
startedAt: process.hrtime.bigint(),
finished: false,
};
done();
},
);
fastify.addHook(
'onResponse',
(request: FastifyRequest, reply: FastifyReply, done: () => void) => {
finish(
request as InstrumentedRequest,
reply.statusCode,
headerValue(reply.getHeader('x-request-id')),
false,
);
done();
},
);
fastify.addHook(
'onRequestAbort',
(request: FastifyRequest, done: () => void) => {
// 499 — клиент закрыл соединение до ответа (конвенция nginx); в SLO
// доступности не считается ошибкой сервиса.
finish(request as InstrumentedRequest, 499, undefined, true);
done();
},
);
}
function finish(
request: InstrumentedRequest,
status: number,
requestId: string | undefined,
aborted: boolean,
): void {
const state = request[STATE];
if (!state || state.finished) return;
state.finished = true;
const seconds = Number(process.hrtime.bigint() - state.startedAt) / 1e9;
instruments().httpDuration.record(seconds, {
'http.request.method': state.method,
'http.route': state.route,
'http.response.status_code': status,
});
const span = state.span;
if (!span) return;
span.setAttribute('http.response.status_code', status);
if (requestId) span.setAttribute('messenger.request_id', requestId);
if (aborted) span.setAttribute('error.type', 'request_aborted');
if (status >= 500)
span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` });
context.with(state.ctx, () =>
appLogger.event(
status >= 500 ? 'ERROR' : 'INFO',
'http.request',
`${state.method} ${state.route} ${status}`,
{
'http.request.method': state.method,
'http.route': state.route,
'http.response.status_code': status,
duration_ms: Math.round(seconds * 10_000) / 10,
request_id: requestId,
aborted: aborted || undefined,
},
),
);
span.end();
}
+35
View File
@@ -0,0 +1,35 @@
import os from 'node:os';
export type ServiceIdentity = {
readonly serviceName: string;
readonly serviceVersion: string;
readonly instanceId: string;
};
function fromEnv(defaultServiceName: string): ServiceIdentity {
return {
serviceName: process.env.OTEL_SERVICE_NAME?.trim() || defaultServiceName,
serviceVersion: process.env.SERVICE_VERSION?.trim() || 'dev',
// В compose у реплик API нет INSTANCE_ID, но HOSTNAME — id контейнера,
// уникальный для каждой реплики: так service.instance.id различает
// api-1 и api-2 во всех трёх сигналах.
instanceId:
process.env.INSTANCE_ID?.trim() ||
process.env.HOSTNAME?.trim() ||
os.hostname(),
};
}
let current = fromEnv('messenger');
/** service.name / service.version / service.instance.id, общие для всех сигналов. */
export function serviceIdentity(): ServiceIdentity {
return current;
}
export function resolveServiceIdentity(
defaultServiceName: string,
): ServiceIdentity {
current = fromEnv(defaultServiceName);
return current;
}
+94
View File
@@ -0,0 +1,94 @@
import {
metrics,
type Counter,
type Histogram,
type Meter,
type MeterProvider,
} from '@opentelemetry/api';
export const METER_NAME = 'messenger';
// Каждый SLO-порог — точная граница бакета (0.25 с для API, 30 с для
// time-to-ready): иначе good-события пришлось бы интерполировать.
export const HTTP_DURATION_BUCKETS = [
0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5, 0.75, 1, 2.5, 5, 10,
30,
];
export const JOB_DURATION_BUCKETS = [
0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 7.5, 10, 15, 20, 30, 45, 60, 120,
];
export const TIME_TO_READY_BUCKETS = [
0.25, 0.5, 1, 2, 3, 5, 7.5, 10, 15, 20, 30, 45, 60, 120, 300, 600,
];
export const WAIT_BUCKETS = [
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
];
export type Instruments = {
readonly httpDuration: Histogram;
readonly jobsProcessed: Counter;
readonly jobDuration: Histogram;
readonly timeToReady: Histogram;
readonly brokerPublished: Counter;
readonly dbConnectionWait: Histogram;
readonly telemetryExport: Counter;
};
let cached: { provider: MeterProvider; instruments: Instruments } | undefined;
export function meter(): Meter {
return metrics.getMeter(METER_NAME, '1.0.0');
}
/**
* Инструменты создаются лениво и пересоздаются, если сменился глобальный
* MeterProvider: созданный до старта SDK инструмент навсегда остался бы no-op.
* Атрибуты — только ограниченные множества (метод, route template, код
* статуса, outcome); id пользователей, чатов, вложений и запросов сюда не
* попадают никогда.
*/
export function instruments(): Instruments {
const provider = metrics.getMeterProvider();
if (cached?.provider === provider) return cached.instruments;
const m = provider.getMeter(METER_NAME, '1.0.0');
const created: Instruments = {
httpDuration: m.createHistogram('http.server.request.duration', {
unit: 's',
description:
'HTTP server request duration by method, route template and status code',
advice: { explicitBucketBoundaries: HTTP_DURATION_BUCKETS },
}),
jobsProcessed: m.createCounter('messenger.jobs.processed', {
unit: '{job}',
description: 'Job deliveries handled by the worker, by outcome',
}),
jobDuration: m.createHistogram('messenger.job.duration', {
unit: 's',
description:
'Duration of one job attempt from claim to commit, by outcome',
advice: { explicitBucketBoundaries: JOB_DURATION_BUCKETS },
}),
timeToReady: m.createHistogram('messenger.attachment.time_to_ready', {
unit: 's',
description:
'Time from durable acceptance of an upload to the ready commit (includes queueing and retries)',
advice: { explicitBucketBoundaries: TIME_TO_READY_BUCKETS },
}),
brokerPublished: m.createCounter('messenger.broker.published', {
unit: '{message}',
description: 'Messages published to the broker, by outcome',
}),
dbConnectionWait: m.createHistogram('db.client.connection.wait_time', {
unit: 's',
description: 'Time spent waiting for a connection from the pool',
advice: { explicitBucketBoundaries: WAIT_BUCKETS },
}),
telemetryExport: m.createCounter('messenger.telemetry.export.items', {
unit: '{item}',
description:
'Spans and log records handed to the OTLP exporter, by outcome (failed = lost)',
}),
};
cached = { provider, instruments: created };
return created;
}
+217
View File
@@ -0,0 +1,217 @@
import type { LoggerService, LogLevel } from '@nestjs/common';
import { context, trace } from '@opentelemetry/api';
import {
logs,
SeverityNumber,
type AnyValueMap,
type Logger as OtelLogger,
type LoggerProvider,
} from '@opentelemetry/api-logs';
import { serviceIdentity } from './identity.js';
import { redact } from './redact.js';
export type Severity = 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' | 'FATAL';
export type LogAttributes = Record<
string,
string | number | boolean | null | undefined
>;
const SEVERITY_NUMBER: Record<Severity, SeverityNumber> = {
TRACE: SeverityNumber.TRACE,
DEBUG: SeverityNumber.DEBUG,
INFO: SeverityNumber.INFO,
WARN: SeverityNumber.WARN,
ERROR: SeverityNumber.ERROR,
FATAL: SeverityNumber.FATAL,
};
const RANK: Record<Severity, number> = {
TRACE: 1,
DEBUG: 5,
INFO: 9,
WARN: 13,
ERROR: 17,
FATAL: 21,
};
const NEST_LEVEL: Record<LogLevel, Severity> = {
verbose: 'TRACE',
debug: 'DEBUG',
log: 'INFO',
warn: 'WARN',
error: 'ERROR',
fatal: 'FATAL',
};
const MAX_TEXT = 4000;
function parseSeverity(raw: string | undefined): Severity {
switch ((raw ?? '').trim().toLowerCase()) {
case 'trace':
case 'verbose':
return 'TRACE';
case 'debug':
return 'DEBUG';
case 'warn':
case 'warning':
return 'WARN';
case 'error':
return 'ERROR';
default:
return 'INFO';
}
}
function stringify(value: unknown): string {
if (typeof value === 'string') return value;
if (value instanceof Error) return `${value.name}: ${value.message}`;
try {
return JSON.stringify(value) ?? String(value);
} catch {
return String(value);
}
}
/**
* Структурный логгер для Nest и событий приложения. Каждая запись уходит:
* - в stdout JSON-строкой (docker logs остаются читаемыми без Loki);
* - в OTLP logs с контекстом активного span, поэтому в Loki у записи есть
* trace_id/span_id и из неё можно перейти в Tempo.
* Текст и строковые атрибуты проходят redact() и обрезаются до 4000 символов.
*/
export class StructuredLogger implements LoggerService {
private minRank = RANK[parseSeverity(process.env.LOG_LEVEL)];
private otel: { provider: LoggerProvider; logger: OtelLogger } | undefined;
log(message: unknown, ...optionalParams: unknown[]): void {
this.fromNest('INFO', message, optionalParams);
}
error(message: unknown, ...optionalParams: unknown[]): void {
this.fromNest('ERROR', message, optionalParams);
}
warn(message: unknown, ...optionalParams: unknown[]): void {
this.fromNest('WARN', message, optionalParams);
}
debug(message: unknown, ...optionalParams: unknown[]): void {
this.fromNest('DEBUG', message, optionalParams);
}
verbose(message: unknown, ...optionalParams: unknown[]): void {
this.fromNest('TRACE', message, optionalParams);
}
fatal(message: unknown, ...optionalParams: unknown[]): void {
this.fromNest('FATAL', message, optionalParams);
}
setLogLevels(levels: LogLevel[]): void {
const ranks = levels
.map((level) => NEST_LEVEL[level])
.filter((severity): severity is Severity => severity !== undefined)
.map((severity) => RANK[severity]);
if (ranks.length > 0) this.minRank = Math.min(...ranks);
}
/** Событие приложения: `event` — стабильное имя для запросов в Loki. */
event(
severity: Severity,
event: string,
message: string,
attributes: LogAttributes = {},
): void {
this.write(severity, event, undefined, message, attributes);
}
/**
* Конвенция Nest: контекст — последний строковый параметр, для error перед
* ним может идти stack.
*/
private fromNest(
severity: Severity,
message: unknown,
params: unknown[],
): void {
if (RANK[severity] < this.minRank) return;
const rest = [...params];
let contextName: string | undefined;
if (rest.length > 0 && typeof rest[rest.length - 1] === 'string')
contextName = rest.pop() as string;
let stack: string | undefined;
if (severity === 'ERROR' || severity === 'FATAL') {
const index = rest.findIndex(
(param) => typeof param === 'string' && param.includes('\n'),
);
if (index >= 0) stack = rest.splice(index, 1)[0] as string;
if (!stack && message instanceof Error) stack = message.stack;
}
let text = stringify(message);
if (rest.length > 0) text = `${text} ${rest.map(stringify).join(' ')}`;
this.write(
severity,
'log',
contextName,
text,
stack ? { 'exception.stacktrace': stack } : {},
);
}
private write(
severity: Severity,
event: string,
contextName: string | undefined,
message: string,
attributes: LogAttributes,
): void {
if (RANK[severity] < this.minRank) return;
const identity = serviceIdentity();
const active = context.active();
const spanContext = trace.getSpan(active)?.spanContext();
let ids: { trace_id: string; span_id: string } | undefined;
if (spanContext && trace.isSpanContextValid(spanContext))
ids = { trace_id: spanContext.traceId, span_id: spanContext.spanId };
const body = redact(message).slice(0, MAX_TEXT);
const clean: Record<string, string | number | boolean> = {};
for (const [key, value] of Object.entries(attributes)) {
if (value === undefined || value === null) continue;
clean[key] =
typeof value === 'string' ? redact(value).slice(0, MAX_TEXT) : value;
}
process.stdout.write(
`${JSON.stringify({
timestamp: new Date().toISOString(),
severity,
'service.name': identity.serviceName,
'service.instance.id': identity.instanceId,
event,
...(contextName ? { context: contextName } : {}),
message: body,
...ids,
...clean,
})}\n`,
);
this.otelLogger().emit({
severityNumber: SEVERITY_NUMBER[severity],
severityText: severity,
body,
attributes: {
'event.name': event,
...(contextName ? { 'log.context': contextName } : {}),
...clean,
} as AnyValueMap,
context: active,
});
}
private otelLogger(): OtelLogger {
const provider = logs.getLoggerProvider();
if (this.otel?.provider !== provider)
this.otel = { provider, logger: provider.getLogger('messenger', '1.0.0') };
return this.otel.logger;
}
}
/** Единственный экземпляр: передаётся в NestFactory и используется для событий. */
export const appLogger = new StructuredLogger();
+164
View File
@@ -0,0 +1,164 @@
import {
Injectable,
Logger,
type OnModuleDestroy,
type OnModuleInit,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import type { AppConfig } from '../config/app-config.schema.js';
import { DatabaseClient } from '../database/database.client.js';
import { JOB_QUEUE } from '../jobs/broker.topology.js';
import { meter } from './instruments.js';
type QueueSnapshot = {
readonly takenAt: number;
readonly ready: number;
readonly delayed: number;
readonly processing: number;
readonly oldestAgeSeconds: number;
readonly outboxPending: number;
readonly outboxOldestAgeSeconds: number;
};
// Источник истины о состоянии задач — PostgreSQL, а не глубина очереди RabbitMQ:
// задача может ждать в outbox, в backoff или под истёкшим lease, не лежа в
// брокере. Возраст самой старой незавершённой задачи — то, что чувствует
// пользователь, ожидающий ready.
const JOBS_SQL = `SELECT
count(*) FILTER (WHERE status = 'queued' AND next_attempt_at <= now())::int AS ready,
count(*) FILTER (WHERE status = 'queued' AND next_attempt_at > now())::int AS delayed,
count(*) FILTER (WHERE status = 'processing')::int AS processing,
COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::float8 AS oldest_age
FROM attachment_jobs
WHERE status IN ('queued', 'processing')`;
const OUTBOX_SQL = `SELECT
count(*)::int AS pending,
COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::float8 AS oldest_age
FROM job_outbox
WHERE published_at IS NULL AND available_at <= now()`;
type JobsRow = {
ready: number;
delayed: number;
processing: number;
oldest_age: number;
};
type OutboxRow = { pending: number; oldest_age: number };
/**
* Метрики очереди задач. Работает в API, а не в worker: в сценарии «worker
* остановлен» именно эти gauge должны продолжать расти. Обе реплики
* публикуют одинаковые значения, в PromQL берётся max(). При ошибке запроса
* старый снимок публикуется не дольше трёх интервалов, затем серия пропадает
* — лучше отсутствие данных, чем устаревший ноль.
*/
@Injectable()
export class QueueStatsService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(QueueStatsService.name);
private readonly intervalMs: number;
private snapshot: QueueSnapshot | undefined;
private timer: NodeJS.Timeout | undefined;
private refreshing = false;
private failures = 0;
constructor(
private readonly database: DatabaseClient,
config: ConfigService<AppConfig>,
) {
this.intervalMs = config.getOrThrow('QUEUE_STATS_INTERVAL_MS');
}
onModuleInit(): void {
const m = meter();
const depth = m.createObservableGauge('messenger.jobs.queue.depth', {
unit: '{job}',
description: 'Unfinished attachment jobs by state (PostgreSQL)',
});
const oldestAge = m.createObservableGauge(
'messenger.jobs.queue.oldest_age',
{
unit: 's',
description:
'Age of the oldest unfinished job since durable acceptance',
},
);
const outboxPending = m.createObservableGauge('messenger.outbox.pending', {
unit: '{message}',
description: 'Outbox rows due for publishing but not yet confirmed',
});
const outboxAge = m.createObservableGauge('messenger.outbox.oldest_age', {
unit: 's',
description: 'Age of the oldest unpublished outbox row',
});
m.addBatchObservableCallback(
(result) => {
const snapshot = this.freshSnapshot();
if (!snapshot) return;
const queue = { 'messaging.destination.name': JOB_QUEUE };
result.observe(depth, snapshot.ready, { ...queue, state: 'ready' });
result.observe(depth, snapshot.delayed, {
...queue,
state: 'delayed',
});
result.observe(depth, snapshot.processing, {
...queue,
state: 'processing',
});
result.observe(oldestAge, snapshot.oldestAgeSeconds, queue);
result.observe(outboxPending, snapshot.outboxPending, queue);
result.observe(outboxAge, snapshot.outboxOldestAgeSeconds, queue);
},
[depth, oldestAge, outboxPending, outboxAge],
);
void this.refresh();
this.timer = setInterval(() => void this.refresh(), this.intervalMs);
this.timer.unref();
}
onModuleDestroy(): void {
if (this.timer) clearInterval(this.timer);
}
private freshSnapshot(): QueueSnapshot | undefined {
if (!this.snapshot) return undefined;
return Date.now() - this.snapshot.takenAt <= this.intervalMs * 3
? this.snapshot
: undefined;
}
private async refresh(): Promise<void> {
if (this.refreshing) return;
this.refreshing = true;
try {
const [jobs, outbox] = await Promise.all([
this.database.query<JobsRow>('telemetry.queue_stats', JOBS_SQL),
this.database.query<OutboxRow>('telemetry.outbox_stats', OUTBOX_SQL),
]);
const j = jobs.rows[0];
const o = outbox.rows[0];
this.snapshot = {
takenAt: Date.now(),
ready: Number(j?.ready ?? 0),
delayed: Number(j?.delayed ?? 0),
processing: Number(j?.processing ?? 0),
oldestAgeSeconds: Number(j?.oldest_age ?? 0),
outboxPending: Number(o?.pending ?? 0),
outboxOldestAgeSeconds: Number(o?.oldest_age ?? 0),
};
if (this.failures > 0)
this.logger.log(
`queue stats recovered after ${this.failures} failed refreshes`,
);
this.failures = 0;
} catch (error) {
this.failures += 1;
if (this.failures === 1 || this.failures % 30 === 0)
this.logger.warn(
`queue stats refresh failed (${this.failures} in a row): ${error instanceof Error ? error.name : 'unknown error'}`,
);
} finally {
this.refreshing = false;
}
}
}
+37
View File
@@ -0,0 +1,37 @@
import { describe, expect, it } from 'vitest';
import { redact } from './redact.js';
describe('redact', () => {
it('masks bearer and basic credentials', () => {
const out = redact(
'Authorization: Bearer abc.def-123 and Basic dXNlcjpwYXNz',
);
expect(out).not.toContain('abc.def-123');
expect(out).not.toContain('dXNlcjpwYXNz');
});
it('masks credentials embedded in connection strings', () => {
expect(
redact('postgresql://messenger:s3cret@postgres:5432/messenger'),
).toBe('postgresql://[REDACTED]@postgres:5432/messenger');
});
it('masks presigned URL credentials and signatures', () => {
const out = redact(
'https://s3.local/b/k?X-Amz-Credential=AKIAEXAMPLE%2F20260101&X-Amz-Signature=deadbeef',
);
expect(out).not.toContain('AKIAEXAMPLE');
expect(out).not.toContain('deadbeef');
});
it('masks password-like fields', () => {
expect(redact('{"password":"hunter2"}')).not.toContain('hunter2');
expect(redact('password=hunter2&x=1')).not.toContain('hunter2');
});
it('keeps ordinary diagnostic text intact', () => {
expect(redact('job retry scheduled attempt=2 code=transient')).toBe(
'job retry scheduled attempt=2 code=transient',
);
});
});
+29
View File
@@ -0,0 +1,29 @@
/**
* Последний рубеж перед stdout и OTLP: если секрет всё же попал в текст
* события (например, в сообщение ошибки драйвера), он не уйдёт в Loki.
* Основная защита — не класть такие данные в события вообще: route template
* вместо URL, коды ошибок вместо сообщений, без тел запросов.
*/
const RULES: ReadonlyArray<readonly [RegExp, string]> = [
[/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/giu, '$1 [REDACTED]'],
[/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/giu, '$1[REDACTED]@'],
[
/\b(X-Amz-(?:Signature|Credential|Security-Token))=[^&\s"']+/giu,
'$1=[REDACTED]',
],
[
/((?:password|passwd|secret|token|authorization|cookie)["']?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;&}]+)/giu,
'$1[REDACTED]',
],
[
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/gu,
'[REDACTED PRIVATE KEY]',
],
];
export function redact(text: string): string {
let result = text;
for (const [pattern, replacement] of RULES)
result = result.replace(pattern, replacement);
return result;
}
+55
View File
@@ -0,0 +1,55 @@
import { monitorEventLoopDelay } from 'node:perf_hooks';
import { meter } from './instruments.js';
let started = false;
/**
* Насыщение процесса: CPU, RSS, heap и задержка event loop. Задержка event
* loop — главный признак насыщения Node.js: CPU-bound работа (sharp, argon2)
* и синхронный JSON на больших ответах растят её раньше, чем latency API.
*/
export function startRuntimeMetrics(): void {
if (started) return;
started = true;
const m = meter();
const loop = monitorEventLoopDelay({ resolution: 20 });
loop.enable();
m.createObservableCounter('process.cpu.time', {
unit: 's',
description: 'CPU time consumed by the process',
}).addCallback((result) => {
const usage = process.cpuUsage();
result.observe(usage.user / 1e6, { 'cpu.mode': 'user' });
result.observe(usage.system / 1e6, { 'cpu.mode': 'system' });
});
m.createObservableUpDownCounter('process.memory.usage', {
unit: 'By',
description: 'Resident set size of the process',
}).addCallback((result) => {
result.observe(process.memoryUsage.rss());
});
m.createObservableUpDownCounter('v8js.memory.heap.used', {
unit: 'By',
description: 'V8 heap in use',
}).addCallback((result) => {
result.observe(process.memoryUsage().heapUsed);
});
const delayP99 = m.createObservableGauge('nodejs.eventloop.delay.p99', {
unit: 's',
description: 'p99 event loop delay since the previous collection',
});
const delayMax = m.createObservableGauge('nodejs.eventloop.delay.max', {
unit: 's',
description: 'Max event loop delay since the previous collection',
});
m.addBatchObservableCallback(
(result) => {
result.observe(delayP99, loop.percentile(99) / 1e9);
result.observe(delayMax, loop.max / 1e9);
loop.reset();
},
[delayP99, delayMax],
);
}
+295
View File
@@ -0,0 +1,295 @@
import {
diag,
DiagLogLevel,
metrics,
type DiagLogger,
} from '@opentelemetry/api';
import { logs } from '@opentelemetry/api-logs';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
import { resourceFromAttributes } from '@opentelemetry/resources';
import {
BatchLogRecordProcessor,
LoggerProvider,
} from '@opentelemetry/sdk-logs';
import {
MeterProvider,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics';
import {
BatchSpanProcessor,
ParentBasedSampler,
TraceIdRatioBasedSampler,
} from '@opentelemetry/sdk-trace-base';
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { countingExporter } from './export-counter.js';
import { resolveServiceIdentity } from './identity.js';
import { redact } from './redact.js';
import { startRuntimeMetrics } from './runtime-metrics.js';
type Providers = {
readonly tracer: NodeTracerProvider;
readonly meter: MeterProvider;
readonly logger: LoggerProvider;
};
let providers: Providers | undefined;
function intFromEnv(
name: string,
fallback: number,
min: number,
max: number,
): number {
const raw = process.env[name]?.trim();
if (!raw) return fallback;
const value = Number(raw);
return Number.isInteger(value) && value >= min && value <= max
? value
: fallback;
}
function ratioFromEnv(name: string, fallback: number): number {
const raw = process.env[name]?.trim();
if (!raw) return fallback;
const value = Number(raw);
return Number.isFinite(value) && value >= 0 && value <= 1 ? value : fallback;
}
/** OTEL_RESOURCE_ATTRIBUTES: `k=v,k2=v2`, значения percent-encoded. */
export function parseResourceAttributes(
raw: string | undefined,
): Record<string, string> {
const attributes: Record<string, string> = {};
if (!raw) return attributes;
for (const pair of raw.split(',')) {
const index = pair.indexOf('=');
if (index <= 0) continue;
const key = pair.slice(0, index).trim();
const value = pair.slice(index + 1).trim();
if (!key) continue;
try {
attributes[key] = decodeURIComponent(value);
} catch {
attributes[key] = value;
}
}
return attributes;
}
export function telemetryEnabled(): boolean {
return (process.env.OTEL_SDK_DISABLED ?? '').trim().toLowerCase() !== 'true';
}
/**
* Явная настройка SDK без auto-instrumentation: приложение — ESM (авто-
* инструментации понадобился бы loader hook), а явные span дают полный
* контроль над атрибутами: SQL-текст, URL и тела не попадают в телеметрию.
*
* Экспорт асинхронный и ограниченный: очереди BatchProcessor фиксированного
* размера, таймаут экспорта 5 с. Недоступный Collector оборачивается потерей
* телеметрии (счётчик messenger.telemetry.export.items{outcome="failed"}), а не
* ростом latency или памяти API.
*/
export function startTelemetry(defaultServiceName: string): void {
if (providers) return;
const identity = resolveServiceIdentity(defaultServiceName);
if (!telemetryEnabled()) return;
diag.setLogger(
new ThrottledDiagLogger(identity.serviceName),
DiagLogLevel.WARN,
);
const resource = resourceFromAttributes({
...parseResourceAttributes(process.env.OTEL_RESOURCE_ATTRIBUTES),
'service.name': identity.serviceName,
'service.version': identity.serviceVersion,
'service.instance.id': identity.instanceId,
});
const exportTimeout = intFromEnv(
'OTEL_EXPORTER_OTLP_TIMEOUT',
5000,
100,
60_000,
);
const tracer = new NodeTracerProvider({
resource,
sampler: new ParentBasedSampler({
root: new TraceIdRatioBasedSampler(
ratioFromEnv('OTEL_TRACES_SAMPLER_ARG', 1),
),
}),
spanProcessors: [
new BatchSpanProcessor(
countingExporter(
'traces',
new OTLPTraceExporter({ timeoutMillis: exportTimeout }),
),
{
maxQueueSize: intFromEnv('OTEL_BSP_MAX_QUEUE_SIZE', 2048, 1, 100_000),
maxExportBatchSize: intFromEnv(
'OTEL_BSP_MAX_EXPORT_BATCH_SIZE',
512,
1,
10_000,
),
scheduledDelayMillis: intFromEnv(
'OTEL_BSP_SCHEDULE_DELAY',
1000,
1,
60_000,
),
exportTimeoutMillis: exportTimeout,
},
),
],
});
// Регистрирует глобальный TracerProvider, AsyncLocalStorage context manager и
// W3C TraceContext + Baggage propagator.
tracer.register();
const interval = intFromEnv(
'OTEL_METRIC_EXPORT_INTERVAL',
10_000,
1000,
600_000,
);
const meter = new MeterProvider({
resource,
readers: [
new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({ timeoutMillis: exportTimeout }),
exportIntervalMillis: interval,
exportTimeoutMillis: Math.min(exportTimeout, interval),
}),
],
});
metrics.setGlobalMeterProvider(meter);
const logger = new LoggerProvider({
resource,
processors: [
new BatchLogRecordProcessor(
countingExporter(
'logs',
new OTLPLogExporter({ timeoutMillis: exportTimeout }),
),
{
maxQueueSize: intFromEnv(
'OTEL_BLRP_MAX_QUEUE_SIZE',
2048,
1,
100_000,
),
maxExportBatchSize: intFromEnv(
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE',
512,
1,
10_000,
),
scheduledDelayMillis: intFromEnv(
'OTEL_BLRP_SCHEDULE_DELAY',
1000,
1,
60_000,
),
exportTimeoutMillis: exportTimeout,
},
),
],
});
logs.setGlobalLoggerProvider(logger);
providers = { tracer, meter, logger };
startRuntimeMetrics();
}
/**
* Сбрасывает буферы при остановке. Ограничено по времени: недоступный
* Collector не должен задерживать graceful shutdown сервиса.
*/
export async function shutdownTelemetry(timeoutMs = 3000): Promise<void> {
const current = providers;
if (!current) return;
providers = undefined;
let timer: NodeJS.Timeout | undefined;
await Promise.race([
Promise.allSettled([
current.tracer.shutdown(),
current.meter.shutdown(),
current.logger.shutdown(),
]),
new Promise<void>((resolve) => {
timer = setTimeout(resolve, timeoutMs);
}),
]);
if (timer) clearTimeout(timer);
}
/**
* Ошибки экспорта при недоступном Collector повторяются каждую секунду по
* каждому сигналу; одинаковые сообщения печатаем не чаще раза в 30 с.
*/
class ThrottledDiagLogger implements DiagLogger {
private readonly lastWritten = new Map<string, number>();
constructor(private readonly serviceName: string) {}
error(message: string, ...args: unknown[]): void {
this.write('ERROR', message, args);
}
warn(message: string, ...args: unknown[]): void {
this.write('WARN', message, args);
}
info(message: string, ...args: unknown[]): void {
this.write('INFO', message, args);
}
debug(message: string, ...args: unknown[]): void {
this.write('DEBUG', message, args);
}
verbose(message: string, ...args: unknown[]): void {
this.write('TRACE', message, args);
}
private write(severity: string, message: string, args: unknown[]): void {
const key = `${severity}:${message}`;
const now = Date.now();
const previous = this.lastWritten.get(key);
if (previous !== undefined && now - previous < 30_000) return;
if (this.lastWritten.size > 200) this.lastWritten.clear();
this.lastWritten.set(key, now);
const detail = args
.map((arg) =>
arg instanceof Error
? `${arg.name}: ${arg.message}`
: typeof arg === 'string'
? arg
: safeJson(arg),
)
.join(' ');
process.stdout.write(
`${JSON.stringify({
timestamp: new Date(now).toISOString(),
severity,
'service.name': this.serviceName,
event: 'otel.sdk',
message: redact(`${message} ${detail}`.trim()).slice(0, 2000),
})}\n`,
);
}
}
function safeJson(value: unknown): string {
try {
return JSON.stringify(value) ?? String(value);
} catch {
return String(value);
}
}
+72
View File
@@ -0,0 +1,72 @@
import {
propagation,
ROOT_CONTEXT,
trace,
TraceFlags,
} from '@opentelemetry/api';
import { W3CTraceContextPropagator } from '@opentelemetry/core';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import {
contextFromTraceFields,
currentTraceFields,
errorType,
traceFieldsFromHeaders,
} from './tracing.js';
const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736';
const SPAN_ID = '00f067aa0ba902b7';
const TRACEPARENT = `00-${TRACE_ID}-${SPAN_ID}-01`;
describe('trace context helpers', () => {
beforeAll(() => {
propagation.setGlobalPropagator(new W3CTraceContextPropagator());
});
afterAll(() => {
propagation.disable();
});
it('serialises the active span context as job message fields', () => {
const ctx = trace.setSpanContext(ROOT_CONTEXT, {
traceId: TRACE_ID,
spanId: SPAN_ID,
traceFlags: TraceFlags.SAMPLED,
});
expect(currentTraceFields(ctx)).toEqual({ traceparent: TRACEPARENT });
});
it('restores the remote parent from job message fields', () => {
const ctx = contextFromTraceFields({ traceparent: TRACEPARENT });
expect(trace.getSpanContext(ctx)).toMatchObject({
traceId: TRACE_ID,
spanId: SPAN_ID,
isRemote: true,
});
});
it('reads AMQP headers delivered as buffers', () => {
expect(
traceFieldsFromHeaders({ traceparent: Buffer.from(TRACEPARENT) }),
).toEqual({ traceparent: TRACEPARENT });
});
it('ignores malformed trace context', () => {
expect(contextFromTraceFields({ traceparent: 'garbage' })).toBe(
ROOT_CONTEXT,
);
expect(traceFieldsFromHeaders({ traceparent: 'nope' })).toEqual({});
expect(currentTraceFields(ROOT_CONTEXT)).toEqual({});
});
it('classifies errors without leaking their messages', () => {
expect(
errorType(
Object.assign(new Error('Key (username)=(alice) already exists'), {
code: '23505',
}),
),
).toBe('23505');
expect(errorType(new TypeError('x'))).toBe('TypeError');
expect(errorType('boom')).toBe('_OTHER');
});
});
+130
View File
@@ -0,0 +1,130 @@
import {
context,
propagation,
ROOT_CONTEXT,
SpanStatusCode,
trace,
type Attributes,
type Context,
type Span,
type SpanKind,
type Tracer,
} from '@opentelemetry/api';
export const TRACER_NAME = 'messenger';
/** Совпадает с полем traceparent в contracts/jobs.schema.json. */
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u;
const MAX_TRACESTATE = 512;
export type TraceFields = {
traceparent?: string;
tracestate?: string;
};
export function tracer(): Tracer {
return trace.getTracer(TRACER_NAME, '1.0.0');
}
/**
* W3C trace context активного span в виде полей job message. Пишется в
* outbox в той же транзакции, что и задача: причинность переживает рестарт
* API, брокера и worker, потому что хранится вместе с данными.
*/
export function currentTraceFields(
ctx: Context = context.active(),
): TraceFields {
const carrier: Record<string, string> = {};
propagation.inject(ctx, carrier);
const fields: TraceFields = {};
if (carrier.traceparent && TRACEPARENT.test(carrier.traceparent))
fields.traceparent = carrier.traceparent;
if (
fields.traceparent &&
carrier.tracestate &&
carrier.tracestate.length <= MAX_TRACESTATE
)
fields.tracestate = carrier.tracestate;
return fields;
}
export function contextFromTraceFields(
fields: TraceFields | null | undefined,
base: Context = ROOT_CONTEXT,
): Context {
if (!fields?.traceparent || !TRACEPARENT.test(fields.traceparent))
return base;
const carrier: Record<string, string> = { traceparent: fields.traceparent };
if (fields.tracestate) carrier.tracestate = fields.tracestate;
return propagation.extract(base, carrier);
}
/** Заголовки AMQP приходят строками или Buffer — нормализуем к TraceFields. */
export function traceFieldsFromHeaders(
headers: Record<string, unknown> | undefined,
): TraceFields {
const read = (key: string): string | undefined => {
const value = headers?.[key];
if (typeof value === 'string') return value;
if (Buffer.isBuffer(value)) return value.toString('utf8');
return undefined;
};
const traceparent = read('traceparent');
if (!traceparent || !TRACEPARENT.test(traceparent)) return {};
const tracestate = read('tracestate');
return tracestate && tracestate.length <= MAX_TRACESTATE
? { traceparent, tracestate }
: { traceparent };
}
/**
* Дочерний span вокруг операции зависимости. Создаётся только при активном
* родителе: опрос outbox, reaper и сбор метрик по таймеру не порождают
* тысячи одиночных корневых трасс.
*/
export async function withChildSpan<T>(
name: string,
kind: SpanKind,
attributes: Attributes,
work: (span: Span | undefined) => Promise<T>,
): Promise<T> {
const parent = trace.getSpan(context.active());
if (!parent || !parent.isRecording()) return work(undefined);
return tracer().startActiveSpan(
name,
{ kind, attributes },
async (span: Span) => {
try {
return await work(span);
} catch (error) {
markSpanError(span, error);
throw error;
} finally {
span.end();
}
},
);
}
/**
* Ошибка на span — только тип/код. Сообщения драйверов могут содержать
* значения из запроса (например, «Key (username)=(alice) already exists»),
* поэтому recordException с текстом не используется.
*/
export function markSpanError(span: Span, error: unknown): void {
const type = errorType(error);
span.setAttribute('error.type', type);
span.setStatus({ code: SpanStatusCode.ERROR, message: type });
}
export function errorType(error: unknown): string {
if (error && typeof error === 'object') {
const code = (error as { code?: unknown }).code;
if (typeof code === 'string' && /^[A-Za-z0-9_.-]{1,40}$/u.test(code))
return code;
const name = (error as { name?: unknown }).name;
if (typeof name === 'string' && /^[A-Za-z0-9_.-]{1,60}$/u.test(name))
return name;
}
return '_OTHER';
}
+4
View File
@@ -0,0 +1,4 @@
// Импортируется первым в worker.main.ts (см. api-bootstrap.ts).
import { startTelemetry } from './sdk.js';
startTelemetry('messenger-worker');