forked from mtusi2026/mtusi2026lab6
lab6: каркас OpenTelemetry — SDK, инструменты метрик, трассировка, структурный логгер, HTTP-хуки, метрики очереди
This commit is contained in:
@@ -35,6 +35,17 @@
|
||||
"@nestjs/core": "^12.0.1",
|
||||
"@nestjs/platform-express": "^12.0.1",
|
||||
"@nestjs/platform-fastify": "^12.0.3",
|
||||
"@opentelemetry/api": "1.9.0",
|
||||
"@opentelemetry/api-logs": "0.203.0",
|
||||
"@opentelemetry/core": "2.0.1",
|
||||
"@opentelemetry/exporter-logs-otlp-proto": "0.203.0",
|
||||
"@opentelemetry/exporter-metrics-otlp-proto": "0.203.0",
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "0.203.0",
|
||||
"@opentelemetry/resources": "2.0.1",
|
||||
"@opentelemetry/sdk-logs": "0.203.0",
|
||||
"@opentelemetry/sdk-metrics": "2.0.1",
|
||||
"@opentelemetry/sdk-trace-base": "2.0.1",
|
||||
"@opentelemetry/sdk-trace-node": "2.0.1",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"amqplib": "^2.2.1",
|
||||
"argon2": "^0.45.1",
|
||||
|
||||
@@ -170,6 +170,14 @@ export const appConfigSchema = z
|
||||
86400,
|
||||
),
|
||||
WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081),
|
||||
// lab6: как часто API пересчитывает глубину и возраст очереди задач для
|
||||
// метрик messenger.jobs.queue.* (один дешёвый агрегат по частичным индексам).
|
||||
QUEUE_STATS_INTERVAL_MS: numeric(
|
||||
'QUEUE_STATS_INTERVAL_MS',
|
||||
1000,
|
||||
600000,
|
||||
10000,
|
||||
),
|
||||
FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0),
|
||||
QUEUED_RECONCILE_GRACE_SECONDS: numeric(
|
||||
'QUEUED_RECONCILE_GRACE_SECONDS',
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
// Импортируется первым в main.ts: ESM выполняет модули в порядке импорта, и
|
||||
// глобальные провайдеры OTel должны быть установлены до первого запроса.
|
||||
import { startTelemetry } from './sdk.js';
|
||||
|
||||
startTelemetry('messenger-api');
|
||||
@@ -0,0 +1,36 @@
|
||||
import { ExportResultCode, type ExportResult } from '@opentelemetry/core';
|
||||
import { instruments } from './instruments.js';
|
||||
|
||||
type BatchExporter<T> = {
|
||||
export(items: T[], resultCallback: (result: ExportResult) => void): void;
|
||||
shutdown(): Promise<void>;
|
||||
forceFlush?(): Promise<void>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Обёртка над OTLP-экспортёром: считает, сколько span и log records ушло и
|
||||
* сколько потеряно. Счётчик кумулятивный, поэтому потери во время простоя
|
||||
* Collector становятся видны в Prometheus сразу после его возвращения.
|
||||
* Переполнение очереди BatchProcessor (drop до экспорта) SDK пишет в diag-лог.
|
||||
*/
|
||||
export function countingExporter<T>(
|
||||
signal: 'traces' | 'logs',
|
||||
inner: BatchExporter<T>,
|
||||
): BatchExporter<T> {
|
||||
return {
|
||||
export(items: T[], resultCallback: (result: ExportResult) => void) {
|
||||
inner.export(items, (result: ExportResult) => {
|
||||
instruments().telemetryExport.add(items.length, {
|
||||
signal,
|
||||
outcome:
|
||||
result.code === ExportResultCode.SUCCESS
|
||||
? 'success'
|
||||
: 'failed',
|
||||
});
|
||||
resultCallback(result);
|
||||
});
|
||||
},
|
||||
shutdown: () => inner.shutdown(),
|
||||
forceFlush: () => inner.forceFlush?.() ?? Promise.resolve(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { isTracedRoute, normalizeMethod, routeTemplate } from './http.js';
|
||||
|
||||
describe('routeTemplate', () => {
|
||||
it('turns Fastify params into low-cardinality templates', () => {
|
||||
expect(routeTemplate('/api/v1/chats/:chatId/messages')).toBe(
|
||||
'/api/v1/chats/{chat_id}/messages',
|
||||
);
|
||||
expect(routeTemplate('/api/v1/attachments/:attachmentId/content')).toBe(
|
||||
'/api/v1/attachments/{attachment_id}/content',
|
||||
);
|
||||
expect(routeTemplate('/api/v1/users/me')).toBe('/api/v1/users/me');
|
||||
});
|
||||
|
||||
it('groups requests that matched no route', () => {
|
||||
expect(routeTemplate(undefined)).toBe('unmatched');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeMethod', () => {
|
||||
it('keeps standard methods and folds the rest', () => {
|
||||
expect(normalizeMethod('get')).toBe('GET');
|
||||
expect(normalizeMethod('DELETE')).toBe('DELETE');
|
||||
expect(normalizeMethod('PROPFIND')).toBe('_OTHER');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isTracedRoute', () => {
|
||||
it('skips probes, internal endpoints and unmatched requests', () => {
|
||||
expect(isTracedRoute('/health/ready')).toBe(false);
|
||||
expect(isTracedRoute('/internal/metrics')).toBe(false);
|
||||
expect(isTracedRoute('unmatched')).toBe(false);
|
||||
expect(isTracedRoute('/api/v1/chats/{chat_id}/messages')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,201 @@
|
||||
import {
|
||||
context,
|
||||
propagation,
|
||||
ROOT_CONTEXT,
|
||||
SpanKind,
|
||||
SpanStatusCode,
|
||||
trace,
|
||||
type Context,
|
||||
type Span,
|
||||
} from '@opentelemetry/api';
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { instruments } from './instruments.js';
|
||||
import { appLogger } from './logger.js';
|
||||
import { tracer } from './tracing.js';
|
||||
|
||||
const STATE = Symbol('messenger.telemetry.http');
|
||||
const KNOWN_METHODS = new Set([
|
||||
'GET',
|
||||
'HEAD',
|
||||
'POST',
|
||||
'PUT',
|
||||
'PATCH',
|
||||
'DELETE',
|
||||
'OPTIONS',
|
||||
]);
|
||||
// Пробы HAProxy (каждые 500 мс на реплику) и внутренние endpoint не трассируются
|
||||
// и не пишут access-log: они дали бы больше половины объёма Tempo и Loki без
|
||||
// диагностической ценности. В метриках они остаются.
|
||||
const UNTRACED_PREFIXES = ['/health', '/internal/'];
|
||||
|
||||
type HttpState = {
|
||||
readonly ctx: Context;
|
||||
readonly span: Span | undefined;
|
||||
readonly method: string;
|
||||
readonly route: string;
|
||||
readonly startedAt: bigint;
|
||||
finished: boolean;
|
||||
};
|
||||
type InstrumentedRequest = FastifyRequest & { [STATE]?: HttpState };
|
||||
type RouteHandler = (
|
||||
this: unknown,
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
) => unknown;
|
||||
|
||||
/** `/api/v1/chats/:chatId/messages` → `/api/v1/chats/{chat_id}/messages`. */
|
||||
export function routeTemplate(url: string | undefined): string {
|
||||
if (!url) return 'unmatched';
|
||||
return url.replace(
|
||||
/:([A-Za-z0-9_]+)/gu,
|
||||
(_match, name: string) =>
|
||||
`{${name.replace(/([a-z0-9])([A-Z])/gu, '$1_$2').toLowerCase()}}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function normalizeMethod(method: string): string {
|
||||
const upper = method.toUpperCase();
|
||||
return KNOWN_METHODS.has(upper) ? upper : '_OTHER';
|
||||
}
|
||||
|
||||
export function isTracedRoute(route: string): boolean {
|
||||
return (
|
||||
route !== 'unmatched' &&
|
||||
!UNTRACED_PREFIXES.some((prefix) => route.startsWith(prefix))
|
||||
);
|
||||
}
|
||||
|
||||
function headerValue(value: unknown): string | undefined {
|
||||
if (typeof value === 'string') return value;
|
||||
if (typeof value === 'number') return String(value);
|
||||
if (Array.isArray(value) && typeof value[0] === 'string') return value[0];
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP-инструментация на хуках Fastify:
|
||||
* - onRequest: SERVER span с родителем из входящего `traceparent`;
|
||||
* - onRoute: обработчик маршрута выполняется внутри контекста span, поэтому
|
||||
* span PostgreSQL/S3/RabbitMQ и логи обработчика становятся его детьми;
|
||||
* - onResponse / onRequestAbort: длительность в гистограмму, access-log, end span.
|
||||
* В метрики попадают только метод, route template и код статуса.
|
||||
* Регистрируется до app.listen(): маршруты Nest добавляются при init.
|
||||
*/
|
||||
export function installHttpTelemetry(fastify: FastifyInstance): void {
|
||||
fastify.addHook('onRoute', (routeOptions) => {
|
||||
const original = routeOptions.handler as unknown as RouteHandler | undefined;
|
||||
if (typeof original !== 'function') return;
|
||||
const wrapped: RouteHandler = function (
|
||||
this: unknown,
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
) {
|
||||
const state = (request as InstrumentedRequest)[STATE];
|
||||
if (!state?.span) return original.call(this, request, reply);
|
||||
return context.with(state.ctx, () =>
|
||||
original.call(this, request, reply),
|
||||
);
|
||||
};
|
||||
(routeOptions as unknown as { handler: RouteHandler }).handler = wrapped;
|
||||
});
|
||||
|
||||
fastify.addHook(
|
||||
'onRequest',
|
||||
(request: FastifyRequest, _reply: FastifyReply, done: () => void) => {
|
||||
const method = normalizeMethod(request.method);
|
||||
const route = routeTemplate(request.routeOptions?.url);
|
||||
let ctx = ROOT_CONTEXT;
|
||||
let span: Span | undefined;
|
||||
if (isTracedRoute(route)) {
|
||||
const parent = propagation.extract(ROOT_CONTEXT, request.headers);
|
||||
span = tracer().startSpan(
|
||||
`${method} ${route}`,
|
||||
{
|
||||
kind: SpanKind.SERVER,
|
||||
attributes: {
|
||||
'http.request.method': method,
|
||||
'http.route': route,
|
||||
'url.scheme': 'http',
|
||||
'network.protocol.version': request.raw.httpVersion,
|
||||
},
|
||||
},
|
||||
parent,
|
||||
);
|
||||
ctx = trace.setSpan(parent, span);
|
||||
}
|
||||
(request as InstrumentedRequest)[STATE] = {
|
||||
ctx,
|
||||
span,
|
||||
method,
|
||||
route,
|
||||
startedAt: process.hrtime.bigint(),
|
||||
finished: false,
|
||||
};
|
||||
done();
|
||||
},
|
||||
);
|
||||
|
||||
fastify.addHook(
|
||||
'onResponse',
|
||||
(request: FastifyRequest, reply: FastifyReply, done: () => void) => {
|
||||
finish(
|
||||
request as InstrumentedRequest,
|
||||
reply.statusCode,
|
||||
headerValue(reply.getHeader('x-request-id')),
|
||||
false,
|
||||
);
|
||||
done();
|
||||
},
|
||||
);
|
||||
|
||||
fastify.addHook(
|
||||
'onRequestAbort',
|
||||
(request: FastifyRequest, done: () => void) => {
|
||||
// 499 — клиент закрыл соединение до ответа (конвенция nginx); в SLO
|
||||
// доступности не считается ошибкой сервиса.
|
||||
finish(request as InstrumentedRequest, 499, undefined, true);
|
||||
done();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function finish(
|
||||
request: InstrumentedRequest,
|
||||
status: number,
|
||||
requestId: string | undefined,
|
||||
aborted: boolean,
|
||||
): void {
|
||||
const state = request[STATE];
|
||||
if (!state || state.finished) return;
|
||||
state.finished = true;
|
||||
const seconds = Number(process.hrtime.bigint() - state.startedAt) / 1e9;
|
||||
instruments().httpDuration.record(seconds, {
|
||||
'http.request.method': state.method,
|
||||
'http.route': state.route,
|
||||
'http.response.status_code': status,
|
||||
});
|
||||
|
||||
const span = state.span;
|
||||
if (!span) return;
|
||||
span.setAttribute('http.response.status_code', status);
|
||||
if (requestId) span.setAttribute('messenger.request_id', requestId);
|
||||
if (aborted) span.setAttribute('error.type', 'request_aborted');
|
||||
if (status >= 500)
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` });
|
||||
context.with(state.ctx, () =>
|
||||
appLogger.event(
|
||||
status >= 500 ? 'ERROR' : 'INFO',
|
||||
'http.request',
|
||||
`${state.method} ${state.route} ${status}`,
|
||||
{
|
||||
'http.request.method': state.method,
|
||||
'http.route': state.route,
|
||||
'http.response.status_code': status,
|
||||
duration_ms: Math.round(seconds * 10_000) / 10,
|
||||
request_id: requestId,
|
||||
aborted: aborted || undefined,
|
||||
},
|
||||
),
|
||||
);
|
||||
span.end();
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import os from 'node:os';
|
||||
|
||||
export type ServiceIdentity = {
|
||||
readonly serviceName: string;
|
||||
readonly serviceVersion: string;
|
||||
readonly instanceId: string;
|
||||
};
|
||||
|
||||
function fromEnv(defaultServiceName: string): ServiceIdentity {
|
||||
return {
|
||||
serviceName: process.env.OTEL_SERVICE_NAME?.trim() || defaultServiceName,
|
||||
serviceVersion: process.env.SERVICE_VERSION?.trim() || 'dev',
|
||||
// В compose у реплик API нет INSTANCE_ID, но HOSTNAME — id контейнера,
|
||||
// уникальный для каждой реплики: так service.instance.id различает
|
||||
// api-1 и api-2 во всех трёх сигналах.
|
||||
instanceId:
|
||||
process.env.INSTANCE_ID?.trim() ||
|
||||
process.env.HOSTNAME?.trim() ||
|
||||
os.hostname(),
|
||||
};
|
||||
}
|
||||
|
||||
let current = fromEnv('messenger');
|
||||
|
||||
/** service.name / service.version / service.instance.id, общие для всех сигналов. */
|
||||
export function serviceIdentity(): ServiceIdentity {
|
||||
return current;
|
||||
}
|
||||
|
||||
export function resolveServiceIdentity(
|
||||
defaultServiceName: string,
|
||||
): ServiceIdentity {
|
||||
current = fromEnv(defaultServiceName);
|
||||
return current;
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import {
|
||||
metrics,
|
||||
type Counter,
|
||||
type Histogram,
|
||||
type Meter,
|
||||
type MeterProvider,
|
||||
} from '@opentelemetry/api';
|
||||
|
||||
export const METER_NAME = 'messenger';
|
||||
|
||||
// Каждый SLO-порог — точная граница бакета (0.25 с для API, 30 с для
|
||||
// time-to-ready): иначе good-события пришлось бы интерполировать.
|
||||
export const HTTP_DURATION_BUCKETS = [
|
||||
0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5, 0.75, 1, 2.5, 5, 10,
|
||||
30,
|
||||
];
|
||||
export const JOB_DURATION_BUCKETS = [
|
||||
0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 7.5, 10, 15, 20, 30, 45, 60, 120,
|
||||
];
|
||||
export const TIME_TO_READY_BUCKETS = [
|
||||
0.25, 0.5, 1, 2, 3, 5, 7.5, 10, 15, 20, 30, 45, 60, 120, 300, 600,
|
||||
];
|
||||
export const WAIT_BUCKETS = [
|
||||
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
|
||||
];
|
||||
|
||||
export type Instruments = {
|
||||
readonly httpDuration: Histogram;
|
||||
readonly jobsProcessed: Counter;
|
||||
readonly jobDuration: Histogram;
|
||||
readonly timeToReady: Histogram;
|
||||
readonly brokerPublished: Counter;
|
||||
readonly dbConnectionWait: Histogram;
|
||||
readonly telemetryExport: Counter;
|
||||
};
|
||||
|
||||
let cached: { provider: MeterProvider; instruments: Instruments } | undefined;
|
||||
|
||||
export function meter(): Meter {
|
||||
return metrics.getMeter(METER_NAME, '1.0.0');
|
||||
}
|
||||
|
||||
/**
|
||||
* Инструменты создаются лениво и пересоздаются, если сменился глобальный
|
||||
* MeterProvider: созданный до старта SDK инструмент навсегда остался бы no-op.
|
||||
* Атрибуты — только ограниченные множества (метод, route template, код
|
||||
* статуса, outcome); id пользователей, чатов, вложений и запросов сюда не
|
||||
* попадают никогда.
|
||||
*/
|
||||
export function instruments(): Instruments {
|
||||
const provider = metrics.getMeterProvider();
|
||||
if (cached?.provider === provider) return cached.instruments;
|
||||
const m = provider.getMeter(METER_NAME, '1.0.0');
|
||||
const created: Instruments = {
|
||||
httpDuration: m.createHistogram('http.server.request.duration', {
|
||||
unit: 's',
|
||||
description:
|
||||
'HTTP server request duration by method, route template and status code',
|
||||
advice: { explicitBucketBoundaries: HTTP_DURATION_BUCKETS },
|
||||
}),
|
||||
jobsProcessed: m.createCounter('messenger.jobs.processed', {
|
||||
unit: '{job}',
|
||||
description: 'Job deliveries handled by the worker, by outcome',
|
||||
}),
|
||||
jobDuration: m.createHistogram('messenger.job.duration', {
|
||||
unit: 's',
|
||||
description:
|
||||
'Duration of one job attempt from claim to commit, by outcome',
|
||||
advice: { explicitBucketBoundaries: JOB_DURATION_BUCKETS },
|
||||
}),
|
||||
timeToReady: m.createHistogram('messenger.attachment.time_to_ready', {
|
||||
unit: 's',
|
||||
description:
|
||||
'Time from durable acceptance of an upload to the ready commit (includes queueing and retries)',
|
||||
advice: { explicitBucketBoundaries: TIME_TO_READY_BUCKETS },
|
||||
}),
|
||||
brokerPublished: m.createCounter('messenger.broker.published', {
|
||||
unit: '{message}',
|
||||
description: 'Messages published to the broker, by outcome',
|
||||
}),
|
||||
dbConnectionWait: m.createHistogram('db.client.connection.wait_time', {
|
||||
unit: 's',
|
||||
description: 'Time spent waiting for a connection from the pool',
|
||||
advice: { explicitBucketBoundaries: WAIT_BUCKETS },
|
||||
}),
|
||||
telemetryExport: m.createCounter('messenger.telemetry.export.items', {
|
||||
unit: '{item}',
|
||||
description:
|
||||
'Spans and log records handed to the OTLP exporter, by outcome (failed = lost)',
|
||||
}),
|
||||
};
|
||||
cached = { provider, instruments: created };
|
||||
return created;
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
import type { LoggerService, LogLevel } from '@nestjs/common';
|
||||
import { context, trace } from '@opentelemetry/api';
|
||||
import {
|
||||
logs,
|
||||
SeverityNumber,
|
||||
type AnyValueMap,
|
||||
type Logger as OtelLogger,
|
||||
type LoggerProvider,
|
||||
} from '@opentelemetry/api-logs';
|
||||
import { serviceIdentity } from './identity.js';
|
||||
import { redact } from './redact.js';
|
||||
|
||||
export type Severity = 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' | 'FATAL';
|
||||
export type LogAttributes = Record<
|
||||
string,
|
||||
string | number | boolean | null | undefined
|
||||
>;
|
||||
|
||||
const SEVERITY_NUMBER: Record<Severity, SeverityNumber> = {
|
||||
TRACE: SeverityNumber.TRACE,
|
||||
DEBUG: SeverityNumber.DEBUG,
|
||||
INFO: SeverityNumber.INFO,
|
||||
WARN: SeverityNumber.WARN,
|
||||
ERROR: SeverityNumber.ERROR,
|
||||
FATAL: SeverityNumber.FATAL,
|
||||
};
|
||||
const RANK: Record<Severity, number> = {
|
||||
TRACE: 1,
|
||||
DEBUG: 5,
|
||||
INFO: 9,
|
||||
WARN: 13,
|
||||
ERROR: 17,
|
||||
FATAL: 21,
|
||||
};
|
||||
const NEST_LEVEL: Record<LogLevel, Severity> = {
|
||||
verbose: 'TRACE',
|
||||
debug: 'DEBUG',
|
||||
log: 'INFO',
|
||||
warn: 'WARN',
|
||||
error: 'ERROR',
|
||||
fatal: 'FATAL',
|
||||
};
|
||||
const MAX_TEXT = 4000;
|
||||
|
||||
function parseSeverity(raw: string | undefined): Severity {
|
||||
switch ((raw ?? '').trim().toLowerCase()) {
|
||||
case 'trace':
|
||||
case 'verbose':
|
||||
return 'TRACE';
|
||||
case 'debug':
|
||||
return 'DEBUG';
|
||||
case 'warn':
|
||||
case 'warning':
|
||||
return 'WARN';
|
||||
case 'error':
|
||||
return 'ERROR';
|
||||
default:
|
||||
return 'INFO';
|
||||
}
|
||||
}
|
||||
|
||||
function stringify(value: unknown): string {
|
||||
if (typeof value === 'string') return value;
|
||||
if (value instanceof Error) return `${value.name}: ${value.message}`;
|
||||
try {
|
||||
return JSON.stringify(value) ?? String(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Структурный логгер для Nest и событий приложения. Каждая запись уходит:
|
||||
* - в stdout JSON-строкой (docker logs остаются читаемыми без Loki);
|
||||
* - в OTLP logs с контекстом активного span, поэтому в Loki у записи есть
|
||||
* trace_id/span_id и из неё можно перейти в Tempo.
|
||||
* Текст и строковые атрибуты проходят redact() и обрезаются до 4000 символов.
|
||||
*/
|
||||
export class StructuredLogger implements LoggerService {
|
||||
private minRank = RANK[parseSeverity(process.env.LOG_LEVEL)];
|
||||
private otel: { provider: LoggerProvider; logger: OtelLogger } | undefined;
|
||||
|
||||
log(message: unknown, ...optionalParams: unknown[]): void {
|
||||
this.fromNest('INFO', message, optionalParams);
|
||||
}
|
||||
|
||||
error(message: unknown, ...optionalParams: unknown[]): void {
|
||||
this.fromNest('ERROR', message, optionalParams);
|
||||
}
|
||||
|
||||
warn(message: unknown, ...optionalParams: unknown[]): void {
|
||||
this.fromNest('WARN', message, optionalParams);
|
||||
}
|
||||
|
||||
debug(message: unknown, ...optionalParams: unknown[]): void {
|
||||
this.fromNest('DEBUG', message, optionalParams);
|
||||
}
|
||||
|
||||
verbose(message: unknown, ...optionalParams: unknown[]): void {
|
||||
this.fromNest('TRACE', message, optionalParams);
|
||||
}
|
||||
|
||||
fatal(message: unknown, ...optionalParams: unknown[]): void {
|
||||
this.fromNest('FATAL', message, optionalParams);
|
||||
}
|
||||
|
||||
setLogLevels(levels: LogLevel[]): void {
|
||||
const ranks = levels
|
||||
.map((level) => NEST_LEVEL[level])
|
||||
.filter((severity): severity is Severity => severity !== undefined)
|
||||
.map((severity) => RANK[severity]);
|
||||
if (ranks.length > 0) this.minRank = Math.min(...ranks);
|
||||
}
|
||||
|
||||
/** Событие приложения: `event` — стабильное имя для запросов в Loki. */
|
||||
event(
|
||||
severity: Severity,
|
||||
event: string,
|
||||
message: string,
|
||||
attributes: LogAttributes = {},
|
||||
): void {
|
||||
this.write(severity, event, undefined, message, attributes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Конвенция Nest: контекст — последний строковый параметр, для error перед
|
||||
* ним может идти stack.
|
||||
*/
|
||||
private fromNest(
|
||||
severity: Severity,
|
||||
message: unknown,
|
||||
params: unknown[],
|
||||
): void {
|
||||
if (RANK[severity] < this.minRank) return;
|
||||
const rest = [...params];
|
||||
let contextName: string | undefined;
|
||||
if (rest.length > 0 && typeof rest[rest.length - 1] === 'string')
|
||||
contextName = rest.pop() as string;
|
||||
let stack: string | undefined;
|
||||
if (severity === 'ERROR' || severity === 'FATAL') {
|
||||
const index = rest.findIndex(
|
||||
(param) => typeof param === 'string' && param.includes('\n'),
|
||||
);
|
||||
if (index >= 0) stack = rest.splice(index, 1)[0] as string;
|
||||
if (!stack && message instanceof Error) stack = message.stack;
|
||||
}
|
||||
let text = stringify(message);
|
||||
if (rest.length > 0) text = `${text} ${rest.map(stringify).join(' ')}`;
|
||||
this.write(
|
||||
severity,
|
||||
'log',
|
||||
contextName,
|
||||
text,
|
||||
stack ? { 'exception.stacktrace': stack } : {},
|
||||
);
|
||||
}
|
||||
|
||||
private write(
|
||||
severity: Severity,
|
||||
event: string,
|
||||
contextName: string | undefined,
|
||||
message: string,
|
||||
attributes: LogAttributes,
|
||||
): void {
|
||||
if (RANK[severity] < this.minRank) return;
|
||||
const identity = serviceIdentity();
|
||||
const active = context.active();
|
||||
const spanContext = trace.getSpan(active)?.spanContext();
|
||||
let ids: { trace_id: string; span_id: string } | undefined;
|
||||
if (spanContext && trace.isSpanContextValid(spanContext))
|
||||
ids = { trace_id: spanContext.traceId, span_id: spanContext.spanId };
|
||||
|
||||
const body = redact(message).slice(0, MAX_TEXT);
|
||||
const clean: Record<string, string | number | boolean> = {};
|
||||
for (const [key, value] of Object.entries(attributes)) {
|
||||
if (value === undefined || value === null) continue;
|
||||
clean[key] =
|
||||
typeof value === 'string' ? redact(value).slice(0, MAX_TEXT) : value;
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
severity,
|
||||
'service.name': identity.serviceName,
|
||||
'service.instance.id': identity.instanceId,
|
||||
event,
|
||||
...(contextName ? { context: contextName } : {}),
|
||||
message: body,
|
||||
...ids,
|
||||
...clean,
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
this.otelLogger().emit({
|
||||
severityNumber: SEVERITY_NUMBER[severity],
|
||||
severityText: severity,
|
||||
body,
|
||||
attributes: {
|
||||
'event.name': event,
|
||||
...(contextName ? { 'log.context': contextName } : {}),
|
||||
...clean,
|
||||
} as AnyValueMap,
|
||||
context: active,
|
||||
});
|
||||
}
|
||||
|
||||
private otelLogger(): OtelLogger {
|
||||
const provider = logs.getLoggerProvider();
|
||||
if (this.otel?.provider !== provider)
|
||||
this.otel = { provider, logger: provider.getLogger('messenger', '1.0.0') };
|
||||
return this.otel.logger;
|
||||
}
|
||||
}
|
||||
|
||||
/** Единственный экземпляр: передаётся в NestFactory и используется для событий. */
|
||||
export const appLogger = new StructuredLogger();
|
||||
@@ -0,0 +1,164 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
type OnModuleDestroy,
|
||||
type OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { JOB_QUEUE } from '../jobs/broker.topology.js';
|
||||
import { meter } from './instruments.js';
|
||||
|
||||
type QueueSnapshot = {
|
||||
readonly takenAt: number;
|
||||
readonly ready: number;
|
||||
readonly delayed: number;
|
||||
readonly processing: number;
|
||||
readonly oldestAgeSeconds: number;
|
||||
readonly outboxPending: number;
|
||||
readonly outboxOldestAgeSeconds: number;
|
||||
};
|
||||
|
||||
// Источник истины о состоянии задач — PostgreSQL, а не глубина очереди RabbitMQ:
|
||||
// задача может ждать в outbox, в backoff или под истёкшим lease, не лежа в
|
||||
// брокере. Возраст самой старой незавершённой задачи — то, что чувствует
|
||||
// пользователь, ожидающий ready.
|
||||
const JOBS_SQL = `SELECT
|
||||
count(*) FILTER (WHERE status = 'queued' AND next_attempt_at <= now())::int AS ready,
|
||||
count(*) FILTER (WHERE status = 'queued' AND next_attempt_at > now())::int AS delayed,
|
||||
count(*) FILTER (WHERE status = 'processing')::int AS processing,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::float8 AS oldest_age
|
||||
FROM attachment_jobs
|
||||
WHERE status IN ('queued', 'processing')`;
|
||||
|
||||
const OUTBOX_SQL = `SELECT
|
||||
count(*)::int AS pending,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::float8 AS oldest_age
|
||||
FROM job_outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()`;
|
||||
|
||||
type JobsRow = {
|
||||
ready: number;
|
||||
delayed: number;
|
||||
processing: number;
|
||||
oldest_age: number;
|
||||
};
|
||||
type OutboxRow = { pending: number; oldest_age: number };
|
||||
|
||||
/**
|
||||
* Метрики очереди задач. Работает в API, а не в worker: в сценарии «worker
|
||||
* остановлен» именно эти gauge должны продолжать расти. Обе реплики
|
||||
* публикуют одинаковые значения, в PromQL берётся max(). При ошибке запроса
|
||||
* старый снимок публикуется не дольше трёх интервалов, затем серия пропадает
|
||||
* — лучше отсутствие данных, чем устаревший ноль.
|
||||
*/
|
||||
@Injectable()
|
||||
export class QueueStatsService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(QueueStatsService.name);
|
||||
private readonly intervalMs: number;
|
||||
private snapshot: QueueSnapshot | undefined;
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private refreshing = false;
|
||||
private failures = 0;
|
||||
|
||||
constructor(
|
||||
private readonly database: DatabaseClient,
|
||||
config: ConfigService<AppConfig>,
|
||||
) {
|
||||
this.intervalMs = config.getOrThrow('QUEUE_STATS_INTERVAL_MS');
|
||||
}
|
||||
|
||||
onModuleInit(): void {
|
||||
const m = meter();
|
||||
const depth = m.createObservableGauge('messenger.jobs.queue.depth', {
|
||||
unit: '{job}',
|
||||
description: 'Unfinished attachment jobs by state (PostgreSQL)',
|
||||
});
|
||||
const oldestAge = m.createObservableGauge(
|
||||
'messenger.jobs.queue.oldest_age',
|
||||
{
|
||||
unit: 's',
|
||||
description:
|
||||
'Age of the oldest unfinished job since durable acceptance',
|
||||
},
|
||||
);
|
||||
const outboxPending = m.createObservableGauge('messenger.outbox.pending', {
|
||||
unit: '{message}',
|
||||
description: 'Outbox rows due for publishing but not yet confirmed',
|
||||
});
|
||||
const outboxAge = m.createObservableGauge('messenger.outbox.oldest_age', {
|
||||
unit: 's',
|
||||
description: 'Age of the oldest unpublished outbox row',
|
||||
});
|
||||
m.addBatchObservableCallback(
|
||||
(result) => {
|
||||
const snapshot = this.freshSnapshot();
|
||||
if (!snapshot) return;
|
||||
const queue = { 'messaging.destination.name': JOB_QUEUE };
|
||||
result.observe(depth, snapshot.ready, { ...queue, state: 'ready' });
|
||||
result.observe(depth, snapshot.delayed, {
|
||||
...queue,
|
||||
state: 'delayed',
|
||||
});
|
||||
result.observe(depth, snapshot.processing, {
|
||||
...queue,
|
||||
state: 'processing',
|
||||
});
|
||||
result.observe(oldestAge, snapshot.oldestAgeSeconds, queue);
|
||||
result.observe(outboxPending, snapshot.outboxPending, queue);
|
||||
result.observe(outboxAge, snapshot.outboxOldestAgeSeconds, queue);
|
||||
},
|
||||
[depth, oldestAge, outboxPending, outboxAge],
|
||||
);
|
||||
void this.refresh();
|
||||
this.timer = setInterval(() => void this.refresh(), this.intervalMs);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
}
|
||||
|
||||
private freshSnapshot(): QueueSnapshot | undefined {
|
||||
if (!this.snapshot) return undefined;
|
||||
return Date.now() - this.snapshot.takenAt <= this.intervalMs * 3
|
||||
? this.snapshot
|
||||
: undefined;
|
||||
}
|
||||
|
||||
private async refresh(): Promise<void> {
|
||||
if (this.refreshing) return;
|
||||
this.refreshing = true;
|
||||
try {
|
||||
const [jobs, outbox] = await Promise.all([
|
||||
this.database.query<JobsRow>('telemetry.queue_stats', JOBS_SQL),
|
||||
this.database.query<OutboxRow>('telemetry.outbox_stats', OUTBOX_SQL),
|
||||
]);
|
||||
const j = jobs.rows[0];
|
||||
const o = outbox.rows[0];
|
||||
this.snapshot = {
|
||||
takenAt: Date.now(),
|
||||
ready: Number(j?.ready ?? 0),
|
||||
delayed: Number(j?.delayed ?? 0),
|
||||
processing: Number(j?.processing ?? 0),
|
||||
oldestAgeSeconds: Number(j?.oldest_age ?? 0),
|
||||
outboxPending: Number(o?.pending ?? 0),
|
||||
outboxOldestAgeSeconds: Number(o?.oldest_age ?? 0),
|
||||
};
|
||||
if (this.failures > 0)
|
||||
this.logger.log(
|
||||
`queue stats recovered after ${this.failures} failed refreshes`,
|
||||
);
|
||||
this.failures = 0;
|
||||
} catch (error) {
|
||||
this.failures += 1;
|
||||
if (this.failures === 1 || this.failures % 30 === 0)
|
||||
this.logger.warn(
|
||||
`queue stats refresh failed (${this.failures} in a row): ${error instanceof Error ? error.name : 'unknown error'}`,
|
||||
);
|
||||
} finally {
|
||||
this.refreshing = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { redact } from './redact.js';
|
||||
|
||||
describe('redact', () => {
|
||||
it('masks bearer and basic credentials', () => {
|
||||
const out = redact(
|
||||
'Authorization: Bearer abc.def-123 and Basic dXNlcjpwYXNz',
|
||||
);
|
||||
expect(out).not.toContain('abc.def-123');
|
||||
expect(out).not.toContain('dXNlcjpwYXNz');
|
||||
});
|
||||
|
||||
it('masks credentials embedded in connection strings', () => {
|
||||
expect(
|
||||
redact('postgresql://messenger:s3cret@postgres:5432/messenger'),
|
||||
).toBe('postgresql://[REDACTED]@postgres:5432/messenger');
|
||||
});
|
||||
|
||||
it('masks presigned URL credentials and signatures', () => {
|
||||
const out = redact(
|
||||
'https://s3.local/b/k?X-Amz-Credential=AKIAEXAMPLE%2F20260101&X-Amz-Signature=deadbeef',
|
||||
);
|
||||
expect(out).not.toContain('AKIAEXAMPLE');
|
||||
expect(out).not.toContain('deadbeef');
|
||||
});
|
||||
|
||||
it('masks password-like fields', () => {
|
||||
expect(redact('{"password":"hunter2"}')).not.toContain('hunter2');
|
||||
expect(redact('password=hunter2&x=1')).not.toContain('hunter2');
|
||||
});
|
||||
|
||||
it('keeps ordinary diagnostic text intact', () => {
|
||||
expect(redact('job retry scheduled attempt=2 code=transient')).toBe(
|
||||
'job retry scheduled attempt=2 code=transient',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* Последний рубеж перед stdout и OTLP: если секрет всё же попал в текст
|
||||
* события (например, в сообщение ошибки драйвера), он не уйдёт в Loki.
|
||||
* Основная защита — не класть такие данные в события вообще: route template
|
||||
* вместо URL, коды ошибок вместо сообщений, без тел запросов.
|
||||
*/
|
||||
const RULES: ReadonlyArray<readonly [RegExp, string]> = [
|
||||
[/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/giu, '$1 [REDACTED]'],
|
||||
[/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/giu, '$1[REDACTED]@'],
|
||||
[
|
||||
/\b(X-Amz-(?:Signature|Credential|Security-Token))=[^&\s"']+/giu,
|
||||
'$1=[REDACTED]',
|
||||
],
|
||||
[
|
||||
/((?:password|passwd|secret|token|authorization|cookie)["']?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;&}]+)/giu,
|
||||
'$1[REDACTED]',
|
||||
],
|
||||
[
|
||||
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/gu,
|
||||
'[REDACTED PRIVATE KEY]',
|
||||
],
|
||||
];
|
||||
|
||||
export function redact(text: string): string {
|
||||
let result = text;
|
||||
for (const [pattern, replacement] of RULES)
|
||||
result = result.replace(pattern, replacement);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||
import { meter } from './instruments.js';
|
||||
|
||||
let started = false;
|
||||
|
||||
/**
|
||||
* Насыщение процесса: CPU, RSS, heap и задержка event loop. Задержка event
|
||||
* loop — главный признак насыщения Node.js: CPU-bound работа (sharp, argon2)
|
||||
* и синхронный JSON на больших ответах растят её раньше, чем latency API.
|
||||
*/
|
||||
export function startRuntimeMetrics(): void {
|
||||
if (started) return;
|
||||
started = true;
|
||||
const m = meter();
|
||||
const loop = monitorEventLoopDelay({ resolution: 20 });
|
||||
loop.enable();
|
||||
|
||||
m.createObservableCounter('process.cpu.time', {
|
||||
unit: 's',
|
||||
description: 'CPU time consumed by the process',
|
||||
}).addCallback((result) => {
|
||||
const usage = process.cpuUsage();
|
||||
result.observe(usage.user / 1e6, { 'cpu.mode': 'user' });
|
||||
result.observe(usage.system / 1e6, { 'cpu.mode': 'system' });
|
||||
});
|
||||
m.createObservableUpDownCounter('process.memory.usage', {
|
||||
unit: 'By',
|
||||
description: 'Resident set size of the process',
|
||||
}).addCallback((result) => {
|
||||
result.observe(process.memoryUsage.rss());
|
||||
});
|
||||
m.createObservableUpDownCounter('v8js.memory.heap.used', {
|
||||
unit: 'By',
|
||||
description: 'V8 heap in use',
|
||||
}).addCallback((result) => {
|
||||
result.observe(process.memoryUsage().heapUsed);
|
||||
});
|
||||
|
||||
const delayP99 = m.createObservableGauge('nodejs.eventloop.delay.p99', {
|
||||
unit: 's',
|
||||
description: 'p99 event loop delay since the previous collection',
|
||||
});
|
||||
const delayMax = m.createObservableGauge('nodejs.eventloop.delay.max', {
|
||||
unit: 's',
|
||||
description: 'Max event loop delay since the previous collection',
|
||||
});
|
||||
m.addBatchObservableCallback(
|
||||
(result) => {
|
||||
result.observe(delayP99, loop.percentile(99) / 1e9);
|
||||
result.observe(delayMax, loop.max / 1e9);
|
||||
loop.reset();
|
||||
},
|
||||
[delayP99, delayMax],
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
import {
|
||||
diag,
|
||||
DiagLogLevel,
|
||||
metrics,
|
||||
type DiagLogger,
|
||||
} from '@opentelemetry/api';
|
||||
import { logs } from '@opentelemetry/api-logs';
|
||||
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
|
||||
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||
import {
|
||||
BatchLogRecordProcessor,
|
||||
LoggerProvider,
|
||||
} from '@opentelemetry/sdk-logs';
|
||||
import {
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from '@opentelemetry/sdk-metrics';
|
||||
import {
|
||||
BatchSpanProcessor,
|
||||
ParentBasedSampler,
|
||||
TraceIdRatioBasedSampler,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
||||
import { countingExporter } from './export-counter.js';
|
||||
import { resolveServiceIdentity } from './identity.js';
|
||||
import { redact } from './redact.js';
|
||||
import { startRuntimeMetrics } from './runtime-metrics.js';
|
||||
|
||||
type Providers = {
|
||||
readonly tracer: NodeTracerProvider;
|
||||
readonly meter: MeterProvider;
|
||||
readonly logger: LoggerProvider;
|
||||
};
|
||||
|
||||
let providers: Providers | undefined;
|
||||
|
||||
function intFromEnv(
|
||||
name: string,
|
||||
fallback: number,
|
||||
min: number,
|
||||
max: number,
|
||||
): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) return fallback;
|
||||
const value = Number(raw);
|
||||
return Number.isInteger(value) && value >= min && value <= max
|
||||
? value
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function ratioFromEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) return fallback;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) && value >= 0 && value <= 1 ? value : fallback;
|
||||
}
|
||||
|
||||
/** OTEL_RESOURCE_ATTRIBUTES: `k=v,k2=v2`, значения percent-encoded. */
|
||||
export function parseResourceAttributes(
|
||||
raw: string | undefined,
|
||||
): Record<string, string> {
|
||||
const attributes: Record<string, string> = {};
|
||||
if (!raw) return attributes;
|
||||
for (const pair of raw.split(',')) {
|
||||
const index = pair.indexOf('=');
|
||||
if (index <= 0) continue;
|
||||
const key = pair.slice(0, index).trim();
|
||||
const value = pair.slice(index + 1).trim();
|
||||
if (!key) continue;
|
||||
try {
|
||||
attributes[key] = decodeURIComponent(value);
|
||||
} catch {
|
||||
attributes[key] = value;
|
||||
}
|
||||
}
|
||||
return attributes;
|
||||
}
|
||||
|
||||
export function telemetryEnabled(): boolean {
|
||||
return (process.env.OTEL_SDK_DISABLED ?? '').trim().toLowerCase() !== 'true';
|
||||
}
|
||||
|
||||
/**
|
||||
* Явная настройка SDK без auto-instrumentation: приложение — ESM (авто-
|
||||
* инструментации понадобился бы loader hook), а явные span дают полный
|
||||
* контроль над атрибутами: SQL-текст, URL и тела не попадают в телеметрию.
|
||||
*
|
||||
* Экспорт асинхронный и ограниченный: очереди BatchProcessor фиксированного
|
||||
* размера, таймаут экспорта 5 с. Недоступный Collector оборачивается потерей
|
||||
* телеметрии (счётчик messenger.telemetry.export.items{outcome="failed"}), а не
|
||||
* ростом latency или памяти API.
|
||||
*/
|
||||
export function startTelemetry(defaultServiceName: string): void {
|
||||
if (providers) return;
|
||||
const identity = resolveServiceIdentity(defaultServiceName);
|
||||
if (!telemetryEnabled()) return;
|
||||
|
||||
diag.setLogger(
|
||||
new ThrottledDiagLogger(identity.serviceName),
|
||||
DiagLogLevel.WARN,
|
||||
);
|
||||
|
||||
const resource = resourceFromAttributes({
|
||||
...parseResourceAttributes(process.env.OTEL_RESOURCE_ATTRIBUTES),
|
||||
'service.name': identity.serviceName,
|
||||
'service.version': identity.serviceVersion,
|
||||
'service.instance.id': identity.instanceId,
|
||||
});
|
||||
const exportTimeout = intFromEnv(
|
||||
'OTEL_EXPORTER_OTLP_TIMEOUT',
|
||||
5000,
|
||||
100,
|
||||
60_000,
|
||||
);
|
||||
|
||||
const tracer = new NodeTracerProvider({
|
||||
resource,
|
||||
sampler: new ParentBasedSampler({
|
||||
root: new TraceIdRatioBasedSampler(
|
||||
ratioFromEnv('OTEL_TRACES_SAMPLER_ARG', 1),
|
||||
),
|
||||
}),
|
||||
spanProcessors: [
|
||||
new BatchSpanProcessor(
|
||||
countingExporter(
|
||||
'traces',
|
||||
new OTLPTraceExporter({ timeoutMillis: exportTimeout }),
|
||||
),
|
||||
{
|
||||
maxQueueSize: intFromEnv('OTEL_BSP_MAX_QUEUE_SIZE', 2048, 1, 100_000),
|
||||
maxExportBatchSize: intFromEnv(
|
||||
'OTEL_BSP_MAX_EXPORT_BATCH_SIZE',
|
||||
512,
|
||||
1,
|
||||
10_000,
|
||||
),
|
||||
scheduledDelayMillis: intFromEnv(
|
||||
'OTEL_BSP_SCHEDULE_DELAY',
|
||||
1000,
|
||||
1,
|
||||
60_000,
|
||||
),
|
||||
exportTimeoutMillis: exportTimeout,
|
||||
},
|
||||
),
|
||||
],
|
||||
});
|
||||
// Регистрирует глобальный TracerProvider, AsyncLocalStorage context manager и
|
||||
// W3C TraceContext + Baggage propagator.
|
||||
tracer.register();
|
||||
|
||||
const interval = intFromEnv(
|
||||
'OTEL_METRIC_EXPORT_INTERVAL',
|
||||
10_000,
|
||||
1000,
|
||||
600_000,
|
||||
);
|
||||
const meter = new MeterProvider({
|
||||
resource,
|
||||
readers: [
|
||||
new PeriodicExportingMetricReader({
|
||||
exporter: new OTLPMetricExporter({ timeoutMillis: exportTimeout }),
|
||||
exportIntervalMillis: interval,
|
||||
exportTimeoutMillis: Math.min(exportTimeout, interval),
|
||||
}),
|
||||
],
|
||||
});
|
||||
metrics.setGlobalMeterProvider(meter);
|
||||
|
||||
const logger = new LoggerProvider({
|
||||
resource,
|
||||
processors: [
|
||||
new BatchLogRecordProcessor(
|
||||
countingExporter(
|
||||
'logs',
|
||||
new OTLPLogExporter({ timeoutMillis: exportTimeout }),
|
||||
),
|
||||
{
|
||||
maxQueueSize: intFromEnv(
|
||||
'OTEL_BLRP_MAX_QUEUE_SIZE',
|
||||
2048,
|
||||
1,
|
||||
100_000,
|
||||
),
|
||||
maxExportBatchSize: intFromEnv(
|
||||
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE',
|
||||
512,
|
||||
1,
|
||||
10_000,
|
||||
),
|
||||
scheduledDelayMillis: intFromEnv(
|
||||
'OTEL_BLRP_SCHEDULE_DELAY',
|
||||
1000,
|
||||
1,
|
||||
60_000,
|
||||
),
|
||||
exportTimeoutMillis: exportTimeout,
|
||||
},
|
||||
),
|
||||
],
|
||||
});
|
||||
logs.setGlobalLoggerProvider(logger);
|
||||
|
||||
providers = { tracer, meter, logger };
|
||||
startRuntimeMetrics();
|
||||
}
|
||||
|
||||
/**
|
||||
* Сбрасывает буферы при остановке. Ограничено по времени: недоступный
|
||||
* Collector не должен задерживать graceful shutdown сервиса.
|
||||
*/
|
||||
export async function shutdownTelemetry(timeoutMs = 3000): Promise<void> {
|
||||
const current = providers;
|
||||
if (!current) return;
|
||||
providers = undefined;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
await Promise.race([
|
||||
Promise.allSettled([
|
||||
current.tracer.shutdown(),
|
||||
current.meter.shutdown(),
|
||||
current.logger.shutdown(),
|
||||
]),
|
||||
new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ошибки экспорта при недоступном Collector повторяются каждую секунду по
|
||||
* каждому сигналу; одинаковые сообщения печатаем не чаще раза в 30 с.
|
||||
*/
|
||||
class ThrottledDiagLogger implements DiagLogger {
|
||||
private readonly lastWritten = new Map<string, number>();
|
||||
|
||||
constructor(private readonly serviceName: string) {}
|
||||
|
||||
error(message: string, ...args: unknown[]): void {
|
||||
this.write('ERROR', message, args);
|
||||
}
|
||||
|
||||
warn(message: string, ...args: unknown[]): void {
|
||||
this.write('WARN', message, args);
|
||||
}
|
||||
|
||||
info(message: string, ...args: unknown[]): void {
|
||||
this.write('INFO', message, args);
|
||||
}
|
||||
|
||||
debug(message: string, ...args: unknown[]): void {
|
||||
this.write('DEBUG', message, args);
|
||||
}
|
||||
|
||||
verbose(message: string, ...args: unknown[]): void {
|
||||
this.write('TRACE', message, args);
|
||||
}
|
||||
|
||||
private write(severity: string, message: string, args: unknown[]): void {
|
||||
const key = `${severity}:${message}`;
|
||||
const now = Date.now();
|
||||
const previous = this.lastWritten.get(key);
|
||||
if (previous !== undefined && now - previous < 30_000) return;
|
||||
if (this.lastWritten.size > 200) this.lastWritten.clear();
|
||||
this.lastWritten.set(key, now);
|
||||
const detail = args
|
||||
.map((arg) =>
|
||||
arg instanceof Error
|
||||
? `${arg.name}: ${arg.message}`
|
||||
: typeof arg === 'string'
|
||||
? arg
|
||||
: safeJson(arg),
|
||||
)
|
||||
.join(' ');
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
timestamp: new Date(now).toISOString(),
|
||||
severity,
|
||||
'service.name': this.serviceName,
|
||||
event: 'otel.sdk',
|
||||
message: redact(`${message} ${detail}`.trim()).slice(0, 2000),
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function safeJson(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value) ?? String(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import {
|
||||
propagation,
|
||||
ROOT_CONTEXT,
|
||||
trace,
|
||||
TraceFlags,
|
||||
} from '@opentelemetry/api';
|
||||
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
contextFromTraceFields,
|
||||
currentTraceFields,
|
||||
errorType,
|
||||
traceFieldsFromHeaders,
|
||||
} from './tracing.js';
|
||||
|
||||
const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736';
|
||||
const SPAN_ID = '00f067aa0ba902b7';
|
||||
const TRACEPARENT = `00-${TRACE_ID}-${SPAN_ID}-01`;
|
||||
|
||||
describe('trace context helpers', () => {
|
||||
beforeAll(() => {
|
||||
propagation.setGlobalPropagator(new W3CTraceContextPropagator());
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
propagation.disable();
|
||||
});
|
||||
|
||||
it('serialises the active span context as job message fields', () => {
|
||||
const ctx = trace.setSpanContext(ROOT_CONTEXT, {
|
||||
traceId: TRACE_ID,
|
||||
spanId: SPAN_ID,
|
||||
traceFlags: TraceFlags.SAMPLED,
|
||||
});
|
||||
expect(currentTraceFields(ctx)).toEqual({ traceparent: TRACEPARENT });
|
||||
});
|
||||
|
||||
it('restores the remote parent from job message fields', () => {
|
||||
const ctx = contextFromTraceFields({ traceparent: TRACEPARENT });
|
||||
expect(trace.getSpanContext(ctx)).toMatchObject({
|
||||
traceId: TRACE_ID,
|
||||
spanId: SPAN_ID,
|
||||
isRemote: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('reads AMQP headers delivered as buffers', () => {
|
||||
expect(
|
||||
traceFieldsFromHeaders({ traceparent: Buffer.from(TRACEPARENT) }),
|
||||
).toEqual({ traceparent: TRACEPARENT });
|
||||
});
|
||||
|
||||
it('ignores malformed trace context', () => {
|
||||
expect(contextFromTraceFields({ traceparent: 'garbage' })).toBe(
|
||||
ROOT_CONTEXT,
|
||||
);
|
||||
expect(traceFieldsFromHeaders({ traceparent: 'nope' })).toEqual({});
|
||||
expect(currentTraceFields(ROOT_CONTEXT)).toEqual({});
|
||||
});
|
||||
|
||||
it('classifies errors without leaking their messages', () => {
|
||||
expect(
|
||||
errorType(
|
||||
Object.assign(new Error('Key (username)=(alice) already exists'), {
|
||||
code: '23505',
|
||||
}),
|
||||
),
|
||||
).toBe('23505');
|
||||
expect(errorType(new TypeError('x'))).toBe('TypeError');
|
||||
expect(errorType('boom')).toBe('_OTHER');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import {
|
||||
context,
|
||||
propagation,
|
||||
ROOT_CONTEXT,
|
||||
SpanStatusCode,
|
||||
trace,
|
||||
type Attributes,
|
||||
type Context,
|
||||
type Span,
|
||||
type SpanKind,
|
||||
type Tracer,
|
||||
} from '@opentelemetry/api';
|
||||
|
||||
export const TRACER_NAME = 'messenger';
|
||||
|
||||
/** Совпадает с полем traceparent в contracts/jobs.schema.json. */
|
||||
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u;
|
||||
const MAX_TRACESTATE = 512;
|
||||
|
||||
export type TraceFields = {
|
||||
traceparent?: string;
|
||||
tracestate?: string;
|
||||
};
|
||||
|
||||
export function tracer(): Tracer {
|
||||
return trace.getTracer(TRACER_NAME, '1.0.0');
|
||||
}
|
||||
|
||||
/**
|
||||
* W3C trace context активного span в виде полей job message. Пишется в
|
||||
* outbox в той же транзакции, что и задача: причинность переживает рестарт
|
||||
* API, брокера и worker, потому что хранится вместе с данными.
|
||||
*/
|
||||
export function currentTraceFields(
|
||||
ctx: Context = context.active(),
|
||||
): TraceFields {
|
||||
const carrier: Record<string, string> = {};
|
||||
propagation.inject(ctx, carrier);
|
||||
const fields: TraceFields = {};
|
||||
if (carrier.traceparent && TRACEPARENT.test(carrier.traceparent))
|
||||
fields.traceparent = carrier.traceparent;
|
||||
if (
|
||||
fields.traceparent &&
|
||||
carrier.tracestate &&
|
||||
carrier.tracestate.length <= MAX_TRACESTATE
|
||||
)
|
||||
fields.tracestate = carrier.tracestate;
|
||||
return fields;
|
||||
}
|
||||
|
||||
export function contextFromTraceFields(
|
||||
fields: TraceFields | null | undefined,
|
||||
base: Context = ROOT_CONTEXT,
|
||||
): Context {
|
||||
if (!fields?.traceparent || !TRACEPARENT.test(fields.traceparent))
|
||||
return base;
|
||||
const carrier: Record<string, string> = { traceparent: fields.traceparent };
|
||||
if (fields.tracestate) carrier.tracestate = fields.tracestate;
|
||||
return propagation.extract(base, carrier);
|
||||
}
|
||||
|
||||
/** Заголовки AMQP приходят строками или Buffer — нормализуем к TraceFields. */
|
||||
export function traceFieldsFromHeaders(
|
||||
headers: Record<string, unknown> | undefined,
|
||||
): TraceFields {
|
||||
const read = (key: string): string | undefined => {
|
||||
const value = headers?.[key];
|
||||
if (typeof value === 'string') return value;
|
||||
if (Buffer.isBuffer(value)) return value.toString('utf8');
|
||||
return undefined;
|
||||
};
|
||||
const traceparent = read('traceparent');
|
||||
if (!traceparent || !TRACEPARENT.test(traceparent)) return {};
|
||||
const tracestate = read('tracestate');
|
||||
return tracestate && tracestate.length <= MAX_TRACESTATE
|
||||
? { traceparent, tracestate }
|
||||
: { traceparent };
|
||||
}
|
||||
|
||||
/**
|
||||
* Дочерний span вокруг операции зависимости. Создаётся только при активном
|
||||
* родителе: опрос outbox, reaper и сбор метрик по таймеру не порождают
|
||||
* тысячи одиночных корневых трасс.
|
||||
*/
|
||||
export async function withChildSpan<T>(
|
||||
name: string,
|
||||
kind: SpanKind,
|
||||
attributes: Attributes,
|
||||
work: (span: Span | undefined) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const parent = trace.getSpan(context.active());
|
||||
if (!parent || !parent.isRecording()) return work(undefined);
|
||||
return tracer().startActiveSpan(
|
||||
name,
|
||||
{ kind, attributes },
|
||||
async (span: Span) => {
|
||||
try {
|
||||
return await work(span);
|
||||
} catch (error) {
|
||||
markSpanError(span, error);
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ошибка на span — только тип/код. Сообщения драйверов могут содержать
|
||||
* значения из запроса (например, «Key (username)=(alice) already exists»),
|
||||
* поэтому recordException с текстом не используется.
|
||||
*/
|
||||
export function markSpanError(span: Span, error: unknown): void {
|
||||
const type = errorType(error);
|
||||
span.setAttribute('error.type', type);
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: type });
|
||||
}
|
||||
|
||||
export function errorType(error: unknown): string {
|
||||
if (error && typeof error === 'object') {
|
||||
const code = (error as { code?: unknown }).code;
|
||||
if (typeof code === 'string' && /^[A-Za-z0-9_.-]{1,40}$/u.test(code))
|
||||
return code;
|
||||
const name = (error as { name?: unknown }).name;
|
||||
if (typeof name === 'string' && /^[A-Za-z0-9_.-]{1,60}$/u.test(name))
|
||||
return name;
|
||||
}
|
||||
return '_OTHER';
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
// Импортируется первым в worker.main.ts (см. api-bootstrap.ts).
|
||||
import { startTelemetry } from './sdk.js';
|
||||
|
||||
startTelemetry('messenger-worker');
|
||||
Reference in New Issue
Block a user