diff --git a/app/package.json b/app/package.json index e9277cd..1db9689 100644 --- a/app/package.json +++ b/app/package.json @@ -35,6 +35,17 @@ "@nestjs/core": "^12.0.1", "@nestjs/platform-express": "^12.0.1", "@nestjs/platform-fastify": "^12.0.3", + "@opentelemetry/api": "1.9.0", + "@opentelemetry/api-logs": "0.203.0", + "@opentelemetry/core": "2.0.1", + "@opentelemetry/exporter-logs-otlp-proto": "0.203.0", + "@opentelemetry/exporter-metrics-otlp-proto": "0.203.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.203.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-logs": "0.203.0", + "@opentelemetry/sdk-metrics": "2.0.1", + "@opentelemetry/sdk-trace-base": "2.0.1", + "@opentelemetry/sdk-trace-node": "2.0.1", "@smithy/node-http-handler": "^4.12.1", "amqplib": "^2.2.1", "argon2": "^0.45.1", diff --git a/app/src/config/app-config.schema.ts b/app/src/config/app-config.schema.ts index c5b4cda..4e0949c 100644 --- a/app/src/config/app-config.schema.ts +++ b/app/src/config/app-config.schema.ts @@ -170,6 +170,14 @@ export const appConfigSchema = z 86400, ), WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081), + // lab6: как часто API пересчитывает глубину и возраст очереди задач для + // метрик messenger.jobs.queue.* (один дешёвый агрегат по частичным индексам). + QUEUE_STATS_INTERVAL_MS: numeric( + 'QUEUE_STATS_INTERVAL_MS', + 1000, + 600000, + 10000, + ), FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0), QUEUED_RECONCILE_GRACE_SECONDS: numeric( 'QUEUED_RECONCILE_GRACE_SECONDS', diff --git a/app/src/telemetry/api-bootstrap.ts b/app/src/telemetry/api-bootstrap.ts new file mode 100644 index 0000000..2671d45 --- /dev/null +++ b/app/src/telemetry/api-bootstrap.ts @@ -0,0 +1,5 @@ +// Импортируется первым в main.ts: ESM выполняет модули в порядке импорта, и +// глобальные провайдеры OTel должны быть установлены до первого запроса. +import { startTelemetry } from './sdk.js'; + +startTelemetry('messenger-api'); diff --git a/app/src/telemetry/export-counter.ts b/app/src/telemetry/export-counter.ts new file mode 100644 index 0000000..3160e90 --- /dev/null +++ b/app/src/telemetry/export-counter.ts @@ -0,0 +1,36 @@ +import { ExportResultCode, type ExportResult } from '@opentelemetry/core'; +import { instruments } from './instruments.js'; + +type BatchExporter = { + export(items: T[], resultCallback: (result: ExportResult) => void): void; + shutdown(): Promise; + forceFlush?(): Promise; +}; + +/** + * Обёртка над OTLP-экспортёром: считает, сколько span и log records ушло и + * сколько потеряно. Счётчик кумулятивный, поэтому потери во время простоя + * Collector становятся видны в Prometheus сразу после его возвращения. + * Переполнение очереди BatchProcessor (drop до экспорта) SDK пишет в diag-лог. + */ +export function countingExporter( + signal: 'traces' | 'logs', + inner: BatchExporter, +): BatchExporter { + return { + export(items: T[], resultCallback: (result: ExportResult) => void) { + inner.export(items, (result: ExportResult) => { + instruments().telemetryExport.add(items.length, { + signal, + outcome: + result.code === ExportResultCode.SUCCESS + ? 'success' + : 'failed', + }); + resultCallback(result); + }); + }, + shutdown: () => inner.shutdown(), + forceFlush: () => inner.forceFlush?.() ?? Promise.resolve(), + }; +} diff --git a/app/src/telemetry/http.spec.ts b/app/src/telemetry/http.spec.ts new file mode 100644 index 0000000..3467510 --- /dev/null +++ b/app/src/telemetry/http.spec.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { isTracedRoute, normalizeMethod, routeTemplate } from './http.js'; + +describe('routeTemplate', () => { + it('turns Fastify params into low-cardinality templates', () => { + expect(routeTemplate('/api/v1/chats/:chatId/messages')).toBe( + '/api/v1/chats/{chat_id}/messages', + ); + expect(routeTemplate('/api/v1/attachments/:attachmentId/content')).toBe( + '/api/v1/attachments/{attachment_id}/content', + ); + expect(routeTemplate('/api/v1/users/me')).toBe('/api/v1/users/me'); + }); + + it('groups requests that matched no route', () => { + expect(routeTemplate(undefined)).toBe('unmatched'); + }); +}); + +describe('normalizeMethod', () => { + it('keeps standard methods and folds the rest', () => { + expect(normalizeMethod('get')).toBe('GET'); + expect(normalizeMethod('DELETE')).toBe('DELETE'); + expect(normalizeMethod('PROPFIND')).toBe('_OTHER'); + }); +}); + +describe('isTracedRoute', () => { + it('skips probes, internal endpoints and unmatched requests', () => { + expect(isTracedRoute('/health/ready')).toBe(false); + expect(isTracedRoute('/internal/metrics')).toBe(false); + expect(isTracedRoute('unmatched')).toBe(false); + expect(isTracedRoute('/api/v1/chats/{chat_id}/messages')).toBe(true); + }); +}); diff --git a/app/src/telemetry/http.ts b/app/src/telemetry/http.ts new file mode 100644 index 0000000..d869c4f --- /dev/null +++ b/app/src/telemetry/http.ts @@ -0,0 +1,201 @@ +import { + context, + propagation, + ROOT_CONTEXT, + SpanKind, + SpanStatusCode, + trace, + type Context, + type Span, +} from '@opentelemetry/api'; +import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; +import { instruments } from './instruments.js'; +import { appLogger } from './logger.js'; +import { tracer } from './tracing.js'; + +const STATE = Symbol('messenger.telemetry.http'); +const KNOWN_METHODS = new Set([ + 'GET', + 'HEAD', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', + 'OPTIONS', +]); +// Пробы HAProxy (каждые 500 мс на реплику) и внутренние endpoint не трассируются +// и не пишут access-log: они дали бы больше половины объёма Tempo и Loki без +// диагностической ценности. В метриках они остаются. +const UNTRACED_PREFIXES = ['/health', '/internal/']; + +type HttpState = { + readonly ctx: Context; + readonly span: Span | undefined; + readonly method: string; + readonly route: string; + readonly startedAt: bigint; + finished: boolean; +}; +type InstrumentedRequest = FastifyRequest & { [STATE]?: HttpState }; +type RouteHandler = ( + this: unknown, + request: FastifyRequest, + reply: FastifyReply, +) => unknown; + +/** `/api/v1/chats/:chatId/messages` → `/api/v1/chats/{chat_id}/messages`. */ +export function routeTemplate(url: string | undefined): string { + if (!url) return 'unmatched'; + return url.replace( + /:([A-Za-z0-9_]+)/gu, + (_match, name: string) => + `{${name.replace(/([a-z0-9])([A-Z])/gu, '$1_$2').toLowerCase()}}`, + ); +} + +export function normalizeMethod(method: string): string { + const upper = method.toUpperCase(); + return KNOWN_METHODS.has(upper) ? upper : '_OTHER'; +} + +export function isTracedRoute(route: string): boolean { + return ( + route !== 'unmatched' && + !UNTRACED_PREFIXES.some((prefix) => route.startsWith(prefix)) + ); +} + +function headerValue(value: unknown): string | undefined { + if (typeof value === 'string') return value; + if (typeof value === 'number') return String(value); + if (Array.isArray(value) && typeof value[0] === 'string') return value[0]; + return undefined; +} + +/** + * HTTP-инструментация на хуках Fastify: + * - onRequest: SERVER span с родителем из входящего `traceparent`; + * - onRoute: обработчик маршрута выполняется внутри контекста span, поэтому + * span PostgreSQL/S3/RabbitMQ и логи обработчика становятся его детьми; + * - onResponse / onRequestAbort: длительность в гистограмму, access-log, end span. + * В метрики попадают только метод, route template и код статуса. + * Регистрируется до app.listen(): маршруты Nest добавляются при init. + */ +export function installHttpTelemetry(fastify: FastifyInstance): void { + fastify.addHook('onRoute', (routeOptions) => { + const original = routeOptions.handler as unknown as RouteHandler | undefined; + if (typeof original !== 'function') return; + const wrapped: RouteHandler = function ( + this: unknown, + request: FastifyRequest, + reply: FastifyReply, + ) { + const state = (request as InstrumentedRequest)[STATE]; + if (!state?.span) return original.call(this, request, reply); + return context.with(state.ctx, () => + original.call(this, request, reply), + ); + }; + (routeOptions as unknown as { handler: RouteHandler }).handler = wrapped; + }); + + fastify.addHook( + 'onRequest', + (request: FastifyRequest, _reply: FastifyReply, done: () => void) => { + const method = normalizeMethod(request.method); + const route = routeTemplate(request.routeOptions?.url); + let ctx = ROOT_CONTEXT; + let span: Span | undefined; + if (isTracedRoute(route)) { + const parent = propagation.extract(ROOT_CONTEXT, request.headers); + span = tracer().startSpan( + `${method} ${route}`, + { + kind: SpanKind.SERVER, + attributes: { + 'http.request.method': method, + 'http.route': route, + 'url.scheme': 'http', + 'network.protocol.version': request.raw.httpVersion, + }, + }, + parent, + ); + ctx = trace.setSpan(parent, span); + } + (request as InstrumentedRequest)[STATE] = { + ctx, + span, + method, + route, + startedAt: process.hrtime.bigint(), + finished: false, + }; + done(); + }, + ); + + fastify.addHook( + 'onResponse', + (request: FastifyRequest, reply: FastifyReply, done: () => void) => { + finish( + request as InstrumentedRequest, + reply.statusCode, + headerValue(reply.getHeader('x-request-id')), + false, + ); + done(); + }, + ); + + fastify.addHook( + 'onRequestAbort', + (request: FastifyRequest, done: () => void) => { + // 499 — клиент закрыл соединение до ответа (конвенция nginx); в SLO + // доступности не считается ошибкой сервиса. + finish(request as InstrumentedRequest, 499, undefined, true); + done(); + }, + ); +} + +function finish( + request: InstrumentedRequest, + status: number, + requestId: string | undefined, + aborted: boolean, +): void { + const state = request[STATE]; + if (!state || state.finished) return; + state.finished = true; + const seconds = Number(process.hrtime.bigint() - state.startedAt) / 1e9; + instruments().httpDuration.record(seconds, { + 'http.request.method': state.method, + 'http.route': state.route, + 'http.response.status_code': status, + }); + + const span = state.span; + if (!span) return; + span.setAttribute('http.response.status_code', status); + if (requestId) span.setAttribute('messenger.request_id', requestId); + if (aborted) span.setAttribute('error.type', 'request_aborted'); + if (status >= 500) + span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` }); + context.with(state.ctx, () => + appLogger.event( + status >= 500 ? 'ERROR' : 'INFO', + 'http.request', + `${state.method} ${state.route} ${status}`, + { + 'http.request.method': state.method, + 'http.route': state.route, + 'http.response.status_code': status, + duration_ms: Math.round(seconds * 10_000) / 10, + request_id: requestId, + aborted: aborted || undefined, + }, + ), + ); + span.end(); +} diff --git a/app/src/telemetry/identity.ts b/app/src/telemetry/identity.ts new file mode 100644 index 0000000..3ec0de5 --- /dev/null +++ b/app/src/telemetry/identity.ts @@ -0,0 +1,35 @@ +import os from 'node:os'; + +export type ServiceIdentity = { + readonly serviceName: string; + readonly serviceVersion: string; + readonly instanceId: string; +}; + +function fromEnv(defaultServiceName: string): ServiceIdentity { + return { + serviceName: process.env.OTEL_SERVICE_NAME?.trim() || defaultServiceName, + serviceVersion: process.env.SERVICE_VERSION?.trim() || 'dev', + // В compose у реплик API нет INSTANCE_ID, но HOSTNAME — id контейнера, + // уникальный для каждой реплики: так service.instance.id различает + // api-1 и api-2 во всех трёх сигналах. + instanceId: + process.env.INSTANCE_ID?.trim() || + process.env.HOSTNAME?.trim() || + os.hostname(), + }; +} + +let current = fromEnv('messenger'); + +/** service.name / service.version / service.instance.id, общие для всех сигналов. */ +export function serviceIdentity(): ServiceIdentity { + return current; +} + +export function resolveServiceIdentity( + defaultServiceName: string, +): ServiceIdentity { + current = fromEnv(defaultServiceName); + return current; +} diff --git a/app/src/telemetry/instruments.ts b/app/src/telemetry/instruments.ts new file mode 100644 index 0000000..7453963 --- /dev/null +++ b/app/src/telemetry/instruments.ts @@ -0,0 +1,94 @@ +import { + metrics, + type Counter, + type Histogram, + type Meter, + type MeterProvider, +} from '@opentelemetry/api'; + +export const METER_NAME = 'messenger'; + +// Каждый SLO-порог — точная граница бакета (0.25 с для API, 30 с для +// time-to-ready): иначе good-события пришлось бы интерполировать. +export const HTTP_DURATION_BUCKETS = [ + 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.5, 0.75, 1, 2.5, 5, 10, + 30, +]; +export const JOB_DURATION_BUCKETS = [ + 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 7.5, 10, 15, 20, 30, 45, 60, 120, +]; +export const TIME_TO_READY_BUCKETS = [ + 0.25, 0.5, 1, 2, 3, 5, 7.5, 10, 15, 20, 30, 45, 60, 120, 300, 600, +]; +export const WAIT_BUCKETS = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, +]; + +export type Instruments = { + readonly httpDuration: Histogram; + readonly jobsProcessed: Counter; + readonly jobDuration: Histogram; + readonly timeToReady: Histogram; + readonly brokerPublished: Counter; + readonly dbConnectionWait: Histogram; + readonly telemetryExport: Counter; +}; + +let cached: { provider: MeterProvider; instruments: Instruments } | undefined; + +export function meter(): Meter { + return metrics.getMeter(METER_NAME, '1.0.0'); +} + +/** + * Инструменты создаются лениво и пересоздаются, если сменился глобальный + * MeterProvider: созданный до старта SDK инструмент навсегда остался бы no-op. + * Атрибуты — только ограниченные множества (метод, route template, код + * статуса, outcome); id пользователей, чатов, вложений и запросов сюда не + * попадают никогда. + */ +export function instruments(): Instruments { + const provider = metrics.getMeterProvider(); + if (cached?.provider === provider) return cached.instruments; + const m = provider.getMeter(METER_NAME, '1.0.0'); + const created: Instruments = { + httpDuration: m.createHistogram('http.server.request.duration', { + unit: 's', + description: + 'HTTP server request duration by method, route template and status code', + advice: { explicitBucketBoundaries: HTTP_DURATION_BUCKETS }, + }), + jobsProcessed: m.createCounter('messenger.jobs.processed', { + unit: '{job}', + description: 'Job deliveries handled by the worker, by outcome', + }), + jobDuration: m.createHistogram('messenger.job.duration', { + unit: 's', + description: + 'Duration of one job attempt from claim to commit, by outcome', + advice: { explicitBucketBoundaries: JOB_DURATION_BUCKETS }, + }), + timeToReady: m.createHistogram('messenger.attachment.time_to_ready', { + unit: 's', + description: + 'Time from durable acceptance of an upload to the ready commit (includes queueing and retries)', + advice: { explicitBucketBoundaries: TIME_TO_READY_BUCKETS }, + }), + brokerPublished: m.createCounter('messenger.broker.published', { + unit: '{message}', + description: 'Messages published to the broker, by outcome', + }), + dbConnectionWait: m.createHistogram('db.client.connection.wait_time', { + unit: 's', + description: 'Time spent waiting for a connection from the pool', + advice: { explicitBucketBoundaries: WAIT_BUCKETS }, + }), + telemetryExport: m.createCounter('messenger.telemetry.export.items', { + unit: '{item}', + description: + 'Spans and log records handed to the OTLP exporter, by outcome (failed = lost)', + }), + }; + cached = { provider, instruments: created }; + return created; +} diff --git a/app/src/telemetry/logger.ts b/app/src/telemetry/logger.ts new file mode 100644 index 0000000..96c8b54 --- /dev/null +++ b/app/src/telemetry/logger.ts @@ -0,0 +1,217 @@ +import type { LoggerService, LogLevel } from '@nestjs/common'; +import { context, trace } from '@opentelemetry/api'; +import { + logs, + SeverityNumber, + type AnyValueMap, + type Logger as OtelLogger, + type LoggerProvider, +} from '@opentelemetry/api-logs'; +import { serviceIdentity } from './identity.js'; +import { redact } from './redact.js'; + +export type Severity = 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' | 'FATAL'; +export type LogAttributes = Record< + string, + string | number | boolean | null | undefined +>; + +const SEVERITY_NUMBER: Record = { + TRACE: SeverityNumber.TRACE, + DEBUG: SeverityNumber.DEBUG, + INFO: SeverityNumber.INFO, + WARN: SeverityNumber.WARN, + ERROR: SeverityNumber.ERROR, + FATAL: SeverityNumber.FATAL, +}; +const RANK: Record = { + TRACE: 1, + DEBUG: 5, + INFO: 9, + WARN: 13, + ERROR: 17, + FATAL: 21, +}; +const NEST_LEVEL: Record = { + verbose: 'TRACE', + debug: 'DEBUG', + log: 'INFO', + warn: 'WARN', + error: 'ERROR', + fatal: 'FATAL', +}; +const MAX_TEXT = 4000; + +function parseSeverity(raw: string | undefined): Severity { + switch ((raw ?? '').trim().toLowerCase()) { + case 'trace': + case 'verbose': + return 'TRACE'; + case 'debug': + return 'DEBUG'; + case 'warn': + case 'warning': + return 'WARN'; + case 'error': + return 'ERROR'; + default: + return 'INFO'; + } +} + +function stringify(value: unknown): string { + if (typeof value === 'string') return value; + if (value instanceof Error) return `${value.name}: ${value.message}`; + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} + +/** + * Структурный логгер для Nest и событий приложения. Каждая запись уходит: + * - в stdout JSON-строкой (docker logs остаются читаемыми без Loki); + * - в OTLP logs с контекстом активного span, поэтому в Loki у записи есть + * trace_id/span_id и из неё можно перейти в Tempo. + * Текст и строковые атрибуты проходят redact() и обрезаются до 4000 символов. + */ +export class StructuredLogger implements LoggerService { + private minRank = RANK[parseSeverity(process.env.LOG_LEVEL)]; + private otel: { provider: LoggerProvider; logger: OtelLogger } | undefined; + + log(message: unknown, ...optionalParams: unknown[]): void { + this.fromNest('INFO', message, optionalParams); + } + + error(message: unknown, ...optionalParams: unknown[]): void { + this.fromNest('ERROR', message, optionalParams); + } + + warn(message: unknown, ...optionalParams: unknown[]): void { + this.fromNest('WARN', message, optionalParams); + } + + debug(message: unknown, ...optionalParams: unknown[]): void { + this.fromNest('DEBUG', message, optionalParams); + } + + verbose(message: unknown, ...optionalParams: unknown[]): void { + this.fromNest('TRACE', message, optionalParams); + } + + fatal(message: unknown, ...optionalParams: unknown[]): void { + this.fromNest('FATAL', message, optionalParams); + } + + setLogLevels(levels: LogLevel[]): void { + const ranks = levels + .map((level) => NEST_LEVEL[level]) + .filter((severity): severity is Severity => severity !== undefined) + .map((severity) => RANK[severity]); + if (ranks.length > 0) this.minRank = Math.min(...ranks); + } + + /** Событие приложения: `event` — стабильное имя для запросов в Loki. */ + event( + severity: Severity, + event: string, + message: string, + attributes: LogAttributes = {}, + ): void { + this.write(severity, event, undefined, message, attributes); + } + + /** + * Конвенция Nest: контекст — последний строковый параметр, для error перед + * ним может идти stack. + */ + private fromNest( + severity: Severity, + message: unknown, + params: unknown[], + ): void { + if (RANK[severity] < this.minRank) return; + const rest = [...params]; + let contextName: string | undefined; + if (rest.length > 0 && typeof rest[rest.length - 1] === 'string') + contextName = rest.pop() as string; + let stack: string | undefined; + if (severity === 'ERROR' || severity === 'FATAL') { + const index = rest.findIndex( + (param) => typeof param === 'string' && param.includes('\n'), + ); + if (index >= 0) stack = rest.splice(index, 1)[0] as string; + if (!stack && message instanceof Error) stack = message.stack; + } + let text = stringify(message); + if (rest.length > 0) text = `${text} ${rest.map(stringify).join(' ')}`; + this.write( + severity, + 'log', + contextName, + text, + stack ? { 'exception.stacktrace': stack } : {}, + ); + } + + private write( + severity: Severity, + event: string, + contextName: string | undefined, + message: string, + attributes: LogAttributes, + ): void { + if (RANK[severity] < this.minRank) return; + const identity = serviceIdentity(); + const active = context.active(); + const spanContext = trace.getSpan(active)?.spanContext(); + let ids: { trace_id: string; span_id: string } | undefined; + if (spanContext && trace.isSpanContextValid(spanContext)) + ids = { trace_id: spanContext.traceId, span_id: spanContext.spanId }; + + const body = redact(message).slice(0, MAX_TEXT); + const clean: Record = {}; + for (const [key, value] of Object.entries(attributes)) { + if (value === undefined || value === null) continue; + clean[key] = + typeof value === 'string' ? redact(value).slice(0, MAX_TEXT) : value; + } + + process.stdout.write( + `${JSON.stringify({ + timestamp: new Date().toISOString(), + severity, + 'service.name': identity.serviceName, + 'service.instance.id': identity.instanceId, + event, + ...(contextName ? { context: contextName } : {}), + message: body, + ...ids, + ...clean, + })}\n`, + ); + + this.otelLogger().emit({ + severityNumber: SEVERITY_NUMBER[severity], + severityText: severity, + body, + attributes: { + 'event.name': event, + ...(contextName ? { 'log.context': contextName } : {}), + ...clean, + } as AnyValueMap, + context: active, + }); + } + + private otelLogger(): OtelLogger { + const provider = logs.getLoggerProvider(); + if (this.otel?.provider !== provider) + this.otel = { provider, logger: provider.getLogger('messenger', '1.0.0') }; + return this.otel.logger; + } +} + +/** Единственный экземпляр: передаётся в NestFactory и используется для событий. */ +export const appLogger = new StructuredLogger(); diff --git a/app/src/telemetry/queue-stats.service.ts b/app/src/telemetry/queue-stats.service.ts new file mode 100644 index 0000000..4f37c01 --- /dev/null +++ b/app/src/telemetry/queue-stats.service.ts @@ -0,0 +1,164 @@ +import { + Injectable, + Logger, + type OnModuleDestroy, + type OnModuleInit, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { DatabaseClient } from '../database/database.client.js'; +import { JOB_QUEUE } from '../jobs/broker.topology.js'; +import { meter } from './instruments.js'; + +type QueueSnapshot = { + readonly takenAt: number; + readonly ready: number; + readonly delayed: number; + readonly processing: number; + readonly oldestAgeSeconds: number; + readonly outboxPending: number; + readonly outboxOldestAgeSeconds: number; +}; + +// Источник истины о состоянии задач — PostgreSQL, а не глубина очереди RabbitMQ: +// задача может ждать в outbox, в backoff или под истёкшим lease, не лежа в +// брокере. Возраст самой старой незавершённой задачи — то, что чувствует +// пользователь, ожидающий ready. +const JOBS_SQL = `SELECT + count(*) FILTER (WHERE status = 'queued' AND next_attempt_at <= now())::int AS ready, + count(*) FILTER (WHERE status = 'queued' AND next_attempt_at > now())::int AS delayed, + count(*) FILTER (WHERE status = 'processing')::int AS processing, + COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::float8 AS oldest_age +FROM attachment_jobs +WHERE status IN ('queued', 'processing')`; + +const OUTBOX_SQL = `SELECT + count(*)::int AS pending, + COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)::float8 AS oldest_age +FROM job_outbox +WHERE published_at IS NULL AND available_at <= now()`; + +type JobsRow = { + ready: number; + delayed: number; + processing: number; + oldest_age: number; +}; +type OutboxRow = { pending: number; oldest_age: number }; + +/** + * Метрики очереди задач. Работает в API, а не в worker: в сценарии «worker + * остановлен» именно эти gauge должны продолжать расти. Обе реплики + * публикуют одинаковые значения, в PromQL берётся max(). При ошибке запроса + * старый снимок публикуется не дольше трёх интервалов, затем серия пропадает + * — лучше отсутствие данных, чем устаревший ноль. + */ +@Injectable() +export class QueueStatsService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(QueueStatsService.name); + private readonly intervalMs: number; + private snapshot: QueueSnapshot | undefined; + private timer: NodeJS.Timeout | undefined; + private refreshing = false; + private failures = 0; + + constructor( + private readonly database: DatabaseClient, + config: ConfigService, + ) { + this.intervalMs = config.getOrThrow('QUEUE_STATS_INTERVAL_MS'); + } + + onModuleInit(): void { + const m = meter(); + const depth = m.createObservableGauge('messenger.jobs.queue.depth', { + unit: '{job}', + description: 'Unfinished attachment jobs by state (PostgreSQL)', + }); + const oldestAge = m.createObservableGauge( + 'messenger.jobs.queue.oldest_age', + { + unit: 's', + description: + 'Age of the oldest unfinished job since durable acceptance', + }, + ); + const outboxPending = m.createObservableGauge('messenger.outbox.pending', { + unit: '{message}', + description: 'Outbox rows due for publishing but not yet confirmed', + }); + const outboxAge = m.createObservableGauge('messenger.outbox.oldest_age', { + unit: 's', + description: 'Age of the oldest unpublished outbox row', + }); + m.addBatchObservableCallback( + (result) => { + const snapshot = this.freshSnapshot(); + if (!snapshot) return; + const queue = { 'messaging.destination.name': JOB_QUEUE }; + result.observe(depth, snapshot.ready, { ...queue, state: 'ready' }); + result.observe(depth, snapshot.delayed, { + ...queue, + state: 'delayed', + }); + result.observe(depth, snapshot.processing, { + ...queue, + state: 'processing', + }); + result.observe(oldestAge, snapshot.oldestAgeSeconds, queue); + result.observe(outboxPending, snapshot.outboxPending, queue); + result.observe(outboxAge, snapshot.outboxOldestAgeSeconds, queue); + }, + [depth, oldestAge, outboxPending, outboxAge], + ); + void this.refresh(); + this.timer = setInterval(() => void this.refresh(), this.intervalMs); + this.timer.unref(); + } + + onModuleDestroy(): void { + if (this.timer) clearInterval(this.timer); + } + + private freshSnapshot(): QueueSnapshot | undefined { + if (!this.snapshot) return undefined; + return Date.now() - this.snapshot.takenAt <= this.intervalMs * 3 + ? this.snapshot + : undefined; + } + + private async refresh(): Promise { + if (this.refreshing) return; + this.refreshing = true; + try { + const [jobs, outbox] = await Promise.all([ + this.database.query('telemetry.queue_stats', JOBS_SQL), + this.database.query('telemetry.outbox_stats', OUTBOX_SQL), + ]); + const j = jobs.rows[0]; + const o = outbox.rows[0]; + this.snapshot = { + takenAt: Date.now(), + ready: Number(j?.ready ?? 0), + delayed: Number(j?.delayed ?? 0), + processing: Number(j?.processing ?? 0), + oldestAgeSeconds: Number(j?.oldest_age ?? 0), + outboxPending: Number(o?.pending ?? 0), + outboxOldestAgeSeconds: Number(o?.oldest_age ?? 0), + }; + if (this.failures > 0) + this.logger.log( + `queue stats recovered after ${this.failures} failed refreshes`, + ); + this.failures = 0; + } catch (error) { + this.failures += 1; + if (this.failures === 1 || this.failures % 30 === 0) + this.logger.warn( + `queue stats refresh failed (${this.failures} in a row): ${error instanceof Error ? error.name : 'unknown error'}`, + ); + } finally { + this.refreshing = false; + } + } +} diff --git a/app/src/telemetry/redact.spec.ts b/app/src/telemetry/redact.spec.ts new file mode 100644 index 0000000..0ee534d --- /dev/null +++ b/app/src/telemetry/redact.spec.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { redact } from './redact.js'; + +describe('redact', () => { + it('masks bearer and basic credentials', () => { + const out = redact( + 'Authorization: Bearer abc.def-123 and Basic dXNlcjpwYXNz', + ); + expect(out).not.toContain('abc.def-123'); + expect(out).not.toContain('dXNlcjpwYXNz'); + }); + + it('masks credentials embedded in connection strings', () => { + expect( + redact('postgresql://messenger:s3cret@postgres:5432/messenger'), + ).toBe('postgresql://[REDACTED]@postgres:5432/messenger'); + }); + + it('masks presigned URL credentials and signatures', () => { + const out = redact( + 'https://s3.local/b/k?X-Amz-Credential=AKIAEXAMPLE%2F20260101&X-Amz-Signature=deadbeef', + ); + expect(out).not.toContain('AKIAEXAMPLE'); + expect(out).not.toContain('deadbeef'); + }); + + it('masks password-like fields', () => { + expect(redact('{"password":"hunter2"}')).not.toContain('hunter2'); + expect(redact('password=hunter2&x=1')).not.toContain('hunter2'); + }); + + it('keeps ordinary diagnostic text intact', () => { + expect(redact('job retry scheduled attempt=2 code=transient')).toBe( + 'job retry scheduled attempt=2 code=transient', + ); + }); +}); diff --git a/app/src/telemetry/redact.ts b/app/src/telemetry/redact.ts new file mode 100644 index 0000000..3e64812 --- /dev/null +++ b/app/src/telemetry/redact.ts @@ -0,0 +1,29 @@ +/** + * Последний рубеж перед stdout и OTLP: если секрет всё же попал в текст + * события (например, в сообщение ошибки драйвера), он не уйдёт в Loki. + * Основная защита — не класть такие данные в события вообще: route template + * вместо URL, коды ошибок вместо сообщений, без тел запросов. + */ +const RULES: ReadonlyArray = [ + [/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/giu, '$1 [REDACTED]'], + [/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/giu, '$1[REDACTED]@'], + [ + /\b(X-Amz-(?:Signature|Credential|Security-Token))=[^&\s"']+/giu, + '$1=[REDACTED]', + ], + [ + /((?:password|passwd|secret|token|authorization|cookie)["']?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;&}]+)/giu, + '$1[REDACTED]', + ], + [ + /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/gu, + '[REDACTED PRIVATE KEY]', + ], +]; + +export function redact(text: string): string { + let result = text; + for (const [pattern, replacement] of RULES) + result = result.replace(pattern, replacement); + return result; +} diff --git a/app/src/telemetry/runtime-metrics.ts b/app/src/telemetry/runtime-metrics.ts new file mode 100644 index 0000000..0fdcf2b --- /dev/null +++ b/app/src/telemetry/runtime-metrics.ts @@ -0,0 +1,55 @@ +import { monitorEventLoopDelay } from 'node:perf_hooks'; +import { meter } from './instruments.js'; + +let started = false; + +/** + * Насыщение процесса: CPU, RSS, heap и задержка event loop. Задержка event + * loop — главный признак насыщения Node.js: CPU-bound работа (sharp, argon2) + * и синхронный JSON на больших ответах растят её раньше, чем latency API. + */ +export function startRuntimeMetrics(): void { + if (started) return; + started = true; + const m = meter(); + const loop = monitorEventLoopDelay({ resolution: 20 }); + loop.enable(); + + m.createObservableCounter('process.cpu.time', { + unit: 's', + description: 'CPU time consumed by the process', + }).addCallback((result) => { + const usage = process.cpuUsage(); + result.observe(usage.user / 1e6, { 'cpu.mode': 'user' }); + result.observe(usage.system / 1e6, { 'cpu.mode': 'system' }); + }); + m.createObservableUpDownCounter('process.memory.usage', { + unit: 'By', + description: 'Resident set size of the process', + }).addCallback((result) => { + result.observe(process.memoryUsage.rss()); + }); + m.createObservableUpDownCounter('v8js.memory.heap.used', { + unit: 'By', + description: 'V8 heap in use', + }).addCallback((result) => { + result.observe(process.memoryUsage().heapUsed); + }); + + const delayP99 = m.createObservableGauge('nodejs.eventloop.delay.p99', { + unit: 's', + description: 'p99 event loop delay since the previous collection', + }); + const delayMax = m.createObservableGauge('nodejs.eventloop.delay.max', { + unit: 's', + description: 'Max event loop delay since the previous collection', + }); + m.addBatchObservableCallback( + (result) => { + result.observe(delayP99, loop.percentile(99) / 1e9); + result.observe(delayMax, loop.max / 1e9); + loop.reset(); + }, + [delayP99, delayMax], + ); +} diff --git a/app/src/telemetry/sdk.ts b/app/src/telemetry/sdk.ts new file mode 100644 index 0000000..58c948a --- /dev/null +++ b/app/src/telemetry/sdk.ts @@ -0,0 +1,295 @@ +import { + diag, + DiagLogLevel, + metrics, + type DiagLogger, +} from '@opentelemetry/api'; +import { logs } from '@opentelemetry/api-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { + BatchLogRecordProcessor, + LoggerProvider, +} from '@opentelemetry/sdk-logs'; +import { + MeterProvider, + PeriodicExportingMetricReader, +} from '@opentelemetry/sdk-metrics'; +import { + BatchSpanProcessor, + ParentBasedSampler, + TraceIdRatioBasedSampler, +} from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { countingExporter } from './export-counter.js'; +import { resolveServiceIdentity } from './identity.js'; +import { redact } from './redact.js'; +import { startRuntimeMetrics } from './runtime-metrics.js'; + +type Providers = { + readonly tracer: NodeTracerProvider; + readonly meter: MeterProvider; + readonly logger: LoggerProvider; +}; + +let providers: Providers | undefined; + +function intFromEnv( + name: string, + fallback: number, + min: number, + max: number, +): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const value = Number(raw); + return Number.isInteger(value) && value >= min && value <= max + ? value + : fallback; +} + +function ratioFromEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const value = Number(raw); + return Number.isFinite(value) && value >= 0 && value <= 1 ? value : fallback; +} + +/** OTEL_RESOURCE_ATTRIBUTES: `k=v,k2=v2`, значения percent-encoded. */ +export function parseResourceAttributes( + raw: string | undefined, +): Record { + const attributes: Record = {}; + if (!raw) return attributes; + for (const pair of raw.split(',')) { + const index = pair.indexOf('='); + if (index <= 0) continue; + const key = pair.slice(0, index).trim(); + const value = pair.slice(index + 1).trim(); + if (!key) continue; + try { + attributes[key] = decodeURIComponent(value); + } catch { + attributes[key] = value; + } + } + return attributes; +} + +export function telemetryEnabled(): boolean { + return (process.env.OTEL_SDK_DISABLED ?? '').trim().toLowerCase() !== 'true'; +} + +/** + * Явная настройка SDK без auto-instrumentation: приложение — ESM (авто- + * инструментации понадобился бы loader hook), а явные span дают полный + * контроль над атрибутами: SQL-текст, URL и тела не попадают в телеметрию. + * + * Экспорт асинхронный и ограниченный: очереди BatchProcessor фиксированного + * размера, таймаут экспорта 5 с. Недоступный Collector оборачивается потерей + * телеметрии (счётчик messenger.telemetry.export.items{outcome="failed"}), а не + * ростом latency или памяти API. + */ +export function startTelemetry(defaultServiceName: string): void { + if (providers) return; + const identity = resolveServiceIdentity(defaultServiceName); + if (!telemetryEnabled()) return; + + diag.setLogger( + new ThrottledDiagLogger(identity.serviceName), + DiagLogLevel.WARN, + ); + + const resource = resourceFromAttributes({ + ...parseResourceAttributes(process.env.OTEL_RESOURCE_ATTRIBUTES), + 'service.name': identity.serviceName, + 'service.version': identity.serviceVersion, + 'service.instance.id': identity.instanceId, + }); + const exportTimeout = intFromEnv( + 'OTEL_EXPORTER_OTLP_TIMEOUT', + 5000, + 100, + 60_000, + ); + + const tracer = new NodeTracerProvider({ + resource, + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler( + ratioFromEnv('OTEL_TRACES_SAMPLER_ARG', 1), + ), + }), + spanProcessors: [ + new BatchSpanProcessor( + countingExporter( + 'traces', + new OTLPTraceExporter({ timeoutMillis: exportTimeout }), + ), + { + maxQueueSize: intFromEnv('OTEL_BSP_MAX_QUEUE_SIZE', 2048, 1, 100_000), + maxExportBatchSize: intFromEnv( + 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE', + 512, + 1, + 10_000, + ), + scheduledDelayMillis: intFromEnv( + 'OTEL_BSP_SCHEDULE_DELAY', + 1000, + 1, + 60_000, + ), + exportTimeoutMillis: exportTimeout, + }, + ), + ], + }); + // Регистрирует глобальный TracerProvider, AsyncLocalStorage context manager и + // W3C TraceContext + Baggage propagator. + tracer.register(); + + const interval = intFromEnv( + 'OTEL_METRIC_EXPORT_INTERVAL', + 10_000, + 1000, + 600_000, + ); + const meter = new MeterProvider({ + resource, + readers: [ + new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ timeoutMillis: exportTimeout }), + exportIntervalMillis: interval, + exportTimeoutMillis: Math.min(exportTimeout, interval), + }), + ], + }); + metrics.setGlobalMeterProvider(meter); + + const logger = new LoggerProvider({ + resource, + processors: [ + new BatchLogRecordProcessor( + countingExporter( + 'logs', + new OTLPLogExporter({ timeoutMillis: exportTimeout }), + ), + { + maxQueueSize: intFromEnv( + 'OTEL_BLRP_MAX_QUEUE_SIZE', + 2048, + 1, + 100_000, + ), + maxExportBatchSize: intFromEnv( + 'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', + 512, + 1, + 10_000, + ), + scheduledDelayMillis: intFromEnv( + 'OTEL_BLRP_SCHEDULE_DELAY', + 1000, + 1, + 60_000, + ), + exportTimeoutMillis: exportTimeout, + }, + ), + ], + }); + logs.setGlobalLoggerProvider(logger); + + providers = { tracer, meter, logger }; + startRuntimeMetrics(); +} + +/** + * Сбрасывает буферы при остановке. Ограничено по времени: недоступный + * Collector не должен задерживать graceful shutdown сервиса. + */ +export async function shutdownTelemetry(timeoutMs = 3000): Promise { + const current = providers; + if (!current) return; + providers = undefined; + let timer: NodeJS.Timeout | undefined; + await Promise.race([ + Promise.allSettled([ + current.tracer.shutdown(), + current.meter.shutdown(), + current.logger.shutdown(), + ]), + new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs); + }), + ]); + if (timer) clearTimeout(timer); +} + +/** + * Ошибки экспорта при недоступном Collector повторяются каждую секунду по + * каждому сигналу; одинаковые сообщения печатаем не чаще раза в 30 с. + */ +class ThrottledDiagLogger implements DiagLogger { + private readonly lastWritten = new Map(); + + constructor(private readonly serviceName: string) {} + + error(message: string, ...args: unknown[]): void { + this.write('ERROR', message, args); + } + + warn(message: string, ...args: unknown[]): void { + this.write('WARN', message, args); + } + + info(message: string, ...args: unknown[]): void { + this.write('INFO', message, args); + } + + debug(message: string, ...args: unknown[]): void { + this.write('DEBUG', message, args); + } + + verbose(message: string, ...args: unknown[]): void { + this.write('TRACE', message, args); + } + + private write(severity: string, message: string, args: unknown[]): void { + const key = `${severity}:${message}`; + const now = Date.now(); + const previous = this.lastWritten.get(key); + if (previous !== undefined && now - previous < 30_000) return; + if (this.lastWritten.size > 200) this.lastWritten.clear(); + this.lastWritten.set(key, now); + const detail = args + .map((arg) => + arg instanceof Error + ? `${arg.name}: ${arg.message}` + : typeof arg === 'string' + ? arg + : safeJson(arg), + ) + .join(' '); + process.stdout.write( + `${JSON.stringify({ + timestamp: new Date(now).toISOString(), + severity, + 'service.name': this.serviceName, + event: 'otel.sdk', + message: redact(`${message} ${detail}`.trim()).slice(0, 2000), + })}\n`, + ); + } +} + +function safeJson(value: unknown): string { + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} diff --git a/app/src/telemetry/tracing.spec.ts b/app/src/telemetry/tracing.spec.ts new file mode 100644 index 0000000..2e4420e --- /dev/null +++ b/app/src/telemetry/tracing.spec.ts @@ -0,0 +1,72 @@ +import { + propagation, + ROOT_CONTEXT, + trace, + TraceFlags, +} from '@opentelemetry/api'; +import { W3CTraceContextPropagator } from '@opentelemetry/core'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { + contextFromTraceFields, + currentTraceFields, + errorType, + traceFieldsFromHeaders, +} from './tracing.js'; + +const TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736'; +const SPAN_ID = '00f067aa0ba902b7'; +const TRACEPARENT = `00-${TRACE_ID}-${SPAN_ID}-01`; + +describe('trace context helpers', () => { + beforeAll(() => { + propagation.setGlobalPropagator(new W3CTraceContextPropagator()); + }); + + afterAll(() => { + propagation.disable(); + }); + + it('serialises the active span context as job message fields', () => { + const ctx = trace.setSpanContext(ROOT_CONTEXT, { + traceId: TRACE_ID, + spanId: SPAN_ID, + traceFlags: TraceFlags.SAMPLED, + }); + expect(currentTraceFields(ctx)).toEqual({ traceparent: TRACEPARENT }); + }); + + it('restores the remote parent from job message fields', () => { + const ctx = contextFromTraceFields({ traceparent: TRACEPARENT }); + expect(trace.getSpanContext(ctx)).toMatchObject({ + traceId: TRACE_ID, + spanId: SPAN_ID, + isRemote: true, + }); + }); + + it('reads AMQP headers delivered as buffers', () => { + expect( + traceFieldsFromHeaders({ traceparent: Buffer.from(TRACEPARENT) }), + ).toEqual({ traceparent: TRACEPARENT }); + }); + + it('ignores malformed trace context', () => { + expect(contextFromTraceFields({ traceparent: 'garbage' })).toBe( + ROOT_CONTEXT, + ); + expect(traceFieldsFromHeaders({ traceparent: 'nope' })).toEqual({}); + expect(currentTraceFields(ROOT_CONTEXT)).toEqual({}); + }); + + it('classifies errors without leaking their messages', () => { + expect( + errorType( + Object.assign(new Error('Key (username)=(alice) already exists'), { + code: '23505', + }), + ), + ).toBe('23505'); + expect(errorType(new TypeError('x'))).toBe('TypeError'); + expect(errorType('boom')).toBe('_OTHER'); + }); +}); diff --git a/app/src/telemetry/tracing.ts b/app/src/telemetry/tracing.ts new file mode 100644 index 0000000..e885a97 --- /dev/null +++ b/app/src/telemetry/tracing.ts @@ -0,0 +1,130 @@ +import { + context, + propagation, + ROOT_CONTEXT, + SpanStatusCode, + trace, + type Attributes, + type Context, + type Span, + type SpanKind, + type Tracer, +} from '@opentelemetry/api'; + +export const TRACER_NAME = 'messenger'; + +/** Совпадает с полем traceparent в contracts/jobs.schema.json. */ +const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u; +const MAX_TRACESTATE = 512; + +export type TraceFields = { + traceparent?: string; + tracestate?: string; +}; + +export function tracer(): Tracer { + return trace.getTracer(TRACER_NAME, '1.0.0'); +} + +/** + * W3C trace context активного span в виде полей job message. Пишется в + * outbox в той же транзакции, что и задача: причинность переживает рестарт + * API, брокера и worker, потому что хранится вместе с данными. + */ +export function currentTraceFields( + ctx: Context = context.active(), +): TraceFields { + const carrier: Record = {}; + propagation.inject(ctx, carrier); + const fields: TraceFields = {}; + if (carrier.traceparent && TRACEPARENT.test(carrier.traceparent)) + fields.traceparent = carrier.traceparent; + if ( + fields.traceparent && + carrier.tracestate && + carrier.tracestate.length <= MAX_TRACESTATE + ) + fields.tracestate = carrier.tracestate; + return fields; +} + +export function contextFromTraceFields( + fields: TraceFields | null | undefined, + base: Context = ROOT_CONTEXT, +): Context { + if (!fields?.traceparent || !TRACEPARENT.test(fields.traceparent)) + return base; + const carrier: Record = { traceparent: fields.traceparent }; + if (fields.tracestate) carrier.tracestate = fields.tracestate; + return propagation.extract(base, carrier); +} + +/** Заголовки AMQP приходят строками или Buffer — нормализуем к TraceFields. */ +export function traceFieldsFromHeaders( + headers: Record | undefined, +): TraceFields { + const read = (key: string): string | undefined => { + const value = headers?.[key]; + if (typeof value === 'string') return value; + if (Buffer.isBuffer(value)) return value.toString('utf8'); + return undefined; + }; + const traceparent = read('traceparent'); + if (!traceparent || !TRACEPARENT.test(traceparent)) return {}; + const tracestate = read('tracestate'); + return tracestate && tracestate.length <= MAX_TRACESTATE + ? { traceparent, tracestate } + : { traceparent }; +} + +/** + * Дочерний span вокруг операции зависимости. Создаётся только при активном + * родителе: опрос outbox, reaper и сбор метрик по таймеру не порождают + * тысячи одиночных корневых трасс. + */ +export async function withChildSpan( + name: string, + kind: SpanKind, + attributes: Attributes, + work: (span: Span | undefined) => Promise, +): Promise { + const parent = trace.getSpan(context.active()); + if (!parent || !parent.isRecording()) return work(undefined); + return tracer().startActiveSpan( + name, + { kind, attributes }, + async (span: Span) => { + try { + return await work(span); + } catch (error) { + markSpanError(span, error); + throw error; + } finally { + span.end(); + } + }, + ); +} + +/** + * Ошибка на span — только тип/код. Сообщения драйверов могут содержать + * значения из запроса (например, «Key (username)=(alice) already exists»), + * поэтому recordException с текстом не используется. + */ +export function markSpanError(span: Span, error: unknown): void { + const type = errorType(error); + span.setAttribute('error.type', type); + span.setStatus({ code: SpanStatusCode.ERROR, message: type }); +} + +export function errorType(error: unknown): string { + if (error && typeof error === 'object') { + const code = (error as { code?: unknown }).code; + if (typeof code === 'string' && /^[A-Za-z0-9_.-]{1,40}$/u.test(code)) + return code; + const name = (error as { name?: unknown }).name; + if (typeof name === 'string' && /^[A-Za-z0-9_.-]{1,60}$/u.test(name)) + return name; + } + return '_OTHER'; +} diff --git a/app/src/telemetry/worker-bootstrap.ts b/app/src/telemetry/worker-bootstrap.ts new file mode 100644 index 0000000..cc3c880 --- /dev/null +++ b/app/src/telemetry/worker-bootstrap.ts @@ -0,0 +1,4 @@ +// Импортируется первым в worker.main.ts (см. api-bootstrap.ts). +import { startTelemetry } from './sdk.js'; + +startTelemetry('messenger-worker');