diff --git a/app/src/app.module.ts b/app/src/app.module.ts index 220c60a..a1b259c 100644 --- a/app/src/app.module.ts +++ b/app/src/app.module.ts @@ -11,6 +11,7 @@ import { MetricsModule } from './metrics/metrics.module.js'; import { UsersModule } from './users/users.module.js'; import { APP_FILTER } from '@nestjs/core'; import { ApiExceptionFilter } from './common/errors/api-exception.filter.js'; +import { QueueStatsService } from './telemetry/queue-stats.service.js'; @Module({ imports: [ @@ -30,6 +31,9 @@ import { ApiExceptionFilter } from './common/errors/api-exception.filter.js'; provide: APP_FILTER, useClass: ApiExceptionFilter, }, + // Queue depth/age gauges are published by the API so that they keep + // reporting while the worker is down. + QueueStatsService, ], }) export class AppModule {} diff --git a/app/src/common/errors/api-exception.filter.ts b/app/src/common/errors/api-exception.filter.ts index e8e01b7..377c115 100644 --- a/app/src/common/errors/api-exception.filter.ts +++ b/app/src/common/errors/api-exception.filter.ts @@ -9,11 +9,20 @@ import { import { HttpAdapterHost } from '@nestjs/core'; import { ConfigService } from '@nestjs/config'; import { randomUUID } from 'node:crypto'; +import { trace } from '@opentelemetry/api'; import type { AppConfig } from '../../config/app-config.schema.js'; import { resolveInstanceId } from '../../config/instance-id.js'; +import { routeTemplate } from '../../telemetry/http.js'; +import { errorType } from '../../telemetry/tracing.js'; import type { ApiErrorBody } from './api-error.js'; -type RequestWithId = { requestId?: string; method?: string; url?: string }; +type RequestWithId = { + requestId?: string; + method?: string; + url?: string; + // Fastify route template: logs must not carry raw URLs (they hold ids). + routeOptions?: { url?: string }; +}; type ExceptionBody = { message?: unknown; challenge?: unknown; @@ -146,11 +155,11 @@ export class ApiExceptionFilter implements ExceptionFilter { status = HttpStatus.SERVICE_UNAVAILABLE; message = 'Service unavailable'; this.logger.warn( - `Dependency outage on ${request.method ?? ''} ${request.url ?? ''}: ${exception instanceof Error ? exception.message : 'unknown error'}`, + `Dependency outage on ${request.method ?? ''} ${routeTemplate(request.routeOptions?.url)}: ${exception instanceof Error ? exception.message : 'unknown error'}`, ); } else { this.logger.error( - `Unhandled exception on ${request.method ?? ''} ${request.url ?? ''}`, + `Unhandled exception on ${request.method ?? ''} ${routeTemplate(request.routeOptions?.url)}`, exception instanceof Error ? exception.stack : String(exception), @@ -176,6 +185,10 @@ export class ApiExceptionFilter implements ExceptionFilter { request_id: requestId, }, }; + if (status >= HttpStatus.INTERNAL_SERVER_ERROR) + trace + .getActiveSpan() + ?.setAttribute('error.type', errorType(exception)); adapter.reply(response, payload, status); } } diff --git a/app/src/database/database.client.ts b/app/src/database/database.client.ts index d992c8a..9238cc9 100644 --- a/app/src/database/database.client.ts +++ b/app/src/database/database.client.ts @@ -1,8 +1,97 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { SpanKind, type Attributes, type Span } from '@opentelemetry/api'; import pg from 'pg'; import { MetricsService } from '../metrics/metrics.service.js'; import type { AppConfig } from '../config/app-config.schema.js'; +import { instruments, meter } from '../telemetry/instruments.js'; +import { errorType, withChildSpan } from '../telemetry/tracing.js'; + +const DB_SYSTEM = 'postgresql'; +const POOL_NAME = 'main'; +const TRANSACTION_CONTROL = + /^\s*(?:BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE)\b/iu; +const SQL_TARGET = + /\b(?:FROM|INTO|UPDATE|JOIN)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?/iu; + +/** First SQL keyword (SELECT, INSERT, WITH, …): a bounded attribute. */ +function sqlOperation(text: string): string { + const match = /^\s*([A-Za-z]+)/u.exec(text); + return match ? match[1].toUpperCase() : 'OTHER'; +} + +/** Best-effort table name for span names; never used as a metric label. */ +function sqlTarget(text: string): string | undefined { + return SQL_TARGET.exec(text)?.[1]?.toLowerCase(); +} + +function queryText(args: readonly unknown[]): string { + const first = args[0]; + if (typeof first === 'string') return first; + if (typeof first === 'object' && first !== null) { + const text = (first as { text?: unknown }).text; + if (typeof text === 'string') return text; + } + return ''; +} + +function isPassThrough(args: readonly unknown[]): boolean { + const first = args[0]; + return ( + typeof args[args.length - 1] === 'function' || + (typeof first === 'object' && + first !== null && + typeof (first as { submit?: unknown }).submit === 'function') + ); +} + +function elapsedSeconds(started: bigint): number { + return Number(process.hrtime.bigint() - started) / 1e9; +} + +function statementAttributes( + label: string, + operation: string, + target: string | undefined, +): Attributes { + return { + 'db.system.name': DB_SYSTEM, + 'db.operation.name': operation, + 'db.query.summary': label, + ...(target ? { 'db.collection.name': target } : {}), + }; +} + +/** + * Inside a traced transaction every statement gets its own CLIENT span + * (`UPDATE attachment_jobs`, `INSERT job_outbox`, …) while repository code + * stays unaware of tracing. Transaction control, callback-style calls and + * submittables (cursors, streams) pass through untouched. SQL text and + * parameter values are never recorded. + */ +function tracedClient(client: pg.PoolClient, label: string): pg.PoolClient { + return new Proxy(client, { + get(target, property) { + const value: unknown = Reflect.get(target, property, target); + if (typeof value !== 'function') return value; + const method = value as (...args: unknown[]) => unknown; + if (property !== 'query') return method.bind(target); + return (...args: unknown[]): unknown => { + const text = queryText(args); + if (isPassThrough(args) || TRANSACTION_CONTROL.test(text)) + return method.apply(target, args); + const operation = sqlOperation(text); + const table = sqlTarget(text); + return withChildSpan( + `${operation} ${table ?? label}`, + SpanKind.CLIENT, + statementAttributes(label, operation, table), + async () => method.apply(target, args), + ); + }; + }, + }); +} @Injectable() export class DatabaseClient implements OnModuleDestroy { @@ -49,15 +138,44 @@ export class DatabaseClient implements OnModuleDestroy { `PostgreSQL health connection error: ${error.message}`, ); }); + this.observePool(this.config.getOrThrow('DATABASE_POOL_MAX')); } + /** + * Same contract as pg.Pool#query — a failed query destroys its connection — + * but the checkout is explicit, so the wait for a pooled connection is + * measured apart from the statement: «the pool is exhausted» and «the + * database is slow» are different incidents with different runbooks. + */ query( label: string, text: string, values: readonly unknown[] = [], ): Promise> { this.metrics.increment(label); - return this.pool.query({ text, values: [...values] }); + return this.observe( + label, + sqlOperation(text), + sqlTarget(text), + async () => { + const client = await this.checkout(); + let failure: Error | undefined; + const onError = (error: Error): void => { + failure = error; + }; + client.on('error', onError); + try { + return await client.query({ text, values: [...values] }); + } catch (error) { + failure ??= + error instanceof Error ? error : new Error('Query failed'); + throw error; + } finally { + client.removeListener('error', onError); + client.release(failure); + } + }, + ); } async health(): Promise { @@ -65,41 +183,143 @@ export class DatabaseClient implements OnModuleDestroy { await this.healthPool.query('SELECT 1'); } - async withTransaction( + withTransaction( work: (client: pg.PoolClient) => Promise, ): Promise { - const client = await this.pool.connect(); - // A server-side idle_in_transaction_session_timeout or a network drop - // can emit 'error' on a checked-out client after pg-pool removed its - // idle listener. Without this listener Node treats it as uncaught. - let fatal: Error | undefined; - const onError = (error: Error): void => { - fatal = error; - }; - client.on('error', onError); - try { - await client.query('BEGIN'); - const result = await work(client); - await client.query('COMMIT'); - return result; - } catch (error) { + return this.transaction('transaction', work); + } + + /** + * A labelled transaction: one span for the unit of work with a child span + * per statement and one db.client.operation.duration sample (pool wait + * included) with db.operation.name = TRANSACTION. + */ + transaction( + label: string, + work: (client: pg.PoolClient) => Promise, + ): Promise { + return this.observe(label, 'TRANSACTION', undefined, async (span) => { + const client = await this.checkout(); + // A server-side idle_in_transaction_session_timeout or a network drop + // can emit 'error' on a checked-out client after pg-pool removed its + // idle listener. Without this listener Node treats it as uncaught. + let fatal: Error | undefined; + const onError = (error: Error): void => { + fatal = error; + }; + client.on('error', onError); try { - await client.query('ROLLBACK'); - } catch (rollbackError) { - this.metrics.increment('database.rollback_error'); - this.logger.warn( - `Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`, + await client.query('BEGIN'); + const result = await work( + span ? tracedClient(client, label) : client, ); + await client.query('COMMIT'); + return result; + } catch (error) { + try { + await client.query('ROLLBACK'); + } catch (rollbackError) { + this.metrics.increment('database.rollback_error'); + this.logger.warn( + `Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`, + ); + } + // Callers must see the operation error, not the rollback error. + throw error; + } finally { + client.removeListener('error', onError); + client.release(fatal); } - // Callers must see the operation error, not the rollback error. - throw error; - } finally { - client.removeListener('error', onError); - client.release(fatal); - } + }); } async onModuleDestroy(): Promise { await Promise.all([this.pool.end(), this.healthPool.end()]); } + + private observe( + label: string, + operation: string, + target: string | undefined, + work: (span: Span | undefined) => Promise, + ): Promise { + const started = process.hrtime.bigint(); + return withChildSpan( + `${operation} ${target ?? label}`, + SpanKind.CLIENT, + statementAttributes(label, operation, target), + async (span) => { + let failure: string | undefined; + try { + return await work(span); + } catch (error) { + failure = errorType(error); + throw error; + } finally { + instruments().dbOperationDuration.record(elapsedSeconds(started), { + 'db.system.name': DB_SYSTEM, + 'db.operation.name': operation, + 'db.query.summary': label, + ...(failure ? { 'error.type': failure } : {}), + }); + } + }, + ); + } + + private async checkout(): Promise { + const started = process.hrtime.bigint(); + let failure: string | undefined; + try { + return await this.pool.connect(); + } catch (error) { + failure = errorType(error); + throw error; + } finally { + instruments().dbConnectionWait.record(elapsedSeconds(started), { + 'db.client.connection.pool.name': POOL_NAME, + ...(failure ? { 'error.type': failure } : {}), + }); + } + } + + /** Pool saturation: used vs max connections and callers queued for one. */ + private observePool(max: number): void { + const m = meter(); + const pool = { 'db.client.connection.pool.name': POOL_NAME }; + const count = m.createObservableUpDownCounter( + 'db.client.connection.count', + { + unit: '{connection}', + description: 'Connections in the main pool by state', + }, + ); + const limit = m.createObservableUpDownCounter('db.client.connection.max', { + unit: '{connection}', + description: 'Maximum size of the main pool (DATABASE_POOL_MAX)', + }); + const pending = m.createObservableUpDownCounter( + 'db.client.connection.pending_requests', + { + unit: '{request}', + description: 'Callers waiting for a connection from the main pool', + }, + ); + m.addBatchObservableCallback( + (result) => { + const idle = this.pool.idleCount; + result.observe(count, this.pool.totalCount - idle, { + ...pool, + 'db.client.connection.state': 'used', + }); + result.observe(count, idle, { + ...pool, + 'db.client.connection.state': 'idle', + }); + result.observe(limit, max, pool); + result.observe(pending, this.pool.waitingCount, pool); + }, + [count, limit, pending], + ); + } } diff --git a/app/src/jobs/broker.client.ts b/app/src/jobs/broker.client.ts index a749e90..e74185c 100644 --- a/app/src/jobs/broker.client.ts +++ b/app/src/jobs/broker.client.ts @@ -5,6 +5,12 @@ import { type ConfirmChannel, type ConsumeMessage, } from 'amqplib'; +import { + context, + propagation, + SpanKind, + type Span, +} from '@opentelemetry/api'; import { Injectable, Logger, @@ -22,6 +28,8 @@ import { JOB_ROUTING_KEY, } from './broker.topology.js'; import type { JobMessage } from './job-message.js'; +import { instruments } from '../telemetry/instruments.js'; +import { markSpanError, tracer } from '../telemetry/tracing.js'; export class BrokerUnavailableError extends ServiceUnavailableException { constructor() { @@ -70,7 +78,58 @@ export class BrokerClient implements OnModuleDestroy { ); } + /** + * PRODUCER span around publish + broker confirm. Its context goes into the + * AMQP headers (W3C traceparent/tracestate), so the worker's CONSUMER span + * becomes its child; the body keeps the context of the transaction that + * wrote the outbox row. + */ async publish(message: JobMessage | Buffer): Promise { + const attributes = { + 'messaging.system': 'rabbitmq', + 'messaging.operation.type': 'send', + 'messaging.operation.name': 'publish', + 'messaging.destination.name': JOB_EXCHANGE, + 'messaging.rabbitmq.destination.routing_key': JOB_ROUTING_KEY, + ...(Buffer.isBuffer(message) + ? {} + : { + 'messaging.message.id': message.job_id, + 'messenger.job.id': message.job_id, + 'messenger.job.attempt': message.attempt, + }), + }; + return tracer().startActiveSpan( + `publish ${JOB_EXCHANGE}`, + { kind: SpanKind.PRODUCER, attributes }, + async (span: Span) => { + const traceHeaders: Record = {}; + propagation.inject(context.active(), traceHeaders); + const labels = { 'messaging.destination.name': JOB_EXCHANGE }; + try { + await this.publishConfirmed(message, traceHeaders); + instruments().brokerPublished.add(1, { + ...labels, + outcome: 'success', + }); + } catch (error) { + instruments().brokerPublished.add(1, { + ...labels, + outcome: 'failed', + }); + markSpanError(span, error); + throw error; + } finally { + span.end(); + } + }, + ); + } + + private async publishConfirmed( + message: JobMessage | Buffer, + traceHeaders: Record, + ): Promise { const channel = await this.withTimeout( this.ensureConnected(), this.publishTimeoutMs, @@ -85,11 +144,11 @@ export class BrokerClient implements OnModuleDestroy { mandatory: true, correlationId, ...(Buffer.isBuffer(message) - ? {} + ? { headers: traceHeaders } : { messageId: message.job_id, type: message.type, - headers: { 'x-attempt': message.attempt }, + headers: { 'x-attempt': message.attempt, ...traceHeaders }, }), }; const { promise, resolve, reject } = Promise.withResolvers(); diff --git a/app/src/jobs/jobs.repository.ts b/app/src/jobs/jobs.repository.ts index 6307ef5..9e0ba44 100644 --- a/app/src/jobs/jobs.repository.ts +++ b/app/src/jobs/jobs.repository.ts @@ -4,6 +4,20 @@ import type { PoolClient } from 'pg'; import type { AttachmentRow, VariantRow } from '../attachments/attachment.js'; import { DatabaseClient } from '../database/database.client.js'; import { buildJobMessage, type JobMessage } from './job-message.js'; +import { currentTraceFields } from '../telemetry/tracing.js'; + +/** + * Every outbox row carries the W3C trace context of the transaction that + * wrote it (optional traceparent/tracestate of contracts/jobs.schema.json), + * so causality survives restarts of the API, the broker and the worker. + * Writers without an active span (lease reaper, reconciler, CLI) produce the + * same message as before. + */ +function tracedJobMessage( + job: Parameters[0], +): JobMessage { + return { ...buildJobMessage(job), ...currentTraceFields() }; +} export type JobStatus = 'queued' | 'processing' | 'succeeded' | 'failed' | 'superseded'; @@ -70,7 +84,7 @@ export class JobsRepository { pipelineVersion: number, maxAttempts: number, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.accept_upload', async (client) => { const accepted = await client.query( `UPDATE attachments SET status = 'queued', size_bytes = $2, sha256 = $3, @@ -104,7 +118,7 @@ export class JobsRepository { [ job.job_id, JSON.stringify( - buildJobMessage({ + tracedJobMessage({ ...job, created_at: job.created_at, }), @@ -121,7 +135,7 @@ export class JobsRepository { workerId: string, leaseSeconds: number, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.claim', async (client) => { // Lock order is always job -> attachment, matching commitSuccess and // commitFailure. A join FOR UPDATE would let the planner choose the // opposite order under a concurrent duplicate delivery. @@ -240,7 +254,7 @@ export class JobsRepository { token: string, variants: VariantWrite[], ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.commit_success', async (client) => { const succeeded = await client.query( `UPDATE attachment_jobs SET status = 'succeeded', lease_owner = NULL, lease_token = NULL, @@ -325,7 +339,7 @@ export class JobsRepository { failure: { kind: 'permanent' | 'transient'; code: string }, delayMs: number, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.commit_failure', async (client) => { const found = await client.query( `SELECT * FROM attachment_jobs WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' @@ -389,7 +403,7 @@ export class JobsRepository { [ job.job_id, JSON.stringify( - buildJobMessage({ ...job, attempt: nextAttempt }), + tracedJobMessage({ ...job, attempt: nextAttempt }), ), delayMs / 1000, ], @@ -412,7 +426,7 @@ export class JobsRepository { limit: number, delayFor: (attempt: number) => number, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.reap_expired_leases', async (client) => { const expired = await client.query( `SELECT * FROM attachment_jobs WHERE status = 'processing' AND lease_until < now() @@ -492,7 +506,7 @@ export class JobsRepository { [ job.job_id, JSON.stringify( - buildJobMessage({ ...job, attempt: nextAttempt }), + tracedJobMessage({ ...job, attempt: nextAttempt }), ), delayMs / 1000, ], @@ -518,7 +532,7 @@ export class JobsRepository { limit: number, graceSeconds: number, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.reconcile_queued', async (client) => { const stuck = await client.query( `SELECT j.* FROM attachment_jobs j WHERE j.status = 'queued' @@ -546,7 +560,7 @@ export class JobsRepository { await client.query( `INSERT INTO job_outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, now())`, - [job.job_id, JSON.stringify(buildJobMessage(job))], + [job.job_id, JSON.stringify(tracedJobMessage(job))], ); await this.event( client, @@ -565,7 +579,7 @@ export class JobsRepository { publish: (message: JobMessage) => Promise, afterPublish: () => Promise, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('outbox.publish_pending', async (client) => { const pending = await client.query<{ id: string; payload: JobMessage; @@ -641,7 +655,7 @@ export class JobsRepository { key: string, removeObject: (key: string) => Promise, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('attachments.clear_staging', async (client) => { // Keep the same job -> attachment lock order as claim/replay, then // serialize cleanup with replay on the attachment row. await client.query( @@ -868,7 +882,7 @@ export class JobsRepository { expectedStagingKey: string | null; }, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.replay', async (client) => { const found = await client.query( `SELECT * FROM attachment_jobs WHERE attachment_id = $1 AND status = 'failed' @@ -945,7 +959,7 @@ export class JobsRepository { VALUES ($1, $2::jsonb, now())`, [ job.job_id, - JSON.stringify(buildJobMessage({ ...job, attempt })), + JSON.stringify(tracedJobMessage({ ...job, attempt })), ], ); await this.event(client, job.job_id, 'replayed', attempt, { @@ -962,7 +976,7 @@ export class JobsRepository { pipelineVersion: number, options: { reason: string; operator: string; maxAttempts: number }, ): Promise { - return this.database.withTransaction(async (client) => { + return this.database.transaction('jobs.reprocess', async (client) => { const attachment = await client.query( `SELECT * FROM attachments WHERE id = $1 FOR UPDATE`, [attachmentId], @@ -1002,7 +1016,7 @@ export class JobsRepository { await client.query( `INSERT INTO job_outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, now())`, - [job.job_id, JSON.stringify(buildJobMessage(job))], + [job.job_id, JSON.stringify(tracedJobMessage(job))], ); await this.event(client, job.job_id, 'created', 1, { reason: options.reason, diff --git a/app/src/jobs/outbox-relay.service.ts b/app/src/jobs/outbox-relay.service.ts index 9fd99f1..06e77cd 100644 --- a/app/src/jobs/outbox-relay.service.ts +++ b/app/src/jobs/outbox-relay.service.ts @@ -4,12 +4,14 @@ import { OnApplicationBootstrap, OnModuleDestroy, } from '@nestjs/common'; +import { context, ROOT_CONTEXT } from '@opentelemetry/api'; import { ConfigService } from '@nestjs/config'; import type { AppConfig } from '../config/app-config.schema.js'; import { MetricsService } from '../metrics/metrics.service.js'; import { BrokerClient } from './broker.client.js'; import { FaultInjector } from './fault-injector.js'; import { JobsRepository } from './jobs.repository.js'; +import { contextFromTraceFields } from '../telemetry/tracing.js'; const BATCH_SIZE = 20; @@ -58,7 +60,10 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { return; } this.running = true; - this.current = this.drain() + // A batch belongs to no particular request or job: each message carries + // its own trace context (see drain). + this.current = context + .with(ROOT_CONTEXT, () => this.drain()) .catch((error: unknown) => { this.metrics.increment('outbox.publish_failed'); this.retryAt = Date.now() + this.retryDelayMs; @@ -93,7 +98,11 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy { async (message) => { if (Date.now() - started > budgetMs) throw new Error('outbox batch deadline exceeded'); - await this.broker.publish(message); + // The PRODUCER span continues the trace of the request that wrote + // the outbox row. + await context.with(contextFromTraceFields(message), () => + this.broker.publish(message), + ); }, () => this.faults.point('relay.after_publish_before_mark'), ); diff --git a/app/src/main.ts b/app/src/main.ts index 8c4a615..11c043d 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -1,3 +1,6 @@ +// Must stay the first import: the OpenTelemetry SDK has to be registered +// before any module creates instruments or handles a request. +import './telemetry/api-bootstrap.js'; import { NestFactory } from '@nestjs/core'; import { HttpAdapterHost } from '@nestjs/core'; import { ConfigService } from '@nestjs/config'; @@ -11,6 +14,9 @@ import { AppModule } from './app.module.js'; import { HealthService } from './health/health.service.js'; import { MetricsService } from './metrics/metrics.service.js'; import { RequestIdInterceptor } from './common/http/request-id.interceptor.js'; +import { installHttpTelemetry } from './telemetry/http.js'; +import { appLogger } from './telemetry/logger.js'; +import { shutdownTelemetry } from './telemetry/sdk.js'; const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments'; const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream'; @@ -18,7 +24,10 @@ const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream'; async function bootstrap(): Promise { const app = await NestFactory.create( AppModule, - new FastifyAdapter({ logger: true }), + // Fastify's pino access log is replaced by the structured http.request + // event (telemetry/http.ts): pino logged raw URLs with identifiers. + new FastifyAdapter({ logger: false }), + { logger: appLogger }, ); // The upload route receives its body as an unread stream whatever the // declared Content-Type: the service checks the type itself, after the @@ -26,6 +35,10 @@ async function bootstrap(): Promise { // its own limits, not Fastify's bodyLimit. The declared type is kept on the // request; all other routes keep Fastify's parsers untouched. const fastify = app.getHttpAdapter().getInstance(); + // Telemetry hooks go first, so every request is measured even if a later + // hook answers it. Routes are registered at init, after the onRoute hook, + // so their handlers run inside the request span. + installHttpTelemetry(fastify); fastify.addHook( 'onRequest', (request: UploadRequest, _reply: unknown, done: () => void) => { @@ -85,6 +98,7 @@ async function bootstrap(): Promise { setTimeout(resolve, drainDelayMs), ); await app.close(); + await shutdownTelemetry(); process.exit(0); }; for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const) diff --git a/app/src/storage/object-storage.ts b/app/src/storage/object-storage.ts index b531a0d..b9275fe 100644 --- a/app/src/storage/object-storage.ts +++ b/app/src/storage/object-storage.ts @@ -7,6 +7,7 @@ import { } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { SpanKind } from '@opentelemetry/api'; import { Injectable, Logger, @@ -16,6 +17,7 @@ import { ConfigService } from '@nestjs/config'; import http from 'node:http'; import { Readable } from 'node:stream'; import type { AppConfig } from '../config/app-config.schema.js'; +import { withChildSpan } from '../telemetry/tracing.js'; export class ObjectStorageUnavailableError extends ServiceUnavailableException { constructor() { @@ -80,6 +82,28 @@ export class ObjectStorage { ...common, endpoint: config.getOrThrow('S3_ENDPOINT'), }); + // One CLIENT span per S3 call (S3.PutObject, S3.GetObject, …), only + // inside a traced request or job. Object keys contain attachment ids + // and are deliberately not recorded. + this.internal.middlewareStack.add( + (next, handlerContext) => async (args) => { + const operation = String( + handlerContext.commandName ?? 'Unknown', + ).replace(/Command$/u, ''); + return withChildSpan( + `S3.${operation}`, + SpanKind.CLIENT, + { + 'rpc.system': 'aws-api', + 'rpc.service': 'S3', + 'rpc.method': operation, + 'aws.s3.bucket': this.bucket, + }, + () => next(args), + ); + }, + { step: 'initialize', name: 'messengerTelemetry' }, + ); // Сеть не использует: только подпись URL для клиента. this.presign = new S3Client({ ...common, diff --git a/app/src/telemetry/instruments.ts b/app/src/telemetry/instruments.ts index 7453963..fff2b7e 100644 --- a/app/src/telemetry/instruments.ts +++ b/app/src/telemetry/instruments.ts @@ -23,6 +23,9 @@ export const TIME_TO_READY_BUCKETS = [ export const WAIT_BUCKETS = [ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, ]; +export const DB_DURATION_BUCKETS = [ + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, +]; export type Instruments = { readonly httpDuration: Histogram; @@ -31,6 +34,7 @@ export type Instruments = { readonly timeToReady: Histogram; readonly brokerPublished: Counter; readonly dbConnectionWait: Histogram; + readonly dbOperationDuration: Histogram; readonly telemetryExport: Counter; }; @@ -83,6 +87,12 @@ export function instruments(): Instruments { description: 'Time spent waiting for a connection from the pool', advice: { explicitBucketBoundaries: WAIT_BUCKETS }, }), + dbOperationDuration: m.createHistogram('db.client.operation.duration', { + unit: 's', + description: + 'PostgreSQL statement or transaction duration by query label, pool wait included', + advice: { explicitBucketBoundaries: DB_DURATION_BUCKETS }, + }), telemetryExport: m.createCounter('messenger.telemetry.export.items', { unit: '{item}', description: diff --git a/app/src/worker.main.ts b/app/src/worker.main.ts index 77dfea6..085c13a 100644 --- a/app/src/worker.main.ts +++ b/app/src/worker.main.ts @@ -1,3 +1,5 @@ +// Must stay the first import (see main.ts). +import './telemetry/worker-bootstrap.js'; import { Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { NestFactory } from '@nestjs/core'; @@ -6,12 +8,16 @@ import type { AppConfig } from './config/app-config.schema.js'; import { BrokerClient } from './jobs/broker.client.js'; import { JobProcessor } from './worker/job-processor.service.js'; import { WorkerModule } from './worker/worker.module.js'; +import { appLogger } from './telemetry/logger.js'; +import { shutdownTelemetry } from './telemetry/sdk.js'; const DRAIN_MS = 30_000; async function bootstrap(): Promise { const app = await NestFactory.createApplicationContext(WorkerModule, { - logger: ['error', 'warn', 'log'], + // Verbosity comes from LOG_LEVEL (default INFO, the former + // error/warn/log set). + logger: appLogger, }); const config = app.get(ConfigService); const broker = app.get(BrokerClient); @@ -56,6 +62,7 @@ async function bootstrap(): Promise { await processor.drain(DRAIN_MS); await new Promise((resolve) => server.close(() => resolve())); await app.close(); + await shutdownTelemetry(); process.exit(0); }; for (const signal of ['SIGTERM', 'SIGINT'] as const) diff --git a/app/src/worker/job-processor.service.ts b/app/src/worker/job-processor.service.ts index da69e65..095683a 100644 --- a/app/src/worker/job-processor.service.ts +++ b/app/src/worker/job-processor.service.ts @@ -1,6 +1,13 @@ import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import type { Channel, ConsumeMessage } from 'amqplib'; +import { + ROOT_CONTEXT, + SpanKind, + SpanStatusCode, + trace, + type Span, +} from '@opentelemetry/api'; import { createHash } from 'node:crypto'; import os from 'node:os'; import type { AttachmentRow } from '../attachments/attachment.js'; @@ -8,6 +15,7 @@ import type { AppConfig } from '../config/app-config.schema.js'; import { ObjectStorage } from '../storage/object-storage.js'; import { retryDelayMs } from '../jobs/backoff.js'; import { BrokerClient } from '../jobs/broker.client.js'; +import { JOB_QUEUE } from '../jobs/broker.topology.js'; import { classifyError, JobTimeoutError, @@ -16,7 +24,11 @@ import { type ErrorClass, } from '../jobs/errors.js'; import { FaultInjector } from '../jobs/fault-injector.js'; -import { parseJobMessage, type JobMessage } from '../jobs/job-message.js'; +import { + JOB_TYPE, + parseJobMessage, + type JobMessage, +} from '../jobs/job-message.js'; import { OutboxRelay } from '../jobs/outbox-relay.service.js'; import { JobsRepository, @@ -25,6 +37,15 @@ import { } from '../jobs/jobs.repository.js'; import { MetricsService } from '../metrics/metrics.service.js'; import { renderVariants, type RenderedVariant } from './pipelines.js'; +import { instruments, meter } from '../telemetry/instruments.js'; +import { appLogger } from '../telemetry/logger.js'; +import { + contextFromTraceFields, + errorType, + markSpanError, + traceFieldsFromHeaders, + tracer, +} from '../telemetry/tracing.js'; const SHUTDOWN_DRAIN_MS = 30_000; const VARIANT_EXTENSIONS = [ @@ -33,6 +54,31 @@ const VARIANT_EXTENSIONS = [ ['optimized', 'webp'], ] as const; +/** Bounded outcome set: the `outcome` label of messenger.jobs.processed. */ +type JobOutcome = + | 'succeeded' + | 'superseded' + | 'lease_lost' + | 'retry' + | 'failed' + | 'commit_failed' + | 'poison' + | 'claim_failed' + | 'skipped' + | 'mismatch' + | 'crashed'; + +// Outcomes where the delivery did not do its job: WARN log, ERROR span status. +const PROBLEM_OUTCOMES: ReadonlySet = new Set([ + 'retry', + 'failed', + 'commit_failed', + 'poison', + 'claim_failed', + 'mismatch', + 'crashed', +]); + function throwIfAborted(signal: AbortSignal): void { if (!signal.aborted) return; throw signal.reason instanceof Error ? signal.reason : new Error('Aborted'); @@ -73,9 +119,38 @@ export class JobProcessor implements OnApplicationBootstrap { } onApplicationBootstrap(): void { - this.broker.consume( - this.config.getOrThrow('WORKER_CONCURRENCY'), - (message, channel) => this.handle(message, channel), + const concurrency = this.config.getOrThrow('WORKER_CONCURRENCY'); + this.observeSaturation(concurrency); + this.broker.consume(concurrency, (message, channel) => + this.handle(message, channel), + ); + } + + /** + * Saturation of this worker: jobs in flight against WORKER_CONCURRENCY + * (= prefetch). Together with messenger.jobs.queue.* it separates «the + * worker is slow» from «the worker is full» from «the worker is gone». + */ + private observeSaturation(concurrency: number): void { + const m = meter(); + const active = m.createObservableUpDownCounter( + 'messenger.worker.jobs.active', + { unit: '{job}', description: 'Jobs this worker is processing now' }, + ); + const capacity = m.createObservableUpDownCounter( + 'messenger.worker.jobs.capacity', + { + unit: '{job}', + description: + 'Jobs this worker may process at once (WORKER_CONCURRENCY)', + }, + ); + m.addBatchObservableCallback( + (result) => { + result.observe(active, this.active); + result.observe(capacity, concurrency); + }, + [active, capacity], ); } @@ -88,15 +163,104 @@ export class JobProcessor implements OnApplicationBootstrap { clearTimeout(timer); } + /** + * One CONSUMER span per delivery. Its parent is the PRODUCER span from the + * AMQP headers; messages without trace headers (published before lab6 or + * replayed raw) fall back to the trace context stored in the body. + */ private async handle( message: ConsumeMessage, channel: Channel, ): Promise { const parsed = parseJobMessage(message.content); + const fromHeaders = traceFieldsFromHeaders( + message.properties.headers as Record | undefined, + ); + const parent = fromHeaders.traceparent + ? contextFromTraceFields(fromHeaders) + : contextFromTraceFields(parsed, ROOT_CONTEXT); + const started = process.hrtime.bigint(); + await tracer().startActiveSpan( + `process ${JOB_QUEUE}`, + { + kind: SpanKind.CONSUMER, + attributes: { + 'messaging.system': 'rabbitmq', + 'messaging.operation.type': 'process', + 'messaging.destination.name': JOB_QUEUE, + 'messaging.rabbitmq.message.redelivered': + message.fields.redelivered, + ...(parsed + ? { + 'messaging.message.id': parsed.job_id, + 'messenger.job.id': parsed.job_id, + 'messenger.attachment.id': parsed.attachment_id, + 'messenger.job.attempt': parsed.attempt, + 'messenger.pipeline.version': parsed.pipeline_version, + } + : {}), + }, + }, + parent, + async (span: Span) => { + let outcome: JobOutcome = 'crashed'; + try { + outcome = await this.dispatch(parsed, message, channel); + } catch (error) { + markSpanError(span, error); + throw error; + } finally { + this.record(span, parsed, outcome, started); + } + }, + ); + } + + private record( + span: Span, + parsed: JobMessage | undefined, + outcome: JobOutcome, + started: bigint, + ): void { + const seconds = Number(process.hrtime.bigint() - started) / 1e9; + const labels = { 'job.type': JOB_TYPE, outcome }; + instruments().jobsProcessed.add(1, labels); + instruments().jobDuration.record(seconds, labels); + if (outcome === 'succeeded' && parsed) { + // From durable acceptance (job created_at) to the ready commit: + // queueing, retries and backoff included. + const ready = (Date.now() - Date.parse(parsed.created_at)) / 1000; + if (Number.isFinite(ready) && ready >= 0) + instruments().timeToReady.record(ready, { 'job.type': JOB_TYPE }); + } + span.setAttribute('messenger.job.outcome', outcome); + const problem = PROBLEM_OUTCOMES.has(outcome); + if (problem && outcome !== 'crashed') + span.setStatus({ code: SpanStatusCode.ERROR, message: outcome }); + appLogger.event( + problem ? 'WARN' : 'INFO', + 'job.processed', + `job ${outcome}`, + { + 'messenger.job.id': parsed?.job_id, + 'messenger.attachment.id': parsed?.attachment_id, + 'messenger.job.attempt': parsed?.attempt, + 'messenger.job.outcome': outcome, + duration_ms: Math.round(seconds * 1000), + }, + ); + span.end(); + } + + private async dispatch( + parsed: JobMessage | undefined, + message: ConsumeMessage, + channel: Channel, + ): Promise { if (!parsed) { this.metrics.increment('worker.messages.poison'); this.safeNack(channel, message, false); - return; + return 'poison'; } const claim = await this.claimWithRetry(parsed); @@ -105,21 +269,24 @@ export class JobProcessor implements OnApplicationBootstrap { // Do not burn the delivery limit in a tight nack loop while the // database is briefly unavailable. this.safeNackAfter(channel, message, true, 5000); - return; + return 'claim_failed'; } if (claim.kind === 'skip') { if (claim.reason === 'mismatch') { this.safeNack(channel, message, false); - return; + return 'mismatch'; } this.metrics.increment(`worker.jobs.skipped.${claim.reason}`); + trace + .getActiveSpan() + ?.setAttribute('messenger.job.skip_reason', claim.reason); this.safeAck(channel, message); - return; + return 'skipped'; } this.begin(); try { - await this.process( + return await this.process( claim.job, claim.attachment, claim.leaseToken, @@ -142,7 +309,7 @@ export class JobProcessor implements OnApplicationBootstrap { leaseToken: string, message: ConsumeMessage, channel: Channel, - ): Promise { + ): Promise { this.metrics.increment('worker.jobs.claimed'); const controller = new AbortController(); const target = { @@ -170,6 +337,7 @@ export class JobProcessor implements OnApplicationBootstrap { Math.max(1000, (this.leaseSeconds * 1000) / 3), ); let committed = false; + let committedOutcome: JobOutcome = 'succeeded'; try { await this.faults.point( @@ -222,6 +390,7 @@ export class JobProcessor implements OnApplicationBootstrap { variants.map((variant) => this.variantWrite(job, variant)), ); committed = true; + committedOutcome = result; this.metrics.increment(`worker.jobs.${result}`); await this.faults.point( 'worker.after_commit_before_ack', @@ -235,13 +404,17 @@ export class JobProcessor implements OnApplicationBootstrap { } else if (result === 'superseded') { await this.cleanupAttemptObjects(job, null); } + return result; } catch (error) { if (committed) { // The durable effect is already committed; a failure at the ack // fault point or a lost reply must not turn into a retry. this.safeAck(channel, message); - return; + return committedOutcome; } + // Only the error type/code reaches the span: driver messages may + // carry request values. + trace.getActiveSpan()?.setAttribute('error.type', errorType(error)); const failure = this.classify(error, controller.signal); const delay = retryDelayMs( job.attempt, @@ -260,12 +433,14 @@ export class JobProcessor implements OnApplicationBootstrap { this.safeAck(channel, message); if (outcome === 'retry' || outcome === 'failed') await this.cleanupAttemptObjects(job, null); + return outcome; } catch (commitError) { this.metrics.increment('worker.jobs.commit_failed'); this.logger.warn( `Failure commit failed: ${commitError instanceof Error ? commitError.message : 'unknown error'}`, ); setTimeout(() => this.safeNack(channel, message, true), 1000); + return 'commit_failed'; } } finally { clearTimeout(timeout);