lab6: инструментирование кода — HTTP, PostgreSQL, RabbitMQ (W3C trace context), S3, обработка задач, структурные логи

This commit is contained in:
Ilya
2026-11-09 20:38:40 +03:00
parent cf87f4dd43
commit 63dba7627c
11 changed files with 613 additions and 64 deletions
+4
View File
@@ -11,6 +11,7 @@ import { MetricsModule } from './metrics/metrics.module.js';
import { UsersModule } from './users/users.module.js';
import { APP_FILTER } from '@nestjs/core';
import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
import { QueueStatsService } from './telemetry/queue-stats.service.js';
@Module({
imports: [
@@ -30,6 +31,9 @@ import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
provide: APP_FILTER,
useClass: ApiExceptionFilter,
},
// Queue depth/age gauges are published by the API so that they keep
// reporting while the worker is down.
QueueStatsService,
],
})
export class AppModule {}
+16 -3
View File
@@ -9,11 +9,20 @@ import {
import { HttpAdapterHost } from '@nestjs/core';
import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'node:crypto';
import { trace } from '@opentelemetry/api';
import type { AppConfig } from '../../config/app-config.schema.js';
import { resolveInstanceId } from '../../config/instance-id.js';
import { routeTemplate } from '../../telemetry/http.js';
import { errorType } from '../../telemetry/tracing.js';
import type { ApiErrorBody } from './api-error.js';
type RequestWithId = { requestId?: string; method?: string; url?: string };
type RequestWithId = {
requestId?: string;
method?: string;
url?: string;
// Fastify route template: logs must not carry raw URLs (they hold ids).
routeOptions?: { url?: string };
};
type ExceptionBody = {
message?: unknown;
challenge?: unknown;
@@ -146,11 +155,11 @@ export class ApiExceptionFilter implements ExceptionFilter {
status = HttpStatus.SERVICE_UNAVAILABLE;
message = 'Service unavailable';
this.logger.warn(
`Dependency outage on ${request.method ?? ''} ${request.url ?? ''}: ${exception instanceof Error ? exception.message : 'unknown error'}`,
`Dependency outage on ${request.method ?? ''} ${routeTemplate(request.routeOptions?.url)}: ${exception instanceof Error ? exception.message : 'unknown error'}`,
);
} else {
this.logger.error(
`Unhandled exception on ${request.method ?? ''} ${request.url ?? ''}`,
`Unhandled exception on ${request.method ?? ''} ${routeTemplate(request.routeOptions?.url)}`,
exception instanceof Error
? exception.stack
: String(exception),
@@ -176,6 +185,10 @@ export class ApiExceptionFilter implements ExceptionFilter {
request_id: requestId,
},
};
if (status >= HttpStatus.INTERNAL_SERVER_ERROR)
trace
.getActiveSpan()
?.setAttribute('error.type', errorType(exception));
adapter.reply(response, payload, status);
}
}
+248 -28
View File
@@ -1,8 +1,97 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { SpanKind, type Attributes, type Span } from '@opentelemetry/api';
import pg from 'pg';
import { MetricsService } from '../metrics/metrics.service.js';
import type { AppConfig } from '../config/app-config.schema.js';
import { instruments, meter } from '../telemetry/instruments.js';
import { errorType, withChildSpan } from '../telemetry/tracing.js';
const DB_SYSTEM = 'postgresql';
const POOL_NAME = 'main';
const TRANSACTION_CONTROL =
/^\s*(?:BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE)\b/iu;
const SQL_TARGET =
/\b(?:FROM|INTO|UPDATE|JOIN)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?/iu;
/** First SQL keyword (SELECT, INSERT, WITH, …): a bounded attribute. */
function sqlOperation(text: string): string {
const match = /^\s*([A-Za-z]+)/u.exec(text);
return match ? match[1].toUpperCase() : 'OTHER';
}
/** Best-effort table name for span names; never used as a metric label. */
function sqlTarget(text: string): string | undefined {
return SQL_TARGET.exec(text)?.[1]?.toLowerCase();
}
function queryText(args: readonly unknown[]): string {
const first = args[0];
if (typeof first === 'string') return first;
if (typeof first === 'object' && first !== null) {
const text = (first as { text?: unknown }).text;
if (typeof text === 'string') return text;
}
return '';
}
function isPassThrough(args: readonly unknown[]): boolean {
const first = args[0];
return (
typeof args[args.length - 1] === 'function' ||
(typeof first === 'object' &&
first !== null &&
typeof (first as { submit?: unknown }).submit === 'function')
);
}
function elapsedSeconds(started: bigint): number {
return Number(process.hrtime.bigint() - started) / 1e9;
}
function statementAttributes(
label: string,
operation: string,
target: string | undefined,
): Attributes {
return {
'db.system.name': DB_SYSTEM,
'db.operation.name': operation,
'db.query.summary': label,
...(target ? { 'db.collection.name': target } : {}),
};
}
/**
* Inside a traced transaction every statement gets its own CLIENT span
* (`UPDATE attachment_jobs`, `INSERT job_outbox`, …) while repository code
* stays unaware of tracing. Transaction control, callback-style calls and
* submittables (cursors, streams) pass through untouched. SQL text and
* parameter values are never recorded.
*/
function tracedClient(client: pg.PoolClient, label: string): pg.PoolClient {
return new Proxy(client, {
get(target, property) {
const value: unknown = Reflect.get(target, property, target);
if (typeof value !== 'function') return value;
const method = value as (...args: unknown[]) => unknown;
if (property !== 'query') return method.bind(target);
return (...args: unknown[]): unknown => {
const text = queryText(args);
if (isPassThrough(args) || TRANSACTION_CONTROL.test(text))
return method.apply(target, args);
const operation = sqlOperation(text);
const table = sqlTarget(text);
return withChildSpan(
`${operation} ${table ?? label}`,
SpanKind.CLIENT,
statementAttributes(label, operation, table),
async () => method.apply(target, args),
);
};
},
});
}
@Injectable()
export class DatabaseClient implements OnModuleDestroy {
@@ -49,15 +138,44 @@ export class DatabaseClient implements OnModuleDestroy {
`PostgreSQL health connection error: ${error.message}`,
);
});
this.observePool(this.config.getOrThrow('DATABASE_POOL_MAX'));
}
/**
* Same contract as pg.Pool#query — a failed query destroys its connection —
* but the checkout is explicit, so the wait for a pooled connection is
* measured apart from the statement: «the pool is exhausted» and «the
* database is slow» are different incidents with different runbooks.
*/
query<T extends pg.QueryResultRow = pg.QueryResultRow>(
label: string,
text: string,
values: readonly unknown[] = [],
): Promise<pg.QueryResult<T>> {
this.metrics.increment(label);
return this.pool.query<T>({ text, values: [...values] });
return this.observe(
label,
sqlOperation(text),
sqlTarget(text),
async () => {
const client = await this.checkout();
let failure: Error | undefined;
const onError = (error: Error): void => {
failure = error;
};
client.on('error', onError);
try {
return await client.query<T>({ text, values: [...values] });
} catch (error) {
failure ??=
error instanceof Error ? error : new Error('Query failed');
throw error;
} finally {
client.removeListener('error', onError);
client.release(failure);
}
},
);
}
async health(): Promise<void> {
@@ -65,41 +183,143 @@ export class DatabaseClient implements OnModuleDestroy {
await this.healthPool.query('SELECT 1');
}
async withTransaction<T>(
withTransaction<T>(
work: (client: pg.PoolClient) => Promise<T>,
): Promise<T> {
const client = await this.pool.connect();
// A server-side idle_in_transaction_session_timeout or a network drop
// can emit 'error' on a checked-out client after pg-pool removed its
// idle listener. Without this listener Node treats it as uncaught.
let fatal: Error | undefined;
const onError = (error: Error): void => {
fatal = error;
};
client.on('error', onError);
try {
await client.query('BEGIN');
const result = await work(client);
await client.query('COMMIT');
return result;
} catch (error) {
return this.transaction('transaction', work);
}
/**
* A labelled transaction: one span for the unit of work with a child span
* per statement and one db.client.operation.duration sample (pool wait
* included) with db.operation.name = TRANSACTION.
*/
transaction<T>(
label: string,
work: (client: pg.PoolClient) => Promise<T>,
): Promise<T> {
return this.observe(label, 'TRANSACTION', undefined, async (span) => {
const client = await this.checkout();
// A server-side idle_in_transaction_session_timeout or a network drop
// can emit 'error' on a checked-out client after pg-pool removed its
// idle listener. Without this listener Node treats it as uncaught.
let fatal: Error | undefined;
const onError = (error: Error): void => {
fatal = error;
};
client.on('error', onError);
try {
await client.query('ROLLBACK');
} catch (rollbackError) {
this.metrics.increment('database.rollback_error');
this.logger.warn(
`Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`,
await client.query('BEGIN');
const result = await work(
span ? tracedClient(client, label) : client,
);
await client.query('COMMIT');
return result;
} catch (error) {
try {
await client.query('ROLLBACK');
} catch (rollbackError) {
this.metrics.increment('database.rollback_error');
this.logger.warn(
`Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`,
);
}
// Callers must see the operation error, not the rollback error.
throw error;
} finally {
client.removeListener('error', onError);
client.release(fatal);
}
// Callers must see the operation error, not the rollback error.
throw error;
} finally {
client.removeListener('error', onError);
client.release(fatal);
}
});
}
async onModuleDestroy(): Promise<void> {
await Promise.all([this.pool.end(), this.healthPool.end()]);
}
private observe<T>(
label: string,
operation: string,
target: string | undefined,
work: (span: Span | undefined) => Promise<T>,
): Promise<T> {
const started = process.hrtime.bigint();
return withChildSpan(
`${operation} ${target ?? label}`,
SpanKind.CLIENT,
statementAttributes(label, operation, target),
async (span) => {
let failure: string | undefined;
try {
return await work(span);
} catch (error) {
failure = errorType(error);
throw error;
} finally {
instruments().dbOperationDuration.record(elapsedSeconds(started), {
'db.system.name': DB_SYSTEM,
'db.operation.name': operation,
'db.query.summary': label,
...(failure ? { 'error.type': failure } : {}),
});
}
},
);
}
private async checkout(): Promise<pg.PoolClient> {
const started = process.hrtime.bigint();
let failure: string | undefined;
try {
return await this.pool.connect();
} catch (error) {
failure = errorType(error);
throw error;
} finally {
instruments().dbConnectionWait.record(elapsedSeconds(started), {
'db.client.connection.pool.name': POOL_NAME,
...(failure ? { 'error.type': failure } : {}),
});
}
}
/** Pool saturation: used vs max connections and callers queued for one. */
private observePool(max: number): void {
const m = meter();
const pool = { 'db.client.connection.pool.name': POOL_NAME };
const count = m.createObservableUpDownCounter(
'db.client.connection.count',
{
unit: '{connection}',
description: 'Connections in the main pool by state',
},
);
const limit = m.createObservableUpDownCounter('db.client.connection.max', {
unit: '{connection}',
description: 'Maximum size of the main pool (DATABASE_POOL_MAX)',
});
const pending = m.createObservableUpDownCounter(
'db.client.connection.pending_requests',
{
unit: '{request}',
description: 'Callers waiting for a connection from the main pool',
},
);
m.addBatchObservableCallback(
(result) => {
const idle = this.pool.idleCount;
result.observe(count, this.pool.totalCount - idle, {
...pool,
'db.client.connection.state': 'used',
});
result.observe(count, idle, {
...pool,
'db.client.connection.state': 'idle',
});
result.observe(limit, max, pool);
result.observe(pending, this.pool.waitingCount, pool);
},
[count, limit, pending],
);
}
}
+61 -2
View File
@@ -5,6 +5,12 @@ import {
type ConfirmChannel,
type ConsumeMessage,
} from 'amqplib';
import {
context,
propagation,
SpanKind,
type Span,
} from '@opentelemetry/api';
import {
Injectable,
Logger,
@@ -22,6 +28,8 @@ import {
JOB_ROUTING_KEY,
} from './broker.topology.js';
import type { JobMessage } from './job-message.js';
import { instruments } from '../telemetry/instruments.js';
import { markSpanError, tracer } from '../telemetry/tracing.js';
export class BrokerUnavailableError extends ServiceUnavailableException {
constructor() {
@@ -70,7 +78,58 @@ export class BrokerClient implements OnModuleDestroy {
);
}
/**
* PRODUCER span around publish + broker confirm. Its context goes into the
* AMQP headers (W3C traceparent/tracestate), so the worker's CONSUMER span
* becomes its child; the body keeps the context of the transaction that
* wrote the outbox row.
*/
async publish(message: JobMessage | Buffer): Promise<void> {
const attributes = {
'messaging.system': 'rabbitmq',
'messaging.operation.type': 'send',
'messaging.operation.name': 'publish',
'messaging.destination.name': JOB_EXCHANGE,
'messaging.rabbitmq.destination.routing_key': JOB_ROUTING_KEY,
...(Buffer.isBuffer(message)
? {}
: {
'messaging.message.id': message.job_id,
'messenger.job.id': message.job_id,
'messenger.job.attempt': message.attempt,
}),
};
return tracer().startActiveSpan(
`publish ${JOB_EXCHANGE}`,
{ kind: SpanKind.PRODUCER, attributes },
async (span: Span) => {
const traceHeaders: Record<string, string> = {};
propagation.inject(context.active(), traceHeaders);
const labels = { 'messaging.destination.name': JOB_EXCHANGE };
try {
await this.publishConfirmed(message, traceHeaders);
instruments().brokerPublished.add(1, {
...labels,
outcome: 'success',
});
} catch (error) {
instruments().brokerPublished.add(1, {
...labels,
outcome: 'failed',
});
markSpanError(span, error);
throw error;
} finally {
span.end();
}
},
);
}
private async publishConfirmed(
message: JobMessage | Buffer,
traceHeaders: Record<string, string>,
): Promise<void> {
const channel = await this.withTimeout(
this.ensureConnected(),
this.publishTimeoutMs,
@@ -85,11 +144,11 @@ export class BrokerClient implements OnModuleDestroy {
mandatory: true,
correlationId,
...(Buffer.isBuffer(message)
? {}
? { headers: traceHeaders }
: {
messageId: message.job_id,
type: message.type,
headers: { 'x-attempt': message.attempt },
headers: { 'x-attempt': message.attempt, ...traceHeaders },
}),
};
const { promise, resolve, reject } = Promise.withResolvers<void>();
+30 -16
View File
@@ -4,6 +4,20 @@ import type { PoolClient } from 'pg';
import type { AttachmentRow, VariantRow } from '../attachments/attachment.js';
import { DatabaseClient } from '../database/database.client.js';
import { buildJobMessage, type JobMessage } from './job-message.js';
import { currentTraceFields } from '../telemetry/tracing.js';
/**
* Every outbox row carries the W3C trace context of the transaction that
* wrote it (optional traceparent/tracestate of contracts/jobs.schema.json),
* so causality survives restarts of the API, the broker and the worker.
* Writers without an active span (lease reaper, reconciler, CLI) produce the
* same message as before.
*/
function tracedJobMessage(
job: Parameters<typeof buildJobMessage>[0],
): JobMessage {
return { ...buildJobMessage(job), ...currentTraceFields() };
}
export type JobStatus =
'queued' | 'processing' | 'succeeded' | 'failed' | 'superseded';
@@ -70,7 +84,7 @@ export class JobsRepository {
pipelineVersion: number,
maxAttempts: number,
): Promise<AttachmentRow | undefined> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.accept_upload', async (client) => {
const accepted = await client.query<AttachmentRow>(
`UPDATE attachments
SET status = 'queued', size_bytes = $2, sha256 = $3,
@@ -104,7 +118,7 @@ export class JobsRepository {
[
job.job_id,
JSON.stringify(
buildJobMessage({
tracedJobMessage({
...job,
created_at: job.created_at,
}),
@@ -121,7 +135,7 @@ export class JobsRepository {
workerId: string,
leaseSeconds: number,
): Promise<ClaimResult> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.claim', async (client) => {
// Lock order is always job -> attachment, matching commitSuccess and
// commitFailure. A join FOR UPDATE would let the planner choose the
// opposite order under a concurrent duplicate delivery.
@@ -240,7 +254,7 @@ export class JobsRepository {
token: string,
variants: VariantWrite[],
): Promise<CommitSuccessResult> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.commit_success', async (client) => {
const succeeded = await client.query<AttachmentJobRow>(
`UPDATE attachment_jobs
SET status = 'succeeded', lease_owner = NULL, lease_token = NULL,
@@ -325,7 +339,7 @@ export class JobsRepository {
failure: { kind: 'permanent' | 'transient'; code: string },
delayMs: number,
): Promise<CommitFailureResult> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.commit_failure', async (client) => {
const found = await client.query<AttachmentJobRow>(
`SELECT * FROM attachment_jobs
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
@@ -389,7 +403,7 @@ export class JobsRepository {
[
job.job_id,
JSON.stringify(
buildJobMessage({ ...job, attempt: nextAttempt }),
tracedJobMessage({ ...job, attempt: nextAttempt }),
),
delayMs / 1000,
],
@@ -412,7 +426,7 @@ export class JobsRepository {
limit: number,
delayFor: (attempt: number) => number,
): Promise<number> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.reap_expired_leases', async (client) => {
const expired = await client.query<AttachmentJobRow>(
`SELECT * FROM attachment_jobs
WHERE status = 'processing' AND lease_until < now()
@@ -492,7 +506,7 @@ export class JobsRepository {
[
job.job_id,
JSON.stringify(
buildJobMessage({ ...job, attempt: nextAttempt }),
tracedJobMessage({ ...job, attempt: nextAttempt }),
),
delayMs / 1000,
],
@@ -518,7 +532,7 @@ export class JobsRepository {
limit: number,
graceSeconds: number,
): Promise<number> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.reconcile_queued', async (client) => {
const stuck = await client.query<AttachmentJobRow>(
`SELECT j.* FROM attachment_jobs j
WHERE j.status = 'queued'
@@ -546,7 +560,7 @@ export class JobsRepository {
await client.query(
`INSERT INTO job_outbox (job_id, payload, available_at)
VALUES ($1, $2::jsonb, now())`,
[job.job_id, JSON.stringify(buildJobMessage(job))],
[job.job_id, JSON.stringify(tracedJobMessage(job))],
);
await this.event(
client,
@@ -565,7 +579,7 @@ export class JobsRepository {
publish: (message: JobMessage) => Promise<void>,
afterPublish: () => Promise<void>,
): Promise<number> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('outbox.publish_pending', async (client) => {
const pending = await client.query<{
id: string;
payload: JobMessage;
@@ -641,7 +655,7 @@ export class JobsRepository {
key: string,
removeObject: (key: string) => Promise<void>,
): Promise<boolean> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('attachments.clear_staging', async (client) => {
// Keep the same job -> attachment lock order as claim/replay, then
// serialize cleanup with replay on the attachment row.
await client.query(
@@ -868,7 +882,7 @@ export class JobsRepository {
expectedStagingKey: string | null;
},
): Promise<ReplayResult> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.replay', async (client) => {
const found = await client.query<AttachmentJobRow>(
`SELECT * FROM attachment_jobs
WHERE attachment_id = $1 AND status = 'failed'
@@ -945,7 +959,7 @@ export class JobsRepository {
VALUES ($1, $2::jsonb, now())`,
[
job.job_id,
JSON.stringify(buildJobMessage({ ...job, attempt })),
JSON.stringify(tracedJobMessage({ ...job, attempt })),
],
);
await this.event(client, job.job_id, 'replayed', attempt, {
@@ -962,7 +976,7 @@ export class JobsRepository {
pipelineVersion: number,
options: { reason: string; operator: string; maxAttempts: number },
): Promise<ReprocessResult> {
return this.database.withTransaction(async (client) => {
return this.database.transaction('jobs.reprocess', async (client) => {
const attachment = await client.query<AttachmentRow>(
`SELECT * FROM attachments WHERE id = $1 FOR UPDATE`,
[attachmentId],
@@ -1002,7 +1016,7 @@ export class JobsRepository {
await client.query(
`INSERT INTO job_outbox (job_id, payload, available_at)
VALUES ($1, $2::jsonb, now())`,
[job.job_id, JSON.stringify(buildJobMessage(job))],
[job.job_id, JSON.stringify(tracedJobMessage(job))],
);
await this.event(client, job.job_id, 'created', 1, {
reason: options.reason,
+11 -2
View File
@@ -4,12 +4,14 @@ import {
OnApplicationBootstrap,
OnModuleDestroy,
} from '@nestjs/common';
import { context, ROOT_CONTEXT } from '@opentelemetry/api';
import { ConfigService } from '@nestjs/config';
import type { AppConfig } from '../config/app-config.schema.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { BrokerClient } from './broker.client.js';
import { FaultInjector } from './fault-injector.js';
import { JobsRepository } from './jobs.repository.js';
import { contextFromTraceFields } from '../telemetry/tracing.js';
const BATCH_SIZE = 20;
@@ -58,7 +60,10 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy {
return;
}
this.running = true;
this.current = this.drain()
// A batch belongs to no particular request or job: each message carries
// its own trace context (see drain).
this.current = context
.with(ROOT_CONTEXT, () => this.drain())
.catch((error: unknown) => {
this.metrics.increment('outbox.publish_failed');
this.retryAt = Date.now() + this.retryDelayMs;
@@ -93,7 +98,11 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy {
async (message) => {
if (Date.now() - started > budgetMs)
throw new Error('outbox batch deadline exceeded');
await this.broker.publish(message);
// The PRODUCER span continues the trace of the request that wrote
// the outbox row.
await context.with(contextFromTraceFields(message), () =>
this.broker.publish(message),
);
},
() => this.faults.point('relay.after_publish_before_mark'),
);
+15 -1
View File
@@ -1,3 +1,6 @@
// Must stay the first import: the OpenTelemetry SDK has to be registered
// before any module creates instruments or handles a request.
import './telemetry/api-bootstrap.js';
import { NestFactory } from '@nestjs/core';
import { HttpAdapterHost } from '@nestjs/core';
import { ConfigService } from '@nestjs/config';
@@ -11,6 +14,9 @@ import { AppModule } from './app.module.js';
import { HealthService } from './health/health.service.js';
import { MetricsService } from './metrics/metrics.service.js';
import { RequestIdInterceptor } from './common/http/request-id.interceptor.js';
import { installHttpTelemetry } from './telemetry/http.js';
import { appLogger } from './telemetry/logger.js';
import { shutdownTelemetry } from './telemetry/sdk.js';
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream';
@@ -18,7 +24,10 @@ const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream';
async function bootstrap(): Promise<void> {
const app = await NestFactory.create<NestFastifyApplication>(
AppModule,
new FastifyAdapter({ logger: true }),
// Fastify's pino access log is replaced by the structured http.request
// event (telemetry/http.ts): pino logged raw URLs with identifiers.
new FastifyAdapter({ logger: false }),
{ logger: appLogger },
);
// The upload route receives its body as an unread stream whatever the
// declared Content-Type: the service checks the type itself, after the
@@ -26,6 +35,10 @@ async function bootstrap(): Promise<void> {
// its own limits, not Fastify's bodyLimit. The declared type is kept on the
// request; all other routes keep Fastify's parsers untouched.
const fastify = app.getHttpAdapter().getInstance();
// Telemetry hooks go first, so every request is measured even if a later
// hook answers it. Routes are registered at init, after the onRoute hook,
// so their handlers run inside the request span.
installHttpTelemetry(fastify);
fastify.addHook(
'onRequest',
(request: UploadRequest, _reply: unknown, done: () => void) => {
@@ -85,6 +98,7 @@ async function bootstrap(): Promise<void> {
setTimeout(resolve, drainDelayMs),
);
await app.close();
await shutdownTelemetry();
process.exit(0);
};
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const)
+24
View File
@@ -7,6 +7,7 @@ import {
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { NodeHttpHandler } from '@smithy/node-http-handler';
import { SpanKind } from '@opentelemetry/api';
import {
Injectable,
Logger,
@@ -16,6 +17,7 @@ import { ConfigService } from '@nestjs/config';
import http from 'node:http';
import { Readable } from 'node:stream';
import type { AppConfig } from '../config/app-config.schema.js';
import { withChildSpan } from '../telemetry/tracing.js';
export class ObjectStorageUnavailableError extends ServiceUnavailableException {
constructor() {
@@ -80,6 +82,28 @@ export class ObjectStorage {
...common,
endpoint: config.getOrThrow('S3_ENDPOINT'),
});
// One CLIENT span per S3 call (S3.PutObject, S3.GetObject, …), only
// inside a traced request or job. Object keys contain attachment ids
// and are deliberately not recorded.
this.internal.middlewareStack.add(
(next, handlerContext) => async (args) => {
const operation = String(
handlerContext.commandName ?? 'Unknown',
).replace(/Command$/u, '');
return withChildSpan(
`S3.${operation}`,
SpanKind.CLIENT,
{
'rpc.system': 'aws-api',
'rpc.service': 'S3',
'rpc.method': operation,
'aws.s3.bucket': this.bucket,
},
() => next(args),
);
},
{ step: 'initialize', name: 'messengerTelemetry' },
);
// Сеть не использует: только подпись URL для клиента.
this.presign = new S3Client({
...common,
+10
View File
@@ -23,6 +23,9 @@ export const TIME_TO_READY_BUCKETS = [
export const WAIT_BUCKETS = [
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
];
export const DB_DURATION_BUCKETS = [
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
];
export type Instruments = {
readonly httpDuration: Histogram;
@@ -31,6 +34,7 @@ export type Instruments = {
readonly timeToReady: Histogram;
readonly brokerPublished: Counter;
readonly dbConnectionWait: Histogram;
readonly dbOperationDuration: Histogram;
readonly telemetryExport: Counter;
};
@@ -83,6 +87,12 @@ export function instruments(): Instruments {
description: 'Time spent waiting for a connection from the pool',
advice: { explicitBucketBoundaries: WAIT_BUCKETS },
}),
dbOperationDuration: m.createHistogram('db.client.operation.duration', {
unit: 's',
description:
'PostgreSQL statement or transaction duration by query label, pool wait included',
advice: { explicitBucketBoundaries: DB_DURATION_BUCKETS },
}),
telemetryExport: m.createCounter('messenger.telemetry.export.items', {
unit: '{item}',
description:
+8 -1
View File
@@ -1,3 +1,5 @@
// Must stay the first import (see main.ts).
import './telemetry/worker-bootstrap.js';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { NestFactory } from '@nestjs/core';
@@ -6,12 +8,16 @@ import type { AppConfig } from './config/app-config.schema.js';
import { BrokerClient } from './jobs/broker.client.js';
import { JobProcessor } from './worker/job-processor.service.js';
import { WorkerModule } from './worker/worker.module.js';
import { appLogger } from './telemetry/logger.js';
import { shutdownTelemetry } from './telemetry/sdk.js';
const DRAIN_MS = 30_000;
async function bootstrap(): Promise<void> {
const app = await NestFactory.createApplicationContext(WorkerModule, {
logger: ['error', 'warn', 'log'],
// Verbosity comes from LOG_LEVEL (default INFO, the former
// error/warn/log set).
logger: appLogger,
});
const config = app.get(ConfigService<AppConfig>);
const broker = app.get(BrokerClient);
@@ -56,6 +62,7 @@ async function bootstrap(): Promise<void> {
await processor.drain(DRAIN_MS);
await new Promise<void>((resolve) => server.close(() => resolve()));
await app.close();
await shutdownTelemetry();
process.exit(0);
};
for (const signal of ['SIGTERM', 'SIGINT'] as const)
+186 -11
View File
@@ -1,6 +1,13 @@
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import type { Channel, ConsumeMessage } from 'amqplib';
import {
ROOT_CONTEXT,
SpanKind,
SpanStatusCode,
trace,
type Span,
} from '@opentelemetry/api';
import { createHash } from 'node:crypto';
import os from 'node:os';
import type { AttachmentRow } from '../attachments/attachment.js';
@@ -8,6 +15,7 @@ import type { AppConfig } from '../config/app-config.schema.js';
import { ObjectStorage } from '../storage/object-storage.js';
import { retryDelayMs } from '../jobs/backoff.js';
import { BrokerClient } from '../jobs/broker.client.js';
import { JOB_QUEUE } from '../jobs/broker.topology.js';
import {
classifyError,
JobTimeoutError,
@@ -16,7 +24,11 @@ import {
type ErrorClass,
} from '../jobs/errors.js';
import { FaultInjector } from '../jobs/fault-injector.js';
import { parseJobMessage, type JobMessage } from '../jobs/job-message.js';
import {
JOB_TYPE,
parseJobMessage,
type JobMessage,
} from '../jobs/job-message.js';
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
import {
JobsRepository,
@@ -25,6 +37,15 @@ import {
} from '../jobs/jobs.repository.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { renderVariants, type RenderedVariant } from './pipelines.js';
import { instruments, meter } from '../telemetry/instruments.js';
import { appLogger } from '../telemetry/logger.js';
import {
contextFromTraceFields,
errorType,
markSpanError,
traceFieldsFromHeaders,
tracer,
} from '../telemetry/tracing.js';
const SHUTDOWN_DRAIN_MS = 30_000;
const VARIANT_EXTENSIONS = [
@@ -33,6 +54,31 @@ const VARIANT_EXTENSIONS = [
['optimized', 'webp'],
] as const;
/** Bounded outcome set: the `outcome` label of messenger.jobs.processed. */
type JobOutcome =
| 'succeeded'
| 'superseded'
| 'lease_lost'
| 'retry'
| 'failed'
| 'commit_failed'
| 'poison'
| 'claim_failed'
| 'skipped'
| 'mismatch'
| 'crashed';
// Outcomes where the delivery did not do its job: WARN log, ERROR span status.
const PROBLEM_OUTCOMES: ReadonlySet<JobOutcome> = new Set<JobOutcome>([
'retry',
'failed',
'commit_failed',
'poison',
'claim_failed',
'mismatch',
'crashed',
]);
function throwIfAborted(signal: AbortSignal): void {
if (!signal.aborted) return;
throw signal.reason instanceof Error ? signal.reason : new Error('Aborted');
@@ -73,9 +119,38 @@ export class JobProcessor implements OnApplicationBootstrap {
}
onApplicationBootstrap(): void {
this.broker.consume(
this.config.getOrThrow('WORKER_CONCURRENCY'),
(message, channel) => this.handle(message, channel),
const concurrency = this.config.getOrThrow('WORKER_CONCURRENCY');
this.observeSaturation(concurrency);
this.broker.consume(concurrency, (message, channel) =>
this.handle(message, channel),
);
}
/**
* Saturation of this worker: jobs in flight against WORKER_CONCURRENCY
* (= prefetch). Together with messenger.jobs.queue.* it separates «the
* worker is slow» from «the worker is full» from «the worker is gone».
*/
private observeSaturation(concurrency: number): void {
const m = meter();
const active = m.createObservableUpDownCounter(
'messenger.worker.jobs.active',
{ unit: '{job}', description: 'Jobs this worker is processing now' },
);
const capacity = m.createObservableUpDownCounter(
'messenger.worker.jobs.capacity',
{
unit: '{job}',
description:
'Jobs this worker may process at once (WORKER_CONCURRENCY)',
},
);
m.addBatchObservableCallback(
(result) => {
result.observe(active, this.active);
result.observe(capacity, concurrency);
},
[active, capacity],
);
}
@@ -88,15 +163,104 @@ export class JobProcessor implements OnApplicationBootstrap {
clearTimeout(timer);
}
/**
* One CONSUMER span per delivery. Its parent is the PRODUCER span from the
* AMQP headers; messages without trace headers (published before lab6 or
* replayed raw) fall back to the trace context stored in the body.
*/
private async handle(
message: ConsumeMessage,
channel: Channel,
): Promise<void> {
const parsed = parseJobMessage(message.content);
const fromHeaders = traceFieldsFromHeaders(
message.properties.headers as Record<string, unknown> | undefined,
);
const parent = fromHeaders.traceparent
? contextFromTraceFields(fromHeaders)
: contextFromTraceFields(parsed, ROOT_CONTEXT);
const started = process.hrtime.bigint();
await tracer().startActiveSpan(
`process ${JOB_QUEUE}`,
{
kind: SpanKind.CONSUMER,
attributes: {
'messaging.system': 'rabbitmq',
'messaging.operation.type': 'process',
'messaging.destination.name': JOB_QUEUE,
'messaging.rabbitmq.message.redelivered':
message.fields.redelivered,
...(parsed
? {
'messaging.message.id': parsed.job_id,
'messenger.job.id': parsed.job_id,
'messenger.attachment.id': parsed.attachment_id,
'messenger.job.attempt': parsed.attempt,
'messenger.pipeline.version': parsed.pipeline_version,
}
: {}),
},
},
parent,
async (span: Span) => {
let outcome: JobOutcome = 'crashed';
try {
outcome = await this.dispatch(parsed, message, channel);
} catch (error) {
markSpanError(span, error);
throw error;
} finally {
this.record(span, parsed, outcome, started);
}
},
);
}
private record(
span: Span,
parsed: JobMessage | undefined,
outcome: JobOutcome,
started: bigint,
): void {
const seconds = Number(process.hrtime.bigint() - started) / 1e9;
const labels = { 'job.type': JOB_TYPE, outcome };
instruments().jobsProcessed.add(1, labels);
instruments().jobDuration.record(seconds, labels);
if (outcome === 'succeeded' && parsed) {
// From durable acceptance (job created_at) to the ready commit:
// queueing, retries and backoff included.
const ready = (Date.now() - Date.parse(parsed.created_at)) / 1000;
if (Number.isFinite(ready) && ready >= 0)
instruments().timeToReady.record(ready, { 'job.type': JOB_TYPE });
}
span.setAttribute('messenger.job.outcome', outcome);
const problem = PROBLEM_OUTCOMES.has(outcome);
if (problem && outcome !== 'crashed')
span.setStatus({ code: SpanStatusCode.ERROR, message: outcome });
appLogger.event(
problem ? 'WARN' : 'INFO',
'job.processed',
`job ${outcome}`,
{
'messenger.job.id': parsed?.job_id,
'messenger.attachment.id': parsed?.attachment_id,
'messenger.job.attempt': parsed?.attempt,
'messenger.job.outcome': outcome,
duration_ms: Math.round(seconds * 1000),
},
);
span.end();
}
private async dispatch(
parsed: JobMessage | undefined,
message: ConsumeMessage,
channel: Channel,
): Promise<JobOutcome> {
if (!parsed) {
this.metrics.increment('worker.messages.poison');
this.safeNack(channel, message, false);
return;
return 'poison';
}
const claim = await this.claimWithRetry(parsed);
@@ -105,21 +269,24 @@ export class JobProcessor implements OnApplicationBootstrap {
// Do not burn the delivery limit in a tight nack loop while the
// database is briefly unavailable.
this.safeNackAfter(channel, message, true, 5000);
return;
return 'claim_failed';
}
if (claim.kind === 'skip') {
if (claim.reason === 'mismatch') {
this.safeNack(channel, message, false);
return;
return 'mismatch';
}
this.metrics.increment(`worker.jobs.skipped.${claim.reason}`);
trace
.getActiveSpan()
?.setAttribute('messenger.job.skip_reason', claim.reason);
this.safeAck(channel, message);
return;
return 'skipped';
}
this.begin();
try {
await this.process(
return await this.process(
claim.job,
claim.attachment,
claim.leaseToken,
@@ -142,7 +309,7 @@ export class JobProcessor implements OnApplicationBootstrap {
leaseToken: string,
message: ConsumeMessage,
channel: Channel,
): Promise<void> {
): Promise<JobOutcome> {
this.metrics.increment('worker.jobs.claimed');
const controller = new AbortController();
const target = {
@@ -170,6 +337,7 @@ export class JobProcessor implements OnApplicationBootstrap {
Math.max(1000, (this.leaseSeconds * 1000) / 3),
);
let committed = false;
let committedOutcome: JobOutcome = 'succeeded';
try {
await this.faults.point(
@@ -222,6 +390,7 @@ export class JobProcessor implements OnApplicationBootstrap {
variants.map((variant) => this.variantWrite(job, variant)),
);
committed = true;
committedOutcome = result;
this.metrics.increment(`worker.jobs.${result}`);
await this.faults.point(
'worker.after_commit_before_ack',
@@ -235,13 +404,17 @@ export class JobProcessor implements OnApplicationBootstrap {
} else if (result === 'superseded') {
await this.cleanupAttemptObjects(job, null);
}
return result;
} catch (error) {
if (committed) {
// The durable effect is already committed; a failure at the ack
// fault point or a lost reply must not turn into a retry.
this.safeAck(channel, message);
return;
return committedOutcome;
}
// Only the error type/code reaches the span: driver messages may
// carry request values.
trace.getActiveSpan()?.setAttribute('error.type', errorType(error));
const failure = this.classify(error, controller.signal);
const delay = retryDelayMs(
job.attempt,
@@ -260,12 +433,14 @@ export class JobProcessor implements OnApplicationBootstrap {
this.safeAck(channel, message);
if (outcome === 'retry' || outcome === 'failed')
await this.cleanupAttemptObjects(job, null);
return outcome;
} catch (commitError) {
this.metrics.increment('worker.jobs.commit_failed');
this.logger.warn(
`Failure commit failed: ${commitError instanceof Error ? commitError.message : 'unknown error'}`,
);
setTimeout(() => this.safeNack(channel, message, true), 1000);
return 'commit_failed';
}
} finally {
clearTimeout(timeout);