lab6: инструментирование кода — HTTP, PostgreSQL, RabbitMQ (W3C trace context), S3, обработка задач, структурные логи
This commit is contained in:
@@ -11,6 +11,7 @@ import { MetricsModule } from './metrics/metrics.module.js';
|
||||
import { UsersModule } from './users/users.module.js';
|
||||
import { APP_FILTER } from '@nestjs/core';
|
||||
import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
|
||||
import { QueueStatsService } from './telemetry/queue-stats.service.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -30,6 +31,9 @@ import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
|
||||
provide: APP_FILTER,
|
||||
useClass: ApiExceptionFilter,
|
||||
},
|
||||
// Queue depth/age gauges are published by the API so that they keep
|
||||
// reporting while the worker is down.
|
||||
QueueStatsService,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -9,11 +9,20 @@ import {
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { trace } from '@opentelemetry/api';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
import { resolveInstanceId } from '../../config/instance-id.js';
|
||||
import { routeTemplate } from '../../telemetry/http.js';
|
||||
import { errorType } from '../../telemetry/tracing.js';
|
||||
import type { ApiErrorBody } from './api-error.js';
|
||||
|
||||
type RequestWithId = { requestId?: string; method?: string; url?: string };
|
||||
type RequestWithId = {
|
||||
requestId?: string;
|
||||
method?: string;
|
||||
url?: string;
|
||||
// Fastify route template: logs must not carry raw URLs (they hold ids).
|
||||
routeOptions?: { url?: string };
|
||||
};
|
||||
type ExceptionBody = {
|
||||
message?: unknown;
|
||||
challenge?: unknown;
|
||||
@@ -146,11 +155,11 @@ export class ApiExceptionFilter implements ExceptionFilter {
|
||||
status = HttpStatus.SERVICE_UNAVAILABLE;
|
||||
message = 'Service unavailable';
|
||||
this.logger.warn(
|
||||
`Dependency outage on ${request.method ?? ''} ${request.url ?? ''}: ${exception instanceof Error ? exception.message : 'unknown error'}`,
|
||||
`Dependency outage on ${request.method ?? ''} ${routeTemplate(request.routeOptions?.url)}: ${exception instanceof Error ? exception.message : 'unknown error'}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.error(
|
||||
`Unhandled exception on ${request.method ?? ''} ${request.url ?? ''}`,
|
||||
`Unhandled exception on ${request.method ?? ''} ${routeTemplate(request.routeOptions?.url)}`,
|
||||
exception instanceof Error
|
||||
? exception.stack
|
||||
: String(exception),
|
||||
@@ -176,6 +185,10 @@ export class ApiExceptionFilter implements ExceptionFilter {
|
||||
request_id: requestId,
|
||||
},
|
||||
};
|
||||
if (status >= HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
trace
|
||||
.getActiveSpan()
|
||||
?.setAttribute('error.type', errorType(exception));
|
||||
adapter.reply(response, payload, status);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,97 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { SpanKind, type Attributes, type Span } from '@opentelemetry/api';
|
||||
import pg from 'pg';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { instruments, meter } from '../telemetry/instruments.js';
|
||||
import { errorType, withChildSpan } from '../telemetry/tracing.js';
|
||||
|
||||
const DB_SYSTEM = 'postgresql';
|
||||
const POOL_NAME = 'main';
|
||||
const TRANSACTION_CONTROL =
|
||||
/^\s*(?:BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE)\b/iu;
|
||||
const SQL_TARGET =
|
||||
/\b(?:FROM|INTO|UPDATE|JOIN)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?/iu;
|
||||
|
||||
/** First SQL keyword (SELECT, INSERT, WITH, …): a bounded attribute. */
|
||||
function sqlOperation(text: string): string {
|
||||
const match = /^\s*([A-Za-z]+)/u.exec(text);
|
||||
return match ? match[1].toUpperCase() : 'OTHER';
|
||||
}
|
||||
|
||||
/** Best-effort table name for span names; never used as a metric label. */
|
||||
function sqlTarget(text: string): string | undefined {
|
||||
return SQL_TARGET.exec(text)?.[1]?.toLowerCase();
|
||||
}
|
||||
|
||||
function queryText(args: readonly unknown[]): string {
|
||||
const first = args[0];
|
||||
if (typeof first === 'string') return first;
|
||||
if (typeof first === 'object' && first !== null) {
|
||||
const text = (first as { text?: unknown }).text;
|
||||
if (typeof text === 'string') return text;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function isPassThrough(args: readonly unknown[]): boolean {
|
||||
const first = args[0];
|
||||
return (
|
||||
typeof args[args.length - 1] === 'function' ||
|
||||
(typeof first === 'object' &&
|
||||
first !== null &&
|
||||
typeof (first as { submit?: unknown }).submit === 'function')
|
||||
);
|
||||
}
|
||||
|
||||
function elapsedSeconds(started: bigint): number {
|
||||
return Number(process.hrtime.bigint() - started) / 1e9;
|
||||
}
|
||||
|
||||
function statementAttributes(
|
||||
label: string,
|
||||
operation: string,
|
||||
target: string | undefined,
|
||||
): Attributes {
|
||||
return {
|
||||
'db.system.name': DB_SYSTEM,
|
||||
'db.operation.name': operation,
|
||||
'db.query.summary': label,
|
||||
...(target ? { 'db.collection.name': target } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Inside a traced transaction every statement gets its own CLIENT span
|
||||
* (`UPDATE attachment_jobs`, `INSERT job_outbox`, …) while repository code
|
||||
* stays unaware of tracing. Transaction control, callback-style calls and
|
||||
* submittables (cursors, streams) pass through untouched. SQL text and
|
||||
* parameter values are never recorded.
|
||||
*/
|
||||
function tracedClient(client: pg.PoolClient, label: string): pg.PoolClient {
|
||||
return new Proxy(client, {
|
||||
get(target, property) {
|
||||
const value: unknown = Reflect.get(target, property, target);
|
||||
if (typeof value !== 'function') return value;
|
||||
const method = value as (...args: unknown[]) => unknown;
|
||||
if (property !== 'query') return method.bind(target);
|
||||
return (...args: unknown[]): unknown => {
|
||||
const text = queryText(args);
|
||||
if (isPassThrough(args) || TRANSACTION_CONTROL.test(text))
|
||||
return method.apply(target, args);
|
||||
const operation = sqlOperation(text);
|
||||
const table = sqlTarget(text);
|
||||
return withChildSpan(
|
||||
`${operation} ${table ?? label}`,
|
||||
SpanKind.CLIENT,
|
||||
statementAttributes(label, operation, table),
|
||||
async () => method.apply(target, args),
|
||||
);
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DatabaseClient implements OnModuleDestroy {
|
||||
@@ -49,15 +138,44 @@ export class DatabaseClient implements OnModuleDestroy {
|
||||
`PostgreSQL health connection error: ${error.message}`,
|
||||
);
|
||||
});
|
||||
this.observePool(this.config.getOrThrow('DATABASE_POOL_MAX'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Same contract as pg.Pool#query — a failed query destroys its connection —
|
||||
* but the checkout is explicit, so the wait for a pooled connection is
|
||||
* measured apart from the statement: «the pool is exhausted» and «the
|
||||
* database is slow» are different incidents with different runbooks.
|
||||
*/
|
||||
query<T extends pg.QueryResultRow = pg.QueryResultRow>(
|
||||
label: string,
|
||||
text: string,
|
||||
values: readonly unknown[] = [],
|
||||
): Promise<pg.QueryResult<T>> {
|
||||
this.metrics.increment(label);
|
||||
return this.pool.query<T>({ text, values: [...values] });
|
||||
return this.observe(
|
||||
label,
|
||||
sqlOperation(text),
|
||||
sqlTarget(text),
|
||||
async () => {
|
||||
const client = await this.checkout();
|
||||
let failure: Error | undefined;
|
||||
const onError = (error: Error): void => {
|
||||
failure = error;
|
||||
};
|
||||
client.on('error', onError);
|
||||
try {
|
||||
return await client.query<T>({ text, values: [...values] });
|
||||
} catch (error) {
|
||||
failure ??=
|
||||
error instanceof Error ? error : new Error('Query failed');
|
||||
throw error;
|
||||
} finally {
|
||||
client.removeListener('error', onError);
|
||||
client.release(failure);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async health(): Promise<void> {
|
||||
@@ -65,41 +183,143 @@ export class DatabaseClient implements OnModuleDestroy {
|
||||
await this.healthPool.query('SELECT 1');
|
||||
}
|
||||
|
||||
async withTransaction<T>(
|
||||
withTransaction<T>(
|
||||
work: (client: pg.PoolClient) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const client = await this.pool.connect();
|
||||
// A server-side idle_in_transaction_session_timeout or a network drop
|
||||
// can emit 'error' on a checked-out client after pg-pool removed its
|
||||
// idle listener. Without this listener Node treats it as uncaught.
|
||||
let fatal: Error | undefined;
|
||||
const onError = (error: Error): void => {
|
||||
fatal = error;
|
||||
};
|
||||
client.on('error', onError);
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const result = await work(client);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
return this.transaction('transaction', work);
|
||||
}
|
||||
|
||||
/**
|
||||
* A labelled transaction: one span for the unit of work with a child span
|
||||
* per statement and one db.client.operation.duration sample (pool wait
|
||||
* included) with db.operation.name = TRANSACTION.
|
||||
*/
|
||||
transaction<T>(
|
||||
label: string,
|
||||
work: (client: pg.PoolClient) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return this.observe(label, 'TRANSACTION', undefined, async (span) => {
|
||||
const client = await this.checkout();
|
||||
// A server-side idle_in_transaction_session_timeout or a network drop
|
||||
// can emit 'error' on a checked-out client after pg-pool removed its
|
||||
// idle listener. Without this listener Node treats it as uncaught.
|
||||
let fatal: Error | undefined;
|
||||
const onError = (error: Error): void => {
|
||||
fatal = error;
|
||||
};
|
||||
client.on('error', onError);
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch (rollbackError) {
|
||||
this.metrics.increment('database.rollback_error');
|
||||
this.logger.warn(
|
||||
`Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`,
|
||||
await client.query('BEGIN');
|
||||
const result = await work(
|
||||
span ? tracedClient(client, label) : client,
|
||||
);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch (rollbackError) {
|
||||
this.metrics.increment('database.rollback_error');
|
||||
this.logger.warn(
|
||||
`Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
// Callers must see the operation error, not the rollback error.
|
||||
throw error;
|
||||
} finally {
|
||||
client.removeListener('error', onError);
|
||||
client.release(fatal);
|
||||
}
|
||||
// Callers must see the operation error, not the rollback error.
|
||||
throw error;
|
||||
} finally {
|
||||
client.removeListener('error', onError);
|
||||
client.release(fatal);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
await Promise.all([this.pool.end(), this.healthPool.end()]);
|
||||
}
|
||||
|
||||
private observe<T>(
|
||||
label: string,
|
||||
operation: string,
|
||||
target: string | undefined,
|
||||
work: (span: Span | undefined) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const started = process.hrtime.bigint();
|
||||
return withChildSpan(
|
||||
`${operation} ${target ?? label}`,
|
||||
SpanKind.CLIENT,
|
||||
statementAttributes(label, operation, target),
|
||||
async (span) => {
|
||||
let failure: string | undefined;
|
||||
try {
|
||||
return await work(span);
|
||||
} catch (error) {
|
||||
failure = errorType(error);
|
||||
throw error;
|
||||
} finally {
|
||||
instruments().dbOperationDuration.record(elapsedSeconds(started), {
|
||||
'db.system.name': DB_SYSTEM,
|
||||
'db.operation.name': operation,
|
||||
'db.query.summary': label,
|
||||
...(failure ? { 'error.type': failure } : {}),
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async checkout(): Promise<pg.PoolClient> {
|
||||
const started = process.hrtime.bigint();
|
||||
let failure: string | undefined;
|
||||
try {
|
||||
return await this.pool.connect();
|
||||
} catch (error) {
|
||||
failure = errorType(error);
|
||||
throw error;
|
||||
} finally {
|
||||
instruments().dbConnectionWait.record(elapsedSeconds(started), {
|
||||
'db.client.connection.pool.name': POOL_NAME,
|
||||
...(failure ? { 'error.type': failure } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Pool saturation: used vs max connections and callers queued for one. */
|
||||
private observePool(max: number): void {
|
||||
const m = meter();
|
||||
const pool = { 'db.client.connection.pool.name': POOL_NAME };
|
||||
const count = m.createObservableUpDownCounter(
|
||||
'db.client.connection.count',
|
||||
{
|
||||
unit: '{connection}',
|
||||
description: 'Connections in the main pool by state',
|
||||
},
|
||||
);
|
||||
const limit = m.createObservableUpDownCounter('db.client.connection.max', {
|
||||
unit: '{connection}',
|
||||
description: 'Maximum size of the main pool (DATABASE_POOL_MAX)',
|
||||
});
|
||||
const pending = m.createObservableUpDownCounter(
|
||||
'db.client.connection.pending_requests',
|
||||
{
|
||||
unit: '{request}',
|
||||
description: 'Callers waiting for a connection from the main pool',
|
||||
},
|
||||
);
|
||||
m.addBatchObservableCallback(
|
||||
(result) => {
|
||||
const idle = this.pool.idleCount;
|
||||
result.observe(count, this.pool.totalCount - idle, {
|
||||
...pool,
|
||||
'db.client.connection.state': 'used',
|
||||
});
|
||||
result.observe(count, idle, {
|
||||
...pool,
|
||||
'db.client.connection.state': 'idle',
|
||||
});
|
||||
result.observe(limit, max, pool);
|
||||
result.observe(pending, this.pool.waitingCount, pool);
|
||||
},
|
||||
[count, limit, pending],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,12 @@ import {
|
||||
type ConfirmChannel,
|
||||
type ConsumeMessage,
|
||||
} from 'amqplib';
|
||||
import {
|
||||
context,
|
||||
propagation,
|
||||
SpanKind,
|
||||
type Span,
|
||||
} from '@opentelemetry/api';
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
@@ -22,6 +28,8 @@ import {
|
||||
JOB_ROUTING_KEY,
|
||||
} from './broker.topology.js';
|
||||
import type { JobMessage } from './job-message.js';
|
||||
import { instruments } from '../telemetry/instruments.js';
|
||||
import { markSpanError, tracer } from '../telemetry/tracing.js';
|
||||
|
||||
export class BrokerUnavailableError extends ServiceUnavailableException {
|
||||
constructor() {
|
||||
@@ -70,7 +78,58 @@ export class BrokerClient implements OnModuleDestroy {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* PRODUCER span around publish + broker confirm. Its context goes into the
|
||||
* AMQP headers (W3C traceparent/tracestate), so the worker's CONSUMER span
|
||||
* becomes its child; the body keeps the context of the transaction that
|
||||
* wrote the outbox row.
|
||||
*/
|
||||
async publish(message: JobMessage | Buffer): Promise<void> {
|
||||
const attributes = {
|
||||
'messaging.system': 'rabbitmq',
|
||||
'messaging.operation.type': 'send',
|
||||
'messaging.operation.name': 'publish',
|
||||
'messaging.destination.name': JOB_EXCHANGE,
|
||||
'messaging.rabbitmq.destination.routing_key': JOB_ROUTING_KEY,
|
||||
...(Buffer.isBuffer(message)
|
||||
? {}
|
||||
: {
|
||||
'messaging.message.id': message.job_id,
|
||||
'messenger.job.id': message.job_id,
|
||||
'messenger.job.attempt': message.attempt,
|
||||
}),
|
||||
};
|
||||
return tracer().startActiveSpan(
|
||||
`publish ${JOB_EXCHANGE}`,
|
||||
{ kind: SpanKind.PRODUCER, attributes },
|
||||
async (span: Span) => {
|
||||
const traceHeaders: Record<string, string> = {};
|
||||
propagation.inject(context.active(), traceHeaders);
|
||||
const labels = { 'messaging.destination.name': JOB_EXCHANGE };
|
||||
try {
|
||||
await this.publishConfirmed(message, traceHeaders);
|
||||
instruments().brokerPublished.add(1, {
|
||||
...labels,
|
||||
outcome: 'success',
|
||||
});
|
||||
} catch (error) {
|
||||
instruments().brokerPublished.add(1, {
|
||||
...labels,
|
||||
outcome: 'failed',
|
||||
});
|
||||
markSpanError(span, error);
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async publishConfirmed(
|
||||
message: JobMessage | Buffer,
|
||||
traceHeaders: Record<string, string>,
|
||||
): Promise<void> {
|
||||
const channel = await this.withTimeout(
|
||||
this.ensureConnected(),
|
||||
this.publishTimeoutMs,
|
||||
@@ -85,11 +144,11 @@ export class BrokerClient implements OnModuleDestroy {
|
||||
mandatory: true,
|
||||
correlationId,
|
||||
...(Buffer.isBuffer(message)
|
||||
? {}
|
||||
? { headers: traceHeaders }
|
||||
: {
|
||||
messageId: message.job_id,
|
||||
type: message.type,
|
||||
headers: { 'x-attempt': message.attempt },
|
||||
headers: { 'x-attempt': message.attempt, ...traceHeaders },
|
||||
}),
|
||||
};
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
|
||||
@@ -4,6 +4,20 @@ import type { PoolClient } from 'pg';
|
||||
import type { AttachmentRow, VariantRow } from '../attachments/attachment.js';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { buildJobMessage, type JobMessage } from './job-message.js';
|
||||
import { currentTraceFields } from '../telemetry/tracing.js';
|
||||
|
||||
/**
|
||||
* Every outbox row carries the W3C trace context of the transaction that
|
||||
* wrote it (optional traceparent/tracestate of contracts/jobs.schema.json),
|
||||
* so causality survives restarts of the API, the broker and the worker.
|
||||
* Writers without an active span (lease reaper, reconciler, CLI) produce the
|
||||
* same message as before.
|
||||
*/
|
||||
function tracedJobMessage(
|
||||
job: Parameters<typeof buildJobMessage>[0],
|
||||
): JobMessage {
|
||||
return { ...buildJobMessage(job), ...currentTraceFields() };
|
||||
}
|
||||
|
||||
export type JobStatus =
|
||||
'queued' | 'processing' | 'succeeded' | 'failed' | 'superseded';
|
||||
@@ -70,7 +84,7 @@ export class JobsRepository {
|
||||
pipelineVersion: number,
|
||||
maxAttempts: number,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.accept_upload', async (client) => {
|
||||
const accepted = await client.query<AttachmentRow>(
|
||||
`UPDATE attachments
|
||||
SET status = 'queued', size_bytes = $2, sha256 = $3,
|
||||
@@ -104,7 +118,7 @@ export class JobsRepository {
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(
|
||||
buildJobMessage({
|
||||
tracedJobMessage({
|
||||
...job,
|
||||
created_at: job.created_at,
|
||||
}),
|
||||
@@ -121,7 +135,7 @@ export class JobsRepository {
|
||||
workerId: string,
|
||||
leaseSeconds: number,
|
||||
): Promise<ClaimResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.claim', async (client) => {
|
||||
// Lock order is always job -> attachment, matching commitSuccess and
|
||||
// commitFailure. A join FOR UPDATE would let the planner choose the
|
||||
// opposite order under a concurrent duplicate delivery.
|
||||
@@ -240,7 +254,7 @@ export class JobsRepository {
|
||||
token: string,
|
||||
variants: VariantWrite[],
|
||||
): Promise<CommitSuccessResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.commit_success', async (client) => {
|
||||
const succeeded = await client.query<AttachmentJobRow>(
|
||||
`UPDATE attachment_jobs
|
||||
SET status = 'succeeded', lease_owner = NULL, lease_token = NULL,
|
||||
@@ -325,7 +339,7 @@ export class JobsRepository {
|
||||
failure: { kind: 'permanent' | 'transient'; code: string },
|
||||
delayMs: number,
|
||||
): Promise<CommitFailureResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.commit_failure', async (client) => {
|
||||
const found = await client.query<AttachmentJobRow>(
|
||||
`SELECT * FROM attachment_jobs
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
@@ -389,7 +403,7 @@ export class JobsRepository {
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(
|
||||
buildJobMessage({ ...job, attempt: nextAttempt }),
|
||||
tracedJobMessage({ ...job, attempt: nextAttempt }),
|
||||
),
|
||||
delayMs / 1000,
|
||||
],
|
||||
@@ -412,7 +426,7 @@ export class JobsRepository {
|
||||
limit: number,
|
||||
delayFor: (attempt: number) => number,
|
||||
): Promise<number> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.reap_expired_leases', async (client) => {
|
||||
const expired = await client.query<AttachmentJobRow>(
|
||||
`SELECT * FROM attachment_jobs
|
||||
WHERE status = 'processing' AND lease_until < now()
|
||||
@@ -492,7 +506,7 @@ export class JobsRepository {
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(
|
||||
buildJobMessage({ ...job, attempt: nextAttempt }),
|
||||
tracedJobMessage({ ...job, attempt: nextAttempt }),
|
||||
),
|
||||
delayMs / 1000,
|
||||
],
|
||||
@@ -518,7 +532,7 @@ export class JobsRepository {
|
||||
limit: number,
|
||||
graceSeconds: number,
|
||||
): Promise<number> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.reconcile_queued', async (client) => {
|
||||
const stuck = await client.query<AttachmentJobRow>(
|
||||
`SELECT j.* FROM attachment_jobs j
|
||||
WHERE j.status = 'queued'
|
||||
@@ -546,7 +560,7 @@ export class JobsRepository {
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb, now())`,
|
||||
[job.job_id, JSON.stringify(buildJobMessage(job))],
|
||||
[job.job_id, JSON.stringify(tracedJobMessage(job))],
|
||||
);
|
||||
await this.event(
|
||||
client,
|
||||
@@ -565,7 +579,7 @@ export class JobsRepository {
|
||||
publish: (message: JobMessage) => Promise<void>,
|
||||
afterPublish: () => Promise<void>,
|
||||
): Promise<number> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('outbox.publish_pending', async (client) => {
|
||||
const pending = await client.query<{
|
||||
id: string;
|
||||
payload: JobMessage;
|
||||
@@ -641,7 +655,7 @@ export class JobsRepository {
|
||||
key: string,
|
||||
removeObject: (key: string) => Promise<void>,
|
||||
): Promise<boolean> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('attachments.clear_staging', async (client) => {
|
||||
// Keep the same job -> attachment lock order as claim/replay, then
|
||||
// serialize cleanup with replay on the attachment row.
|
||||
await client.query(
|
||||
@@ -868,7 +882,7 @@ export class JobsRepository {
|
||||
expectedStagingKey: string | null;
|
||||
},
|
||||
): Promise<ReplayResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.replay', async (client) => {
|
||||
const found = await client.query<AttachmentJobRow>(
|
||||
`SELECT * FROM attachment_jobs
|
||||
WHERE attachment_id = $1 AND status = 'failed'
|
||||
@@ -945,7 +959,7 @@ export class JobsRepository {
|
||||
VALUES ($1, $2::jsonb, now())`,
|
||||
[
|
||||
job.job_id,
|
||||
JSON.stringify(buildJobMessage({ ...job, attempt })),
|
||||
JSON.stringify(tracedJobMessage({ ...job, attempt })),
|
||||
],
|
||||
);
|
||||
await this.event(client, job.job_id, 'replayed', attempt, {
|
||||
@@ -962,7 +976,7 @@ export class JobsRepository {
|
||||
pipelineVersion: number,
|
||||
options: { reason: string; operator: string; maxAttempts: number },
|
||||
): Promise<ReprocessResult> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
return this.database.transaction('jobs.reprocess', async (client) => {
|
||||
const attachment = await client.query<AttachmentRow>(
|
||||
`SELECT * FROM attachments WHERE id = $1 FOR UPDATE`,
|
||||
[attachmentId],
|
||||
@@ -1002,7 +1016,7 @@ export class JobsRepository {
|
||||
await client.query(
|
||||
`INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
VALUES ($1, $2::jsonb, now())`,
|
||||
[job.job_id, JSON.stringify(buildJobMessage(job))],
|
||||
[job.job_id, JSON.stringify(tracedJobMessage(job))],
|
||||
);
|
||||
await this.event(client, job.job_id, 'created', 1, {
|
||||
reason: options.reason,
|
||||
|
||||
@@ -4,12 +4,14 @@ import {
|
||||
OnApplicationBootstrap,
|
||||
OnModuleDestroy,
|
||||
} from '@nestjs/common';
|
||||
import { context, ROOT_CONTEXT } from '@opentelemetry/api';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { FaultInjector } from './fault-injector.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
import { contextFromTraceFields } from '../telemetry/tracing.js';
|
||||
|
||||
const BATCH_SIZE = 20;
|
||||
|
||||
@@ -58,7 +60,10 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy {
|
||||
return;
|
||||
}
|
||||
this.running = true;
|
||||
this.current = this.drain()
|
||||
// A batch belongs to no particular request or job: each message carries
|
||||
// its own trace context (see drain).
|
||||
this.current = context
|
||||
.with(ROOT_CONTEXT, () => this.drain())
|
||||
.catch((error: unknown) => {
|
||||
this.metrics.increment('outbox.publish_failed');
|
||||
this.retryAt = Date.now() + this.retryDelayMs;
|
||||
@@ -93,7 +98,11 @@ export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy {
|
||||
async (message) => {
|
||||
if (Date.now() - started > budgetMs)
|
||||
throw new Error('outbox batch deadline exceeded');
|
||||
await this.broker.publish(message);
|
||||
// The PRODUCER span continues the trace of the request that wrote
|
||||
// the outbox row.
|
||||
await context.with(contextFromTraceFields(message), () =>
|
||||
this.broker.publish(message),
|
||||
);
|
||||
},
|
||||
() => this.faults.point('relay.after_publish_before_mark'),
|
||||
);
|
||||
|
||||
+15
-1
@@ -1,3 +1,6 @@
|
||||
// Must stay the first import: the OpenTelemetry SDK has to be registered
|
||||
// before any module creates instruments or handles a request.
|
||||
import './telemetry/api-bootstrap.js';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
@@ -11,6 +14,9 @@ import { AppModule } from './app.module.js';
|
||||
import { HealthService } from './health/health.service.js';
|
||||
import { MetricsService } from './metrics/metrics.service.js';
|
||||
import { RequestIdInterceptor } from './common/http/request-id.interceptor.js';
|
||||
import { installHttpTelemetry } from './telemetry/http.js';
|
||||
import { appLogger } from './telemetry/logger.js';
|
||||
import { shutdownTelemetry } from './telemetry/sdk.js';
|
||||
|
||||
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
|
||||
const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream';
|
||||
@@ -18,7 +24,10 @@ const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream';
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
AppModule,
|
||||
new FastifyAdapter({ logger: true }),
|
||||
// Fastify's pino access log is replaced by the structured http.request
|
||||
// event (telemetry/http.ts): pino logged raw URLs with identifiers.
|
||||
new FastifyAdapter({ logger: false }),
|
||||
{ logger: appLogger },
|
||||
);
|
||||
// The upload route receives its body as an unread stream whatever the
|
||||
// declared Content-Type: the service checks the type itself, after the
|
||||
@@ -26,6 +35,10 @@ async function bootstrap(): Promise<void> {
|
||||
// its own limits, not Fastify's bodyLimit. The declared type is kept on the
|
||||
// request; all other routes keep Fastify's parsers untouched.
|
||||
const fastify = app.getHttpAdapter().getInstance();
|
||||
// Telemetry hooks go first, so every request is measured even if a later
|
||||
// hook answers it. Routes are registered at init, after the onRoute hook,
|
||||
// so their handlers run inside the request span.
|
||||
installHttpTelemetry(fastify);
|
||||
fastify.addHook(
|
||||
'onRequest',
|
||||
(request: UploadRequest, _reply: unknown, done: () => void) => {
|
||||
@@ -85,6 +98,7 @@ async function bootstrap(): Promise<void> {
|
||||
setTimeout(resolve, drainDelayMs),
|
||||
);
|
||||
await app.close();
|
||||
await shutdownTelemetry();
|
||||
process.exit(0);
|
||||
};
|
||||
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const)
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
||||
import { SpanKind } from '@opentelemetry/api';
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
@@ -16,6 +17,7 @@ import { ConfigService } from '@nestjs/config';
|
||||
import http from 'node:http';
|
||||
import { Readable } from 'node:stream';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { withChildSpan } from '../telemetry/tracing.js';
|
||||
|
||||
export class ObjectStorageUnavailableError extends ServiceUnavailableException {
|
||||
constructor() {
|
||||
@@ -80,6 +82,28 @@ export class ObjectStorage {
|
||||
...common,
|
||||
endpoint: config.getOrThrow('S3_ENDPOINT'),
|
||||
});
|
||||
// One CLIENT span per S3 call (S3.PutObject, S3.GetObject, …), only
|
||||
// inside a traced request or job. Object keys contain attachment ids
|
||||
// and are deliberately not recorded.
|
||||
this.internal.middlewareStack.add(
|
||||
(next, handlerContext) => async (args) => {
|
||||
const operation = String(
|
||||
handlerContext.commandName ?? 'Unknown',
|
||||
).replace(/Command$/u, '');
|
||||
return withChildSpan(
|
||||
`S3.${operation}`,
|
||||
SpanKind.CLIENT,
|
||||
{
|
||||
'rpc.system': 'aws-api',
|
||||
'rpc.service': 'S3',
|
||||
'rpc.method': operation,
|
||||
'aws.s3.bucket': this.bucket,
|
||||
},
|
||||
() => next(args),
|
||||
);
|
||||
},
|
||||
{ step: 'initialize', name: 'messengerTelemetry' },
|
||||
);
|
||||
// Сеть не использует: только подпись URL для клиента.
|
||||
this.presign = new S3Client({
|
||||
...common,
|
||||
|
||||
@@ -23,6 +23,9 @@ export const TIME_TO_READY_BUCKETS = [
|
||||
export const WAIT_BUCKETS = [
|
||||
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
|
||||
];
|
||||
export const DB_DURATION_BUCKETS = [
|
||||
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
|
||||
];
|
||||
|
||||
export type Instruments = {
|
||||
readonly httpDuration: Histogram;
|
||||
@@ -31,6 +34,7 @@ export type Instruments = {
|
||||
readonly timeToReady: Histogram;
|
||||
readonly brokerPublished: Counter;
|
||||
readonly dbConnectionWait: Histogram;
|
||||
readonly dbOperationDuration: Histogram;
|
||||
readonly telemetryExport: Counter;
|
||||
};
|
||||
|
||||
@@ -83,6 +87,12 @@ export function instruments(): Instruments {
|
||||
description: 'Time spent waiting for a connection from the pool',
|
||||
advice: { explicitBucketBoundaries: WAIT_BUCKETS },
|
||||
}),
|
||||
dbOperationDuration: m.createHistogram('db.client.operation.duration', {
|
||||
unit: 's',
|
||||
description:
|
||||
'PostgreSQL statement or transaction duration by query label, pool wait included',
|
||||
advice: { explicitBucketBoundaries: DB_DURATION_BUCKETS },
|
||||
}),
|
||||
telemetryExport: m.createCounter('messenger.telemetry.export.items', {
|
||||
unit: '{item}',
|
||||
description:
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// Must stay the first import (see main.ts).
|
||||
import './telemetry/worker-bootstrap.js';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
@@ -6,12 +8,16 @@ import type { AppConfig } from './config/app-config.schema.js';
|
||||
import { BrokerClient } from './jobs/broker.client.js';
|
||||
import { JobProcessor } from './worker/job-processor.service.js';
|
||||
import { WorkerModule } from './worker/worker.module.js';
|
||||
import { appLogger } from './telemetry/logger.js';
|
||||
import { shutdownTelemetry } from './telemetry/sdk.js';
|
||||
|
||||
const DRAIN_MS = 30_000;
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.createApplicationContext(WorkerModule, {
|
||||
logger: ['error', 'warn', 'log'],
|
||||
// Verbosity comes from LOG_LEVEL (default INFO, the former
|
||||
// error/warn/log set).
|
||||
logger: appLogger,
|
||||
});
|
||||
const config = app.get(ConfigService<AppConfig>);
|
||||
const broker = app.get(BrokerClient);
|
||||
@@ -56,6 +62,7 @@ async function bootstrap(): Promise<void> {
|
||||
await processor.drain(DRAIN_MS);
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
await app.close();
|
||||
await shutdownTelemetry();
|
||||
process.exit(0);
|
||||
};
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const)
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { Channel, ConsumeMessage } from 'amqplib';
|
||||
import {
|
||||
ROOT_CONTEXT,
|
||||
SpanKind,
|
||||
SpanStatusCode,
|
||||
trace,
|
||||
type Span,
|
||||
} from '@opentelemetry/api';
|
||||
import { createHash } from 'node:crypto';
|
||||
import os from 'node:os';
|
||||
import type { AttachmentRow } from '../attachments/attachment.js';
|
||||
@@ -8,6 +15,7 @@ import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { retryDelayMs } from '../jobs/backoff.js';
|
||||
import { BrokerClient } from '../jobs/broker.client.js';
|
||||
import { JOB_QUEUE } from '../jobs/broker.topology.js';
|
||||
import {
|
||||
classifyError,
|
||||
JobTimeoutError,
|
||||
@@ -16,7 +24,11 @@ import {
|
||||
type ErrorClass,
|
||||
} from '../jobs/errors.js';
|
||||
import { FaultInjector } from '../jobs/fault-injector.js';
|
||||
import { parseJobMessage, type JobMessage } from '../jobs/job-message.js';
|
||||
import {
|
||||
JOB_TYPE,
|
||||
parseJobMessage,
|
||||
type JobMessage,
|
||||
} from '../jobs/job-message.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import {
|
||||
JobsRepository,
|
||||
@@ -25,6 +37,15 @@ import {
|
||||
} from '../jobs/jobs.repository.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { renderVariants, type RenderedVariant } from './pipelines.js';
|
||||
import { instruments, meter } from '../telemetry/instruments.js';
|
||||
import { appLogger } from '../telemetry/logger.js';
|
||||
import {
|
||||
contextFromTraceFields,
|
||||
errorType,
|
||||
markSpanError,
|
||||
traceFieldsFromHeaders,
|
||||
tracer,
|
||||
} from '../telemetry/tracing.js';
|
||||
|
||||
const SHUTDOWN_DRAIN_MS = 30_000;
|
||||
const VARIANT_EXTENSIONS = [
|
||||
@@ -33,6 +54,31 @@ const VARIANT_EXTENSIONS = [
|
||||
['optimized', 'webp'],
|
||||
] as const;
|
||||
|
||||
/** Bounded outcome set: the `outcome` label of messenger.jobs.processed. */
|
||||
type JobOutcome =
|
||||
| 'succeeded'
|
||||
| 'superseded'
|
||||
| 'lease_lost'
|
||||
| 'retry'
|
||||
| 'failed'
|
||||
| 'commit_failed'
|
||||
| 'poison'
|
||||
| 'claim_failed'
|
||||
| 'skipped'
|
||||
| 'mismatch'
|
||||
| 'crashed';
|
||||
|
||||
// Outcomes where the delivery did not do its job: WARN log, ERROR span status.
|
||||
const PROBLEM_OUTCOMES: ReadonlySet<JobOutcome> = new Set<JobOutcome>([
|
||||
'retry',
|
||||
'failed',
|
||||
'commit_failed',
|
||||
'poison',
|
||||
'claim_failed',
|
||||
'mismatch',
|
||||
'crashed',
|
||||
]);
|
||||
|
||||
function throwIfAborted(signal: AbortSignal): void {
|
||||
if (!signal.aborted) return;
|
||||
throw signal.reason instanceof Error ? signal.reason : new Error('Aborted');
|
||||
@@ -73,9 +119,38 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.broker.consume(
|
||||
this.config.getOrThrow('WORKER_CONCURRENCY'),
|
||||
(message, channel) => this.handle(message, channel),
|
||||
const concurrency = this.config.getOrThrow('WORKER_CONCURRENCY');
|
||||
this.observeSaturation(concurrency);
|
||||
this.broker.consume(concurrency, (message, channel) =>
|
||||
this.handle(message, channel),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saturation of this worker: jobs in flight against WORKER_CONCURRENCY
|
||||
* (= prefetch). Together with messenger.jobs.queue.* it separates «the
|
||||
* worker is slow» from «the worker is full» from «the worker is gone».
|
||||
*/
|
||||
private observeSaturation(concurrency: number): void {
|
||||
const m = meter();
|
||||
const active = m.createObservableUpDownCounter(
|
||||
'messenger.worker.jobs.active',
|
||||
{ unit: '{job}', description: 'Jobs this worker is processing now' },
|
||||
);
|
||||
const capacity = m.createObservableUpDownCounter(
|
||||
'messenger.worker.jobs.capacity',
|
||||
{
|
||||
unit: '{job}',
|
||||
description:
|
||||
'Jobs this worker may process at once (WORKER_CONCURRENCY)',
|
||||
},
|
||||
);
|
||||
m.addBatchObservableCallback(
|
||||
(result) => {
|
||||
result.observe(active, this.active);
|
||||
result.observe(capacity, concurrency);
|
||||
},
|
||||
[active, capacity],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -88,15 +163,104 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* One CONSUMER span per delivery. Its parent is the PRODUCER span from the
|
||||
* AMQP headers; messages without trace headers (published before lab6 or
|
||||
* replayed raw) fall back to the trace context stored in the body.
|
||||
*/
|
||||
private async handle(
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
): Promise<void> {
|
||||
const parsed = parseJobMessage(message.content);
|
||||
const fromHeaders = traceFieldsFromHeaders(
|
||||
message.properties.headers as Record<string, unknown> | undefined,
|
||||
);
|
||||
const parent = fromHeaders.traceparent
|
||||
? contextFromTraceFields(fromHeaders)
|
||||
: contextFromTraceFields(parsed, ROOT_CONTEXT);
|
||||
const started = process.hrtime.bigint();
|
||||
await tracer().startActiveSpan(
|
||||
`process ${JOB_QUEUE}`,
|
||||
{
|
||||
kind: SpanKind.CONSUMER,
|
||||
attributes: {
|
||||
'messaging.system': 'rabbitmq',
|
||||
'messaging.operation.type': 'process',
|
||||
'messaging.destination.name': JOB_QUEUE,
|
||||
'messaging.rabbitmq.message.redelivered':
|
||||
message.fields.redelivered,
|
||||
...(parsed
|
||||
? {
|
||||
'messaging.message.id': parsed.job_id,
|
||||
'messenger.job.id': parsed.job_id,
|
||||
'messenger.attachment.id': parsed.attachment_id,
|
||||
'messenger.job.attempt': parsed.attempt,
|
||||
'messenger.pipeline.version': parsed.pipeline_version,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
parent,
|
||||
async (span: Span) => {
|
||||
let outcome: JobOutcome = 'crashed';
|
||||
try {
|
||||
outcome = await this.dispatch(parsed, message, channel);
|
||||
} catch (error) {
|
||||
markSpanError(span, error);
|
||||
throw error;
|
||||
} finally {
|
||||
this.record(span, parsed, outcome, started);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private record(
|
||||
span: Span,
|
||||
parsed: JobMessage | undefined,
|
||||
outcome: JobOutcome,
|
||||
started: bigint,
|
||||
): void {
|
||||
const seconds = Number(process.hrtime.bigint() - started) / 1e9;
|
||||
const labels = { 'job.type': JOB_TYPE, outcome };
|
||||
instruments().jobsProcessed.add(1, labels);
|
||||
instruments().jobDuration.record(seconds, labels);
|
||||
if (outcome === 'succeeded' && parsed) {
|
||||
// From durable acceptance (job created_at) to the ready commit:
|
||||
// queueing, retries and backoff included.
|
||||
const ready = (Date.now() - Date.parse(parsed.created_at)) / 1000;
|
||||
if (Number.isFinite(ready) && ready >= 0)
|
||||
instruments().timeToReady.record(ready, { 'job.type': JOB_TYPE });
|
||||
}
|
||||
span.setAttribute('messenger.job.outcome', outcome);
|
||||
const problem = PROBLEM_OUTCOMES.has(outcome);
|
||||
if (problem && outcome !== 'crashed')
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: outcome });
|
||||
appLogger.event(
|
||||
problem ? 'WARN' : 'INFO',
|
||||
'job.processed',
|
||||
`job ${outcome}`,
|
||||
{
|
||||
'messenger.job.id': parsed?.job_id,
|
||||
'messenger.attachment.id': parsed?.attachment_id,
|
||||
'messenger.job.attempt': parsed?.attempt,
|
||||
'messenger.job.outcome': outcome,
|
||||
duration_ms: Math.round(seconds * 1000),
|
||||
},
|
||||
);
|
||||
span.end();
|
||||
}
|
||||
|
||||
private async dispatch(
|
||||
parsed: JobMessage | undefined,
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
): Promise<JobOutcome> {
|
||||
if (!parsed) {
|
||||
this.metrics.increment('worker.messages.poison');
|
||||
this.safeNack(channel, message, false);
|
||||
return;
|
||||
return 'poison';
|
||||
}
|
||||
|
||||
const claim = await this.claimWithRetry(parsed);
|
||||
@@ -105,21 +269,24 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
// Do not burn the delivery limit in a tight nack loop while the
|
||||
// database is briefly unavailable.
|
||||
this.safeNackAfter(channel, message, true, 5000);
|
||||
return;
|
||||
return 'claim_failed';
|
||||
}
|
||||
if (claim.kind === 'skip') {
|
||||
if (claim.reason === 'mismatch') {
|
||||
this.safeNack(channel, message, false);
|
||||
return;
|
||||
return 'mismatch';
|
||||
}
|
||||
this.metrics.increment(`worker.jobs.skipped.${claim.reason}`);
|
||||
trace
|
||||
.getActiveSpan()
|
||||
?.setAttribute('messenger.job.skip_reason', claim.reason);
|
||||
this.safeAck(channel, message);
|
||||
return;
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
this.begin();
|
||||
try {
|
||||
await this.process(
|
||||
return await this.process(
|
||||
claim.job,
|
||||
claim.attachment,
|
||||
claim.leaseToken,
|
||||
@@ -142,7 +309,7 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
leaseToken: string,
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
): Promise<void> {
|
||||
): Promise<JobOutcome> {
|
||||
this.metrics.increment('worker.jobs.claimed');
|
||||
const controller = new AbortController();
|
||||
const target = {
|
||||
@@ -170,6 +337,7 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
Math.max(1000, (this.leaseSeconds * 1000) / 3),
|
||||
);
|
||||
let committed = false;
|
||||
let committedOutcome: JobOutcome = 'succeeded';
|
||||
|
||||
try {
|
||||
await this.faults.point(
|
||||
@@ -222,6 +390,7 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
variants.map((variant) => this.variantWrite(job, variant)),
|
||||
);
|
||||
committed = true;
|
||||
committedOutcome = result;
|
||||
this.metrics.increment(`worker.jobs.${result}`);
|
||||
await this.faults.point(
|
||||
'worker.after_commit_before_ack',
|
||||
@@ -235,13 +404,17 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
} else if (result === 'superseded') {
|
||||
await this.cleanupAttemptObjects(job, null);
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (committed) {
|
||||
// The durable effect is already committed; a failure at the ack
|
||||
// fault point or a lost reply must not turn into a retry.
|
||||
this.safeAck(channel, message);
|
||||
return;
|
||||
return committedOutcome;
|
||||
}
|
||||
// Only the error type/code reaches the span: driver messages may
|
||||
// carry request values.
|
||||
trace.getActiveSpan()?.setAttribute('error.type', errorType(error));
|
||||
const failure = this.classify(error, controller.signal);
|
||||
const delay = retryDelayMs(
|
||||
job.attempt,
|
||||
@@ -260,12 +433,14 @@ export class JobProcessor implements OnApplicationBootstrap {
|
||||
this.safeAck(channel, message);
|
||||
if (outcome === 'retry' || outcome === 'failed')
|
||||
await this.cleanupAttemptObjects(job, null);
|
||||
return outcome;
|
||||
} catch (commitError) {
|
||||
this.metrics.increment('worker.jobs.commit_failed');
|
||||
this.logger.warn(
|
||||
`Failure commit failed: ${commitError instanceof Error ? commitError.message : 'unknown error'}`,
|
||||
);
|
||||
setTimeout(() => this.safeNack(channel, message, true), 1000);
|
||||
return 'commit_failed';
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
|
||||
Reference in New Issue
Block a user