feat: add queue, pool, saturation metrics, slo and alert rules

This commit is contained in:
2026-11-20 13:24:31 +03:00
parent 307b658bfb
commit 3171835fff
10 changed files with 511 additions and 55 deletions
+4 -1
View File
@@ -1,13 +1,16 @@
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { Prisma, PrismaClient } from '@prisma/client';
import { readFileSync } from 'fs';
import { registerDbPool } from '../telemetry/db-pool';
const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']);
@Injectable()
export class PrismaService extends PrismaClient implements OnModuleDestroy {
constructor() {
super({ datasourceUrl: readDatabaseUrl() });
const url = readDatabaseUrl();
super({ datasourceUrl: url });
registerDbPool(this, url);
}
async onModuleDestroy(): Promise<void> {
+12 -2
View File
@@ -1,5 +1,5 @@
import { Logger } from '@nestjs/common';
import { SpanStatusCode, trace } from '@opentelemetry/api';
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import {
@@ -7,7 +7,7 @@ import {
ObjectTooLargeError,
S3Service,
} from '../storage/s3.service';
import { JobMessage, PIPELINE_VERSION } from './job-message';
import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message';
import { markFailed, retryOrFail } from './job-store';
import {
InvalidImageError,
@@ -52,6 +52,13 @@ class PermanentError extends Error {
class TimeoutError extends Error {}
const tracer = trace.getTracer('messenger-worker');
const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', {
description: 'From durable upload acceptance to the committed ready state',
unit: 's',
advice: {
explicitBucketBoundaries: [0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600],
},
});
export class JobProcessor {
private readonly logger = new Logger('JobProcessor');
@@ -225,6 +232,9 @@ export class JobProcessor {
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
return 'skipped';
}
timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, {
'job.type': JOB_TYPE,
});
await this.hooks.afterCommit?.();
await this.removeStaging(attachment);
return 'ready';
+43 -1
View File
@@ -1,6 +1,14 @@
import './telemetry/register';
import { Logger } from '@nestjs/common';
import { propagation, ROOT_CONTEXT, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
import {
metrics,
propagation,
ROOT_CONTEXT,
Span,
SpanKind,
SpanStatusCode,
trace,
} from '@opentelemetry/api';
import { ConfirmChannel } from 'amqplib';
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
import { PrismaService } from './db/prisma.service';
@@ -22,6 +30,40 @@ const prisma = new PrismaService();
const tracer = trace.getTracer('messenger-relay');
let stopping = false;
const meter = metrics.getMeter('messenger-relay');
const queueDepth = meter.createObservableGauge('jobs.queue.depth', {
description: 'Attachment jobs by state (source of truth: attachment_jobs)',
});
const oldestAge = meter.createObservableGauge('jobs.queue.oldest_age', {
description: 'Age of the oldest due queued job (since accepted or retry became due)',
unit: 's',
});
meter.addBatchObservableCallback(
async (result) => {
const [row] = await prisma.$queryRaw<
{ queued: number; due: number; processing: number; oldest: number }[]
>`
SELECT
count(*) FILTER (WHERE status = 'queued')::int AS queued,
count(*) FILTER (WHERE status = 'queued' AND next_attempt_at <= now())::int AS due,
count(*) FILTER (WHERE status = 'processing')::int AS processing,
COALESCE(EXTRACT(EPOCH FROM now() - min(next_attempt_at)
FILTER (WHERE status = 'queued' AND next_attempt_at <= now())), 0)::float8 AS oldest
FROM attachment_jobs
WHERE status IN ('queued', 'processing')`.catch(() => []);
if (!row) {
return;
}
const queue = { 'messaging.destination.name': JOB_QUEUE };
result.observe(queueDepth, row.queued, { ...queue, state: 'queued' });
result.observe(queueDepth, row.due, { ...queue, state: 'due' });
result.observe(queueDepth, row.processing, { ...queue, state: 'processing' });
result.observe(oldestAge, row.oldest, queue);
},
[queueDepth, oldestAge],
);
function startPublishSpan(payload: JobMessage, headers: Record<string, string>): Span {
const parent = propagation.extract(ROOT_CONTEXT, payload);
const span = tracer.startSpan(
+52
View File
@@ -0,0 +1,52 @@
import { metrics } from '@opentelemetry/api';
import { PrismaClient } from '@prisma/client';
interface Pool {
client: PrismaClient;
max: number;
}
const pools: Pool[] = [];
const meter = metrics.getMeter('messenger-db');
const connections = meter.createObservableGauge('db.client.connection.count', {
description: 'Prisma pool connections by state',
});
const pending = meter.createObservableGauge('db.client.connection.pending_requests', {
description: 'Queries waiting for a free pool connection',
});
const max = meter.createObservableGauge('db.client.connection.max', {
description: 'Configured pool size (connection_limit)',
});
meter.addBatchObservableCallback(
async (result) => {
for (const pool of pools) {
let gauges: Record<string, number>;
try {
const snapshot = await pool.client.$metrics.json();
gauges = Object.fromEntries(snapshot.gauges.map((gauge) => [gauge.key, gauge.value]));
} catch {
continue;
}
result.observe(connections, gauges.prisma_pool_connections_busy ?? 0, {
'db.client.connection.state': 'used',
});
result.observe(connections, gauges.prisma_pool_connections_idle ?? 0, {
'db.client.connection.state': 'idle',
});
result.observe(pending, gauges.prisma_client_queries_wait ?? 0);
result.observe(max, pool.max);
}
},
[connections, pending, max],
);
export function registerDbPool(client: PrismaClient, url: string): void {
let limit = 0;
try {
limit = Number(new URL(url).searchParams.get('connection_limit') ?? 0);
} catch {
limit = 0;
}
pools.push({ client, max: limit });
}
+19 -1
View File
@@ -1,4 +1,4 @@
import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
import { Attributes, Context, Link, metrics, SpanKind } from '@opentelemetry/api';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
@@ -133,6 +133,24 @@ export function startTelemetry(): void {
],
});
sdk.start();
registerProcessMetrics();
}
function registerProcessMetrics(): void {
const meter = metrics.getMeter('messenger-process');
meter
.createObservableCounter('process.cpu.time', { description: 'Process CPU time', unit: 's' })
.addCallback((result) => {
const usage = process.cpuUsage();
result.observe(usage.user / 1e6, { 'cpu.mode': 'user' });
result.observe(usage.system / 1e6, { 'cpu.mode': 'system' });
});
meter
.createObservableGauge('process.memory.usage', { description: 'Resident set size', unit: 'By' })
.addCallback((result) => result.observe(process.memoryUsage.rss()));
meter
.createObservableGauge('process.memory.heap_used', { description: 'V8 heap used', unit: 'By' })
.addCallback((result) => result.observe(process.memoryUsage().heapUsed));
}
export async function shutdownTelemetry(): Promise<void> {
+6
View File
@@ -45,6 +45,12 @@ const jobDuration = meter.createHistogram('jobs.duration', {
unit: 's',
advice: { explicitBucketBoundaries: JOB_BUCKETS },
});
meter
.createObservableGauge('worker.jobs.busy', { description: 'Deliveries being handled now' })
.addCallback((result) => result.observe(inFlight));
meter
.createObservableGauge('worker.jobs.capacity', { description: 'Concurrent deliveries allowed (prefetch)' })
.addCallback((result) => result.observe(CONCURRENCY));
function parentContext(delivery: ConsumeMessage, message: JobMessage | null): Context {
const headers = delivery.properties.headers ?? {};