feat: add queue, pool, saturation metrics, slo and alert rules

This commit is contained in:
2026-11-20 13:24:31 +03:00
parent 307b658bfb
commit 3171835fff
10 changed files with 511 additions and 55 deletions
+52
View File
@@ -0,0 +1,52 @@
import { metrics } from '@opentelemetry/api';
import { PrismaClient } from '@prisma/client';
interface Pool {
client: PrismaClient;
max: number;
}
const pools: Pool[] = [];
const meter = metrics.getMeter('messenger-db');
const connections = meter.createObservableGauge('db.client.connection.count', {
description: 'Prisma pool connections by state',
});
const pending = meter.createObservableGauge('db.client.connection.pending_requests', {
description: 'Queries waiting for a free pool connection',
});
const max = meter.createObservableGauge('db.client.connection.max', {
description: 'Configured pool size (connection_limit)',
});
meter.addBatchObservableCallback(
async (result) => {
for (const pool of pools) {
let gauges: Record<string, number>;
try {
const snapshot = await pool.client.$metrics.json();
gauges = Object.fromEntries(snapshot.gauges.map((gauge) => [gauge.key, gauge.value]));
} catch {
continue;
}
result.observe(connections, gauges.prisma_pool_connections_busy ?? 0, {
'db.client.connection.state': 'used',
});
result.observe(connections, gauges.prisma_pool_connections_idle ?? 0, {
'db.client.connection.state': 'idle',
});
result.observe(pending, gauges.prisma_client_queries_wait ?? 0);
result.observe(max, pool.max);
}
},
[connections, pending, max],
);
export function registerDbPool(client: PrismaClient, url: string): void {
let limit = 0;
try {
limit = Number(new URL(url).searchParams.get('connection_limit') ?? 0);
} catch {
limit = 0;
}
pools.push({ client, max: limit });
}
+19 -1
View File
@@ -1,4 +1,4 @@
import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
import { Attributes, Context, Link, metrics, SpanKind } from '@opentelemetry/api';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
@@ -133,6 +133,24 @@ export function startTelemetry(): void {
],
});
sdk.start();
registerProcessMetrics();
}
function registerProcessMetrics(): void {
const meter = metrics.getMeter('messenger-process');
meter
.createObservableCounter('process.cpu.time', { description: 'Process CPU time', unit: 's' })
.addCallback((result) => {
const usage = process.cpuUsage();
result.observe(usage.user / 1e6, { 'cpu.mode': 'user' });
result.observe(usage.system / 1e6, { 'cpu.mode': 'system' });
});
meter
.createObservableGauge('process.memory.usage', { description: 'Resident set size', unit: 'By' })
.addCallback((result) => result.observe(process.memoryUsage.rss()));
meter
.createObservableGauge('process.memory.heap_used', { description: 'V8 heap used', unit: 'By' })
.addCallback((result) => result.observe(process.memoryUsage().heapUsed));
}
export async function shutdownTelemetry(): Promise<void> {