forked from mtusi2026/mtusi2026lab6
feat: add queue, pool, saturation metrics, slo and alert rules
This commit is contained in:
@@ -1,13 +1,16 @@
|
||||
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { Prisma, PrismaClient } from '@prisma/client';
|
||||
import { readFileSync } from 'fs';
|
||||
import { registerDbPool } from '../telemetry/db-pool';
|
||||
|
||||
const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']);
|
||||
|
||||
@Injectable()
|
||||
export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||
constructor() {
|
||||
super({ datasourceUrl: readDatabaseUrl() });
|
||||
const url = readDatabaseUrl();
|
||||
super({ datasourceUrl: url });
|
||||
registerDbPool(this, url);
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
|
||||
+12
-2
@@ -1,5 +1,5 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import {
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
ObjectTooLargeError,
|
||||
S3Service,
|
||||
} from '../storage/s3.service';
|
||||
import { JobMessage, PIPELINE_VERSION } from './job-message';
|
||||
import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message';
|
||||
import { markFailed, retryOrFail } from './job-store';
|
||||
import {
|
||||
InvalidImageError,
|
||||
@@ -52,6 +52,13 @@ class PermanentError extends Error {
|
||||
class TimeoutError extends Error {}
|
||||
|
||||
const tracer = trace.getTracer('messenger-worker');
|
||||
const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', {
|
||||
description: 'From durable upload acceptance to the committed ready state',
|
||||
unit: 's',
|
||||
advice: {
|
||||
explicitBucketBoundaries: [0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600],
|
||||
},
|
||||
});
|
||||
|
||||
export class JobProcessor {
|
||||
private readonly logger = new Logger('JobProcessor');
|
||||
@@ -225,6 +232,9 @@ export class JobProcessor {
|
||||
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
|
||||
return 'skipped';
|
||||
}
|
||||
timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, {
|
||||
'job.type': JOB_TYPE,
|
||||
});
|
||||
await this.hooks.afterCommit?.();
|
||||
await this.removeStaging(attachment);
|
||||
return 'ready';
|
||||
|
||||
+43
-1
@@ -1,6 +1,14 @@
|
||||
import './telemetry/register';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { propagation, ROOT_CONTEXT, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import {
|
||||
metrics,
|
||||
propagation,
|
||||
ROOT_CONTEXT,
|
||||
Span,
|
||||
SpanKind,
|
||||
SpanStatusCode,
|
||||
trace,
|
||||
} from '@opentelemetry/api';
|
||||
import { ConfirmChannel } from 'amqplib';
|
||||
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
||||
import { PrismaService } from './db/prisma.service';
|
||||
@@ -22,6 +30,40 @@ const prisma = new PrismaService();
|
||||
const tracer = trace.getTracer('messenger-relay');
|
||||
let stopping = false;
|
||||
|
||||
const meter = metrics.getMeter('messenger-relay');
|
||||
const queueDepth = meter.createObservableGauge('jobs.queue.depth', {
|
||||
description: 'Attachment jobs by state (source of truth: attachment_jobs)',
|
||||
});
|
||||
const oldestAge = meter.createObservableGauge('jobs.queue.oldest_age', {
|
||||
description: 'Age of the oldest due queued job (since accepted or retry became due)',
|
||||
unit: 's',
|
||||
});
|
||||
|
||||
meter.addBatchObservableCallback(
|
||||
async (result) => {
|
||||
const [row] = await prisma.$queryRaw<
|
||||
{ queued: number; due: number; processing: number; oldest: number }[]
|
||||
>`
|
||||
SELECT
|
||||
count(*) FILTER (WHERE status = 'queued')::int AS queued,
|
||||
count(*) FILTER (WHERE status = 'queued' AND next_attempt_at <= now())::int AS due,
|
||||
count(*) FILTER (WHERE status = 'processing')::int AS processing,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - min(next_attempt_at)
|
||||
FILTER (WHERE status = 'queued' AND next_attempt_at <= now())), 0)::float8 AS oldest
|
||||
FROM attachment_jobs
|
||||
WHERE status IN ('queued', 'processing')`.catch(() => []);
|
||||
if (!row) {
|
||||
return;
|
||||
}
|
||||
const queue = { 'messaging.destination.name': JOB_QUEUE };
|
||||
result.observe(queueDepth, row.queued, { ...queue, state: 'queued' });
|
||||
result.observe(queueDepth, row.due, { ...queue, state: 'due' });
|
||||
result.observe(queueDepth, row.processing, { ...queue, state: 'processing' });
|
||||
result.observe(oldestAge, row.oldest, queue);
|
||||
},
|
||||
[queueDepth, oldestAge],
|
||||
);
|
||||
|
||||
function startPublishSpan(payload: JobMessage, headers: Record<string, string>): Span {
|
||||
const parent = propagation.extract(ROOT_CONTEXT, payload);
|
||||
const span = tracer.startSpan(
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { metrics } from '@opentelemetry/api';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
interface Pool {
|
||||
client: PrismaClient;
|
||||
max: number;
|
||||
}
|
||||
|
||||
const pools: Pool[] = [];
|
||||
const meter = metrics.getMeter('messenger-db');
|
||||
const connections = meter.createObservableGauge('db.client.connection.count', {
|
||||
description: 'Prisma pool connections by state',
|
||||
});
|
||||
const pending = meter.createObservableGauge('db.client.connection.pending_requests', {
|
||||
description: 'Queries waiting for a free pool connection',
|
||||
});
|
||||
const max = meter.createObservableGauge('db.client.connection.max', {
|
||||
description: 'Configured pool size (connection_limit)',
|
||||
});
|
||||
|
||||
meter.addBatchObservableCallback(
|
||||
async (result) => {
|
||||
for (const pool of pools) {
|
||||
let gauges: Record<string, number>;
|
||||
try {
|
||||
const snapshot = await pool.client.$metrics.json();
|
||||
gauges = Object.fromEntries(snapshot.gauges.map((gauge) => [gauge.key, gauge.value]));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
result.observe(connections, gauges.prisma_pool_connections_busy ?? 0, {
|
||||
'db.client.connection.state': 'used',
|
||||
});
|
||||
result.observe(connections, gauges.prisma_pool_connections_idle ?? 0, {
|
||||
'db.client.connection.state': 'idle',
|
||||
});
|
||||
result.observe(pending, gauges.prisma_client_queries_wait ?? 0);
|
||||
result.observe(max, pool.max);
|
||||
}
|
||||
},
|
||||
[connections, pending, max],
|
||||
);
|
||||
|
||||
export function registerDbPool(client: PrismaClient, url: string): void {
|
||||
let limit = 0;
|
||||
try {
|
||||
limit = Number(new URL(url).searchParams.get('connection_limit') ?? 0);
|
||||
} catch {
|
||||
limit = 0;
|
||||
}
|
||||
pools.push({ client, max: limit });
|
||||
}
|
||||
+19
-1
@@ -1,4 +1,4 @@
|
||||
import { Attributes, Context, Link, SpanKind } from '@opentelemetry/api';
|
||||
import { Attributes, Context, Link, metrics, SpanKind } from '@opentelemetry/api';
|
||||
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
|
||||
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
@@ -133,6 +133,24 @@ export function startTelemetry(): void {
|
||||
],
|
||||
});
|
||||
sdk.start();
|
||||
registerProcessMetrics();
|
||||
}
|
||||
|
||||
function registerProcessMetrics(): void {
|
||||
const meter = metrics.getMeter('messenger-process');
|
||||
meter
|
||||
.createObservableCounter('process.cpu.time', { description: 'Process CPU time', unit: 's' })
|
||||
.addCallback((result) => {
|
||||
const usage = process.cpuUsage();
|
||||
result.observe(usage.user / 1e6, { 'cpu.mode': 'user' });
|
||||
result.observe(usage.system / 1e6, { 'cpu.mode': 'system' });
|
||||
});
|
||||
meter
|
||||
.createObservableGauge('process.memory.usage', { description: 'Resident set size', unit: 'By' })
|
||||
.addCallback((result) => result.observe(process.memoryUsage.rss()));
|
||||
meter
|
||||
.createObservableGauge('process.memory.heap_used', { description: 'V8 heap used', unit: 'By' })
|
||||
.addCallback((result) => result.observe(process.memoryUsage().heapUsed));
|
||||
}
|
||||
|
||||
export async function shutdownTelemetry(): Promise<void> {
|
||||
|
||||
@@ -45,6 +45,12 @@ const jobDuration = meter.createHistogram('jobs.duration', {
|
||||
unit: 's',
|
||||
advice: { explicitBucketBoundaries: JOB_BUCKETS },
|
||||
});
|
||||
meter
|
||||
.createObservableGauge('worker.jobs.busy', { description: 'Deliveries being handled now' })
|
||||
.addCallback((result) => result.observe(inFlight));
|
||||
meter
|
||||
.createObservableGauge('worker.jobs.capacity', { description: 'Concurrent deliveries allowed (prefetch)' })
|
||||
.addCallback((result) => result.observe(CONCURRENCY));
|
||||
|
||||
function parentContext(delivery: ConsumeMessage, message: JobMessage | null): Context {
|
||||
const headers = delivery.properties.headers ?? {};
|
||||
|
||||
Reference in New Issue
Block a user