import { Logger } from '@nestjs/common'; import { Channel, ConsumeMessage } from 'amqplib'; import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker'; import { PrismaService } from './db/prisma.service'; import { parseJobMessage } from './jobs/job-message'; import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor'; import { S3Service } from './storage/s3.service'; const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2); const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000); const DRAIN_TIMEOUT_MS = Number(process.env.WORKER_DRAIN_TIMEOUT_MS ?? 25000); const FAULT = process.env.WORKER_FAULT; const logger = new Logger('Worker'); const prisma = new PrismaService(); const s3 = new S3Service(); const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks()); let stopping = false; let inFlight = 0; function faultHooks(): ProcessorHooks { const crash = (point: string) => async () => { logger.warn(`fault injection: worker killed ${point}`); process.exit(137); }; if (FAULT === 'exit-after-store') { return { afterStore: crash('after storing objects, before commit and ack') }; } if (FAULT === 'exit-after-commit') { return { afterCommit: crash('after commit, before ack') }; } return {}; } async function handle(channel: Channel, delivery: ConsumeMessage): Promise { inFlight++; try { const message = parseJobMessage(delivery.content); if (!message) { logger.warn('malformed job message moved to dead letter queue'); channel.nack(delivery, false, false); return; } const outcome = await processor.handle(message); logger.log( `job ${message.job_id} attempt ${message.attempt}` + `${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`, ); channel.ack(delivery); } catch (error) { const reason = error instanceof Error ? error.name : 'unknown error'; logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`); await sleep(INFRA_RETRY_MS); try { channel.nack(delivery, false, true); } catch { logger.warn('channel closed, broker will redeliver unacked message'); } } finally { inFlight--; } } async function run(): Promise { while (!stopping) { const connection = await connectWithRetry(logger, () => stopping); if (!connection) { break; } const closed = new Promise((resolve) => { connection.on('close', () => resolve()); connection.on('error', () => resolve()); }); try { const channel = await connection.createChannel(); await assertTopology(channel); await channel.prefetch(CONCURRENCY); const { consumerTag } = await channel.consume( JOB_QUEUE, (delivery) => { if (delivery) { void handle(channel, delivery); } }, { noAck: false }, ); logger.log(`consuming ${JOB_QUEUE} with prefetch ${CONCURRENCY}`); const stop = new Promise((resolve) => { const timer = setInterval(() => { if (stopping) { clearInterval(timer); resolve(); } }, 200); void closed.then(() => { clearInterval(timer); resolve(); }); }); await stop; if (stopping) { await channel.cancel(consumerTag).catch(() => undefined); const deadline = Date.now() + DRAIN_TIMEOUT_MS; while (inFlight > 0 && Date.now() < deadline) { await sleep(100); } logger.log(`drained, ${inFlight} job(s) left unacked for redelivery`); } } catch (error) { const reason = error instanceof Error ? error.name : 'unknown error'; logger.error(`broker channel failed: ${reason}`); } await connection.close().catch(() => undefined); if (!stopping) { logger.warn('broker connection lost, reconnecting'); await sleep(1000); } } } for (const signal of ['SIGTERM', 'SIGINT'] as const) { process.once(signal, () => { stopping = true; }); } run() .then(async () => { await prisma.$disconnect(); s3.onModuleDestroy(); logger.log('stopped'); process.exit(0); }) .catch((error) => { logger.error(error instanceof Error ? error.message : 'fatal error'); process.exit(1); });