Files
mtusi2026lab5/src/worker.ts
T

138 lines
4.1 KiB
TypeScript

import { Logger } from '@nestjs/common';
import { Channel, ConsumeMessage } from 'amqplib';
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
import { PrismaService } from './db/prisma.service';
import { parseJobMessage } from './jobs/job-message';
import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor';
import { S3Service } from './storage/s3.service';
const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2);
const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000);
const DRAIN_TIMEOUT_MS = Number(process.env.WORKER_DRAIN_TIMEOUT_MS ?? 25000);
const FAULT = process.env.WORKER_FAULT;
const logger = new Logger('Worker');
const prisma = new PrismaService();
const s3 = new S3Service();
const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks());
let stopping = false;
let inFlight = 0;
function faultHooks(): ProcessorHooks {
const crash = (point: string) => async () => {
logger.warn(`fault injection: worker killed ${point}`);
process.exit(137);
};
if (FAULT === 'exit-after-store') {
return { afterStore: crash('after storing objects, before commit and ack') };
}
if (FAULT === 'exit-after-commit') {
return { afterCommit: crash('after commit, before ack') };
}
return {};
}
async function handle(channel: Channel, delivery: ConsumeMessage): Promise<void> {
inFlight++;
try {
const message = parseJobMessage(delivery.content);
if (!message) {
logger.warn('malformed job message moved to dead letter queue');
channel.nack(delivery, false, false);
return;
}
const outcome = await processor.handle(message);
logger.log(
`job ${message.job_id} attempt ${message.attempt}` +
`${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`,
);
channel.ack(delivery);
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`);
await sleep(INFRA_RETRY_MS);
try {
channel.nack(delivery, false, true);
} catch {
logger.warn('channel closed, broker will redeliver unacked message');
}
} finally {
inFlight--;
}
}
async function run(): Promise<void> {
while (!stopping) {
const connection = await connectWithRetry(logger, () => stopping);
if (!connection) {
break;
}
const closed = new Promise<void>((resolve) => {
connection.on('close', () => resolve());
connection.on('error', () => resolve());
});
try {
const channel = await connection.createChannel();
await assertTopology(channel);
await channel.prefetch(CONCURRENCY);
const { consumerTag } = await channel.consume(
JOB_QUEUE,
(delivery) => {
if (delivery) {
void handle(channel, delivery);
}
},
{ noAck: false },
);
logger.log(`consuming ${JOB_QUEUE} with prefetch ${CONCURRENCY}`);
const stop = new Promise<void>((resolve) => {
const timer = setInterval(() => {
if (stopping) {
clearInterval(timer);
resolve();
}
}, 200);
void closed.then(() => {
clearInterval(timer);
resolve();
});
});
await stop;
if (stopping) {
await channel.cancel(consumerTag).catch(() => undefined);
const deadline = Date.now() + DRAIN_TIMEOUT_MS;
while (inFlight > 0 && Date.now() < deadline) {
await sleep(100);
}
logger.log(`drained, ${inFlight} job(s) left unacked for redelivery`);
}
} catch (error) {
const reason = error instanceof Error ? error.name : 'unknown error';
logger.error(`broker channel failed: ${reason}`);
}
await connection.close().catch(() => undefined);
if (!stopping) {
logger.warn('broker connection lost, reconnecting');
await sleep(1000);
}
}
}
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
process.once(signal, () => {
stopping = true;
});
}
run()
.then(async () => {
await prisma.$disconnect();
s3.onModuleDestroy();
logger.log('stopped');
process.exit(0);
})
.catch((error) => {
logger.error(error instanceof Error ? error.message : 'fatal error');
process.exit(1);
});