forked from mtusi2026/mtusi2026lab5
138 lines
4.1 KiB
TypeScript
138 lines
4.1 KiB
TypeScript
import { Logger } from '@nestjs/common';
|
|
import { Channel, ConsumeMessage } from 'amqplib';
|
|
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
|
import { PrismaService } from './db/prisma.service';
|
|
import { parseJobMessage } from './jobs/job-message';
|
|
import { JobProcessor, ProcessorHooks, processorOptions } from './jobs/processor';
|
|
import { S3Service } from './storage/s3.service';
|
|
|
|
const CONCURRENCY = Number(process.env.WORKER_CONCURRENCY ?? 2);
|
|
const INFRA_RETRY_MS = Number(process.env.WORKER_INFRA_RETRY_MS ?? 5000);
|
|
const DRAIN_TIMEOUT_MS = Number(process.env.WORKER_DRAIN_TIMEOUT_MS ?? 25000);
|
|
const FAULT = process.env.WORKER_FAULT;
|
|
|
|
const logger = new Logger('Worker');
|
|
const prisma = new PrismaService();
|
|
const s3 = new S3Service();
|
|
const processor = new JobProcessor(prisma, s3, processorOptions(), faultHooks());
|
|
let stopping = false;
|
|
let inFlight = 0;
|
|
|
|
function faultHooks(): ProcessorHooks {
|
|
const crash = (point: string) => async () => {
|
|
logger.warn(`fault injection: worker killed ${point}`);
|
|
process.exit(137);
|
|
};
|
|
if (FAULT === 'exit-after-store') {
|
|
return { afterStore: crash('after storing objects, before commit and ack') };
|
|
}
|
|
if (FAULT === 'exit-after-commit') {
|
|
return { afterCommit: crash('after commit, before ack') };
|
|
}
|
|
return {};
|
|
}
|
|
|
|
async function handle(channel: Channel, delivery: ConsumeMessage): Promise<void> {
|
|
inFlight++;
|
|
try {
|
|
const message = parseJobMessage(delivery.content);
|
|
if (!message) {
|
|
logger.warn('malformed job message moved to dead letter queue');
|
|
channel.nack(delivery, false, false);
|
|
return;
|
|
}
|
|
const outcome = await processor.handle(message);
|
|
logger.log(
|
|
`job ${message.job_id} attempt ${message.attempt}` +
|
|
`${delivery.fields.redelivered ? ' (redelivered)' : ''}: ${outcome}`,
|
|
);
|
|
channel.ack(delivery);
|
|
} catch (error) {
|
|
const reason = error instanceof Error ? error.name : 'unknown error';
|
|
logger.error(`job handling interrupted (${reason}), returning message after ${INFRA_RETRY_MS} ms`);
|
|
await sleep(INFRA_RETRY_MS);
|
|
try {
|
|
channel.nack(delivery, false, true);
|
|
} catch {
|
|
logger.warn('channel closed, broker will redeliver unacked message');
|
|
}
|
|
} finally {
|
|
inFlight--;
|
|
}
|
|
}
|
|
|
|
async function run(): Promise<void> {
|
|
while (!stopping) {
|
|
const connection = await connectWithRetry(logger, () => stopping);
|
|
if (!connection) {
|
|
break;
|
|
}
|
|
const closed = new Promise<void>((resolve) => {
|
|
connection.on('close', () => resolve());
|
|
connection.on('error', () => resolve());
|
|
});
|
|
try {
|
|
const channel = await connection.createChannel();
|
|
await assertTopology(channel);
|
|
await channel.prefetch(CONCURRENCY);
|
|
const { consumerTag } = await channel.consume(
|
|
JOB_QUEUE,
|
|
(delivery) => {
|
|
if (delivery) {
|
|
void handle(channel, delivery);
|
|
}
|
|
},
|
|
{ noAck: false },
|
|
);
|
|
logger.log(`consuming ${JOB_QUEUE} with prefetch ${CONCURRENCY}`);
|
|
const stop = new Promise<void>((resolve) => {
|
|
const timer = setInterval(() => {
|
|
if (stopping) {
|
|
clearInterval(timer);
|
|
resolve();
|
|
}
|
|
}, 200);
|
|
void closed.then(() => {
|
|
clearInterval(timer);
|
|
resolve();
|
|
});
|
|
});
|
|
await stop;
|
|
if (stopping) {
|
|
await channel.cancel(consumerTag).catch(() => undefined);
|
|
const deadline = Date.now() + DRAIN_TIMEOUT_MS;
|
|
while (inFlight > 0 && Date.now() < deadline) {
|
|
await sleep(100);
|
|
}
|
|
logger.log(`drained, ${inFlight} job(s) left unacked for redelivery`);
|
|
}
|
|
} catch (error) {
|
|
const reason = error instanceof Error ? error.name : 'unknown error';
|
|
logger.error(`broker channel failed: ${reason}`);
|
|
}
|
|
await connection.close().catch(() => undefined);
|
|
if (!stopping) {
|
|
logger.warn('broker connection lost, reconnecting');
|
|
await sleep(1000);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
|
process.once(signal, () => {
|
|
stopping = true;
|
|
});
|
|
}
|
|
|
|
run()
|
|
.then(async () => {
|
|
await prisma.$disconnect();
|
|
s3.onModuleDestroy();
|
|
logger.log('stopped');
|
|
process.exit(0);
|
|
})
|
|
.catch((error) => {
|
|
logger.error(error instanceof Error ? error.message : 'fatal error');
|
|
process.exit(1);
|
|
});
|