forked from mtusi2026/mtusi2026lab6
130 lines
3.7 KiB
TypeScript
130 lines
3.7 KiB
TypeScript
import { Logger } from '@nestjs/common';
|
|
import { ConfirmChannel } from 'amqplib';
|
|
import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker';
|
|
import { PrismaService } from './db/prisma.service';
|
|
import { JobMessage } from './jobs/job-message';
|
|
import { recoverExpiredLeases, republishStale } from './jobs/job-store';
|
|
|
|
const BATCH_SIZE = 100;
|
|
const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500);
|
|
const SWEEP_MS = Number(process.env.RELAY_SWEEP_MS ?? 5000);
|
|
const STALE_MS = Number(process.env.JOB_STALE_SECONDS ?? 300) * 1000;
|
|
const FAULT = process.env.RELAY_FAULT;
|
|
|
|
const logger = new Logger('OutboxRelay');
|
|
const prisma = new PrismaService();
|
|
let stopping = false;
|
|
|
|
async function publishBatch(channel: ConfirmChannel): Promise<number> {
|
|
return prisma.$transaction(
|
|
async (tx) => {
|
|
const rows = await tx.$queryRaw<{ id: bigint; payload: JobMessage }[]>`
|
|
SELECT id, payload FROM outbox
|
|
WHERE published_at IS NULL AND available_at <= ${new Date()}
|
|
ORDER BY id
|
|
LIMIT ${BATCH_SIZE}
|
|
FOR UPDATE SKIP LOCKED`;
|
|
if (rows.length === 0) {
|
|
return 0;
|
|
}
|
|
for (const { payload } of rows) {
|
|
channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), {
|
|
persistent: true,
|
|
contentType: 'application/json',
|
|
messageId: payload.job_id,
|
|
type: payload.type,
|
|
});
|
|
}
|
|
await channel.waitForConfirms();
|
|
logger.log(`published ${rows.length} job message(s), broker confirmed`);
|
|
if (FAULT === 'exit-after-confirm') {
|
|
logger.warn('fault injection: exit after broker confirm, before outbox is marked');
|
|
process.exit(137);
|
|
}
|
|
await tx.outboxEntry.updateMany({
|
|
where: { id: { in: rows.map((row) => row.id) } },
|
|
data: { publishedAt: new Date(), publishAttempts: { increment: 1 } },
|
|
});
|
|
return rows.length;
|
|
},
|
|
{ timeout: 15000 },
|
|
);
|
|
}
|
|
|
|
async function sweep(): Promise<void> {
|
|
const recovered = await recoverExpiredLeases(prisma);
|
|
const republished = await republishStale(prisma, STALE_MS);
|
|
if (recovered + republished > 0) {
|
|
logger.log(`lease expired: ${recovered}, stale queued republished: ${republished}`);
|
|
}
|
|
}
|
|
|
|
async function run(): Promise<void> {
|
|
let lastSweep = 0;
|
|
while (!stopping) {
|
|
const connection = await connectWithRetry(logger, () => stopping);
|
|
if (!connection) {
|
|
break;
|
|
}
|
|
let closed = false;
|
|
connection.on('close', () => {
|
|
closed = true;
|
|
});
|
|
connection.on('error', () => {
|
|
closed = true;
|
|
});
|
|
try {
|
|
const channel = await connection.createConfirmChannel();
|
|
channel.on('error', () => {
|
|
closed = true;
|
|
});
|
|
channel.on('close', () => {
|
|
closed = true;
|
|
});
|
|
await assertTopology(channel);
|
|
while (!stopping && !closed) {
|
|
let published = 0;
|
|
try {
|
|
published = await publishBatch(channel);
|
|
if (Date.now() - lastSweep >= SWEEP_MS) {
|
|
await sweep();
|
|
lastSweep = Date.now();
|
|
}
|
|
} catch (error) {
|
|
const reason = error instanceof Error ? error.name : 'unknown error';
|
|
logger.error(`relay pass failed: ${reason}`);
|
|
await sleep(1000);
|
|
}
|
|
if (published < BATCH_SIZE) {
|
|
await sleep(POLL_MS);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
const reason = error instanceof Error ? error.name : 'unknown error';
|
|
logger.error(`broker channel failed: ${reason}`);
|
|
}
|
|
await connection.close().catch(() => undefined);
|
|
if (!stopping) {
|
|
logger.warn('broker connection lost, reconnecting');
|
|
await sleep(1000);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
|
process.once(signal, () => {
|
|
stopping = true;
|
|
});
|
|
}
|
|
|
|
run()
|
|
.then(async () => {
|
|
await prisma.$disconnect();
|
|
logger.log('stopped');
|
|
process.exit(0);
|
|
})
|
|
.catch((error) => {
|
|
logger.error(error instanceof Error ? error.message : 'fatal error');
|
|
process.exit(1);
|
|
});
|