import { Logger } from '@nestjs/common'; import { ConfirmChannel } from 'amqplib'; import { assertTopology, connectWithRetry, JOB_QUEUE, sleep } from './broker/broker'; import { PrismaService } from './db/prisma.service'; import { JobMessage } from './jobs/job-message'; import { recoverExpiredLeases, republishStale } from './jobs/job-store'; const BATCH_SIZE = 100; const POLL_MS = Number(process.env.RELAY_POLL_MS ?? 500); const SWEEP_MS = Number(process.env.RELAY_SWEEP_MS ?? 5000); const STALE_MS = Number(process.env.JOB_STALE_SECONDS ?? 300) * 1000; const FAULT = process.env.RELAY_FAULT; const logger = new Logger('OutboxRelay'); const prisma = new PrismaService(); let stopping = false; async function publishBatch(channel: ConfirmChannel): Promise { return prisma.$transaction( async (tx) => { const rows = await tx.$queryRaw<{ id: bigint; payload: JobMessage }[]>` SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= ${new Date()} ORDER BY id LIMIT ${BATCH_SIZE} FOR UPDATE SKIP LOCKED`; if (rows.length === 0) { return 0; } for (const { payload } of rows) { channel.sendToQueue(JOB_QUEUE, Buffer.from(JSON.stringify(payload)), { persistent: true, contentType: 'application/json', messageId: payload.job_id, type: payload.type, }); } await channel.waitForConfirms(); logger.log(`published ${rows.length} job message(s), broker confirmed`); if (FAULT === 'exit-after-confirm') { logger.warn('fault injection: exit after broker confirm, before outbox is marked'); process.exit(137); } await tx.outboxEntry.updateMany({ where: { id: { in: rows.map((row) => row.id) } }, data: { publishedAt: new Date(), publishAttempts: { increment: 1 } }, }); return rows.length; }, { timeout: 15000 }, ); } async function sweep(): Promise { const recovered = await recoverExpiredLeases(prisma); const republished = await republishStale(prisma, STALE_MS); if (recovered + republished > 0) { logger.log(`lease expired: ${recovered}, stale queued republished: ${republished}`); } } async function run(): Promise { let lastSweep = 0; while (!stopping) { const connection = await connectWithRetry(logger, () => stopping); if (!connection) { break; } let closed = false; connection.on('close', () => { closed = true; }); connection.on('error', () => { closed = true; }); try { const channel = await connection.createConfirmChannel(); channel.on('error', () => { closed = true; }); channel.on('close', () => { closed = true; }); await assertTopology(channel); while (!stopping && !closed) { let published = 0; try { published = await publishBatch(channel); if (Date.now() - lastSweep >= SWEEP_MS) { await sweep(); lastSweep = Date.now(); } } catch (error) { const reason = error instanceof Error ? error.name : 'unknown error'; logger.error(`relay pass failed: ${reason}`); await sleep(1000); } if (published < BATCH_SIZE) { await sleep(POLL_MS); } } } catch (error) { const reason = error instanceof Error ? error.name : 'unknown error'; logger.error(`broker channel failed: ${reason}`); } await connection.close().catch(() => undefined); if (!stopping) { logger.warn('broker connection lost, reconnecting'); await sleep(1000); } } } for (const signal of ['SIGTERM', 'SIGINT'] as const) { process.once(signal, () => { stopping = true; }); } run() .then(async () => { await prisma.$disconnect(); logger.log('stopped'); process.exit(0); }) .catch((error) => { logger.error(error instanceof Error ? error.message : 'fatal error'); process.exit(1); });