// Run through test/run-reconcile.py with distinct writer/scanner credentials. import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; import { randomUUID, createHash } from 'node:crypto'; import { spawn, spawnSync } from 'node:child_process'; import { once } from 'node:events'; import { setTimeout as delay } from 'node:timers/promises'; const require = createRequire(`${process.cwd()}/package.json`); const { PrismaService } = require('./dist/prisma/prisma.service.js'); const { StorageService } = require('./dist/Storage/storage.service.js'); const { ReconcileService } = require('./dist/Attachment/reconcile.service.js'); const db = new PrismaService(); const storage = new StorageService(); const scannerCredentials = JSON.parse( process.env.RECONCILE_TEST_SCANNER_CREDENTIALS, ); delete process.env.RECONCILE_TEST_SCANNER_CREDENTIALS; const writerCredentials = { S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY, }; Object.assign(process.env, scannerCredentials); const scanner = new StorageService(); Object.assign(process.env, writerCredentials); const events = []; const service = new ReconcileService(db, scanner, (event) => events.push(event), ); const bytes = Buffer.from('isolated reconciliation fixture'); const keys = []; let user; let chat; let writer; const originalStat = scanner.stat.bind(scanner); function newKey() { const key = `originals/${chat.id}/${randomUUID()}`; keys.push(key); return key; } function metadata(key) { return { chatId: chat.id, ownerId: user.id, objectKey: key, filename: 'fixture.png', contentType: 'image/png', sizeBytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex'), }; } try { await db.onModuleInit(); user = await db.user.create({ data: { username: `rc_${randomUUID().replaceAll('-', '').slice(0, 20)}`, displayName: 'Reconciliation test', }, }); chat = await db.chat.create({ data: { title: 'Isolated reconciliation fixtures', members: { create: { userId: user.id, position: 0 } }, }, }); const prefix = `originals/${chat.id}/`; const options = { apply: true, graceSeconds: 2, prefix }; const published = newKey(); await storage.put(published, bytes, 'image/png'); await db.attachment.create({ data: metadata(published) }); const orphan = newKey(); // This is a separate Node process with its own DB transaction and S3 client. const writerCode = ` const {PrismaService}=require('./dist/prisma/prisma.service.js'); const {StorageService}=require('./dist/Storage/storage.service.js'); const {tryAttachmentObjectLock,OBJECT_TRANSACTION}=require('./dist/Attachment/object-lock.js'); const db=new PrismaService(); const s3=new StorageService(); (async()=>{ await db.onModuleInit(); await db.$transaction(async tx=>{ if(!await tryAttachmentObjectLock(tx,process.env.RECONCILE_TEST_KEY)) throw Error('lock'); await s3.put(process.env.RECONCILE_TEST_KEY,Buffer.from('orphan fixture'),'image/png'); process.send('written'); await new Promise(()=>{setInterval(()=>{},1000)}); },OBJECT_TRANSACTION); })().catch(()=>process.exit(1));`; writer = spawn(process.execPath, ['-e', writerCode], { cwd: process.cwd(), env: { ...process.env, RECONCILE_TEST_KEY: orphan }, stdio: ['ignore', 'ignore', 'ignore', 'ipc'], }); const written = await Promise.race([ once(writer, 'message').then(([message]) => message), once(writer, 'exit').then(() => { throw new Error('writer exited before PUT'); }), delay(15000, 'timeout', { ref: false }), ]); assert.equal(written, 'written'); // Even when an object looks twenty minutes old, its active writer lock protects it. const whileWriting = await service.run( options, new Date(Date.now() + 1200000), ); assert.equal(whileWriting.lockedObjects, 1); assert.equal(whileWriting.deletedObjects, 0); assert.equal( writer.exitCode, null, 'writer unexpectedly exited before the simulated crash', ); assert.equal(writer.signalCode, null); writer.kill('SIGKILL'); await once(writer, 'exit'); writer = undefined; assert.equal( await db.attachment.findUnique({ where: { objectKey: orphan } }), null, ); const lateReference = newKey(); const renewed = newKey(); await storage.put(lateReference, bytes, 'image/png'); await storage.put(renewed, bytes, 'image/png'); await delay(3100); // An actual refused DB connection must never turn into "metadata absent". const url = new URL(process.env.DATABASE_URL); url.port = '1'; const savedUrl = process.env.DATABASE_URL; process.env.DATABASE_URL = url.toString(); const unavailable = new PrismaService(); process.env.DATABASE_URL = savedUrl; try { await assert.rejects(() => new ReconcileService(unavailable, scanner).run(options), ); assert.ok(await storage.stat(orphan)); } finally { await unavailable.onModuleDestroy(); } const fresh = newKey(); await storage.put(fresh, bytes, 'image/png'); const scanTime = new Date(); let referenced = false; let refreshed = false; scanner.stat = async (key) => { if (key === lateReference && !referenced) { referenced = true; // Simulate a metadata commit between the first DB lookup and the final check. await db.attachment.create({ data: metadata(key) }); } if (key === renewed && !refreshed) { refreshed = true; await storage.put(key, bytes, 'image/png'); } return originalStat(key); }; const dryRun = await service.run({ ...options, apply: false }, scanTime); assert.equal(dryRun.orphanCandidates, 1); assert.equal(dryRun.deletedObjects, 0); assert.ok(await storage.stat(orphan)); const applied = await service.run(options, scanTime); assert.equal(applied.deletedObjects, 1); assert.equal(await storage.stat(orphan), null); for (const key of [published, lateReference, renewed, fresh]) assert.ok(await storage.stat(key)); assert.equal((await service.run(options, scanTime)).deletedObjects, 0); scanner.stat = originalStat; const missing = newKey(); const missingRow = await db.attachment.create({ data: metadata(missing) }); const report = await service.run({ ...options, apply: false }, scanTime); assert.equal(report.missingObjects, 1); assert.ok(await db.attachment.findUnique({ where: { id: missingRow.id } })); assert.ok( events.some( (event) => event.event === 'missing_object' && event.attachmentId === missingRow.id, ), ); const cli = spawnSync( process.execPath, ['dist/Attachment/reconcile.cli.js', '--dry-run', '--prefix', prefix], { encoding: 'utf8', timeout: 30000, env: { ...process.env, ...scannerCredentials }, }, ); assert.equal( cli.status, 2, 'CLI must report missing bytes as an integrity incident', ); const summary = cli.stdout .split('\n') .filter(Boolean) .map((line) => JSON.parse(line)) .find((line) => line.event === 'reconcile_summary'); assert.equal(summary.graceSeconds, 600); assert.equal(summary.mode, 'dry-run'); assert.equal(summary.missingObjects, 1); console.log( 'PASS: real writer crash, DB outage, dry-run, orphan deletion, active/fresh/referenced protection, repeat run and missing-object reporting.', ); } finally { if (writer && writer.exitCode === null && writer.signalCode === null) { writer.kill('SIGKILL'); await once(writer, 'exit'); } scanner.stat = originalStat; for (const key of keys) await storage.remove(key); if (chat) { await db.attachment.deleteMany({ where: { chatId: chat.id } }); await db.chat.delete({ where: { id: chat.id } }); } if (user) await db.user.delete({ where: { id: user.id } }); storage.onModuleDestroy(); scanner.onModuleDestroy(); await db.onModuleDestroy(); }