forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
225 lines
7.8 KiB
JavaScript
225 lines
7.8 KiB
JavaScript
// Run through test/run-reconcile.py with distinct writer/scanner credentials.
|
|
import assert from 'node:assert/strict';
|
|
import { createRequire } from 'node:module';
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { spawn, spawnSync } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
|
|
const require = createRequire(`${process.cwd()}/package.json`);
|
|
const { PrismaService } = require('./dist/prisma/prisma.service.js');
|
|
const { StorageService } = require('./dist/Storage/storage.service.js');
|
|
const { ReconcileService } = require('./dist/Attachment/reconcile.service.js');
|
|
const db = new PrismaService();
|
|
const storage = new StorageService();
|
|
const scannerCredentials = JSON.parse(
|
|
process.env.RECONCILE_TEST_SCANNER_CREDENTIALS,
|
|
);
|
|
delete process.env.RECONCILE_TEST_SCANNER_CREDENTIALS;
|
|
const writerCredentials = {
|
|
S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID,
|
|
S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY,
|
|
};
|
|
Object.assign(process.env, scannerCredentials);
|
|
const scanner = new StorageService();
|
|
Object.assign(process.env, writerCredentials);
|
|
const events = [];
|
|
const service = new ReconcileService(db, scanner, (event) =>
|
|
events.push(event),
|
|
);
|
|
const bytes = Buffer.from('isolated reconciliation fixture');
|
|
const keys = [];
|
|
let user;
|
|
let chat;
|
|
let writer;
|
|
const originalStat = scanner.stat.bind(scanner);
|
|
|
|
function newKey() {
|
|
const key = `originals/${chat.id}/${randomUUID()}`;
|
|
keys.push(key);
|
|
return key;
|
|
}
|
|
|
|
function metadata(key) {
|
|
return {
|
|
chatId: chat.id,
|
|
ownerId: user.id,
|
|
objectKey: key,
|
|
filename: 'fixture.png',
|
|
contentType: 'image/png',
|
|
sizeBytes: bytes.length,
|
|
sha256: createHash('sha256').update(bytes).digest('hex'),
|
|
};
|
|
}
|
|
|
|
try {
|
|
await db.onModuleInit();
|
|
user = await db.user.create({
|
|
data: {
|
|
username: `rc_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
|
displayName: 'Reconciliation test',
|
|
},
|
|
});
|
|
chat = await db.chat.create({
|
|
data: {
|
|
title: 'Isolated reconciliation fixtures',
|
|
members: { create: { userId: user.id, position: 0 } },
|
|
},
|
|
});
|
|
const prefix = `originals/${chat.id}/`;
|
|
const options = { apply: true, graceSeconds: 2, prefix };
|
|
const published = newKey();
|
|
await storage.put(published, bytes, 'image/png');
|
|
await db.attachment.create({ data: metadata(published) });
|
|
const orphan = newKey();
|
|
// This is a separate Node process with its own DB transaction and S3 client.
|
|
const writerCode = `
|
|
const {PrismaService}=require('./dist/prisma/prisma.service.js');
|
|
const {StorageService}=require('./dist/Storage/storage.service.js');
|
|
const {tryAttachmentObjectLock,OBJECT_TRANSACTION}=require('./dist/Attachment/object-lock.js');
|
|
const db=new PrismaService(); const s3=new StorageService();
|
|
(async()=>{ await db.onModuleInit(); await db.$transaction(async tx=>{
|
|
if(!await tryAttachmentObjectLock(tx,process.env.RECONCILE_TEST_KEY)) throw Error('lock');
|
|
await s3.put(process.env.RECONCILE_TEST_KEY,Buffer.from('orphan fixture'),'image/png');
|
|
process.send('written');
|
|
await new Promise(()=>{setInterval(()=>{},1000)});
|
|
},OBJECT_TRANSACTION); })().catch(()=>process.exit(1));`;
|
|
writer = spawn(process.execPath, ['-e', writerCode], {
|
|
cwd: process.cwd(),
|
|
env: { ...process.env, RECONCILE_TEST_KEY: orphan },
|
|
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
});
|
|
const written = await Promise.race([
|
|
once(writer, 'message').then(([message]) => message),
|
|
once(writer, 'exit').then(() => {
|
|
throw new Error('writer exited before PUT');
|
|
}),
|
|
delay(15000, 'timeout', { ref: false }),
|
|
]);
|
|
assert.equal(written, 'written');
|
|
// Even when an object looks twenty minutes old, its active writer lock protects it.
|
|
const whileWriting = await service.run(
|
|
options,
|
|
new Date(Date.now() + 1200000),
|
|
);
|
|
assert.equal(whileWriting.lockedObjects, 1);
|
|
assert.equal(whileWriting.deletedObjects, 0);
|
|
assert.equal(
|
|
writer.exitCode,
|
|
null,
|
|
'writer unexpectedly exited before the simulated crash',
|
|
);
|
|
assert.equal(writer.signalCode, null);
|
|
writer.kill('SIGKILL');
|
|
await once(writer, 'exit');
|
|
writer = undefined;
|
|
assert.equal(
|
|
await db.attachment.findUnique({ where: { objectKey: orphan } }),
|
|
null,
|
|
);
|
|
|
|
const lateReference = newKey();
|
|
const renewed = newKey();
|
|
await storage.put(lateReference, bytes, 'image/png');
|
|
await storage.put(renewed, bytes, 'image/png');
|
|
await delay(3100);
|
|
|
|
// An actual refused DB connection must never turn into "metadata absent".
|
|
const url = new URL(process.env.DATABASE_URL);
|
|
url.port = '1';
|
|
const savedUrl = process.env.DATABASE_URL;
|
|
process.env.DATABASE_URL = url.toString();
|
|
const unavailable = new PrismaService();
|
|
process.env.DATABASE_URL = savedUrl;
|
|
try {
|
|
await assert.rejects(() =>
|
|
new ReconcileService(unavailable, scanner).run(options),
|
|
);
|
|
assert.ok(await storage.stat(orphan));
|
|
} finally {
|
|
await unavailable.onModuleDestroy();
|
|
}
|
|
|
|
const fresh = newKey();
|
|
await storage.put(fresh, bytes, 'image/png');
|
|
const scanTime = new Date();
|
|
let referenced = false;
|
|
let refreshed = false;
|
|
scanner.stat = async (key) => {
|
|
if (key === lateReference && !referenced) {
|
|
referenced = true;
|
|
// Simulate a metadata commit between the first DB lookup and the final check.
|
|
await db.attachment.create({ data: metadata(key) });
|
|
}
|
|
if (key === renewed && !refreshed) {
|
|
refreshed = true;
|
|
await storage.put(key, bytes, 'image/png');
|
|
}
|
|
return originalStat(key);
|
|
};
|
|
const dryRun = await service.run({ ...options, apply: false }, scanTime);
|
|
assert.equal(dryRun.orphanCandidates, 1);
|
|
assert.equal(dryRun.deletedObjects, 0);
|
|
assert.ok(await storage.stat(orphan));
|
|
const applied = await service.run(options, scanTime);
|
|
assert.equal(applied.deletedObjects, 1);
|
|
assert.equal(await storage.stat(orphan), null);
|
|
for (const key of [published, lateReference, renewed, fresh])
|
|
assert.ok(await storage.stat(key));
|
|
assert.equal((await service.run(options, scanTime)).deletedObjects, 0);
|
|
scanner.stat = originalStat;
|
|
|
|
const missing = newKey();
|
|
const missingRow = await db.attachment.create({ data: metadata(missing) });
|
|
const report = await service.run({ ...options, apply: false }, scanTime);
|
|
assert.equal(report.missingObjects, 1);
|
|
assert.ok(await db.attachment.findUnique({ where: { id: missingRow.id } }));
|
|
assert.ok(
|
|
events.some(
|
|
(event) =>
|
|
event.event === 'missing_object' &&
|
|
event.attachmentId === missingRow.id,
|
|
),
|
|
);
|
|
const cli = spawnSync(
|
|
process.execPath,
|
|
['dist/Attachment/reconcile.cli.js', '--dry-run', '--prefix', prefix],
|
|
{
|
|
encoding: 'utf8',
|
|
timeout: 30000,
|
|
env: { ...process.env, ...scannerCredentials },
|
|
},
|
|
);
|
|
assert.equal(
|
|
cli.status,
|
|
2,
|
|
'CLI must report missing bytes as an integrity incident',
|
|
);
|
|
const summary = cli.stdout
|
|
.split('\n')
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line))
|
|
.find((line) => line.event === 'reconcile_summary');
|
|
assert.equal(summary.graceSeconds, 600);
|
|
assert.equal(summary.mode, 'dry-run');
|
|
assert.equal(summary.missingObjects, 1);
|
|
console.log(
|
|
'PASS: real writer crash, DB outage, dry-run, orphan deletion, active/fresh/referenced protection, repeat run and missing-object reporting.',
|
|
);
|
|
} finally {
|
|
if (writer && writer.exitCode === null && writer.signalCode === null) {
|
|
writer.kill('SIGKILL');
|
|
await once(writer, 'exit');
|
|
}
|
|
scanner.stat = originalStat;
|
|
for (const key of keys) await storage.remove(key);
|
|
if (chat) {
|
|
await db.attachment.deleteMany({ where: { chatId: chat.id } });
|
|
await db.chat.delete({ where: { id: chat.id } });
|
|
}
|
|
if (user) await db.user.delete({ where: { id: user.id } });
|
|
storage.onModuleDestroy();
|
|
scanner.onModuleDestroy();
|
|
await db.onModuleDestroy();
|
|
}
|