Files
mtusi2026lab4/test/reconcile.mjs
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

225 lines
7.8 KiB
JavaScript

// Run through test/run-reconcile.py with distinct writer/scanner credentials.
import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
import { randomUUID, createHash } from 'node:crypto';
import { spawn, spawnSync } from 'node:child_process';
import { once } from 'node:events';
import { setTimeout as delay } from 'node:timers/promises';
const require = createRequire(`${process.cwd()}/package.json`);
const { PrismaService } = require('./dist/prisma/prisma.service.js');
const { StorageService } = require('./dist/Storage/storage.service.js');
const { ReconcileService } = require('./dist/Attachment/reconcile.service.js');
const db = new PrismaService();
const storage = new StorageService();
const scannerCredentials = JSON.parse(
process.env.RECONCILE_TEST_SCANNER_CREDENTIALS,
);
delete process.env.RECONCILE_TEST_SCANNER_CREDENTIALS;
const writerCredentials = {
S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID,
S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY,
};
Object.assign(process.env, scannerCredentials);
const scanner = new StorageService();
Object.assign(process.env, writerCredentials);
const events = [];
const service = new ReconcileService(db, scanner, (event) =>
events.push(event),
);
const bytes = Buffer.from('isolated reconciliation fixture');
const keys = [];
let user;
let chat;
let writer;
const originalStat = scanner.stat.bind(scanner);
function newKey() {
const key = `originals/${chat.id}/${randomUUID()}`;
keys.push(key);
return key;
}
function metadata(key) {
return {
chatId: chat.id,
ownerId: user.id,
objectKey: key,
filename: 'fixture.png',
contentType: 'image/png',
sizeBytes: bytes.length,
sha256: createHash('sha256').update(bytes).digest('hex'),
};
}
try {
await db.onModuleInit();
user = await db.user.create({
data: {
username: `rc_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Reconciliation test',
},
});
chat = await db.chat.create({
data: {
title: 'Isolated reconciliation fixtures',
members: { create: { userId: user.id, position: 0 } },
},
});
const prefix = `originals/${chat.id}/`;
const options = { apply: true, graceSeconds: 2, prefix };
const published = newKey();
await storage.put(published, bytes, 'image/png');
await db.attachment.create({ data: metadata(published) });
const orphan = newKey();
// This is a separate Node process with its own DB transaction and S3 client.
const writerCode = `
const {PrismaService}=require('./dist/prisma/prisma.service.js');
const {StorageService}=require('./dist/Storage/storage.service.js');
const {tryAttachmentObjectLock,OBJECT_TRANSACTION}=require('./dist/Attachment/object-lock.js');
const db=new PrismaService(); const s3=new StorageService();
(async()=>{ await db.onModuleInit(); await db.$transaction(async tx=>{
if(!await tryAttachmentObjectLock(tx,process.env.RECONCILE_TEST_KEY)) throw Error('lock');
await s3.put(process.env.RECONCILE_TEST_KEY,Buffer.from('orphan fixture'),'image/png');
process.send('written');
await new Promise(()=>{setInterval(()=>{},1000)});
},OBJECT_TRANSACTION); })().catch(()=>process.exit(1));`;
writer = spawn(process.execPath, ['-e', writerCode], {
cwd: process.cwd(),
env: { ...process.env, RECONCILE_TEST_KEY: orphan },
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
});
const written = await Promise.race([
once(writer, 'message').then(([message]) => message),
once(writer, 'exit').then(() => {
throw new Error('writer exited before PUT');
}),
delay(15000, 'timeout', { ref: false }),
]);
assert.equal(written, 'written');
// Even when an object looks twenty minutes old, its active writer lock protects it.
const whileWriting = await service.run(
options,
new Date(Date.now() + 1200000),
);
assert.equal(whileWriting.lockedObjects, 1);
assert.equal(whileWriting.deletedObjects, 0);
assert.equal(
writer.exitCode,
null,
'writer unexpectedly exited before the simulated crash',
);
assert.equal(writer.signalCode, null);
writer.kill('SIGKILL');
await once(writer, 'exit');
writer = undefined;
assert.equal(
await db.attachment.findUnique({ where: { objectKey: orphan } }),
null,
);
const lateReference = newKey();
const renewed = newKey();
await storage.put(lateReference, bytes, 'image/png');
await storage.put(renewed, bytes, 'image/png');
await delay(3100);
// An actual refused DB connection must never turn into "metadata absent".
const url = new URL(process.env.DATABASE_URL);
url.port = '1';
const savedUrl = process.env.DATABASE_URL;
process.env.DATABASE_URL = url.toString();
const unavailable = new PrismaService();
process.env.DATABASE_URL = savedUrl;
try {
await assert.rejects(() =>
new ReconcileService(unavailable, scanner).run(options),
);
assert.ok(await storage.stat(orphan));
} finally {
await unavailable.onModuleDestroy();
}
const fresh = newKey();
await storage.put(fresh, bytes, 'image/png');
const scanTime = new Date();
let referenced = false;
let refreshed = false;
scanner.stat = async (key) => {
if (key === lateReference && !referenced) {
referenced = true;
// Simulate a metadata commit between the first DB lookup and the final check.
await db.attachment.create({ data: metadata(key) });
}
if (key === renewed && !refreshed) {
refreshed = true;
await storage.put(key, bytes, 'image/png');
}
return originalStat(key);
};
const dryRun = await service.run({ ...options, apply: false }, scanTime);
assert.equal(dryRun.orphanCandidates, 1);
assert.equal(dryRun.deletedObjects, 0);
assert.ok(await storage.stat(orphan));
const applied = await service.run(options, scanTime);
assert.equal(applied.deletedObjects, 1);
assert.equal(await storage.stat(orphan), null);
for (const key of [published, lateReference, renewed, fresh])
assert.ok(await storage.stat(key));
assert.equal((await service.run(options, scanTime)).deletedObjects, 0);
scanner.stat = originalStat;
const missing = newKey();
const missingRow = await db.attachment.create({ data: metadata(missing) });
const report = await service.run({ ...options, apply: false }, scanTime);
assert.equal(report.missingObjects, 1);
assert.ok(await db.attachment.findUnique({ where: { id: missingRow.id } }));
assert.ok(
events.some(
(event) =>
event.event === 'missing_object' &&
event.attachmentId === missingRow.id,
),
);
const cli = spawnSync(
process.execPath,
['dist/Attachment/reconcile.cli.js', '--dry-run', '--prefix', prefix],
{
encoding: 'utf8',
timeout: 30000,
env: { ...process.env, ...scannerCredentials },
},
);
assert.equal(
cli.status,
2,
'CLI must report missing bytes as an integrity incident',
);
const summary = cli.stdout
.split('\n')
.filter(Boolean)
.map((line) => JSON.parse(line))
.find((line) => line.event === 'reconcile_summary');
assert.equal(summary.graceSeconds, 600);
assert.equal(summary.mode, 'dry-run');
assert.equal(summary.missingObjects, 1);
console.log(
'PASS: real writer crash, DB outage, dry-run, orphan deletion, active/fresh/referenced protection, repeat run and missing-object reporting.',
);
} finally {
if (writer && writer.exitCode === null && writer.signalCode === null) {
writer.kill('SIGKILL');
await once(writer, 'exit');
}
scanner.stat = originalStat;
for (const key of keys) await storage.remove(key);
if (chat) {
await db.attachment.deleteMany({ where: { chatId: chat.id } });
await db.chat.delete({ where: { id: chat.id } });
}
if (user) await db.user.delete({ where: { id: user.id } });
storage.onModuleDestroy();
scanner.onModuleDestroy();
await db.onModuleDestroy();
}