Files
mtusi2026lab4/test/upload-interruption.mjs
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

255 lines
7.9 KiB
JavaScript

// Executed inside reconciler: read-only DB/S3 inspection; uploads use HTTP API credentials.
import assert from 'node:assert/strict';
import http from 'node:http';
import { createHash } from 'node:crypto';
import { createRequire } from 'node:module';
import { setTimeout as delay } from 'node:timers/promises';
let input = '';
for await (const part of process.stdin) input += part;
const fixture = JSON.parse(input);
const image = Buffer.from(fixture.image, 'base64');
const require = createRequire(`${process.cwd()}/package.json`);
const { PrismaService } = require('./dist/prisma/prisma.service.js');
const { StorageService } = require('./dist/Storage/storage.service.js');
const db = new PrismaService();
const storage = new StorageService();
const prefix = `originals/${fixture.chatId}/`;
const reports = [];
let phase = 'startup';
function call(url, { method = 'GET', body, token = fixture.ownerToken } = {}) {
return new Promise((resolve, reject) => {
const req = http.request(
url,
{
method,
headers: {
Authorization: `Bearer ${token}`,
...(body
? { 'Content-Type': 'image/png', 'Content-Length': body.length }
: {}),
},
},
(res) => {
const parts = [];
res.on('data', (part) => parts.push(part));
res.on('end', () =>
resolve({
status: res.statusCode,
body: Buffer.concat(parts),
headers: res.headers,
}),
);
res.on('error', reject);
},
);
req.on('error', reject);
req.setTimeout(10000, () => req.destroy(new Error('HTTP probe timed out')));
req.end(body);
});
}
async function contents(expected) {
const rows = await db.attachment.findMany({
where: { chatId: fixture.chatId },
select: { objectKey: true },
});
const keys = [];
for await (const object of storage.objects(prefix)) keys.push(object.key);
assert.equal(rows.length, expected, 'interrupted request created metadata');
assert.equal(keys.length, expected, 'interrupted request left an S3 object');
assert.deepEqual(keys.sort(), rows.map((row) => row.objectKey).sort());
}
async function abort(url, chunked) {
await new Promise((resolve, reject) => {
const req = http.request(url, {
method: 'POST',
headers: {
Authorization: `Bearer ${fixture.ownerToken}`,
'Content-Type': 'image/png',
...(chunked ? {} : { 'Content-Length': image.length + 1000 }),
},
});
let intentional = false;
req.on('error', (error) => {
if (!intentional) reject(error);
});
req.on('response', (res) => {
res.resume();
reject(new Error('partial upload responded before client abort'));
});
req.setTimeout(5000, () =>
req.destroy(new Error('partial write timed out')),
);
// Valid PNG bytes, but incomplete HTTP framing: neither Content-Length nor chunk terminator is satisfied.
req.write(image, () => {
void delay(200).then(() => {
intentional = true;
req.destroy();
resolve();
});
});
});
}
async function deadline(url, timeoutMs, direct) {
const started = performance.now();
const response = await new Promise((resolve, reject) => {
let timer;
let watchdog;
const req = http.request(
url,
{
method: 'POST',
headers: {
Authorization: `Bearer ${fixture.ownerToken}`,
'Content-Type': 'image/png',
'Content-Length': image.length + 100000,
},
},
(res) => {
clearInterval(timer);
const parts = [];
res.on('data', (part) => parts.push(part));
res.on('error', (error) => {
clearTimeout(watchdog);
reject(error);
});
res.on('end', () => {
clearTimeout(watchdog);
req.destroy();
resolve({
status: res.statusCode,
body: Buffer.concat(parts),
headers: res.headers,
});
});
},
);
req.on('error', (error) => {
clearInterval(timer);
clearTimeout(watchdog);
reject(error);
});
watchdog = setTimeout(
() => req.destroy(new Error('upload deadline did not fire')),
timeoutMs + 8000,
);
req.write(image.subarray(0, 16));
timer = setInterval(() => req.write(Buffer.from([0])), 200);
});
const elapsedMs = performance.now() - started;
assert.equal(response.status, 408);
assert.match(response.headers['content-type'], /^application\/json/);
const error = JSON.parse(response.body).error;
assert.equal(error.code, 'invalid_request');
assert.equal(error.request_id, response.headers['x-request-id']);
assert.ok(error.request_id && response.headers['x-instance-id']);
// Connection is hop-by-hop: Nginx owns the frontend keep-alive policy.
if (direct) assert.equal(response.headers.connection, 'close');
assert.ok(
elapsedMs >= timeoutMs - 300 && elapsedMs < timeoutMs + 7000,
'unexpected upload deadline duration',
);
return {
status: response.status,
elapsedMs: Math.round(elapsedMs),
instance: response.headers['x-instance-id'],
connection: response.headers.connection,
};
}
try {
await db.onModuleInit();
await contents(0);
let expected = 0;
for (const target of fixture.targets) {
phase = target.name;
const url = `${target.url}/api/v1/chats/${fixture.chatId}/attachments?filename=interruption.png`;
const control = await call(url, { method: 'POST', body: image });
assert.equal(control.status, 201);
const attachment = JSON.parse(control.body);
assert.equal(
attachment.sha256,
createHash('sha256').update(image).digest('hex'),
);
expected++;
await contents(expected);
for (const chunked of [false, true]) {
phase = `${target.name}:${chunked ? 'chunked' : 'content-length'}-abort`;
await abort(url, chunked);
await delay(300);
await contents(expected);
}
phase = `${target.name}:deadline`;
const timedOut = await deadline(
url,
target.timeoutMs,
target.name !== 'nginx',
);
await delay(300);
await contents(expected);
phase = `${target.name}:recovery`;
const recovery = await call(url, { method: 'POST', body: image });
assert.equal(recovery.status, 201);
expected++;
const downloaded = await call(
`${target.url}/api/v1/attachments/${JSON.parse(recovery.body).id}/content`,
{ token: fixture.readerToken },
);
assert.equal(downloaded.status, 200);
assert.deepEqual(downloaded.body, image);
await contents(expected);
reports.push({
target: target.name,
contentLengthAbort: 'passed',
chunkedAbort: 'passed',
deadline: timedOut,
normalUploadAfterAbort: 'passed',
databaseAndS3Match: true,
});
console.log(
`PASS: ${target.name}: both client aborts leave no bytes/metadata; slow upload gets 408; next upload/download succeeds`,
);
}
console.log(
JSON.stringify({
event: 'upload_interruption_summary',
status: 'passed',
targets: reports,
}),
);
} catch (error) {
console.log(
JSON.stringify({
event: 'upload_interruption_summary',
status: 'failed',
phase,
errorType: error.name,
assertion:
error.code === 'ERR_ASSERTION'
? {
message: error.message.split('\n')[0],
actual: ['number', 'boolean', 'string'].includes(
typeof error.actual,
)
? String(error.actual).slice(0, 80)
: undefined,
expected: ['number', 'boolean', 'string'].includes(
typeof error.expected,
)
? String(error.expected).slice(0, 80)
: undefined,
}
: undefined,
targets: reports,
}),
);
process.exitCode = 1;
} finally {
storage.onModuleDestroy();
await db.onModuleDestroy();
}