feat: add attachment upload with streaming and image validation

This commit is contained in:
2026-11-05 15:05:36 +03:00
parent d960734dae
commit fa0211eecb
20 changed files with 3252 additions and 15 deletions
+358
View File
@@ -0,0 +1,358 @@
import { INestApplication } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import Ajv2020 from 'ajv/dist/2020';
import { createHash, randomUUID } from 'crypto';
import { readdirSync } from 'fs';
import { connect } from 'net';
import { tmpdir } from 'os';
import request from 'supertest';
import { PrismaService } from '../src/db/prisma.service';
import { loadDocument } from '../src/docs/swagger';
import { S3Service } from '../src/storage/s3.service';
import { bearer, createTestApp, registerUser, TestUser } from './helpers';
import {
corruptedJpegs,
corruptedPngs,
jpeg,
png,
pngHeaderOnly,
pngOfExactSize,
SAMPLE_PNG,
} from './images';
const MAX_BYTES = 10 * 1024 * 1024;
describe('Загрузка вложений (e2e)', () => {
let app: INestApplication;
let server: any;
let alice: TestUser;
let eve: TestUser;
let chatId: string;
const upload = (
body: Buffer | undefined,
{
type = 'image/png',
filename = 'sample.png',
user = alice,
chat = chatId,
}: {
type?: string | null;
filename?: string | null;
user?: TestUser | null;
chat?: string;
} = {},
) => {
const query = filename === null ? '' : `?filename=${encodeURIComponent(filename)}`;
const req = request(server).post(`/api/v1/chats/${chat}/attachments${query}`);
if (user) {
req.set('Authorization', bearer(user));
}
if (type) {
req.set('Content-Type', type);
}
return body ? req.send(body) : req;
};
const expectError = (res: request.Response, status: number, code: string) => {
expect({ status: res.status, code: res.body.error?.code }).toEqual({ status, code });
expect(res.headers['x-request-id']).toBeTruthy();
expect(res.headers['x-instance-id']).toBeTruthy();
expect(res.body.error.request_id).toBe(res.headers['x-request-id']);
};
beforeAll(async () => {
({ app, server } = await createTestApp());
alice = await registerUser(server, 'att_alice');
eve = await registerUser(server, 'att_eve');
const chat = await request(server)
.post('/api/v1/chats')
.set('Authorization', bearer(alice))
.send({ title: 'Files', member_ids: [alice.id] })
.expect(201);
chatId = chat.body.id;
});
afterAll(async () => {
await app.close();
});
describe('успешная загрузка', () => {
it('возвращает 201 Attachment по контракту и сохраняет метаданные', async () => {
const res = await upload(SAMPLE_PNG).expect(201);
const document: any = loadDocument();
const ajv = new Ajv2020({ strict: false, validateFormats: false });
ajv.addSchema(document, 'contract');
const validate = ajv.compile({ $ref: 'contract#/components/schemas/Attachment' });
expect({ valid: validate(res.body), errors: validate.errors }).toEqual({
valid: true,
errors: null,
});
expect(res.body).toMatchObject({
chat_id: chatId,
owner_id: alice.id,
filename: 'sample.png',
content_type: 'image/png',
size_bytes: SAMPLE_PNG.length,
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
status: 'ready',
variants: [],
error_code: null,
});
const row = await app.get(PrismaService).attachment.findUnique({
where: { id: res.body.id },
});
expect(row?.objectKey).toBe(`attachments/${res.body.id}`);
expect(JSON.stringify(res.body)).not.toContain('attachments/');
});
it('принимает реальный JPEG', async () => {
const res = await upload(await jpeg(), {
type: 'image/jpeg',
filename: 'фото 1.jpg',
}).expect(201);
expect(res.body).toMatchObject({
content_type: 'image/jpeg',
filename: 'фото 1.jpg',
});
});
it('игнорирует параметры и регистр Content-Type', async () => {
await upload(SAMPLE_PNG, { type: 'IMAGE/PNG; charset=binary' }).expect(201);
});
it('принимает файл ровно 10 MiB, filename ровно 255 символов и PNG с байтами после IEND', async () => {
const exact = pngOfExactSize(MAX_BYTES);
expect(exact.length).toBe(MAX_BYTES);
const res = await upload(exact).expect(201);
expect(res.body.size_bytes).toBe(MAX_BYTES);
await upload(SAMPLE_PNG, { filename: `${'ф'.repeat(251)}.png` }).expect(201);
await upload(Buffer.concat([png(4, 4), Buffer.from('tail')])).expect(201);
});
it('принимает ровно 40 млн пикселей', async () => {
const res = await upload(await bigPng(8000, 5000));
expect(res.status).toBe(201);
});
});
describe('порядок проверок и доступ', () => {
it('без токена — 401 раньше любых проверок содержимого', async () => {
const cases: [string | null, string | null][] = [
['image/png', 'a.png'],
['text/plain', 'a.png'],
['application/json', 'a.png'],
['image/png', '../a.png'],
[null, null],
];
for (const [type, filename] of cases) {
const res = await upload(Buffer.from('{bad'), { type, filename, user: null });
expectError(res, 401, 'unauthorized');
expect(res.headers['www-authenticate']).toBe('Bearer realm="messenger"');
}
});
it('посторонний и несуществующий чат — одинаковый 404, чужой файл не сохраняется', async () => {
const prisma = app.get(PrismaService);
const before = await prisma.attachment.count();
expectError(await upload(SAMPLE_PNG, { user: eve }), 404, 'not_found');
expectError(await upload(SAMPLE_PNG, { chat: randomUUID() }), 404, 'not_found');
expect(await prisma.attachment.count()).toBe(before);
});
it('неверный UUID чата — 400', async () => {
expectError(await upload(SAMPLE_PNG, { chat: 'not-a-uuid' }), 400, 'invalid_request');
});
});
describe('filename', () => {
const invalid = {
'отсутствует': null,
'пустой': '',
'только пробелы': ' ',
'256 символов': 'x'.repeat(256),
'путь через /': '../x.png',
'путь через \\': 'a\\b.png',
'NUL': 'a\u0000b.png',
'перевод строки': 'a\nb.png',
'DEL': 'bad\u007f.png',
'C1 U+0085': 'c1\u0085.png',
'разделитель строк U+2028': 'a
b.png',
};
it.each(Object.entries(invalid))('%s — 400', async (_name, filename) => {
expectError(await upload(SAMPLE_PNG, { filename }), 400, 'invalid_request');
});
it('повторённый параметр — 400', async () => {
const res = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png&filename=b.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG);
expectError(res, 400, 'invalid_request');
});
});
describe('тип содержимого', () => {
it.each(['text/plain', 'application/json', 'application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'image/webp'])(
'Content-Type %s — 415',
async (type) => {
expectError(await upload(SAMPLE_PNG, { type }), 415, 'unsupported_media_type');
},
);
it('без Content-Type — 415', async () => {
expectError(await upload(SAMPLE_PNG, { type: null }), 415, 'unsupported_media_type');
});
it('подмена MIME в обе стороны и чужие форматы — 415', async () => {
const realJpeg = await jpeg();
expectError(await upload(SAMPLE_PNG, { type: 'image/jpeg' }), 415, 'unsupported_media_type');
expectError(await upload(realJpeg, { type: 'image/png' }), 415, 'unsupported_media_type');
for (const body of [
Buffer.from('просто текст'),
Buffer.from('GIF89a\u0001\u0000\u0001\u0000'),
Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" width="1" height="1"/>'),
]) {
expectError(await upload(body), 415, 'unsupported_media_type');
}
});
it('пустое тело — 400', async () => {
expectError(await upload(undefined), 400, 'invalid_request');
expectError(await upload(Buffer.alloc(0)), 400, 'invalid_request');
});
});
describe('повреждённые изображения', () => {
it.each(Object.entries(corruptedPngs()))('PNG: %s — 415', async (_name, body) => {
expectError(await upload(body), 415, 'unsupported_media_type');
});
it('JPEG: все повреждения — 415', async () => {
for (const body of Object.values(corruptedJpegs(await jpeg()))) {
expectError(await upload(body, { type: 'image/jpeg' }), 415, 'unsupported_media_type');
}
});
});
describe('лимиты', () => {
it('10 MiB + 1 байт — 413 и без чтения объявленного тела', async () => {
expectError(await upload(pngOfExactSize(MAX_BYTES + 1)), 413, 'payload_too_large');
});
it('превышение при chunked без Content-Length — 413', async () => {
const res = await new Promise<request.Response>((resolve, reject) => {
const req = upload(undefined);
req.set('Transfer-Encoding', 'chunked');
req.write(pngOfExactSize(MAX_BYTES));
req.write(Buffer.alloc(1024));
req.end((error, response) => (response ? resolve(response) : reject(error)));
});
expectError(res, 413, 'payload_too_large');
});
it('превышение лимита пикселей — 413 до декодирования', async () => {
for (const body of [pngHeaderOnly(8001, 5000), pngHeaderOnly(20000, 10000)]) {
expectError(await upload(body), 413, 'payload_too_large');
}
});
it('оборванная загрузка не оставляет временных файлов и строк в БД', async () => {
const prisma = app.get(PrismaService);
const before = await prisma.attachment.count();
const { port } = server.address();
await new Promise<void>((resolve, reject) => {
const socket = connect(port, '127.0.0.1', () => {
socket.write(
`POST /api/v1/chats/${chatId}/attachments?filename=a.png HTTP/1.1\r\n` +
`Host: test\r\nAuthorization: ${bearer(alice)}\r\n` +
`Content-Type: image/png\r\nContent-Length: ${SAMPLE_PNG.length}\r\n\r\n`,
);
socket.write(SAMPLE_PNG.subarray(0, 1000));
setTimeout(() => {
socket.destroy();
resolve();
}, 300);
});
socket.on('error', reject);
});
await new Promise((resolve) => setTimeout(resolve, 500));
expect(await prisma.attachment.count()).toBe(before);
expect(activeTempDirs()).toEqual([]);
});
});
describe('отказы', () => {
it('S3 недоступен — 503 с заголовками, метаданные не создаются', async () => {
const { app: down, server: downServer } = await createTestApp({
S3_ENDPOINT: 'http://127.0.0.1:9',
});
try {
const prisma = app.get(PrismaService);
const before = await prisma.attachment.count();
const started = Date.now();
const res = await request(downServer)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG);
expectError(res, 503, 'unavailable');
expect(Date.now() - started).toBeLessThan(3000);
expect(await prisma.attachment.count()).toBe(before);
} finally {
await down.close();
}
});
it('явный отказ INSERT после PUT удаляет объект', async () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);
const remove = jest.spyOn(s3, 'delete');
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
new Prisma.PrismaClientKnownRequestError('rejected', {
code: 'P2003',
clientVersion: 'test',
}),
);
try {
const res = await upload(SAMPLE_PNG);
expect(res.status).toBe(500);
expect(remove).toHaveBeenCalledTimes(1);
} finally {
create.mockRestore();
remove.mockRestore();
}
});
it('при неясном исходе COMMIT объект не удаляется', async () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);
const remove = jest.spyOn(s3, 'delete');
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
new Prisma.PrismaClientKnownRequestError('connection lost', {
code: 'P1017',
clientVersion: 'test',
}),
);
try {
expectError(await upload(SAMPLE_PNG), 503, 'unavailable');
expect(remove).not.toHaveBeenCalled();
} finally {
create.mockRestore();
remove.mockRestore();
}
});
});
});
function activeTempDirs(): string[] {
return readdirSync(tmpdir()).filter((name) => name.startsWith('messenger-upload-'));
}
async function bigPng(width: number, height: number): Promise<Buffer> {
const { default: sharp } = await import('sharp');
return sharp({ create: { width, height, channels: 3, background: '#336699' } })
.png({ compressionLevel: 9 })
.toBuffer();
}
+24 -1
View File
@@ -1,5 +1,28 @@
import {
BucketAlreadyOwnedByYou,
CreateBucketCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { execSync } from 'child_process';
export default function globalSetup(): void {
export default async function globalSetup(): Promise<void> {
execSync('npx prisma migrate deploy', { stdio: 'inherit' });
const client = new S3Client({
endpoint: process.env.S3_ENDPOINT,
region: 'us-east-1',
forcePathStyle: true,
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY_ID!,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY!,
},
});
try {
await client.send(new CreateBucketCommand({ Bucket: process.env.S3_BUCKET! }));
} catch (error) {
if (!(error instanceof BucketAlreadyOwnedByYou)) {
throw error;
}
} finally {
client.destroy();
}
}
+147
View File
@@ -0,0 +1,147 @@
import { readFileSync } from 'fs';
import { join } from 'path';
import sharp from 'sharp';
import { deflateSync } from 'zlib';
const CRC_TABLE = Array.from({ length: 256 }, (_, n) => {
let c = n;
for (let k = 0; k < 8; k++) {
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
}
return c >>> 0;
});
function crc32(data: Buffer): number {
let c = 0xffffffff;
for (const byte of data) {
c = CRC_TABLE[(c ^ byte) & 255] ^ (c >>> 8);
}
return (c ^ 0xffffffff) >>> 0;
}
export function pngChunk(type: string, data: Buffer): Buffer {
const name = Buffer.from(type, 'latin1');
const length = Buffer.alloc(4);
length.writeUInt32BE(data.length);
const crc = Buffer.alloc(4);
crc.writeUInt32BE(crc32(Buffer.concat([name, data])));
return Buffer.concat([length, name, data, crc]);
}
const SIGNATURE = Buffer.from('89504e470d0a1a0a', 'hex');
const END = pngChunk('IEND', Buffer.alloc(0));
function header(width: number, height: number, colorType = 2): Buffer {
const data = Buffer.alloc(13);
data.writeUInt32BE(width, 0);
data.writeUInt32BE(height, 4);
data[8] = 8;
data[9] = colorType;
return pngChunk('IHDR', data);
}
function rows(width: number, height: number, filter = 0): Buffer {
const stride = width * 3 + 1;
const raw = Buffer.alloc(stride * height, 7);
for (let y = 0; y < height; y++) {
raw[y * stride] = filter;
}
return raw;
}
export function png(width: number, height: number, extra: Buffer[] = []): Buffer {
return Buffer.concat([
SIGNATURE,
header(width, height),
...extra,
pngChunk('IDAT', deflateSync(rows(width, height))),
END,
]);
}
export function pngOfExactSize(size: number): Buffer {
const base = png(4, 4);
const padding = size - base.length - 12;
return png(4, 4, [pngChunk('abCd', Buffer.alloc(padding))]);
}
export function pngHeaderOnly(width: number, height: number): Buffer {
return Buffer.concat([
SIGNATURE,
header(width, height),
pngChunk('IDAT', deflateSync(Buffer.alloc(16))),
END,
]);
}
export const SAMPLE_PNG = readFileSync(
join(__dirname, '..', 'tests', 'fixtures', 'sample.png'),
);
export const JPEG_WITHOUT_DATA = Buffer.from(
'ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9',
'hex',
);
export async function jpeg(): Promise<Buffer> {
const noise = Buffer.alloc(320 * 200 * 3);
for (let i = 0; i < noise.length; i++) {
noise[i] = (i * 7919) & 255;
}
return sharp(noise, { raw: { width: 320, height: 200, channels: 3 } })
.jpeg()
.toBuffer();
}
export function corruptedPngs(): Record<string, Buffer> {
const lastRowFilter = rows(64, 64);
lastRowFilter[63 * (64 * 3 + 1)] = 5;
const flipped = Buffer.from(SAMPLE_PNG);
flipped[100] ^= 255;
return {
'первые 33 байта sample.png': SAMPLE_PNG.subarray(0, 33),
'половина файла': SAMPLE_PNG.subarray(0, SAMPLE_PNG.length >> 1),
'без последних 20 байт': SAMPLE_PNG.subarray(0, SAMPLE_PNG.length - 20),
'без IEND': SAMPLE_PNG.subarray(0, SAMPLE_PNG.length - 12),
'неверный CRC': flipped,
'фильтр строки 5': pngFromRaw(1, 1, rows(1, 1, 5)),
'фильтр строки 255': pngFromRaw(1, 1, rows(1, 1, 255)),
'неверный фильтр в последней строке': pngFromRaw(64, 64, lastRowFilter),
'мусор в IDAT с верным CRC': Buffer.concat([
SIGNATURE,
header(64, 64),
pngChunk('IDAT', Buffer.alloc(40, 0xab)),
END,
]),
'данных меньше, чем строк': pngFromRaw(64, 64, rows(64, 10)),
'палитра на 257 записей': Buffer.concat([
SIGNATURE,
header(1, 1, 3),
pngChunk('PLTE', Buffer.alloc(771)),
pngChunk('IDAT', deflateSync(Buffer.from([0, 0]))),
END,
]),
};
}
export function corruptedJpegs(valid: Buffer): Record<string, Buffer> {
const garbled = Buffer.from(valid);
for (let i = 800; i < garbled.length - 2; i += 3) {
garbled[i] = 0x55;
}
return {
'27 байт без таблиц и данных': JPEG_WITHOUT_DATA,
'половина файла': valid.subarray(0, valid.length >> 1),
'без последних 20 байт': valid.subarray(0, valid.length - 20),
'испорченные данные скана': garbled,
};
}
function pngFromRaw(width: number, height: number, raw: Buffer): Buffer {
return Buffer.concat([
SIGNATURE,
header(width, height),
pngChunk('IDAT', deflateSync(raw)),
END,
]);
}