forked from mtusi2026/mtusi2026lab6
344 lines
12 KiB
TypeScript
344 lines
12 KiB
TypeScript
import { INestApplication } from '@nestjs/common';
|
|
import { createHash, randomUUID } from 'crypto';
|
|
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
|
|
import { tmpdir } from 'os';
|
|
import { join } from 'path';
|
|
import request from 'supertest';
|
|
import { reconcile } from '../src/attachments/reconciler';
|
|
import { PrismaService } from '../src/db/prisma.service';
|
|
import { replayJob } from '../src/jobs/job-store';
|
|
import { S3Service } from '../src/storage/s3.service';
|
|
import {
|
|
bearer,
|
|
createTestApp,
|
|
processAttachment,
|
|
registerUser,
|
|
TestUser,
|
|
} from './helpers';
|
|
import { SAMPLE_PNG } from './images';
|
|
|
|
const HOUR_MS = 60 * 60 * 1000;
|
|
|
|
describe('Сверка объектов S3 с метаданными (e2e)', () => {
|
|
let app: INestApplication;
|
|
let server: any;
|
|
let prisma: PrismaService;
|
|
let s3: S3Service;
|
|
let alice: TestUser;
|
|
let chatId: string;
|
|
let dir: string;
|
|
|
|
const putObject = async (key: string): Promise<void> => {
|
|
const path = join(dir, randomUUID());
|
|
writeFileSync(path, SAMPLE_PNG);
|
|
await s3.putFile({
|
|
key,
|
|
path,
|
|
size: SAMPLE_PNG.length,
|
|
contentType: 'image/png',
|
|
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
|
|
});
|
|
};
|
|
|
|
const exists = async (key: string): Promise<boolean> => {
|
|
const page = await s3.listObjects(key, 1);
|
|
return page.objects.some((object) => object.key === key);
|
|
};
|
|
|
|
const scope = () => `reconcile-test/${randomUUID()}/`;
|
|
const later = () => new Date(Date.now() + 2 * HOUR_MS);
|
|
|
|
beforeAll(async () => {
|
|
({ app, server } = await createTestApp());
|
|
prisma = app.get(PrismaService);
|
|
s3 = app.get(S3Service);
|
|
dir = mkdtempSync(join(tmpdir(), 'reconcile-test-'));
|
|
alice = await registerUser(server, 'rec_alice');
|
|
const chat = await request(server)
|
|
.post('/api/v1/chats')
|
|
.set('Authorization', bearer(alice))
|
|
.send({ title: 'Reconcile', member_ids: [alice.id] })
|
|
.expect(201);
|
|
chatId = chat.body.id;
|
|
});
|
|
|
|
afterAll(async () => {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
await app.close();
|
|
});
|
|
|
|
it('dry-run сообщает о сироте и ничего не удаляет, apply удаляет, повторный запуск ничего не делает', async () => {
|
|
const prefix = scope();
|
|
const key = `${prefix}orphan`;
|
|
await putObject(key);
|
|
const options = { graceMs: HOUR_MS, prefix, now: later() };
|
|
|
|
const dry = await reconcile(prisma, s3, { ...options, apply: false });
|
|
expect(dry).toMatchObject({ mode: 'dry-run', scanned: 1, orphans: 1, deleted: 0, orphan_keys: [key] });
|
|
expect(await exists(key)).toBe(true);
|
|
|
|
const applied = await reconcile(prisma, s3, { ...options, apply: true });
|
|
expect(applied).toMatchObject({ mode: 'apply', orphans: 1, deleted: 1, failed: 0 });
|
|
expect(await exists(key)).toBe(false);
|
|
|
|
const again = await reconcile(prisma, s3, { ...options, apply: true });
|
|
expect(again).toMatchObject({ scanned: 0, orphans: 0, deleted: 0 });
|
|
});
|
|
|
|
it('не трогает свежий объект внутри grace period', async () => {
|
|
const prefix = scope();
|
|
const key = `${prefix}fresh`;
|
|
await putObject(key);
|
|
const report = await reconcile(prisma, s3, { apply: true, graceMs: HOUR_MS, prefix });
|
|
expect(report).toMatchObject({ scanned: 1, fresh_skipped: 1, orphans: 0, deleted: 0 });
|
|
expect(await exists(key)).toBe(true);
|
|
});
|
|
|
|
it('не трогает старый объект, на который есть метаданные', async () => {
|
|
const prefix = scope();
|
|
const key = `${prefix}referenced`;
|
|
await putObject(key);
|
|
await prisma.attachment.create({
|
|
data: {
|
|
chatId,
|
|
ownerId: alice.id,
|
|
filename: 'a.png',
|
|
contentType: 'image/png',
|
|
sizeBytes: SAMPLE_PNG.length,
|
|
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
|
|
objectKey: key,
|
|
status: 'ready',
|
|
},
|
|
});
|
|
const report = await reconcile(prisma, s3, {
|
|
apply: true,
|
|
graceMs: HOUR_MS,
|
|
prefix,
|
|
now: later(),
|
|
});
|
|
expect(report).toMatchObject({ scanned: 1, referenced: 1, orphans: 0, deleted: 0 });
|
|
expect(await exists(key)).toBe(true);
|
|
});
|
|
|
|
it('при отказе PostgreSQL проход прерывается и ничего не удаляет', async () => {
|
|
const prefix = scope();
|
|
const key = `${prefix}orphan`;
|
|
await putObject(key);
|
|
const broken = {
|
|
attachment: {
|
|
findMany: jest.fn().mockRejectedValue(new Error('database down')),
|
|
},
|
|
} as unknown as PrismaService;
|
|
await expect(
|
|
reconcile(broken, s3, { apply: true, graceMs: HOUR_MS, prefix, now: later() }),
|
|
).rejects.toThrow('database down');
|
|
expect(await exists(key)).toBe(true);
|
|
});
|
|
|
|
it('сбой удаления считается и повторяется на следующем проходе', async () => {
|
|
const prefix = scope();
|
|
const key = `${prefix}orphan`;
|
|
await putObject(key);
|
|
const options = { apply: true, graceMs: HOUR_MS, prefix, now: later() };
|
|
const remove = jest.spyOn(s3, 'delete').mockRejectedValueOnce(new Error('s3 down'));
|
|
try {
|
|
const failed = await reconcile(prisma, s3, options);
|
|
expect(failed).toMatchObject({ orphans: 1, deleted: 0, failed: 1 });
|
|
expect(await exists(key)).toBe(true);
|
|
} finally {
|
|
remove.mockRestore();
|
|
}
|
|
const retried = await reconcile(prisma, s3, options);
|
|
expect(retried).toMatchObject({ orphans: 1, deleted: 1, failed: 0 });
|
|
expect(await exists(key)).toBe(false);
|
|
});
|
|
|
|
it('обходит все страницы листинга', async () => {
|
|
const prefix = scope();
|
|
await Promise.all([1, 2, 3].map((n) => putObject(`${prefix}${n}`)));
|
|
const report = await reconcile(prisma, s3, {
|
|
apply: true,
|
|
graceMs: HOUR_MS,
|
|
prefix,
|
|
now: later(),
|
|
pageSize: 1,
|
|
});
|
|
expect(report).toMatchObject({ scanned: 3, orphans: 3, deleted: 3 });
|
|
});
|
|
|
|
it('реальная загрузка остаётся доступной после сверки всего bucket префикса вложений', async () => {
|
|
const upload = await request(server)
|
|
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
|
.set('Authorization', bearer(alice))
|
|
.set('Content-Type', 'image/png')
|
|
.send(SAMPLE_PNG)
|
|
.expect(202);
|
|
expect(await processAttachment(app, upload.body.id)).toBe('ready');
|
|
const report = await reconcile(prisma, s3, { apply: true, graceMs: HOUR_MS });
|
|
expect(report.fresh_skipped).toBeGreaterThanOrEqual(4);
|
|
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
|
|
await request(server)
|
|
.get(`/api/v1/attachments/${upload.body.id}/content?variant=${variant}`)
|
|
.set('Authorization', bearer(alice))
|
|
.expect(200);
|
|
}
|
|
});
|
|
|
|
it('после grace period объекты вложения и staging в очереди не считаются сиротами', async () => {
|
|
const queued = await request(server)
|
|
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
|
.set('Authorization', bearer(alice))
|
|
.set('Content-Type', 'image/png')
|
|
.send(SAMPLE_PNG)
|
|
.expect(202);
|
|
const ready = await request(server)
|
|
.post(`/api/v1/chats/${chatId}/attachments?filename=b.png`)
|
|
.set('Authorization', bearer(alice))
|
|
.set('Content-Type', 'image/png')
|
|
.send(SAMPLE_PNG)
|
|
.expect(202);
|
|
expect(await processAttachment(app, ready.body.id)).toBe('ready');
|
|
for (const id of [queued.body.id, ready.body.id]) {
|
|
for (const prefix of [`attachments/${id}/`, `staging/${id}`]) {
|
|
const report = await reconcile(prisma, s3, {
|
|
apply: true,
|
|
graceMs: HOUR_MS,
|
|
prefix,
|
|
now: later(),
|
|
});
|
|
expect(report).toMatchObject({ orphans: 0, deleted: 0 });
|
|
}
|
|
}
|
|
expect(await exists(`staging/${queued.body.id}`)).toBe(true);
|
|
expect(await exists(`staging/${ready.body.id}`)).toBe(false);
|
|
});
|
|
|
|
it('staging failed-вложения удаляется после срока хранения', async () => {
|
|
const res = await request(server)
|
|
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
|
.set('Authorization', bearer(alice))
|
|
.set('Content-Type', 'image/png')
|
|
.send(SAMPLE_PNG)
|
|
.expect(202);
|
|
await prisma.attachment.update({
|
|
where: { id: res.body.id },
|
|
data: { status: 'failed', errorCode: 'invalid_image' },
|
|
});
|
|
const options = { graceMs: HOUR_MS, prefix: `staging/${res.body.id}`, now: later() };
|
|
const kept = await reconcile(prisma, s3, {
|
|
...options,
|
|
apply: true,
|
|
stagingRetentionMs: 3 * HOUR_MS,
|
|
});
|
|
expect(kept.staging_expired).toBe(0);
|
|
expect(await exists(`staging/${res.body.id}`)).toBe(true);
|
|
const expired = await reconcile(prisma, s3, {
|
|
...options,
|
|
apply: true,
|
|
stagingRetentionMs: HOUR_MS,
|
|
});
|
|
expect(expired.staging_expired).toBeGreaterThanOrEqual(1);
|
|
expect(await exists(`staging/${res.body.id}`)).toBe(false);
|
|
const row = await prisma.attachment.findUniqueOrThrow({ where: { id: res.body.id } });
|
|
expect(row.stagingKey).toBeNull();
|
|
});
|
|
|
|
it('объекты вариантов без строк у существующего вложения не удаляются: replay после сверки доводит до ready', async () => {
|
|
const res = await request(server)
|
|
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
|
.set('Authorization', bearer(alice))
|
|
.set('Content-Type', 'image/png')
|
|
.send(SAMPLE_PNG)
|
|
.expect(202);
|
|
const id = res.body.id;
|
|
await expect(
|
|
processAttachment(app, id, {
|
|
afterStore: async () => {
|
|
throw new Error('worker crashed');
|
|
},
|
|
}),
|
|
).rejects.toThrow('worker crashed');
|
|
const job = await prisma.attachmentJob.findFirstOrThrow({ where: { attachmentId: id } });
|
|
await prisma.attachmentJob.update({
|
|
where: { id: job.id },
|
|
data: { status: 'failed', errorCode: 'retry_exhausted', leaseOwner: null, leaseUntil: null },
|
|
});
|
|
await prisma.attachment.update({
|
|
where: { id },
|
|
data: { status: 'failed', errorCode: 'retry_exhausted' },
|
|
});
|
|
expect(await prisma.attachmentVariant.count({ where: { attachmentId: id } })).toBe(0);
|
|
const report = await reconcile(prisma, s3, {
|
|
apply: true,
|
|
graceMs: HOUR_MS,
|
|
prefix: `attachments/${id}/`,
|
|
now: later(),
|
|
});
|
|
expect(report).toMatchObject({ scanned: 4, referenced: 4, orphans: 0, deleted: 0 });
|
|
expect(await replayJob(prisma, job.id, 'test-operator')).toBe('replayed');
|
|
expect(await processAttachment(app, id)).toBe('ready');
|
|
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
|
|
await request(server)
|
|
.get(`/api/v1/attachments/${id}/content?variant=${variant}`)
|
|
.set('Authorization', bearer(alice))
|
|
.expect(200);
|
|
}
|
|
});
|
|
|
|
it('staging ready-вложения, оставшийся после сбоя worker, удаляется без ожидания срока хранения', async () => {
|
|
const res = await request(server)
|
|
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
|
.set('Authorization', bearer(alice))
|
|
.set('Content-Type', 'image/png')
|
|
.send(SAMPLE_PNG)
|
|
.expect(202);
|
|
await expect(
|
|
processAttachment(app, res.body.id, {
|
|
afterCommit: async () => {
|
|
throw new Error('worker crashed');
|
|
},
|
|
}),
|
|
).rejects.toThrow('worker crashed');
|
|
expect(await exists(`staging/${res.body.id}`)).toBe(true);
|
|
const report = await reconcile(prisma, s3, {
|
|
apply: true,
|
|
graceMs: HOUR_MS,
|
|
prefix: `staging/${res.body.id}`,
|
|
stagingRetentionMs: 24 * HOUR_MS,
|
|
});
|
|
expect(report.staging_expired).toBeGreaterThanOrEqual(1);
|
|
expect(await exists(`staging/${res.body.id}`)).toBe(false);
|
|
await request(server)
|
|
.get(`/api/v1/attachments/${res.body.id}/content`)
|
|
.set('Authorization', bearer(alice))
|
|
.expect(200);
|
|
});
|
|
});
|
|
|
|
describe('Готовность при отказе S3 (e2e)', () => {
|
|
it('ready — 503 с заголовками за короткое время, live и core — 200', async () => {
|
|
const { app, server } = await createTestApp({ S3_ENDPOINT: 'http://127.0.0.1:9' });
|
|
try {
|
|
const started = Date.now();
|
|
const res = await request(server).get('/health/ready').expect(503);
|
|
expect(Date.now() - started).toBeLessThan(2000);
|
|
expect(res.body.error.code).toBe('unavailable');
|
|
expect(res.body.error.request_id).toBe(res.headers['x-request-id']);
|
|
expect(res.headers['x-instance-id']).toBeTruthy();
|
|
await request(server).get('/health/live').expect(200);
|
|
await request(server).get('/health/core').expect(200);
|
|
} finally {
|
|
await app.close();
|
|
}
|
|
});
|
|
|
|
it('при рабочем S3 ready — 200', async () => {
|
|
const { app, server } = await createTestApp();
|
|
try {
|
|
await request(server).get('/health/ready').expect(200);
|
|
} finally {
|
|
await app.close();
|
|
}
|
|
});
|
|
});
|