Files
mtusi2026lab6/test/reconciler.e2e-spec.ts
T

344 lines
12 KiB
TypeScript

import { INestApplication } from '@nestjs/common';
import { createHash, randomUUID } from 'crypto';
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import request from 'supertest';
import { reconcile } from '../src/attachments/reconciler';
import { PrismaService } from '../src/db/prisma.service';
import { replayJob } from '../src/jobs/job-store';
import { S3Service } from '../src/storage/s3.service';
import {
bearer,
createTestApp,
processAttachment,
registerUser,
TestUser,
} from './helpers';
import { SAMPLE_PNG } from './images';
const HOUR_MS = 60 * 60 * 1000;
describe('Сверка объектов S3 с метаданными (e2e)', () => {
let app: INestApplication;
let server: any;
let prisma: PrismaService;
let s3: S3Service;
let alice: TestUser;
let chatId: string;
let dir: string;
const putObject = async (key: string): Promise<void> => {
const path = join(dir, randomUUID());
writeFileSync(path, SAMPLE_PNG);
await s3.putFile({
key,
path,
size: SAMPLE_PNG.length,
contentType: 'image/png',
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
});
};
const exists = async (key: string): Promise<boolean> => {
const page = await s3.listObjects(key, 1);
return page.objects.some((object) => object.key === key);
};
const scope = () => `reconcile-test/${randomUUID()}/`;
const later = () => new Date(Date.now() + 2 * HOUR_MS);
beforeAll(async () => {
({ app, server } = await createTestApp());
prisma = app.get(PrismaService);
s3 = app.get(S3Service);
dir = mkdtempSync(join(tmpdir(), 'reconcile-test-'));
alice = await registerUser(server, 'rec_alice');
const chat = await request(server)
.post('/api/v1/chats')
.set('Authorization', bearer(alice))
.send({ title: 'Reconcile', member_ids: [alice.id] })
.expect(201);
chatId = chat.body.id;
});
afterAll(async () => {
rmSync(dir, { recursive: true, force: true });
await app.close();
});
it('dry-run сообщает о сироте и ничего не удаляет, apply удаляет, повторный запуск ничего не делает', async () => {
const prefix = scope();
const key = `${prefix}orphan`;
await putObject(key);
const options = { graceMs: HOUR_MS, prefix, now: later() };
const dry = await reconcile(prisma, s3, { ...options, apply: false });
expect(dry).toMatchObject({ mode: 'dry-run', scanned: 1, orphans: 1, deleted: 0, orphan_keys: [key] });
expect(await exists(key)).toBe(true);
const applied = await reconcile(prisma, s3, { ...options, apply: true });
expect(applied).toMatchObject({ mode: 'apply', orphans: 1, deleted: 1, failed: 0 });
expect(await exists(key)).toBe(false);
const again = await reconcile(prisma, s3, { ...options, apply: true });
expect(again).toMatchObject({ scanned: 0, orphans: 0, deleted: 0 });
});
it('не трогает свежий объект внутри grace period', async () => {
const prefix = scope();
const key = `${prefix}fresh`;
await putObject(key);
const report = await reconcile(prisma, s3, { apply: true, graceMs: HOUR_MS, prefix });
expect(report).toMatchObject({ scanned: 1, fresh_skipped: 1, orphans: 0, deleted: 0 });
expect(await exists(key)).toBe(true);
});
it('не трогает старый объект, на который есть метаданные', async () => {
const prefix = scope();
const key = `${prefix}referenced`;
await putObject(key);
await prisma.attachment.create({
data: {
chatId,
ownerId: alice.id,
filename: 'a.png',
contentType: 'image/png',
sizeBytes: SAMPLE_PNG.length,
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
objectKey: key,
status: 'ready',
},
});
const report = await reconcile(prisma, s3, {
apply: true,
graceMs: HOUR_MS,
prefix,
now: later(),
});
expect(report).toMatchObject({ scanned: 1, referenced: 1, orphans: 0, deleted: 0 });
expect(await exists(key)).toBe(true);
});
it('при отказе PostgreSQL проход прерывается и ничего не удаляет', async () => {
const prefix = scope();
const key = `${prefix}orphan`;
await putObject(key);
const broken = {
attachment: {
findMany: jest.fn().mockRejectedValue(new Error('database down')),
},
} as unknown as PrismaService;
await expect(
reconcile(broken, s3, { apply: true, graceMs: HOUR_MS, prefix, now: later() }),
).rejects.toThrow('database down');
expect(await exists(key)).toBe(true);
});
it('сбой удаления считается и повторяется на следующем проходе', async () => {
const prefix = scope();
const key = `${prefix}orphan`;
await putObject(key);
const options = { apply: true, graceMs: HOUR_MS, prefix, now: later() };
const remove = jest.spyOn(s3, 'delete').mockRejectedValueOnce(new Error('s3 down'));
try {
const failed = await reconcile(prisma, s3, options);
expect(failed).toMatchObject({ orphans: 1, deleted: 0, failed: 1 });
expect(await exists(key)).toBe(true);
} finally {
remove.mockRestore();
}
const retried = await reconcile(prisma, s3, options);
expect(retried).toMatchObject({ orphans: 1, deleted: 1, failed: 0 });
expect(await exists(key)).toBe(false);
});
it('обходит все страницы листинга', async () => {
const prefix = scope();
await Promise.all([1, 2, 3].map((n) => putObject(`${prefix}${n}`)));
const report = await reconcile(prisma, s3, {
apply: true,
graceMs: HOUR_MS,
prefix,
now: later(),
pageSize: 1,
});
expect(report).toMatchObject({ scanned: 3, orphans: 3, deleted: 3 });
});
it('реальная загрузка остаётся доступной после сверки всего bucket префикса вложений', async () => {
const upload = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
expect(await processAttachment(app, upload.body.id)).toBe('ready');
const report = await reconcile(prisma, s3, { apply: true, graceMs: HOUR_MS });
expect(report.fresh_skipped).toBeGreaterThanOrEqual(4);
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
await request(server)
.get(`/api/v1/attachments/${upload.body.id}/content?variant=${variant}`)
.set('Authorization', bearer(alice))
.expect(200);
}
});
it('после grace period объекты вложения и staging в очереди не считаются сиротами', async () => {
const queued = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
const ready = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=b.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
expect(await processAttachment(app, ready.body.id)).toBe('ready');
for (const id of [queued.body.id, ready.body.id]) {
for (const prefix of [`attachments/${id}/`, `staging/${id}`]) {
const report = await reconcile(prisma, s3, {
apply: true,
graceMs: HOUR_MS,
prefix,
now: later(),
});
expect(report).toMatchObject({ orphans: 0, deleted: 0 });
}
}
expect(await exists(`staging/${queued.body.id}`)).toBe(true);
expect(await exists(`staging/${ready.body.id}`)).toBe(false);
});
it('staging failed-вложения удаляется после срока хранения', async () => {
const res = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
await prisma.attachment.update({
where: { id: res.body.id },
data: { status: 'failed', errorCode: 'invalid_image' },
});
const options = { graceMs: HOUR_MS, prefix: `staging/${res.body.id}`, now: later() };
const kept = await reconcile(prisma, s3, {
...options,
apply: true,
stagingRetentionMs: 3 * HOUR_MS,
});
expect(kept.staging_expired).toBe(0);
expect(await exists(`staging/${res.body.id}`)).toBe(true);
const expired = await reconcile(prisma, s3, {
...options,
apply: true,
stagingRetentionMs: HOUR_MS,
});
expect(expired.staging_expired).toBeGreaterThanOrEqual(1);
expect(await exists(`staging/${res.body.id}`)).toBe(false);
const row = await prisma.attachment.findUniqueOrThrow({ where: { id: res.body.id } });
expect(row.stagingKey).toBeNull();
});
it('объекты вариантов без строк у существующего вложения не удаляются: replay после сверки доводит до ready', async () => {
const res = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
const id = res.body.id;
await expect(
processAttachment(app, id, {
afterStore: async () => {
throw new Error('worker crashed');
},
}),
).rejects.toThrow('worker crashed');
const job = await prisma.attachmentJob.findFirstOrThrow({ where: { attachmentId: id } });
await prisma.attachmentJob.update({
where: { id: job.id },
data: { status: 'failed', errorCode: 'retry_exhausted', leaseOwner: null, leaseUntil: null },
});
await prisma.attachment.update({
where: { id },
data: { status: 'failed', errorCode: 'retry_exhausted' },
});
expect(await prisma.attachmentVariant.count({ where: { attachmentId: id } })).toBe(0);
const report = await reconcile(prisma, s3, {
apply: true,
graceMs: HOUR_MS,
prefix: `attachments/${id}/`,
now: later(),
});
expect(report).toMatchObject({ scanned: 4, referenced: 4, orphans: 0, deleted: 0 });
expect(await replayJob(prisma, job.id, 'test-operator')).toBe('replayed');
expect(await processAttachment(app, id)).toBe('ready');
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
await request(server)
.get(`/api/v1/attachments/${id}/content?variant=${variant}`)
.set('Authorization', bearer(alice))
.expect(200);
}
});
it('staging ready-вложения, оставшийся после сбоя worker, удаляется без ожидания срока хранения', async () => {
const res = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
await expect(
processAttachment(app, res.body.id, {
afterCommit: async () => {
throw new Error('worker crashed');
},
}),
).rejects.toThrow('worker crashed');
expect(await exists(`staging/${res.body.id}`)).toBe(true);
const report = await reconcile(prisma, s3, {
apply: true,
graceMs: HOUR_MS,
prefix: `staging/${res.body.id}`,
stagingRetentionMs: 24 * HOUR_MS,
});
expect(report.staging_expired).toBeGreaterThanOrEqual(1);
expect(await exists(`staging/${res.body.id}`)).toBe(false);
await request(server)
.get(`/api/v1/attachments/${res.body.id}/content`)
.set('Authorization', bearer(alice))
.expect(200);
});
});
describe('Готовность при отказе S3 (e2e)', () => {
it('ready — 503 с заголовками за короткое время, live и core — 200', async () => {
const { app, server } = await createTestApp({ S3_ENDPOINT: 'http://127.0.0.1:9' });
try {
const started = Date.now();
const res = await request(server).get('/health/ready').expect(503);
expect(Date.now() - started).toBeLessThan(2000);
expect(res.body.error.code).toBe('unavailable');
expect(res.body.error.request_id).toBe(res.headers['x-request-id']);
expect(res.headers['x-instance-id']).toBeTruthy();
await request(server).get('/health/live').expect(200);
await request(server).get('/health/core').expect(200);
} finally {
await app.close();
}
});
it('при рабочем S3 ready — 200', async () => {
const { app, server } = await createTestApp();
try {
await request(server).get('/health/ready').expect(200);
} finally {
await app.close();
}
});
});