feat: accept uploads into staging with jobs, outbox and job processor

This commit is contained in:
2026-11-15 15:21:46 +03:00
parent 19dda6e286
commit f4793b50cd
18 changed files with 1657 additions and 95 deletions
@@ -0,0 +1,97 @@
ALTER TABLE "attachments"
ADD COLUMN "staging_key" VARCHAR(128),
ADD COLUMN "status" VARCHAR(16) NOT NULL DEFAULT 'ready',
ADD COLUMN "error_code" VARCHAR(64);
ALTER TABLE "attachments" ALTER COLUMN "status" DROP DEFAULT;
ALTER TABLE "attachments"
ADD CONSTRAINT "attachments_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
ADD CONSTRAINT "attachments_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL));
CREATE UNIQUE INDEX "attachments_staging_key_key" ON "attachments"("staging_key");
CREATE TABLE "attachment_variants" (
"attachment_id" UUID NOT NULL,
"pipeline_version" INTEGER NOT NULL,
"name" VARCHAR(16) NOT NULL,
"content_type" VARCHAR(32) NOT NULL,
"object_key" VARCHAR(128) NOT NULL,
"size_bytes" INTEGER NOT NULL,
"width" INTEGER NOT NULL,
"height" INTEGER NOT NULL,
"sha256" CHAR(64) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "attachment_variants_pkey" PRIMARY KEY ("attachment_id","pipeline_version","name"),
CONSTRAINT "attachment_variants_name_allowed" CHECK ("name" IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT "attachment_variants_content_type_allowed" CHECK ("content_type" IN ('image/jpeg', 'image/png', 'image/webp')),
CONSTRAINT "attachment_variants_positive" CHECK ("pipeline_version" >= 1 AND "size_bytes" >= 1 AND "width" >= 1 AND "height" >= 1),
CONSTRAINT "attachment_variants_sha256_format" CHECK ("sha256" ~ '^[a-f0-9]{64}$')
);
CREATE TABLE "attachment_jobs" (
"id" UUID NOT NULL,
"attachment_id" UUID NOT NULL,
"pipeline_version" INTEGER NOT NULL,
"status" VARCHAR(16) NOT NULL,
"attempt" INTEGER NOT NULL,
"max_attempts" INTEGER NOT NULL,
"next_attempt_at" TIMESTAMPTZ(3) NOT NULL,
"lease_owner" UUID,
"lease_until" TIMESTAMPTZ(3),
"error_code" VARCHAR(64),
"last_error" VARCHAR(200),
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"finished_at" TIMESTAMPTZ(3),
CONSTRAINT "attachment_jobs_pkey" PRIMARY KEY ("id"),
CONSTRAINT "attachment_jobs_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT "attachment_jobs_attempt_range" CHECK ("pipeline_version" >= 1 AND "attempt" >= 1 AND "attempt" <= "max_attempts"),
CONSTRAINT "attachment_jobs_lease_when_processing" CHECK ("status" <> 'processing' OR ("lease_owner" IS NOT NULL AND "lease_until" IS NOT NULL)),
CONSTRAINT "attachment_jobs_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL))
);
CREATE TABLE "job_events" (
"id" BIGSERIAL NOT NULL,
"job_id" UUID NOT NULL,
"attempt" INTEGER NOT NULL,
"event" VARCHAR(32) NOT NULL,
"detail" VARCHAR(200),
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "job_events_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "outbox" (
"id" BIGSERIAL NOT NULL,
"job_id" UUID NOT NULL,
"payload" JSONB NOT NULL,
"available_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"published_at" TIMESTAMPTZ(3),
"publish_attempts" INTEGER NOT NULL DEFAULT 0,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "outbox_pkey" PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX "attachment_variants_object_key_key" ON "attachment_variants"("object_key");
CREATE UNIQUE INDEX "attachment_jobs_attachment_id_pipeline_version_key" ON "attachment_jobs"("attachment_id", "pipeline_version");
CREATE INDEX "attachment_jobs_status_lease_until_idx" ON "attachment_jobs"("status", "lease_until");
CREATE INDEX "job_events_job_id_idx" ON "job_events"("job_id");
CREATE INDEX "outbox_published_at_available_at_idx" ON "outbox"("published_at", "available_at");
CREATE INDEX "outbox_job_id_idx" ON "outbox"("job_id");
ALTER TABLE "attachment_variants" ADD CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "attachment_jobs" ADD CONSTRAINT "attachment_jobs_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "job_events" ADD CONSTRAINT "job_events_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "outbox" ADD CONSTRAINT "outbox_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+74
View File
@@ -78,17 +78,91 @@ model Attachment {
sizeBytes Int @map("size_bytes")
sha256 String @db.Char(64)
objectKey String @unique @map("object_key") @db.VarChar(128)
stagingKey String? @unique @map("staging_key") @db.VarChar(128)
status String @db.VarChar(16)
errorCode String? @map("error_code") @db.VarChar(64)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
owner User @relation(fields: [ownerId], references: [id], onDelete: Restrict)
messages MessageAttachment[]
variants AttachmentVariant[]
jobs AttachmentJob[]
@@index([chatId])
@@index([ownerId])
@@map("attachments")
}
model AttachmentVariant {
attachmentId String @map("attachment_id") @db.Uuid
pipelineVersion Int @map("pipeline_version")
name String @db.VarChar(16)
contentType String @map("content_type") @db.VarChar(32)
objectKey String @unique @map("object_key") @db.VarChar(128)
sizeBytes Int @map("size_bytes")
width Int
height Int
sha256 String @db.Char(64)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
@@id([attachmentId, pipelineVersion, name])
@@map("attachment_variants")
}
model AttachmentJob {
id String @id @db.Uuid
attachmentId String @map("attachment_id") @db.Uuid
pipelineVersion Int @map("pipeline_version")
status String @db.VarChar(16)
attempt Int
maxAttempts Int @map("max_attempts")
nextAttemptAt DateTime @map("next_attempt_at") @db.Timestamptz(3)
leaseOwner String? @map("lease_owner") @db.Uuid
leaseUntil DateTime? @map("lease_until") @db.Timestamptz(3)
errorCode String? @map("error_code") @db.VarChar(64)
lastError String? @map("last_error") @db.VarChar(200)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
finishedAt DateTime? @map("finished_at") @db.Timestamptz(3)
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
events JobEvent[]
outbox OutboxEntry[]
@@unique([attachmentId, pipelineVersion])
@@index([status, leaseUntil])
@@map("attachment_jobs")
}
model JobEvent {
id BigInt @id @default(autoincrement())
jobId String @map("job_id") @db.Uuid
attempt Int
event String @db.VarChar(32)
detail String? @db.VarChar(200)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
@@index([jobId])
@@map("job_events")
}
model OutboxEntry {
id BigInt @id @default(autoincrement())
jobId String @map("job_id") @db.Uuid
payload Json
availableAt DateTime @default(now()) @map("available_at") @db.Timestamptz(3)
publishedAt DateTime? @map("published_at") @db.Timestamptz(3)
publishAttempts Int @default(0) @map("publish_attempts")
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
@@index([publishedAt, availableAt])
@@index([jobId])
@@map("outbox")
}
model MessageAttachment {
messageId String @map("message_id") @db.Uuid
position Int @db.SmallInt
+6 -6
View File
@@ -26,7 +26,7 @@ export class AttachmentsController {
constructor(private readonly attachments: AttachmentsService) {}
@Post('api/v1/chats/:chatId/attachments')
@HttpCode(201)
@HttpCode(202)
async upload(
@Param('chatId') chatId: string,
@Query('filename') filename: unknown,
@@ -61,19 +61,19 @@ export class AttachmentsController {
@CurrentUser() user: User,
@Res() reply: FastifyReply,
): Promise<void> {
const { attachment, content } = await this.attachments.openContent(
const download = await this.attachments.openContent(
attachmentId,
user.id,
variant,
);
await reply
.header('content-type', attachment.contentType)
.header('content-length', attachment.sizeBytes)
.header('content-type', download.contentType)
.header('content-length', download.sizeBytes)
.header('x-content-type-options', 'nosniff')
.header(
'content-disposition',
`inline; filename*=UTF-8''${encodeURIComponent(attachment.filename)}`,
`inline; filename*=UTF-8''${encodeURIComponent(download.filename)}`,
)
.send(content);
.send(download.content);
}
}
+74 -32
View File
@@ -1,5 +1,5 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Attachment, Prisma } from '@prisma/client';
import { Attachment, AttachmentVariant, Prisma } from '@prisma/client';
import { randomUUID } from 'crypto';
import { mkdtemp, readdir, rm, stat } from 'fs/promises';
import { tmpdir } from 'os';
@@ -10,6 +10,8 @@ import { codePointLength } from '../common/dto/code-point-length.validator';
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { isDatabaseUnavailable, PrismaService } from '../db/prisma.service';
import { PIPELINE_VERSION } from '../jobs/job-message';
import { enqueueJob } from '../jobs/job-store';
import { S3Service } from '../storage/s3.service';
import { assertValidImage, isSupportedContentType } from './image-validation';
import { receiveBody } from './upload-body';
@@ -18,7 +20,16 @@ const TEMP_PREFIX = 'messenger-upload-';
const STALE_TEMP_MS = 60 * 60 * 1000;
const MAX_FILENAME_CODE_POINTS = 255;
const FORBIDDEN_FILENAME = /[/\\\p{Cc}\p{Zl}\p{Zp}]/u;
const KNOWN_VARIANTS = new Set(['thumbnail', 'crop', 'optimized']);
const KNOWN_VARIANTS = new Set(['original', 'thumbnail', 'crop', 'optimized']);
export type AttachmentWithVariants = Attachment & { variants: AttachmentVariant[] };
export interface DownloadableContent {
filename: string;
contentType: string;
sizeBytes: number;
content: Readable;
}
export interface UploadInput {
chatId: string;
@@ -36,6 +47,7 @@ export class AttachmentsService implements OnModuleInit {
private readonly maxPixels = Number(process.env.MAX_IMAGE_PIXELS ?? 40000000);
private readonly maxConcurrent = Number(process.env.MAX_CONCURRENT_UPLOADS ?? 16);
private readonly bodyTimeoutMs = Number(process.env.UPLOAD_BODY_TIMEOUT_MS ?? 60000);
private readonly tempRoot = process.env.UPLOAD_TMP_DIR ?? tmpdir();
private activeUploads = 0;
constructor(
@@ -47,7 +59,7 @@ export class AttachmentsService implements OnModuleInit {
await this.removeStaleTempDirs();
}
async upload(input: UploadInput): Promise<Attachment> {
async upload(input: UploadInput): Promise<AttachmentWithVariants> {
await assertChatMember(this.prisma, input.chatId, input.ownerId);
const filename = parseFilename(input.filename);
const contentType = parseContentType(input.contentType);
@@ -58,7 +70,7 @@ export class AttachmentsService implements OnModuleInit {
throw new ApiError('unavailable', 'too many concurrent uploads');
}
this.activeUploads++;
const dir = await mkdtemp(join(tmpdir(), TEMP_PREFIX));
const dir = await mkdtemp(join(this.tempRoot, TEMP_PREFIX));
try {
return await this.store(input, filename, contentType, join(dir, 'body'));
} finally {
@@ -67,7 +79,10 @@ export class AttachmentsService implements OnModuleInit {
}
}
async getVisible(attachmentId: string, userId: string): Promise<Attachment> {
async getVisible(
attachmentId: string,
userId: string,
): Promise<AttachmentWithVariants> {
if (!isUuid(attachmentId)) {
throw new ApiError('invalid_request', 'attachmentId must be a valid UUID');
}
@@ -76,6 +91,7 @@ export class AttachmentsService implements OnModuleInit {
id: attachmentId,
chat: { members: { some: { userId } } },
},
include: { variants: { where: { pipelineVersion: PIPELINE_VERSION } } },
});
if (!row) {
throw new ApiError('not_found', 'attachment not found');
@@ -87,10 +103,30 @@ export class AttachmentsService implements OnModuleInit {
attachmentId: string,
userId: string,
variant: unknown,
): Promise<{ attachment: Attachment; content: Readable }> {
): Promise<DownloadableContent> {
const attachment = await this.getVisible(attachmentId, userId);
assertOriginalVariant(variant);
return { attachment, content: await this.s3.getStream(attachment.objectKey) };
const name = parseVariant(variant);
if (attachment.status !== 'ready') {
throw new ApiError('conflict', `attachment is ${attachment.status}`);
}
if (name === 'original') {
return {
filename: attachment.filename,
contentType: attachment.contentType,
sizeBytes: attachment.sizeBytes,
content: await this.s3.getStream(attachment.objectKey),
};
}
const stored = attachment.variants.find((v) => v.name === name);
if (!stored) {
throw new ApiError('not_found', 'attachment variant not found');
}
return {
filename: attachment.filename,
contentType: stored.contentType,
sizeBytes: stored.sizeBytes,
content: await this.s3.getStream(stored.objectKey),
};
}
private async store(
@@ -98,7 +134,7 @@ export class AttachmentsService implements OnModuleInit {
filename: string,
contentType: string,
path: string,
): Promise<Attachment> {
): Promise<AttachmentWithVariants> {
if (!input.body) {
throw new ApiError('invalid_request', 'request body must not be empty');
}
@@ -113,32 +149,38 @@ export class AttachmentsService implements OnModuleInit {
}
await assertValidImage(path, size, contentType, this.maxPixels);
const id = randomUUID();
const objectKey = `attachments/${id}`;
await this.s3.putFile({ key: objectKey, path, size, contentType, sha256 });
const stagingKey = `staging/${id}`;
await this.s3.putFile({ key: stagingKey, path, size, contentType, sha256 });
const now = new Date();
try {
return await this.prisma.attachment.create({
data: {
id,
chatId: input.chatId,
ownerId: input.ownerId,
filename,
contentType,
sizeBytes: size,
sha256,
objectKey,
createdAt: now,
updatedAt: now,
},
return await this.prisma.$transaction(async (tx) => {
const attachment = await tx.attachment.create({
data: {
id,
chatId: input.chatId,
ownerId: input.ownerId,
filename,
contentType,
sizeBytes: size,
sha256,
objectKey: `attachments/${id}/original`,
stagingKey,
status: 'queued',
createdAt: now,
updatedAt: now,
},
});
await enqueueJob(tx, id, now);
return { ...attachment, variants: [] };
});
} catch (error) {
if (isDefiniteRejection(error)) {
await this.s3.delete(objectKey).catch(() => {
this.logger.warn(`compensation failed for ${objectKey}`);
await this.s3.delete(stagingKey).catch(() => {
this.logger.warn(`compensation failed for ${stagingKey}`);
});
}
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
this.logger.error(`metadata insert failed for ${objectKey}`);
this.logger.error(`metadata insert failed for ${stagingKey}`);
throw new ApiError('unavailable', 'database unavailable');
}
throw error;
@@ -146,7 +188,7 @@ export class AttachmentsService implements OnModuleInit {
}
private async removeStaleTempDirs(): Promise<void> {
const root = tmpdir();
const root = this.tempRoot;
const names = await readdir(root);
for (const name of names.filter((n) => n.startsWith(TEMP_PREFIX))) {
const path = join(root, name);
@@ -158,12 +200,12 @@ export class AttachmentsService implements OnModuleInit {
}
}
function assertOriginalVariant(raw: unknown): void {
if (raw === undefined || raw === 'original') {
return;
function parseVariant(raw: unknown): string {
if (raw === undefined) {
return 'original';
}
if (typeof raw === 'string' && KNOWN_VARIANTS.has(raw)) {
throw new ApiError('not_found', 'attachment variant not found');
return raw;
}
throw new ApiError('invalid_request', 'unknown attachment variant');
}
+30 -8
View File
@@ -1,4 +1,15 @@
import { Attachment } from '@prisma/client';
import { Attachment, AttachmentVariant } from '@prisma/client';
const VARIANT_ORDER = ['thumbnail', 'crop', 'optimized'];
export interface VariantResponse {
name: string;
content_type: string;
size_bytes: number;
width: number;
height: number;
sha256: string;
}
export interface AttachmentResponse {
id: string;
@@ -8,14 +19,18 @@ export interface AttachmentResponse {
content_type: string;
size_bytes: number;
sha256: string;
status: 'ready';
variants: never[];
error_code: null;
status: string;
variants: VariantResponse[];
error_code: string | null;
created_at: string;
updated_at: string;
}
export function toAttachmentResponse(row: Attachment): AttachmentResponse {
export function toAttachmentResponse(
row: Attachment & { variants: AttachmentVariant[] },
): AttachmentResponse {
const variants = row.status === 'ready' ? [...row.variants] : [];
variants.sort((a, b) => VARIANT_ORDER.indexOf(a.name) - VARIANT_ORDER.indexOf(b.name));
return {
id: row.id,
chat_id: row.chatId,
@@ -24,9 +39,16 @@ export function toAttachmentResponse(row: Attachment): AttachmentResponse {
content_type: row.contentType,
size_bytes: row.sizeBytes,
sha256: row.sha256,
status: 'ready',
variants: [],
error_code: null,
status: row.status,
variants: variants.map((variant) => ({
name: variant.name,
content_type: variant.contentType,
size_bytes: variant.sizeBytes,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
})),
error_code: row.errorCode,
created_at: row.createdAt.toISOString(),
updated_at: row.updatedAt.toISOString(),
};
+75 -10
View File
@@ -2,10 +2,14 @@ import { PrismaClient } from '@prisma/client';
import { S3Service } from '../storage/s3.service';
const MAX_REPORTED_KEYS = 20;
const DEFAULT_PREFIXES = ['attachments/', 'staging/'];
type ReconcilePrisma = Pick<PrismaClient, 'attachment' | 'attachmentVariant'>;
export interface ReconcileOptions {
apply: boolean;
graceMs: number;
stagingRetentionMs?: number;
prefix?: string;
now?: Date;
pageSize?: number;
@@ -14,6 +18,7 @@ export interface ReconcileOptions {
export interface ReconcileReport {
mode: 'apply' | 'dry-run';
grace_seconds: number;
staging_expired: number;
scanned: number;
fresh_skipped: number;
referenced: number;
@@ -25,7 +30,7 @@ export interface ReconcileReport {
}
export async function reconcile(
prisma: Pick<PrismaClient, 'attachment'>,
prisma: ReconcilePrisma,
s3: S3Service,
options: ReconcileOptions,
): Promise<ReconcileReport> {
@@ -34,6 +39,7 @@ export async function reconcile(
const report: ReconcileReport = {
mode: options.apply ? 'apply' : 'dry-run',
grace_seconds: options.graceMs / 1000,
staging_expired: 0,
scanned: 0,
fresh_skipped: 0,
referenced: 0,
@@ -43,22 +49,83 @@ export async function reconcile(
orphan_keys: [],
duration_ms: 0,
};
if (options.stagingRetentionMs !== undefined) {
report.staging_expired = await expireFailedStaging(
prisma,
s3,
(options.now ?? new Date()).getTime() - options.stagingRetentionMs,
options.apply,
);
}
for (const prefix of options.prefix ? [options.prefix] : DEFAULT_PREFIXES) {
await reconcilePrefix(prisma, s3, prefix, cutoff, options, report);
}
report.duration_ms = Date.now() - started;
return report;
}
async function expireFailedStaging(
prisma: ReconcilePrisma,
s3: S3Service,
before: number,
apply: boolean,
): Promise<number> {
const rows = await prisma.attachment.findMany({
where: {
status: 'failed',
stagingKey: { not: null },
updatedAt: { lt: new Date(before) },
},
select: { id: true, stagingKey: true },
take: 500,
});
if (!apply) {
return rows.length;
}
let expired = 0;
for (const row of rows) {
try {
await s3.delete(row.stagingKey!);
await prisma.attachment.update({
where: { id: row.id },
data: { stagingKey: null },
});
expired++;
} catch {
continue;
}
}
return expired;
}
async function reconcilePrefix(
prisma: ReconcilePrisma,
s3: S3Service,
prefix: string,
cutoff: number,
options: ReconcileOptions,
report: ReconcileReport,
): Promise<void> {
let token: string | undefined;
do {
const page = await s3.listObjects(
options.prefix ?? 'attachments/',
options.pageSize ?? 500,
token,
);
const page = await s3.listObjects(prefix, options.pageSize ?? 500, token);
token = page.nextToken;
report.scanned += page.objects.length;
const old = page.objects.filter((object) => object.lastModified.getTime() < cutoff);
report.fresh_skipped += page.objects.length - old.length;
const keys = old.map((object) => object.key);
const rows = await prisma.attachment.findMany({
where: { objectKey: { in: old.map((object) => object.key) } },
where: { OR: [{ objectKey: { in: keys } }, { stagingKey: { in: keys } }] },
select: { objectKey: true, stagingKey: true },
});
const variants = await prisma.attachmentVariant.findMany({
where: { objectKey: { in: keys } },
select: { objectKey: true },
});
const known = new Set(rows.map((row) => row.objectKey));
const known = new Set([
...rows.flatMap((row) => [row.objectKey, row.stagingKey]),
...variants.map((row) => row.objectKey),
]);
for (const { key } of old) {
if (known.has(key)) {
report.referenced++;
@@ -76,6 +143,4 @@ export async function reconcile(
}
}
} while (token);
report.duration_ms = Date.now() - started;
return report;
}
+69
View File
@@ -0,0 +1,69 @@
import { validate as isUuid } from '../common/uuid';
export const PIPELINE_VERSION = 1;
export const JOB_TYPE = 'attachment.process.v1';
export interface JobMessage {
schema_version: 1;
job_id: string;
type: typeof JOB_TYPE;
attachment_id: string;
pipeline_version: number;
attempt: number;
created_at: string;
}
const FIELDS = [
'schema_version',
'job_id',
'type',
'attachment_id',
'pipeline_version',
'attempt',
'created_at',
];
export function jobMessage(
job: { id: string; attachmentId: string; pipelineVersion: number; attempt: number },
createdAt: Date,
): JobMessage {
return {
schema_version: 1,
job_id: job.id,
type: JOB_TYPE,
attachment_id: job.attachmentId,
pipeline_version: job.pipelineVersion,
attempt: job.attempt,
created_at: createdAt.toISOString(),
};
}
export function parseJobMessage(raw: Buffer): JobMessage | null {
let value: any;
try {
value = JSON.parse(raw.toString('utf8'));
} catch {
return null;
}
if (
typeof value !== 'object' ||
value === null ||
Array.isArray(value) ||
Object.keys(value).some((key) => !FIELDS.includes(key)) ||
value.schema_version !== 1 ||
value.type !== JOB_TYPE ||
typeof value.job_id !== 'string' ||
!isUuid(value.job_id) ||
typeof value.attachment_id !== 'string' ||
!isUuid(value.attachment_id) ||
!Number.isInteger(value.pipeline_version) ||
value.pipeline_version < 1 ||
!Number.isInteger(value.attempt) ||
value.attempt < 1 ||
typeof value.created_at !== 'string' ||
Number.isNaN(Date.parse(value.created_at))
) {
return null;
}
return value as JobMessage;
}
+217
View File
@@ -0,0 +1,217 @@
import { AttachmentJob, Prisma, PrismaClient } from '@prisma/client';
import { randomUUID } from 'crypto';
import { jobMessage, PIPELINE_VERSION } from './job-message';
type Tx = Prisma.TransactionClient;
export const RETRY_EXHAUSTED = 'retry_exhausted';
export function maxAttempts(): number {
return Number(process.env.JOB_MAX_ATTEMPTS ?? 5);
}
export function backoffMs(attempt: number, random = Math.random): number {
const base = Number(process.env.JOB_RETRY_BASE_MS ?? 1000);
const cap = Number(process.env.JOB_RETRY_MAX_MS ?? 30000);
const delay = Math.min(cap, base * 2 ** (attempt - 1));
return Math.round(delay / 2 + random() * (delay / 2));
}
export async function enqueueJob(
tx: Tx,
attachmentId: string,
now: Date,
): Promise<AttachmentJob> {
const job = await tx.attachmentJob.create({
data: {
id: randomUUID(),
attachmentId,
pipelineVersion: PIPELINE_VERSION,
status: 'queued',
attempt: 1,
maxAttempts: maxAttempts(),
nextAttemptAt: now,
createdAt: now,
updatedAt: now,
},
});
await tx.jobEvent.create({
data: { jobId: job.id, attempt: 1, event: 'enqueued', createdAt: now },
});
await publishLater(tx, job, now);
return job;
}
export async function scheduleRetry(
tx: Tx,
job: AttachmentJob,
reason: string,
now: Date,
event = 'retry_scheduled',
): Promise<AttachmentJob> {
const delay = backoffMs(job.attempt);
const next = await tx.attachmentJob.update({
where: { id: job.id },
data: {
status: 'queued',
attempt: job.attempt + 1,
nextAttemptAt: new Date(now.getTime() + delay),
leaseOwner: null,
leaseUntil: null,
lastError: reason,
},
});
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'queued' },
});
await tx.jobEvent.create({
data: {
jobId: job.id,
attempt: job.attempt,
event,
detail: `${reason}; next attempt ${next.attempt} in ${delay} ms`,
createdAt: now,
},
});
await publishLater(tx, next, next.nextAttemptAt);
return next;
}
export async function markFailed(
tx: Tx,
job: AttachmentJob,
errorCode: string,
reason: string,
now: Date,
): Promise<void> {
await tx.attachmentJob.update({
where: { id: job.id },
data: {
status: 'failed',
errorCode,
lastError: reason,
leaseOwner: null,
leaseUntil: null,
finishedAt: now,
},
});
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'failed', errorCode },
});
await tx.jobEvent.create({
data: {
jobId: job.id,
attempt: job.attempt,
event: 'failed',
detail: `${errorCode}: ${reason}`,
createdAt: now,
},
});
}
export async function retryOrFail(
tx: Tx,
job: AttachmentJob,
reason: string,
now: Date,
event?: string,
): Promise<'retry' | 'failed'> {
if (job.attempt >= job.maxAttempts) {
await markFailed(tx, job, RETRY_EXHAUSTED, reason, now);
return 'failed';
}
await scheduleRetry(tx, job, reason, now, event);
return 'retry';
}
export async function recoverExpiredLeases(
prisma: PrismaClient,
now = new Date(),
limit = 100,
): Promise<number> {
const expired = await prisma.attachmentJob.findMany({
where: { status: 'processing', leaseUntil: { lt: now } },
orderBy: { leaseUntil: 'asc' },
take: limit,
});
let recovered = 0;
for (const candidate of expired) {
await prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: {
id: candidate.id,
status: 'processing',
leaseOwner: candidate.leaseOwner,
leaseUntil: { lt: now },
},
data: { leaseOwner: null, leaseUntil: null, status: 'queued' },
});
if (taken.count === 0) {
return;
}
await retryOrFail(tx, candidate, 'lease expired', now, 'lease_expired');
recovered++;
});
}
return recovered;
}
export type ReplayResult = 'replayed' | 'already_ready' | 'not_failed' | 'not_found';
export async function replayJob(
prisma: PrismaClient,
jobId: string,
actor: string,
now = new Date(),
): Promise<ReplayResult> {
return prisma.$transaction(async (tx) => {
const job = await tx.attachmentJob.findUnique({ where: { id: jobId } });
if (!job) {
return 'not_found';
}
if (job.status === 'ready') {
return 'already_ready';
}
const taken = await tx.attachmentJob.updateMany({
where: { id: jobId, status: 'failed' },
data: {
status: 'queued',
attempt: 1,
errorCode: null,
nextAttemptAt: now,
finishedAt: null,
},
});
if (taken.count === 0) {
return 'not_failed';
}
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'queued', errorCode: null },
});
await tx.jobEvent.create({
data: {
jobId,
attempt: 1,
event: 'replayed',
detail: `by ${actor}; previous ${job.errorCode} after attempt ${job.attempt}`,
createdAt: now,
},
});
await publishLater(tx, { ...job, attempt: 1 }, now);
return 'replayed';
});
}
async function publishLater(tx: Tx, job: AttachmentJob, availableAt: Date): Promise<void> {
await tx.outboxEntry.create({
data: {
jobId: job.id,
payload: jobMessage(job, availableAt) as unknown as Prisma.InputJsonObject,
availableAt,
createdAt: new Date(),
},
});
}
+255
View File
@@ -0,0 +1,255 @@
import { Logger } from '@nestjs/common';
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import {
ObjectMissingError,
ObjectTooLargeError,
S3Service,
} from '../storage/s3.service';
import { JobMessage, PIPELINE_VERSION } from './job-message';
import { markFailed, retryOrFail } from './job-store';
import { InvalidImageError, RenderedVariant, renderVariants } from './variants';
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
export interface ProcessorOptions {
timeoutMs: number;
leaseMs: number;
maxBytes: number;
maxPixels: number;
}
export function processorOptions(): ProcessorOptions {
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
return {
timeoutMs,
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
};
}
class PermanentError extends Error {
constructor(
readonly code: string,
reason: string,
) {
super(reason);
}
}
class TimeoutError extends Error {}
export class JobProcessor {
private readonly logger = new Logger('JobProcessor');
constructor(
private readonly prisma: PrismaClient,
private readonly s3: S3Service,
private readonly options: ProcessorOptions = processorOptions(),
private readonly hooks: { afterStore?: () => Promise<void> } = {},
) {}
async handle(message: JobMessage): Promise<Outcome> {
const job = await this.prisma.attachmentJob.findUnique({
where: { id: message.job_id },
include: { attachment: true },
});
if (
!job ||
job.attachmentId !== message.attachment_id ||
job.pipelineVersion !== message.pipeline_version ||
job.pipelineVersion !== PIPELINE_VERSION
) {
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
return 'skipped';
}
const lease = await this.claim(job);
if (!lease) {
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
return 'skipped';
}
let variants: RenderedVariant[];
try {
variants = await this.withTimeout(this.produce(job.attachment));
} catch (error) {
return this.recordFailure(lease, error);
}
await this.hooks.afterStore?.();
return this.commit(lease, job.attachment, variants);
}
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
const now = new Date();
const owner = randomUUID();
return this.prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
data: {
status: 'processing',
leaseOwner: owner,
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
},
});
if (taken.count === 0) {
return null;
}
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'processing' },
});
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
await tx.jobEvent.create({
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
});
return leased;
});
}
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
if (!attachment.stagingKey) {
throw new PermanentError('staging_missing', 'staging object was already removed');
}
let original: Buffer;
try {
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
} catch (error) {
if (error instanceof ObjectMissingError) {
throw new PermanentError('staging_missing', 'staging object not found');
}
if (error instanceof ObjectTooLargeError) {
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
}
throw error;
}
const sha256 = createHash('sha256').update(original).digest('hex');
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
}
let variants: RenderedVariant[];
try {
variants = await renderVariants(original, this.options.maxPixels);
} catch (error) {
if (error instanceof InvalidImageError) {
throw new PermanentError('invalid_image', 'image cannot be decoded');
}
throw error;
}
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
for (const variant of variants) {
await this.s3.putBuffer(
variantKey(attachment.id, variant),
variant.body,
variant.contentType,
variant.sha256,
);
}
return variants;
}
private async commit(
lease: AttachmentJob,
attachment: Attachment,
variants: RenderedVariant[],
): Promise<Outcome> {
const now = new Date();
const committed = await this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.updateMany({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
data: {
status: 'ready',
leaseOwner: null,
leaseUntil: null,
lastError: null,
finishedAt: now,
},
});
if (owned.count === 0) {
return false;
}
await tx.attachmentVariant.createMany({
data: variants.map((variant) => ({
attachmentId: attachment.id,
pipelineVersion: PIPELINE_VERSION,
name: variant.name,
contentType: variant.contentType,
objectKey: variantKey(attachment.id, variant),
sizeBytes: variant.body.length,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
createdAt: now,
})),
skipDuplicates: true,
});
await tx.attachment.update({
where: { id: attachment.id },
data: { status: 'ready', errorCode: null },
});
await tx.jobEvent.create({
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
});
return true;
});
if (!committed) {
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
return 'skipped';
}
await this.removeStaging(attachment);
return 'ready';
}
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
const now = new Date();
const reason =
error instanceof PermanentError
? error.message
: error instanceof TimeoutError
? 'processing timed out'
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
return this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.count({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
});
if (owned === 0) {
return 'skipped' as const;
}
if (error instanceof PermanentError) {
await markFailed(tx, lease, error.code, reason, now);
return 'failed' as const;
}
return retryOrFail(tx, lease, reason, now);
});
}
private async removeStaging(attachment: Attachment): Promise<void> {
if (!attachment.stagingKey) {
return;
}
try {
await this.s3.delete(attachment.stagingKey);
await this.prisma.attachment.update({
where: { id: attachment.id },
data: { stagingKey: null },
});
} catch {
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
}
}
private withTimeout<T>(work: Promise<T>): Promise<T> {
let timer: NodeJS.Timeout;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
});
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
}
}
export function variantKey(
attachmentId: string,
variant: Pick<RenderedVariant, 'name' | 'extension'>,
): string {
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
}
+96
View File
@@ -0,0 +1,96 @@
import { createHash } from 'crypto';
import sharp, { Sharp } from 'sharp';
sharp.cache(false);
sharp.concurrency(1);
const WHITE = { r: 255, g: 255, b: 255 };
const JPEG_QUALITY = 82;
const WEBP_QUALITY = 80;
export interface RenderedVariant {
name: 'thumbnail' | 'crop' | 'optimized';
contentType: string;
extension: string;
body: Buffer;
width: number;
height: number;
sha256: string;
}
export class InvalidImageError extends Error {
constructor(reason: string) {
super(reason);
this.name = 'InvalidImageError';
}
}
export function fitInside(
width: number,
height: number,
box: number,
): { width: number; height: number } {
const scale = Math.min(box / width, box / height, 1);
return {
width: Math.max(1, Math.floor(width * scale)),
height: Math.max(1, Math.floor(height * scale)),
};
}
export async function renderVariants(
original: Buffer,
maxPixels: number,
): Promise<RenderedVariant[]> {
const open = (): Sharp =>
sharp(original, { failOn: 'error', limitInputPixels: maxPixels }).rotate();
try {
const meta = await sharp(original, { limitInputPixels: maxPixels }).metadata();
if (!meta.width || !meta.height) {
throw new InvalidImageError('image has no dimensions');
}
const swapped = (meta.orientation ?? 1) >= 5;
const width = swapped ? meta.height : meta.width;
const height = swapped ? meta.width : meta.height;
const thumb = fitInside(width, height, 256);
const large = fitInside(width, height, 1280);
const thumbnail = open()
.resize(thumb.width, thumb.height, { fit: 'fill' })
.flatten({ background: WHITE })
.jpeg({ quality: JPEG_QUALITY });
const crop = open()
.resize(128, 128, { fit: 'cover', position: 'centre' })
.flatten({ background: WHITE })
.jpeg({ quality: JPEG_QUALITY });
const optimized = open()
.resize(large.width, large.height, { fit: 'fill' })
.webp({ quality: WEBP_QUALITY, effort: 4 });
return [
await finish('thumbnail', 'image/jpeg', 'jpg', thumbnail),
await finish('crop', 'image/jpeg', 'jpg', crop),
await finish('optimized', 'image/webp', 'webp', optimized),
];
} catch (error) {
if (error instanceof InvalidImageError) {
throw error;
}
throw new InvalidImageError(error instanceof Error ? error.message : 'decode failed');
}
}
async function finish(
name: RenderedVariant['name'],
contentType: string,
extension: string,
pipeline: Sharp,
): Promise<RenderedVariant> {
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
return {
name,
contentType,
extension,
body: data,
width: info.width,
height: info.height,
sha256: createHash('sha256').update(data).digest('hex'),
};
}
+6 -2
View File
@@ -166,12 +166,16 @@ export class MessagesService {
if (attachmentIds.length === 0) {
return;
}
const found = await this.prisma.attachment.count({
const found = await this.prisma.attachment.findMany({
where: { id: { in: attachmentIds }, chatId },
select: { status: true },
});
if (found !== attachmentIds.length) {
if (found.length !== attachmentIds.length) {
throw new ApiError('not_found', 'attachment not found');
}
if (found.some((row) => row.status === 'failed')) {
throw new ApiError('conflict', 'failed attachment cannot be attached');
}
}
}
+6
View File
@@ -5,6 +5,7 @@ import { S3Service } from './storage/s3.service';
const DEFAULT_GRACE_SECONDS = 3600;
const DEFAULT_INTERVAL_SECONDS = 300;
const DEFAULT_STAGING_RETENTION_SECONDS = 86400;
function option(name: string, fallback: number): number {
const prefix = `--${name}=`;
@@ -23,6 +24,10 @@ async function main(): Promise<void> {
const loop = process.argv.includes('--loop');
const graceSeconds = option('grace-seconds', DEFAULT_GRACE_SECONDS);
const intervalSeconds = option('interval-seconds', DEFAULT_INTERVAL_SECONDS);
const stagingRetentionSeconds = option(
'staging-retention-seconds',
Number(process.env.STAGING_RETENTION_SECONDS ?? DEFAULT_STAGING_RETENTION_SECONDS),
);
const prisma = new PrismaService();
const s3 = new S3Service();
do {
@@ -30,6 +35,7 @@ async function main(): Promise<void> {
const report = await reconcile(prisma, s3, {
apply,
graceMs: graceSeconds * 1000,
stagingRetentionMs: stagingRetentionSeconds * 1000,
});
process.stdout.write(`${JSON.stringify(report)}\n`);
} catch (error) {
+77
View File
@@ -76,6 +76,69 @@ export class S3Service implements OnModuleDestroy {
);
}
async putBuffer(
key: string,
body: Buffer,
contentType: string,
sha256: string,
): Promise<void> {
await this.guard(`put ${key}`, () =>
this.client.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: key,
Body: body,
ContentLength: body.length,
ContentType: contentType,
ChecksumSHA256: Buffer.from(sha256, 'hex').toString('base64'),
}),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
),
);
}
async getBuffer(key: string, maxBytes: number): Promise<Buffer> {
let body: Readable;
try {
const response = await this.client.send(
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
);
if ((response.ContentLength ?? 0) > maxBytes) {
throw new ObjectTooLargeError(key);
}
body = response.Body as Readable;
} catch (error) {
if (error instanceof ObjectTooLargeError) {
throw error;
}
const name = error instanceof Error ? error.name : 'unknown error';
this.logger.error(`get ${key} failed: ${name}`);
if (name === 'NoSuchKey') {
throw new ObjectMissingError(key);
}
throw new ApiError('unavailable', 'object storage unavailable');
}
const chunks: Buffer[] = [];
let size = 0;
try {
for await (const chunk of body) {
size += chunk.length;
if (size > maxBytes) {
body.destroy();
throw new ObjectTooLargeError(key);
}
chunks.push(chunk);
}
} catch (error) {
if (error instanceof ObjectTooLargeError) {
throw error;
}
throw new ApiError('unavailable', 'object storage unavailable');
}
return Buffer.concat(chunks);
}
async getStream(key: string): Promise<Readable> {
const deadline = new AbortController();
const timer = setTimeout(() => deadline.abort(), IDLE_TIMEOUT_MS);
@@ -158,6 +221,20 @@ export class S3Service implements OnModuleDestroy {
}
}
export class ObjectMissingError extends Error {
constructor(key: string) {
super(`object ${key} is missing from storage`);
this.name = 'ObjectMissingError';
}
}
export class ObjectTooLargeError extends Error {
constructor(key: string) {
super(`object ${key} exceeds size limit`);
this.name = 'ObjectTooLargeError';
}
}
function readSecret(name: string): string {
const file = process.env[`${name}_FILE`];
const value = file ? readFileSync(file, 'utf8').trim() : process.env[name];
+19 -4
View File
@@ -3,7 +3,13 @@ import { createHash, randomUUID } from 'crypto';
import request from 'supertest';
import { PrismaService } from '../src/db/prisma.service';
import { S3Service } from '../src/storage/s3.service';
import { bearer, createTestApp, registerUser, TestUser } from './helpers';
import {
bearer,
createTestApp,
processAttachment,
registerUser,
TestUser,
} from './helpers';
import { jpeg, SAMPLE_PNG } from './images';
describe('Скачивание и привязка вложений (e2e)', () => {
@@ -35,8 +41,13 @@ describe('Скачивание и привязка вложений (e2e)', () =
.set('Authorization', bearer(user))
.set('Content-Type', type)
.send(body)
.expect(201);
return res.body;
.expect(202);
expect(await processAttachment(app, res.body.id)).toBe('ready');
const ready = await request(server)
.get(`/api/v1/attachments/${res.body.id}`)
.set('Authorization', bearer(user))
.expect(200);
return ready.body;
};
const get = (path: string, user: TestUser | null = bob) => {
@@ -134,7 +145,11 @@ describe('Скачивание и привязка вложений (e2e)', () =
it('variant: original явно, отсутствующий 404, неизвестный и повторённый 400', async () => {
const { id } = await upload();
await get(`/api/v1/attachments/${id}/content?variant=original`).expect(200);
await get(`/api/v1/attachments/${id}/content?variant=thumbnail`).expect(404);
await get(`/api/v1/attachments/${id}/content?variant=thumbnail`).expect(200);
await app.get(PrismaService).attachmentVariant.deleteMany({
where: { attachmentId: id, name: 'crop' },
});
await get(`/api/v1/attachments/${id}/content?variant=crop`).expect(404);
await get(`/api/v1/attachments/${id}/content?variant=nonsense`).expect(400);
await get(`/api/v1/attachments/${id}/content?variant=`).expect(400);
await get(`/api/v1/attachments/${id}/content?variant=original&variant=original`).expect(400);
+52 -27
View File
@@ -2,9 +2,10 @@ import { INestApplication } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import Ajv2020 from 'ajv/dist/2020';
import { createHash, randomUUID } from 'crypto';
import { readdirSync } from 'fs';
import { mkdtempSync, readdirSync, rmSync } from 'fs';
import { connect } from 'net';
import { tmpdir } from 'os';
import { join } from 'path';
import request from 'supertest';
import { PrismaService } from '../src/db/prisma.service';
import { loadDocument } from '../src/docs/swagger';
@@ -61,8 +62,13 @@ describe('Загрузка вложений (e2e)', () => {
expect(res.body.error.request_id).toBe(res.headers['x-request-id']);
};
const uploadTmp = mkdtempSync(join(tmpdir(), 'attachments-test-'));
const activeTempDirs = (): string[] =>
readdirSync(uploadTmp).filter((name) => name.startsWith('messenger-upload-'));
beforeAll(async () => {
({ app, server } = await createTestApp());
({ app, server } = await createTestApp({ UPLOAD_TMP_DIR: uploadTmp }));
alice = await registerUser(server, 'att_alice');
eve = await registerUser(server, 'att_eve');
const chat = await request(server)
@@ -75,11 +81,12 @@ describe('Загрузка вложений (e2e)', () => {
afterAll(async () => {
await app.close();
rmSync(uploadTmp, { recursive: true, force: true });
});
describe('успешная загрузка', () => {
it('возвращает 201 Attachment по контракту и сохраняет метаданные', async () => {
const res = await upload(SAMPLE_PNG).expect(201);
it('возвращает 202 Attachment queued по контракту, staging и задание сохранены до ответа', async () => {
const res = await upload(SAMPLE_PNG).expect(202);
const document: any = loadDocument();
const ajv = new Ajv2020({ strict: false, validateFormats: false });
ajv.addSchema(document, 'contract');
@@ -95,22 +102,42 @@ describe('Загрузка вложений (e2e)', () => {
content_type: 'image/png',
size_bytes: SAMPLE_PNG.length,
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
status: 'ready',
status: 'queued',
variants: [],
error_code: null,
});
const row = await app.get(PrismaService).attachment.findUnique({
const prisma = app.get(PrismaService);
const row = await prisma.attachment.findUnique({
where: { id: res.body.id },
include: { jobs: { include: { outbox: true } } },
});
expect(row?.objectKey).toBe(`attachments/${res.body.id}`);
expect(JSON.stringify(res.body)).not.toContain('attachments/');
expect(row).toMatchObject({
status: 'queued',
objectKey: `attachments/${res.body.id}/original`,
stagingKey: `staging/${res.body.id}`,
});
expect(row!.jobs).toHaveLength(1);
expect(row!.jobs[0]).toMatchObject({ status: 'queued', attempt: 1, pipelineVersion: 1 });
expect(row!.jobs[0].outbox).toHaveLength(1);
expect(row!.jobs[0].outbox[0].payload).toEqual({
schema_version: 1,
job_id: row!.jobs[0].id,
type: 'attachment.process.v1',
attachment_id: res.body.id,
pipeline_version: 1,
attempt: 1,
created_at: expect.any(String),
});
const staged = await app.get(S3Service).getBuffer(`staging/${res.body.id}`, MAX_BYTES);
expect(Buffer.compare(staged, SAMPLE_PNG)).toBe(0);
expect(JSON.stringify(res.body)).not.toMatch(/attachments\/|staging\//);
});
it('принимает реальный JPEG', async () => {
const res = await upload(await jpeg(), {
type: 'image/jpeg',
filename: 'фото 1.jpg',
}).expect(201);
}).expect(202);
expect(res.body).toMatchObject({
content_type: 'image/jpeg',
filename: 'фото 1.jpg',
@@ -118,21 +145,21 @@ describe('Загрузка вложений (e2e)', () => {
});
it('игнорирует параметры и регистр Content-Type', async () => {
await upload(SAMPLE_PNG, { type: 'IMAGE/PNG; charset=binary' }).expect(201);
await upload(SAMPLE_PNG, { type: 'IMAGE/PNG; charset=binary' }).expect(202);
});
it('принимает файл ровно 10 MiB, filename ровно 255 символов и PNG с байтами после IEND', async () => {
const exact = pngOfExactSize(MAX_BYTES);
expect(exact.length).toBe(MAX_BYTES);
const res = await upload(exact).expect(201);
const res = await upload(exact).expect(202);
expect(res.body.size_bytes).toBe(MAX_BYTES);
await upload(SAMPLE_PNG, { filename: `${'ф'.repeat(251)}.png` }).expect(201);
await upload(Buffer.concat([png(4, 4), Buffer.from('tail')])).expect(201);
await upload(SAMPLE_PNG, { filename: `${'ф'.repeat(251)}.png` }).expect(202);
await upload(Buffer.concat([png(4, 4), Buffer.from('tail')])).expect(202);
});
it('принимает ровно 40 млн пикселей', async () => {
const res = await upload(await bigPng(8000, 5000));
expect(res.status).toBe(201);
expect(res.status).toBe(202);
});
});
@@ -154,10 +181,10 @@ describe('Загрузка вложений (e2e)', () => {
it('посторонний и несуществующий чат — одинаковый 404, чужой файл не сохраняется', async () => {
const prisma = app.get(PrismaService);
const before = await prisma.attachment.count();
const before = await prisma.attachment.count({ where: { chatId } });
expectError(await upload(SAMPLE_PNG, { user: eve }), 404, 'not_found');
expectError(await upload(SAMPLE_PNG, { chat: randomUUID() }), 404, 'not_found');
expect(await prisma.attachment.count()).toBe(before);
expect(await prisma.attachment.count({ where: { chatId } })).toBe(before);
});
it('неверный UUID чата — 400', async () => {
@@ -260,7 +287,7 @@ describe('Загрузка вложений (e2e)', () => {
it('оборванная загрузка не оставляет временных файлов и строк в БД', async () => {
const prisma = app.get(PrismaService);
const before = await prisma.attachment.count();
const before = await prisma.attachment.count({ where: { chatId } });
const { port } = server.address();
await new Promise<void>((resolve, reject) => {
const socket = connect(port, '127.0.0.1', () => {
@@ -278,7 +305,7 @@ describe('Загрузка вложений (e2e)', () => {
socket.on('error', reject);
});
await new Promise((resolve) => setTimeout(resolve, 500));
expect(await prisma.attachment.count()).toBe(before);
expect(await prisma.attachment.count({ where: { chatId } })).toBe(before);
expect(activeTempDirs()).toEqual([]);
});
});
@@ -290,7 +317,7 @@ describe('Загрузка вложений (e2e)', () => {
});
try {
const prisma = app.get(PrismaService);
const before = await prisma.attachment.count();
const before = await prisma.attachment.count({ where: { chatId } });
const started = Date.now();
const res = await request(downServer)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
@@ -299,17 +326,17 @@ describe('Загрузка вложений (e2e)', () => {
.send(SAMPLE_PNG);
expectError(res, 503, 'unavailable');
expect(Date.now() - started).toBeLessThan(3000);
expect(await prisma.attachment.count()).toBe(before);
expect(await prisma.attachment.count({ where: { chatId } })).toBe(before);
} finally {
await down.close();
}
});
it('явный отказ INSERT после PUT удаляет объект', async () => {
it('явный отказ транзакции после PUT удаляет объект staging', async () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);
const remove = jest.spyOn(s3, 'delete');
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
const create = jest.spyOn(prisma, '$transaction').mockRejectedValueOnce(
new Prisma.PrismaClientKnownRequestError('rejected', {
code: 'P2003',
clientVersion: 'test',
@@ -319,6 +346,7 @@ describe('Загрузка вложений (e2e)', () => {
const res = await upload(SAMPLE_PNG);
expect(res.status).toBe(500);
expect(remove).toHaveBeenCalledTimes(1);
expect(remove.mock.calls[0][0]).toMatch(/^staging\//);
} finally {
create.mockRestore();
remove.mockRestore();
@@ -329,7 +357,7 @@ describe('Загрузка вложений (e2e)', () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);
const remove = jest.spyOn(s3, 'delete');
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
const create = jest.spyOn(prisma, '$transaction').mockRejectedValueOnce(
new Prisma.PrismaClientUnknownRequestError('server closed the connection', {
clientVersion: 'test',
}),
@@ -347,7 +375,7 @@ describe('Загрузка вложений (e2e)', () => {
const prisma = app.get(PrismaService);
const s3 = app.get(S3Service);
const remove = jest.spyOn(s3, 'delete');
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
const create = jest.spyOn(prisma, '$transaction').mockRejectedValueOnce(
new Prisma.PrismaClientKnownRequestError('connection lost', {
code: 'P1017',
clientVersion: 'test',
@@ -364,9 +392,6 @@ describe('Загрузка вложений (e2e)', () => {
});
});
function activeTempDirs(): string[] {
return readdirSync(tmpdir()).filter((name) => name.startsWith('messenger-upload-'));
}
async function bigPng(width: number, height: number): Promise<Buffer> {
const { default: sharp } = await import('sharp');
+33
View File
@@ -8,6 +8,15 @@ import { randomUUID } from 'crypto';
import request from 'supertest';
import { AppModule } from '../src/app.module';
import { configureApp } from '../src/configure-app';
import { PrismaService } from '../src/db/prisma.service';
import { parseJobMessage } from '../src/jobs/job-message';
import {
JobProcessor,
Outcome,
ProcessorOptions,
processorOptions,
} from '../src/jobs/processor';
import { S3Service } from '../src/storage/s3.service';
export const PASSWORD = 'Correct-Horse-Battery-42';
@@ -43,6 +52,30 @@ export async function createTestApp(
return { app, server: app.getHttpServer() };
}
export async function processAttachment(
app: INestApplication,
attachmentId: string,
hooks: { afterStore?: () => Promise<void> } = {},
options: Partial<ProcessorOptions> = {},
): Promise<Outcome> {
const prisma = app.get(PrismaService);
const entry = await prisma.outboxEntry.findFirstOrThrow({
where: { job: { attachmentId } },
orderBy: { id: 'desc' },
});
const message = parseJobMessage(Buffer.from(JSON.stringify(entry.payload)));
if (!message) {
throw new Error('outbox payload does not match jobs.schema.json');
}
const processor = new JobProcessor(
prisma,
app.get(S3Service),
{ ...processorOptions(), ...options },
hooks,
);
return processor.handle(message);
}
export function basic(username: string, password: string): string {
return `Basic ${Buffer.from(`${username}:${password}`, 'utf8').toString('base64')}`;
}
+396
View File
@@ -0,0 +1,396 @@
import { INestApplication } from '@nestjs/common';
import { createHash, randomUUID } from 'crypto';
import sharp from 'sharp';
import request from 'supertest';
import { ApiError } from '../src/common/filters/api-error';
import { PrismaService } from '../src/db/prisma.service';
import { jobMessage, parseJobMessage } from '../src/jobs/job-message';
import { backoffMs, recoverExpiredLeases, replayJob } from '../src/jobs/job-store';
import { JobProcessor, processorOptions } from '../src/jobs/processor';
import { fitInside } from '../src/jobs/variants';
import { S3Service } from '../src/storage/s3.service';
import {
bearer,
createTestApp,
processAttachment,
registerUser,
TestUser,
} from './helpers';
import { SAMPLE_PNG } from './images';
const MAX_BYTES = 10 * 1024 * 1024;
describe('Фоновая обработка вложений (e2e)', () => {
let app: INestApplication;
let server: any;
let prisma: PrismaService;
let s3: S3Service;
let alice: TestUser;
let eve: TestUser;
let chatId: string;
const upload = async (body: Buffer = SAMPLE_PNG, type = 'image/png') => {
const res = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', type)
.send(body)
.expect(202);
return res.body.id as string;
};
const metadata = async (id: string) =>
(await request(server).get(`/api/v1/attachments/${id}`).set('Authorization', bearer(alice)).expect(200)).body;
const content = (id: string, variant = 'original', user = alice) =>
request(server)
.get(`/api/v1/attachments/${id}/content?variant=${variant}`)
.set('Authorization', bearer(user))
.buffer(true)
.parse(binaryParser);
const job = (attachmentId: string) =>
prisma.attachmentJob.findFirstOrThrow({ where: { attachmentId } });
const makeDue = (attachmentId: string) =>
prisma.attachmentJob.updateMany({ where: { attachmentId }, data: { nextAttemptAt: new Date() } });
const events = async (attachmentId: string) =>
(
await prisma.jobEvent.findMany({
where: { job: { attachmentId } },
orderBy: { id: 'asc' },
})
).map((event) => event.event);
const sendMessage = (attachmentIds: string[]) =>
request(server)
.post(`/api/v1/chats/${chatId}/messages`)
.set('Authorization', bearer(alice))
.set('Idempotency-Key', randomUUID())
.send({ text: '', attachment_ids: attachmentIds });
beforeAll(async () => {
({ app, server } = await createTestApp());
prisma = app.get(PrismaService);
s3 = app.get(S3Service);
alice = await registerUser(server, 'job_alice');
eve = await registerUser(server, 'job_eve');
const chat = await request(server)
.post('/api/v1/chats')
.set('Authorization', bearer(alice))
.send({ title: 'Jobs', member_ids: [alice.id] })
.expect(201);
chatId = chat.body.id;
});
afterAll(async () => {
await app.close();
});
describe('до обработки', () => {
it('queued: метаданные доступны, content любого варианта — 409, сообщение с вложением создаётся', async () => {
const id = await upload();
expect(await metadata(id)).toMatchObject({ status: 'queued', variants: [], error_code: null });
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
const res = await content(id, variant).expect(409);
expect(JSON.parse(res.body.toString()).error.code).toBe('conflict');
}
await content(id, 'original', eve).expect(404);
await content(id, 'nonsense').expect(400);
const message = await sendMessage([id]).expect(201);
expect(message.body.attachment_ids).toEqual([id]);
});
});
describe('результат worker', () => {
it('ready: оригинал неизменен, thumbnail 256×160 JPEG, crop 128×128 JPEG, optimized WebP без увеличения', async () => {
const id = await upload();
expect(await processAttachment(app, id)).toBe('ready');
const meta = await metadata(id);
expect(meta.status).toBe('ready');
expect(meta.error_code).toBeNull();
expect(meta.variants.map((v: any) => [v.name, v.content_type, v.width, v.height])).toEqual([
['thumbnail', 'image/jpeg', 256, 160],
['crop', 'image/jpeg', 128, 128],
['optimized', 'image/webp', 320, 200],
]);
const original = await content(id).expect(200);
expect(Buffer.compare(original.body, SAMPLE_PNG)).toBe(0);
for (const variant of meta.variants) {
const res = await content(id, variant.name).expect(200);
expect(res.headers['content-type']).toBe(variant.content_type);
expect(res.body.length).toBe(variant.size_bytes);
expect(createHash('sha256').update(res.body).digest('hex')).toBe(variant.sha256);
const actual = await sharp(res.body).metadata();
expect([actual.format, actual.width, actual.height, actual.exif]).toEqual([
variant.content_type.split('/')[1],
variant.width,
variant.height,
undefined,
]);
}
const row = await prisma.attachment.findUniqueOrThrow({ where: { id } });
expect(row.stagingKey).toBeNull();
await expect(s3.getBuffer(`staging/${id}`, MAX_BYTES)).rejects.toThrow('missing');
expect(await events(id)).toEqual(['enqueued', 'processing', 'ready']);
});
it('маленький исходник не увеличивается в thumbnail/optimized, crop — увеличивается', async () => {
const small = await sharp({ create: { width: 100, height: 50, channels: 3, background: '#123456' } })
.png()
.toBuffer();
const id = await upload(small);
expect(await processAttachment(app, id)).toBe('ready');
const sizes = (await metadata(id)).variants.map((v: any) => [v.name, v.width, v.height]);
expect(sizes).toEqual([
['thumbnail', 100, 50],
['crop', 128, 128],
['optimized', 100, 50],
]);
});
it('прозрачность компонуется на белый, EXIF orientation применяется', async () => {
const transparent = await sharp({
create: { width: 40, height: 20, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } },
})
.png()
.toBuffer();
const alphaId = await upload(transparent);
expect(await processAttachment(app, alphaId)).toBe('ready');
const thumb = await content(alphaId, 'thumbnail').expect(200);
const { data } = await sharp(thumb.body).raw().toBuffer({ resolveWithObject: true });
expect(Math.min(...data)).toBeGreaterThanOrEqual(250);
const rotated = await sharp({ create: { width: 320, height: 200, channels: 3, background: '#808080' } })
.jpeg()
.withMetadata({ orientation: 6 })
.toBuffer();
const rotatedId = await upload(rotated, 'image/jpeg');
expect(await processAttachment(app, rotatedId)).toBe('ready');
const thumbnail = (await metadata(rotatedId)).variants.find((v: any) => v.name === 'thumbnail');
expect([thumbnail.width, thumbnail.height]).toEqual([160, 256]);
});
it('размеры округляются вниз, минимум 1 px', () => {
expect(fitInside(320, 200, 256)).toEqual({ width: 256, height: 160 });
expect(fitInside(1000, 3, 256)).toEqual({ width: 256, height: 1 });
expect(fitInside(333, 1000, 256)).toEqual({ width: 85, height: 256 });
expect(fitInside(10, 10, 256)).toEqual({ width: 10, height: 10 });
});
});
describe('идемпотентность и повторная доставка', () => {
it('повтор уже ready-задания — no-op: второго набора вариантов нет, объекты не меняются', async () => {
const id = await upload();
expect(await processAttachment(app, id)).toBe('ready');
const before = await prisma.attachmentVariant.findMany({ where: { attachmentId: id } });
const put = jest.spyOn(s3, 'putBuffer');
try {
expect(await processAttachment(app, id)).toBe('skipped');
expect(put).not.toHaveBeenCalled();
} finally {
put.mockRestore();
}
const after = await prisma.attachmentVariant.findMany({ where: { attachmentId: id } });
expect(after).toEqual(before);
expect(after).toHaveLength(3);
});
it('пока lease действует, второй обработчик того же job не запускается', async () => {
const id = await upload();
let concurrent: string | undefined;
const outcome = await processAttachment(app, id, {
afterStore: async () => {
concurrent = await processAttachment(app, id);
},
});
expect([outcome, concurrent]).toEqual(['ready', 'skipped']);
});
it('worker упал после записи объектов, до COMMIT/ack: lease истекает, повтор завершает тот же результат', async () => {
const id = await upload();
await expect(
processAttachment(app, id, {
afterStore: async () => {
throw new Error('worker crashed');
},
}),
).rejects.toThrow('worker crashed');
expect(await job(id)).toMatchObject({ status: 'processing', attempt: 1 });
expect((await metadata(id)).status).toBe('processing');
const stored = await s3.getBuffer(`attachments/${id}/v1/thumbnail.jpg`, MAX_BYTES);
expect(await recoverExpiredLeases(prisma)).toBe(0);
const future = new Date(Date.now() + 10 * 60 * 1000);
expect(await recoverExpiredLeases(prisma, future)).toBeGreaterThanOrEqual(1);
expect(await job(id)).toMatchObject({ status: 'queued', attempt: 2 });
await makeDue(id);
expect(await processAttachment(app, id)).toBe('ready');
const variants = await prisma.attachmentVariant.findMany({ where: { attachmentId: id } });
expect(variants).toHaveLength(3);
const thumbnail = variants.find((v) => v.name === 'thumbnail')!;
expect(thumbnail.sha256).toBe(createHash('sha256').update(stored).digest('hex'));
expect(await events(id)).toEqual(['enqueued', 'processing', 'lease_expired', 'processing', 'ready']);
});
it('устаревшее сообщение, чужой attachment_id и другая pipeline_version игнорируются', async () => {
const id = await upload();
const row = await job(id);
const processor = new JobProcessor(prisma, s3, processorOptions());
const base = jobMessage(row, new Date());
expect(await processor.handle({ ...base, attachment_id: randomUUID() })).toBe('skipped');
expect(await processor.handle({ ...base, pipeline_version: 2 })).toBe('skipped');
expect(await processor.handle({ ...base, job_id: randomUUID() })).toBe('skipped');
expect((await job(id)).status).toBe('queued');
});
});
describe('ошибки, повторы и replay', () => {
it('временная ошибка S3: повтор с backoff, до срока задание не берётся, затем ready', async () => {
const id = await upload();
const put = jest
.spyOn(s3, 'putBuffer')
.mockRejectedValueOnce(new ApiError('unavailable', 'object storage unavailable'));
try {
expect(await processAttachment(app, id)).toBe('retry');
} finally {
put.mockRestore();
}
const retried = await job(id);
expect(retried).toMatchObject({ status: 'queued', attempt: 2, lastError: 'temporary failure: ApiError' });
expect(retried.nextAttemptAt.getTime()).toBeGreaterThan(Date.now());
expect((await metadata(id)).status).toBe('queued');
const outbox = await prisma.outboxEntry.findMany({ where: { jobId: retried.id }, orderBy: { id: 'asc' } });
expect(outbox).toHaveLength(2);
expect(outbox[1].availableAt).toEqual(retried.nextAttemptAt);
expect((outbox[1].payload as any).attempt).toBe(2);
expect((outbox[1].payload as any).job_id).toBe(retried.id);
expect(await processAttachment(app, id)).toBe('skipped');
await makeDue(id);
expect(await processAttachment(app, id)).toBe('ready');
expect(await events(id)).toEqual(['enqueued', 'processing', 'retry_scheduled', 'processing', 'ready']);
});
it('попытки исчерпаны — failed retry_exhausted; replay явно возвращает в очередь и доводит до ready', async () => {
const id = await upload();
await prisma.attachmentJob.updateMany({ where: { attachmentId: id }, data: { maxAttempts: 2 } });
const put = jest
.spyOn(s3, 'putBuffer')
.mockRejectedValue(new ApiError('unavailable', 'object storage unavailable'));
try {
expect(await processAttachment(app, id)).toBe('retry');
await makeDue(id);
expect(await processAttachment(app, id)).toBe('failed');
} finally {
put.mockRestore();
}
expect(await metadata(id)).toMatchObject({ status: 'failed', error_code: 'retry_exhausted', variants: [] });
const failed = await job(id);
expect(failed).toMatchObject({ status: 'failed', errorCode: 'retry_exhausted', attempt: 2 });
await sendMessage([id]).expect(409);
expect(await replayJob(prisma, failed.id, 'test-operator')).toBe('replayed');
expect(await job(id)).toMatchObject({ status: 'queued', attempt: 1, errorCode: null });
expect(await metadata(id)).toMatchObject({ status: 'queued', error_code: null });
expect(await processAttachment(app, id)).toBe('ready');
const outboxBefore = await prisma.outboxEntry.count({ where: { jobId: failed.id } });
expect(await replayJob(prisma, failed.id, 'test-operator')).toBe('already_ready');
expect(await prisma.outboxEntry.count({ where: { jobId: failed.id } })).toBe(outboxBefore);
expect(await replayJob(prisma, randomUUID(), 'test-operator')).toBe('not_found');
const replayed = await prisma.jobEvent.findFirstOrThrow({ where: { jobId: failed.id, event: 'replayed' } });
expect(replayed.detail).toBe('by test-operator; previous retry_exhausted after attempt 2');
});
it('replay задания в очереди не создаёт дубль', async () => {
const id = await upload();
expect(await replayJob(prisma, (await job(id)).id, 'test-operator')).toBe('not_failed');
expect(await prisma.outboxEntry.count({ where: { job: { attachmentId: id } } })).toBe(1);
});
it('нераспаковываемое изображение — постоянная ошибка: сразу failed invalid_image без повторов', async () => {
const id = await upload();
const garbage = Buffer.concat([SAMPLE_PNG.subarray(0, 64), Buffer.alloc(512, 7)]);
const sha256 = createHash('sha256').update(garbage).digest('hex');
await s3.putBuffer(`staging/${id}`, garbage, 'image/png', sha256);
await prisma.attachment.update({ where: { id }, data: { sizeBytes: garbage.length, sha256 } });
expect(await processAttachment(app, id)).toBe('failed');
expect(await metadata(id)).toMatchObject({ status: 'failed', error_code: 'invalid_image', variants: [] });
expect(await job(id)).toMatchObject({ status: 'failed', attempt: 1, errorCode: 'invalid_image' });
await content(id).expect(409);
await sendMessage([id]).expect(409);
expect(await prisma.attachment.findUniqueOrThrow({ where: { id } })).toMatchObject({
stagingKey: `staging/${id}`,
});
});
it('staging подменён — failed checksum_mismatch; staging пропал — failed staging_missing', async () => {
const changed = await upload();
const other = await sharp({ create: { width: 8, height: 8, channels: 3, background: '#ff0000' } })
.png()
.toBuffer();
await s3.putBuffer(`staging/${changed}`, other, 'image/png', createHash('sha256').update(other).digest('hex'));
expect(await processAttachment(app, changed)).toBe('failed');
expect((await metadata(changed)).error_code).toBe('checksum_mismatch');
const missing = await upload();
await s3.delete(`staging/${missing}`);
expect(await processAttachment(app, missing)).toBe('failed');
expect((await metadata(missing)).error_code).toBe('staging_missing');
});
it('превышение времени обработки — временная ошибка с повтором', async () => {
const id = await upload();
const slow = jest
.spyOn(s3, 'getBuffer')
.mockImplementationOnce(() => new Promise((resolve) => setTimeout(() => resolve(SAMPLE_PNG), 500)));
try {
expect(await processAttachment(app, id, {}, { timeoutMs: 50 })).toBe('retry');
} finally {
slow.mockRestore();
}
expect((await job(id)).lastError).toBe('processing timed out');
});
});
describe('формат сообщения и backoff', () => {
it('сообщение соответствует jobs.schema.json, лишние/неверные поля отклоняются', () => {
const valid = jobMessage(
{ id: randomUUID(), attachmentId: randomUUID(), pipelineVersion: 1, attempt: 1 },
new Date(),
);
const encode = (value: unknown) => Buffer.from(JSON.stringify(value));
expect(parseJobMessage(encode(valid))).toEqual(valid);
for (const bad of [
{ ...valid, url: 'http://evil' },
{ ...valid, job_id: 'nope' },
{ ...valid, type: 'other' },
{ ...valid, attempt: 0 },
{ ...valid, schema_version: 2 },
{ ...valid, created_at: 'yesterday' },
[valid],
]) {
expect(parseJobMessage(encode(bad))).toBeNull();
}
expect(parseJobMessage(Buffer.from('{bad'))).toBeNull();
});
it('backoff растёт экспоненциально с jitter и ограничен сверху', () => {
expect(backoffMs(1, () => 0)).toBe(500);
expect(backoffMs(1, () => 1)).toBe(1000);
expect(backoffMs(3, () => 0)).toBe(2000);
expect(backoffMs(3, () => 1)).toBe(4000);
expect(backoffMs(20, () => 1)).toBe(30000);
expect(backoffMs(20, () => 0)).toBe(15000);
});
});
});
function binaryParser(res: any, done: (error: Error | null, body: Buffer) => void): void {
const chunks: Buffer[] = [];
res.on('data', (chunk: Buffer) => chunks.push(chunk));
res.on('end', () => done(null, Buffer.concat(chunks)));
}
+75 -6
View File
@@ -7,7 +7,13 @@ import request from 'supertest';
import { reconcile } from '../src/attachments/reconciler';
import { PrismaService } from '../src/db/prisma.service';
import { S3Service } from '../src/storage/s3.service';
import { bearer, createTestApp, registerUser, TestUser } from './helpers';
import {
bearer,
createTestApp,
processAttachment,
registerUser,
TestUser,
} from './helpers';
import { SAMPLE_PNG } from './images';
const HOUR_MS = 60 * 60 * 1000;
@@ -100,6 +106,7 @@ describe('Сверка объектов S3 с метаданными (e2e)', ()
sizeBytes: SAMPLE_PNG.length,
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
objectKey: key,
status: 'ready',
},
});
const report = await reconcile(prisma, s3, {
@@ -164,13 +171,75 @@ describe('Сверка объектов S3 с метаданными (e2e)', ()
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(201);
.expect(202);
expect(await processAttachment(app, upload.body.id)).toBe('ready');
const report = await reconcile(prisma, s3, { apply: true, graceMs: HOUR_MS });
expect(report.fresh_skipped).toBeGreaterThanOrEqual(1);
await request(server)
.get(`/api/v1/attachments/${upload.body.id}/content`)
expect(report.fresh_skipped).toBeGreaterThanOrEqual(4);
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
await request(server)
.get(`/api/v1/attachments/${upload.body.id}/content?variant=${variant}`)
.set('Authorization', bearer(alice))
.expect(200);
}
});
it('после grace period объекты вложения и staging в очереди не считаются сиротами', async () => {
const queued = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.expect(200);
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
const ready = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=b.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
expect(await processAttachment(app, ready.body.id)).toBe('ready');
for (const id of [queued.body.id, ready.body.id]) {
for (const prefix of [`attachments/${id}/`, `staging/${id}`]) {
const report = await reconcile(prisma, s3, {
apply: true,
graceMs: HOUR_MS,
prefix,
now: later(),
});
expect(report).toMatchObject({ orphans: 0, deleted: 0 });
}
}
expect(await exists(`staging/${queued.body.id}`)).toBe(true);
expect(await exists(`staging/${ready.body.id}`)).toBe(false);
});
it('staging failed-вложения удаляется после срока хранения', async () => {
const res = await request(server)
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
.set('Authorization', bearer(alice))
.set('Content-Type', 'image/png')
.send(SAMPLE_PNG)
.expect(202);
await prisma.attachment.update({
where: { id: res.body.id },
data: { status: 'failed', errorCode: 'invalid_image' },
});
const options = { graceMs: HOUR_MS, prefix: `staging/${res.body.id}`, now: later() };
const kept = await reconcile(prisma, s3, {
...options,
apply: true,
stagingRetentionMs: 3 * HOUR_MS,
});
expect(kept.staging_expired).toBe(0);
expect(await exists(`staging/${res.body.id}`)).toBe(true);
const expired = await reconcile(prisma, s3, {
...options,
apply: true,
stagingRetentionMs: HOUR_MS,
});
expect(expired.staging_expired).toBeGreaterThanOrEqual(1);
expect(await exists(`staging/${res.body.id}`)).toBe(false);
const row = await prisma.attachment.findUniqueOrThrow({ where: { id: res.body.id } });
expect(row.stagingKey).toBeNull();
});
});