forked from mtusi2026/mtusi2026lab5
feat: accept uploads into staging with jobs, outbox and job processor
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
ALTER TABLE "attachments"
|
||||
ADD COLUMN "staging_key" VARCHAR(128),
|
||||
ADD COLUMN "status" VARCHAR(16) NOT NULL DEFAULT 'ready',
|
||||
ADD COLUMN "error_code" VARCHAR(64);
|
||||
|
||||
ALTER TABLE "attachments" ALTER COLUMN "status" DROP DEFAULT;
|
||||
|
||||
ALTER TABLE "attachments"
|
||||
ADD CONSTRAINT "attachments_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD CONSTRAINT "attachments_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL));
|
||||
|
||||
CREATE UNIQUE INDEX "attachments_staging_key_key" ON "attachments"("staging_key");
|
||||
|
||||
CREATE TABLE "attachment_variants" (
|
||||
"attachment_id" UUID NOT NULL,
|
||||
"pipeline_version" INTEGER NOT NULL,
|
||||
"name" VARCHAR(16) NOT NULL,
|
||||
"content_type" VARCHAR(32) NOT NULL,
|
||||
"object_key" VARCHAR(128) NOT NULL,
|
||||
"size_bytes" INTEGER NOT NULL,
|
||||
"width" INTEGER NOT NULL,
|
||||
"height" INTEGER NOT NULL,
|
||||
"sha256" CHAR(64) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "attachment_variants_pkey" PRIMARY KEY ("attachment_id","pipeline_version","name"),
|
||||
CONSTRAINT "attachment_variants_name_allowed" CHECK ("name" IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT "attachment_variants_content_type_allowed" CHECK ("content_type" IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
CONSTRAINT "attachment_variants_positive" CHECK ("pipeline_version" >= 1 AND "size_bytes" >= 1 AND "width" >= 1 AND "height" >= 1),
|
||||
CONSTRAINT "attachment_variants_sha256_format" CHECK ("sha256" ~ '^[a-f0-9]{64}$')
|
||||
);
|
||||
|
||||
CREATE TABLE "attachment_jobs" (
|
||||
"id" UUID NOT NULL,
|
||||
"attachment_id" UUID NOT NULL,
|
||||
"pipeline_version" INTEGER NOT NULL,
|
||||
"status" VARCHAR(16) NOT NULL,
|
||||
"attempt" INTEGER NOT NULL,
|
||||
"max_attempts" INTEGER NOT NULL,
|
||||
"next_attempt_at" TIMESTAMPTZ(3) NOT NULL,
|
||||
"lease_owner" UUID,
|
||||
"lease_until" TIMESTAMPTZ(3),
|
||||
"error_code" VARCHAR(64),
|
||||
"last_error" VARCHAR(200),
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"finished_at" TIMESTAMPTZ(3),
|
||||
|
||||
CONSTRAINT "attachment_jobs_pkey" PRIMARY KEY ("id"),
|
||||
CONSTRAINT "attachment_jobs_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT "attachment_jobs_attempt_range" CHECK ("pipeline_version" >= 1 AND "attempt" >= 1 AND "attempt" <= "max_attempts"),
|
||||
CONSTRAINT "attachment_jobs_lease_when_processing" CHECK ("status" <> 'processing' OR ("lease_owner" IS NOT NULL AND "lease_until" IS NOT NULL)),
|
||||
CONSTRAINT "attachment_jobs_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE TABLE "job_events" (
|
||||
"id" BIGSERIAL NOT NULL,
|
||||
"job_id" UUID NOT NULL,
|
||||
"attempt" INTEGER NOT NULL,
|
||||
"event" VARCHAR(32) NOT NULL,
|
||||
"detail" VARCHAR(200),
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "job_events_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE TABLE "outbox" (
|
||||
"id" BIGSERIAL NOT NULL,
|
||||
"job_id" UUID NOT NULL,
|
||||
"payload" JSONB NOT NULL,
|
||||
"available_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"published_at" TIMESTAMPTZ(3),
|
||||
"publish_attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "outbox_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "attachment_variants_object_key_key" ON "attachment_variants"("object_key");
|
||||
|
||||
CREATE UNIQUE INDEX "attachment_jobs_attachment_id_pipeline_version_key" ON "attachment_jobs"("attachment_id", "pipeline_version");
|
||||
|
||||
CREATE INDEX "attachment_jobs_status_lease_until_idx" ON "attachment_jobs"("status", "lease_until");
|
||||
|
||||
CREATE INDEX "job_events_job_id_idx" ON "job_events"("job_id");
|
||||
|
||||
CREATE INDEX "outbox_published_at_available_at_idx" ON "outbox"("published_at", "available_at");
|
||||
|
||||
CREATE INDEX "outbox_job_id_idx" ON "outbox"("job_id");
|
||||
|
||||
ALTER TABLE "attachment_variants" ADD CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "attachment_jobs" ADD CONSTRAINT "attachment_jobs_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "job_events" ADD CONSTRAINT "job_events_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "outbox" ADD CONSTRAINT "outbox_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -78,17 +78,91 @@ model Attachment {
|
||||
sizeBytes Int @map("size_bytes")
|
||||
sha256 String @db.Char(64)
|
||||
objectKey String @unique @map("object_key") @db.VarChar(128)
|
||||
stagingKey String? @unique @map("staging_key") @db.VarChar(128)
|
||||
status String @db.VarChar(16)
|
||||
errorCode String? @map("error_code") @db.VarChar(64)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
owner User @relation(fields: [ownerId], references: [id], onDelete: Restrict)
|
||||
messages MessageAttachment[]
|
||||
variants AttachmentVariant[]
|
||||
jobs AttachmentJob[]
|
||||
|
||||
@@index([chatId])
|
||||
@@index([ownerId])
|
||||
@@map("attachments")
|
||||
}
|
||||
|
||||
model AttachmentVariant {
|
||||
attachmentId String @map("attachment_id") @db.Uuid
|
||||
pipelineVersion Int @map("pipeline_version")
|
||||
name String @db.VarChar(16)
|
||||
contentType String @map("content_type") @db.VarChar(32)
|
||||
objectKey String @unique @map("object_key") @db.VarChar(128)
|
||||
sizeBytes Int @map("size_bytes")
|
||||
width Int
|
||||
height Int
|
||||
sha256 String @db.Char(64)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@id([attachmentId, pipelineVersion, name])
|
||||
@@map("attachment_variants")
|
||||
}
|
||||
|
||||
model AttachmentJob {
|
||||
id String @id @db.Uuid
|
||||
attachmentId String @map("attachment_id") @db.Uuid
|
||||
pipelineVersion Int @map("pipeline_version")
|
||||
status String @db.VarChar(16)
|
||||
attempt Int
|
||||
maxAttempts Int @map("max_attempts")
|
||||
nextAttemptAt DateTime @map("next_attempt_at") @db.Timestamptz(3)
|
||||
leaseOwner String? @map("lease_owner") @db.Uuid
|
||||
leaseUntil DateTime? @map("lease_until") @db.Timestamptz(3)
|
||||
errorCode String? @map("error_code") @db.VarChar(64)
|
||||
lastError String? @map("last_error") @db.VarChar(200)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
|
||||
finishedAt DateTime? @map("finished_at") @db.Timestamptz(3)
|
||||
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
|
||||
events JobEvent[]
|
||||
outbox OutboxEntry[]
|
||||
|
||||
@@unique([attachmentId, pipelineVersion])
|
||||
@@index([status, leaseUntil])
|
||||
@@map("attachment_jobs")
|
||||
}
|
||||
|
||||
model JobEvent {
|
||||
id BigInt @id @default(autoincrement())
|
||||
jobId String @map("job_id") @db.Uuid
|
||||
attempt Int
|
||||
event String @db.VarChar(32)
|
||||
detail String? @db.VarChar(200)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([jobId])
|
||||
@@map("job_events")
|
||||
}
|
||||
|
||||
model OutboxEntry {
|
||||
id BigInt @id @default(autoincrement())
|
||||
jobId String @map("job_id") @db.Uuid
|
||||
payload Json
|
||||
availableAt DateTime @default(now()) @map("available_at") @db.Timestamptz(3)
|
||||
publishedAt DateTime? @map("published_at") @db.Timestamptz(3)
|
||||
publishAttempts Int @default(0) @map("publish_attempts")
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([publishedAt, availableAt])
|
||||
@@index([jobId])
|
||||
@@map("outbox")
|
||||
}
|
||||
|
||||
model MessageAttachment {
|
||||
messageId String @map("message_id") @db.Uuid
|
||||
position Int @db.SmallInt
|
||||
|
||||
@@ -26,7 +26,7 @@ export class AttachmentsController {
|
||||
constructor(private readonly attachments: AttachmentsService) {}
|
||||
|
||||
@Post('api/v1/chats/:chatId/attachments')
|
||||
@HttpCode(201)
|
||||
@HttpCode(202)
|
||||
async upload(
|
||||
@Param('chatId') chatId: string,
|
||||
@Query('filename') filename: unknown,
|
||||
@@ -61,19 +61,19 @@ export class AttachmentsController {
|
||||
@CurrentUser() user: User,
|
||||
@Res() reply: FastifyReply,
|
||||
): Promise<void> {
|
||||
const { attachment, content } = await this.attachments.openContent(
|
||||
const download = await this.attachments.openContent(
|
||||
attachmentId,
|
||||
user.id,
|
||||
variant,
|
||||
);
|
||||
await reply
|
||||
.header('content-type', attachment.contentType)
|
||||
.header('content-length', attachment.sizeBytes)
|
||||
.header('content-type', download.contentType)
|
||||
.header('content-length', download.sizeBytes)
|
||||
.header('x-content-type-options', 'nosniff')
|
||||
.header(
|
||||
'content-disposition',
|
||||
`inline; filename*=UTF-8''${encodeURIComponent(attachment.filename)}`,
|
||||
`inline; filename*=UTF-8''${encodeURIComponent(download.filename)}`,
|
||||
)
|
||||
.send(content);
|
||||
.send(download.content);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { Attachment, Prisma } from '@prisma/client';
|
||||
import { Attachment, AttachmentVariant, Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { mkdtemp, readdir, rm, stat } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -10,6 +10,8 @@ import { codePointLength } from '../common/dto/code-point-length.validator';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { isDatabaseUnavailable, PrismaService } from '../db/prisma.service';
|
||||
import { PIPELINE_VERSION } from '../jobs/job-message';
|
||||
import { enqueueJob } from '../jobs/job-store';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
import { assertValidImage, isSupportedContentType } from './image-validation';
|
||||
import { receiveBody } from './upload-body';
|
||||
@@ -18,7 +20,16 @@ const TEMP_PREFIX = 'messenger-upload-';
|
||||
const STALE_TEMP_MS = 60 * 60 * 1000;
|
||||
const MAX_FILENAME_CODE_POINTS = 255;
|
||||
const FORBIDDEN_FILENAME = /[/\\\p{Cc}\p{Zl}\p{Zp}]/u;
|
||||
const KNOWN_VARIANTS = new Set(['thumbnail', 'crop', 'optimized']);
|
||||
const KNOWN_VARIANTS = new Set(['original', 'thumbnail', 'crop', 'optimized']);
|
||||
|
||||
export type AttachmentWithVariants = Attachment & { variants: AttachmentVariant[] };
|
||||
|
||||
export interface DownloadableContent {
|
||||
filename: string;
|
||||
contentType: string;
|
||||
sizeBytes: number;
|
||||
content: Readable;
|
||||
}
|
||||
|
||||
export interface UploadInput {
|
||||
chatId: string;
|
||||
@@ -36,6 +47,7 @@ export class AttachmentsService implements OnModuleInit {
|
||||
private readonly maxPixels = Number(process.env.MAX_IMAGE_PIXELS ?? 40000000);
|
||||
private readonly maxConcurrent = Number(process.env.MAX_CONCURRENT_UPLOADS ?? 16);
|
||||
private readonly bodyTimeoutMs = Number(process.env.UPLOAD_BODY_TIMEOUT_MS ?? 60000);
|
||||
private readonly tempRoot = process.env.UPLOAD_TMP_DIR ?? tmpdir();
|
||||
private activeUploads = 0;
|
||||
|
||||
constructor(
|
||||
@@ -47,7 +59,7 @@ export class AttachmentsService implements OnModuleInit {
|
||||
await this.removeStaleTempDirs();
|
||||
}
|
||||
|
||||
async upload(input: UploadInput): Promise<Attachment> {
|
||||
async upload(input: UploadInput): Promise<AttachmentWithVariants> {
|
||||
await assertChatMember(this.prisma, input.chatId, input.ownerId);
|
||||
const filename = parseFilename(input.filename);
|
||||
const contentType = parseContentType(input.contentType);
|
||||
@@ -58,7 +70,7 @@ export class AttachmentsService implements OnModuleInit {
|
||||
throw new ApiError('unavailable', 'too many concurrent uploads');
|
||||
}
|
||||
this.activeUploads++;
|
||||
const dir = await mkdtemp(join(tmpdir(), TEMP_PREFIX));
|
||||
const dir = await mkdtemp(join(this.tempRoot, TEMP_PREFIX));
|
||||
try {
|
||||
return await this.store(input, filename, contentType, join(dir, 'body'));
|
||||
} finally {
|
||||
@@ -67,7 +79,10 @@ export class AttachmentsService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async getVisible(attachmentId: string, userId: string): Promise<Attachment> {
|
||||
async getVisible(
|
||||
attachmentId: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentWithVariants> {
|
||||
if (!isUuid(attachmentId)) {
|
||||
throw new ApiError('invalid_request', 'attachmentId must be a valid UUID');
|
||||
}
|
||||
@@ -76,6 +91,7 @@ export class AttachmentsService implements OnModuleInit {
|
||||
id: attachmentId,
|
||||
chat: { members: { some: { userId } } },
|
||||
},
|
||||
include: { variants: { where: { pipelineVersion: PIPELINE_VERSION } } },
|
||||
});
|
||||
if (!row) {
|
||||
throw new ApiError('not_found', 'attachment not found');
|
||||
@@ -87,10 +103,30 @@ export class AttachmentsService implements OnModuleInit {
|
||||
attachmentId: string,
|
||||
userId: string,
|
||||
variant: unknown,
|
||||
): Promise<{ attachment: Attachment; content: Readable }> {
|
||||
): Promise<DownloadableContent> {
|
||||
const attachment = await this.getVisible(attachmentId, userId);
|
||||
assertOriginalVariant(variant);
|
||||
return { attachment, content: await this.s3.getStream(attachment.objectKey) };
|
||||
const name = parseVariant(variant);
|
||||
if (attachment.status !== 'ready') {
|
||||
throw new ApiError('conflict', `attachment is ${attachment.status}`);
|
||||
}
|
||||
if (name === 'original') {
|
||||
return {
|
||||
filename: attachment.filename,
|
||||
contentType: attachment.contentType,
|
||||
sizeBytes: attachment.sizeBytes,
|
||||
content: await this.s3.getStream(attachment.objectKey),
|
||||
};
|
||||
}
|
||||
const stored = attachment.variants.find((v) => v.name === name);
|
||||
if (!stored) {
|
||||
throw new ApiError('not_found', 'attachment variant not found');
|
||||
}
|
||||
return {
|
||||
filename: attachment.filename,
|
||||
contentType: stored.contentType,
|
||||
sizeBytes: stored.sizeBytes,
|
||||
content: await this.s3.getStream(stored.objectKey),
|
||||
};
|
||||
}
|
||||
|
||||
private async store(
|
||||
@@ -98,7 +134,7 @@ export class AttachmentsService implements OnModuleInit {
|
||||
filename: string,
|
||||
contentType: string,
|
||||
path: string,
|
||||
): Promise<Attachment> {
|
||||
): Promise<AttachmentWithVariants> {
|
||||
if (!input.body) {
|
||||
throw new ApiError('invalid_request', 'request body must not be empty');
|
||||
}
|
||||
@@ -113,32 +149,38 @@ export class AttachmentsService implements OnModuleInit {
|
||||
}
|
||||
await assertValidImage(path, size, contentType, this.maxPixels);
|
||||
const id = randomUUID();
|
||||
const objectKey = `attachments/${id}`;
|
||||
await this.s3.putFile({ key: objectKey, path, size, contentType, sha256 });
|
||||
const stagingKey = `staging/${id}`;
|
||||
await this.s3.putFile({ key: stagingKey, path, size, contentType, sha256 });
|
||||
const now = new Date();
|
||||
try {
|
||||
return await this.prisma.attachment.create({
|
||||
data: {
|
||||
id,
|
||||
chatId: input.chatId,
|
||||
ownerId: input.ownerId,
|
||||
filename,
|
||||
contentType,
|
||||
sizeBytes: size,
|
||||
sha256,
|
||||
objectKey,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
return await this.prisma.$transaction(async (tx) => {
|
||||
const attachment = await tx.attachment.create({
|
||||
data: {
|
||||
id,
|
||||
chatId: input.chatId,
|
||||
ownerId: input.ownerId,
|
||||
filename,
|
||||
contentType,
|
||||
sizeBytes: size,
|
||||
sha256,
|
||||
objectKey: `attachments/${id}/original`,
|
||||
stagingKey,
|
||||
status: 'queued',
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await enqueueJob(tx, id, now);
|
||||
return { ...attachment, variants: [] };
|
||||
});
|
||||
} catch (error) {
|
||||
if (isDefiniteRejection(error)) {
|
||||
await this.s3.delete(objectKey).catch(() => {
|
||||
this.logger.warn(`compensation failed for ${objectKey}`);
|
||||
await this.s3.delete(stagingKey).catch(() => {
|
||||
this.logger.warn(`compensation failed for ${stagingKey}`);
|
||||
});
|
||||
}
|
||||
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
|
||||
this.logger.error(`metadata insert failed for ${objectKey}`);
|
||||
this.logger.error(`metadata insert failed for ${stagingKey}`);
|
||||
throw new ApiError('unavailable', 'database unavailable');
|
||||
}
|
||||
throw error;
|
||||
@@ -146,7 +188,7 @@ export class AttachmentsService implements OnModuleInit {
|
||||
}
|
||||
|
||||
private async removeStaleTempDirs(): Promise<void> {
|
||||
const root = tmpdir();
|
||||
const root = this.tempRoot;
|
||||
const names = await readdir(root);
|
||||
for (const name of names.filter((n) => n.startsWith(TEMP_PREFIX))) {
|
||||
const path = join(root, name);
|
||||
@@ -158,12 +200,12 @@ export class AttachmentsService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
function assertOriginalVariant(raw: unknown): void {
|
||||
if (raw === undefined || raw === 'original') {
|
||||
return;
|
||||
function parseVariant(raw: unknown): string {
|
||||
if (raw === undefined) {
|
||||
return 'original';
|
||||
}
|
||||
if (typeof raw === 'string' && KNOWN_VARIANTS.has(raw)) {
|
||||
throw new ApiError('not_found', 'attachment variant not found');
|
||||
return raw;
|
||||
}
|
||||
throw new ApiError('invalid_request', 'unknown attachment variant');
|
||||
}
|
||||
|
||||
@@ -1,4 +1,15 @@
|
||||
import { Attachment } from '@prisma/client';
|
||||
import { Attachment, AttachmentVariant } from '@prisma/client';
|
||||
|
||||
const VARIANT_ORDER = ['thumbnail', 'crop', 'optimized'];
|
||||
|
||||
export interface VariantResponse {
|
||||
name: string;
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface AttachmentResponse {
|
||||
id: string;
|
||||
@@ -8,14 +19,18 @@ export interface AttachmentResponse {
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
sha256: string;
|
||||
status: 'ready';
|
||||
variants: never[];
|
||||
error_code: null;
|
||||
status: string;
|
||||
variants: VariantResponse[];
|
||||
error_code: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export function toAttachmentResponse(row: Attachment): AttachmentResponse {
|
||||
export function toAttachmentResponse(
|
||||
row: Attachment & { variants: AttachmentVariant[] },
|
||||
): AttachmentResponse {
|
||||
const variants = row.status === 'ready' ? [...row.variants] : [];
|
||||
variants.sort((a, b) => VARIANT_ORDER.indexOf(a.name) - VARIANT_ORDER.indexOf(b.name));
|
||||
return {
|
||||
id: row.id,
|
||||
chat_id: row.chatId,
|
||||
@@ -24,9 +39,16 @@ export function toAttachmentResponse(row: Attachment): AttachmentResponse {
|
||||
content_type: row.contentType,
|
||||
size_bytes: row.sizeBytes,
|
||||
sha256: row.sha256,
|
||||
status: 'ready',
|
||||
variants: [],
|
||||
error_code: null,
|
||||
status: row.status,
|
||||
variants: variants.map((variant) => ({
|
||||
name: variant.name,
|
||||
content_type: variant.contentType,
|
||||
size_bytes: variant.sizeBytes,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
})),
|
||||
error_code: row.errorCode,
|
||||
created_at: row.createdAt.toISOString(),
|
||||
updated_at: row.updatedAt.toISOString(),
|
||||
};
|
||||
|
||||
@@ -2,10 +2,14 @@ import { PrismaClient } from '@prisma/client';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
|
||||
const MAX_REPORTED_KEYS = 20;
|
||||
const DEFAULT_PREFIXES = ['attachments/', 'staging/'];
|
||||
|
||||
type ReconcilePrisma = Pick<PrismaClient, 'attachment' | 'attachmentVariant'>;
|
||||
|
||||
export interface ReconcileOptions {
|
||||
apply: boolean;
|
||||
graceMs: number;
|
||||
stagingRetentionMs?: number;
|
||||
prefix?: string;
|
||||
now?: Date;
|
||||
pageSize?: number;
|
||||
@@ -14,6 +18,7 @@ export interface ReconcileOptions {
|
||||
export interface ReconcileReport {
|
||||
mode: 'apply' | 'dry-run';
|
||||
grace_seconds: number;
|
||||
staging_expired: number;
|
||||
scanned: number;
|
||||
fresh_skipped: number;
|
||||
referenced: number;
|
||||
@@ -25,7 +30,7 @@ export interface ReconcileReport {
|
||||
}
|
||||
|
||||
export async function reconcile(
|
||||
prisma: Pick<PrismaClient, 'attachment'>,
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
options: ReconcileOptions,
|
||||
): Promise<ReconcileReport> {
|
||||
@@ -34,6 +39,7 @@ export async function reconcile(
|
||||
const report: ReconcileReport = {
|
||||
mode: options.apply ? 'apply' : 'dry-run',
|
||||
grace_seconds: options.graceMs / 1000,
|
||||
staging_expired: 0,
|
||||
scanned: 0,
|
||||
fresh_skipped: 0,
|
||||
referenced: 0,
|
||||
@@ -43,22 +49,83 @@ export async function reconcile(
|
||||
orphan_keys: [],
|
||||
duration_ms: 0,
|
||||
};
|
||||
if (options.stagingRetentionMs !== undefined) {
|
||||
report.staging_expired = await expireFailedStaging(
|
||||
prisma,
|
||||
s3,
|
||||
(options.now ?? new Date()).getTime() - options.stagingRetentionMs,
|
||||
options.apply,
|
||||
);
|
||||
}
|
||||
for (const prefix of options.prefix ? [options.prefix] : DEFAULT_PREFIXES) {
|
||||
await reconcilePrefix(prisma, s3, prefix, cutoff, options, report);
|
||||
}
|
||||
report.duration_ms = Date.now() - started;
|
||||
return report;
|
||||
}
|
||||
|
||||
async function expireFailedStaging(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
before: number,
|
||||
apply: boolean,
|
||||
): Promise<number> {
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: {
|
||||
status: 'failed',
|
||||
stagingKey: { not: null },
|
||||
updatedAt: { lt: new Date(before) },
|
||||
},
|
||||
select: { id: true, stagingKey: true },
|
||||
take: 500,
|
||||
});
|
||||
if (!apply) {
|
||||
return rows.length;
|
||||
}
|
||||
let expired = 0;
|
||||
for (const row of rows) {
|
||||
try {
|
||||
await s3.delete(row.stagingKey!);
|
||||
await prisma.attachment.update({
|
||||
where: { id: row.id },
|
||||
data: { stagingKey: null },
|
||||
});
|
||||
expired++;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return expired;
|
||||
}
|
||||
|
||||
async function reconcilePrefix(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
prefix: string,
|
||||
cutoff: number,
|
||||
options: ReconcileOptions,
|
||||
report: ReconcileReport,
|
||||
): Promise<void> {
|
||||
let token: string | undefined;
|
||||
do {
|
||||
const page = await s3.listObjects(
|
||||
options.prefix ?? 'attachments/',
|
||||
options.pageSize ?? 500,
|
||||
token,
|
||||
);
|
||||
const page = await s3.listObjects(prefix, options.pageSize ?? 500, token);
|
||||
token = page.nextToken;
|
||||
report.scanned += page.objects.length;
|
||||
const old = page.objects.filter((object) => object.lastModified.getTime() < cutoff);
|
||||
report.fresh_skipped += page.objects.length - old.length;
|
||||
const keys = old.map((object) => object.key);
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: { objectKey: { in: old.map((object) => object.key) } },
|
||||
where: { OR: [{ objectKey: { in: keys } }, { stagingKey: { in: keys } }] },
|
||||
select: { objectKey: true, stagingKey: true },
|
||||
});
|
||||
const variants = await prisma.attachmentVariant.findMany({
|
||||
where: { objectKey: { in: keys } },
|
||||
select: { objectKey: true },
|
||||
});
|
||||
const known = new Set(rows.map((row) => row.objectKey));
|
||||
const known = new Set([
|
||||
...rows.flatMap((row) => [row.objectKey, row.stagingKey]),
|
||||
...variants.map((row) => row.objectKey),
|
||||
]);
|
||||
for (const { key } of old) {
|
||||
if (known.has(key)) {
|
||||
report.referenced++;
|
||||
@@ -76,6 +143,4 @@ export async function reconcile(
|
||||
}
|
||||
}
|
||||
} while (token);
|
||||
report.duration_ms = Date.now() - started;
|
||||
return report;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
|
||||
export const PIPELINE_VERSION = 1;
|
||||
export const JOB_TYPE = 'attachment.process.v1';
|
||||
|
||||
export interface JobMessage {
|
||||
schema_version: 1;
|
||||
job_id: string;
|
||||
type: typeof JOB_TYPE;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
const FIELDS = [
|
||||
'schema_version',
|
||||
'job_id',
|
||||
'type',
|
||||
'attachment_id',
|
||||
'pipeline_version',
|
||||
'attempt',
|
||||
'created_at',
|
||||
];
|
||||
|
||||
export function jobMessage(
|
||||
job: { id: string; attachmentId: string; pipelineVersion: number; attempt: number },
|
||||
createdAt: Date,
|
||||
): JobMessage {
|
||||
return {
|
||||
schema_version: 1,
|
||||
job_id: job.id,
|
||||
type: JOB_TYPE,
|
||||
attachment_id: job.attachmentId,
|
||||
pipeline_version: job.pipelineVersion,
|
||||
attempt: job.attempt,
|
||||
created_at: createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseJobMessage(raw: Buffer): JobMessage | null {
|
||||
let value: any;
|
||||
try {
|
||||
value = JSON.parse(raw.toString('utf8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
typeof value !== 'object' ||
|
||||
value === null ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).some((key) => !FIELDS.includes(key)) ||
|
||||
value.schema_version !== 1 ||
|
||||
value.type !== JOB_TYPE ||
|
||||
typeof value.job_id !== 'string' ||
|
||||
!isUuid(value.job_id) ||
|
||||
typeof value.attachment_id !== 'string' ||
|
||||
!isUuid(value.attachment_id) ||
|
||||
!Number.isInteger(value.pipeline_version) ||
|
||||
value.pipeline_version < 1 ||
|
||||
!Number.isInteger(value.attempt) ||
|
||||
value.attempt < 1 ||
|
||||
typeof value.created_at !== 'string' ||
|
||||
Number.isNaN(Date.parse(value.created_at))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return value as JobMessage;
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
import { AttachmentJob, Prisma, PrismaClient } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { jobMessage, PIPELINE_VERSION } from './job-message';
|
||||
|
||||
type Tx = Prisma.TransactionClient;
|
||||
|
||||
export const RETRY_EXHAUSTED = 'retry_exhausted';
|
||||
|
||||
export function maxAttempts(): number {
|
||||
return Number(process.env.JOB_MAX_ATTEMPTS ?? 5);
|
||||
}
|
||||
|
||||
export function backoffMs(attempt: number, random = Math.random): number {
|
||||
const base = Number(process.env.JOB_RETRY_BASE_MS ?? 1000);
|
||||
const cap = Number(process.env.JOB_RETRY_MAX_MS ?? 30000);
|
||||
const delay = Math.min(cap, base * 2 ** (attempt - 1));
|
||||
return Math.round(delay / 2 + random() * (delay / 2));
|
||||
}
|
||||
|
||||
export async function enqueueJob(
|
||||
tx: Tx,
|
||||
attachmentId: string,
|
||||
now: Date,
|
||||
): Promise<AttachmentJob> {
|
||||
const job = await tx.attachmentJob.create({
|
||||
data: {
|
||||
id: randomUUID(),
|
||||
attachmentId,
|
||||
pipelineVersion: PIPELINE_VERSION,
|
||||
status: 'queued',
|
||||
attempt: 1,
|
||||
maxAttempts: maxAttempts(),
|
||||
nextAttemptAt: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: 1, event: 'enqueued', createdAt: now },
|
||||
});
|
||||
await publishLater(tx, job, now);
|
||||
return job;
|
||||
}
|
||||
|
||||
export async function scheduleRetry(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
reason: string,
|
||||
now: Date,
|
||||
event = 'retry_scheduled',
|
||||
): Promise<AttachmentJob> {
|
||||
const delay = backoffMs(job.attempt);
|
||||
const next = await tx.attachmentJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'queued',
|
||||
attempt: job.attempt + 1,
|
||||
nextAttemptAt: new Date(now.getTime() + delay),
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: reason,
|
||||
},
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'queued' },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
attempt: job.attempt,
|
||||
event,
|
||||
detail: `${reason}; next attempt ${next.attempt} in ${delay} ms`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
await publishLater(tx, next, next.nextAttemptAt);
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function markFailed(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
errorCode: string,
|
||||
reason: string,
|
||||
now: Date,
|
||||
): Promise<void> {
|
||||
await tx.attachmentJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'failed',
|
||||
errorCode,
|
||||
lastError: reason,
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
finishedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'failed', errorCode },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
attempt: job.attempt,
|
||||
event: 'failed',
|
||||
detail: `${errorCode}: ${reason}`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function retryOrFail(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
reason: string,
|
||||
now: Date,
|
||||
event?: string,
|
||||
): Promise<'retry' | 'failed'> {
|
||||
if (job.attempt >= job.maxAttempts) {
|
||||
await markFailed(tx, job, RETRY_EXHAUSTED, reason, now);
|
||||
return 'failed';
|
||||
}
|
||||
await scheduleRetry(tx, job, reason, now, event);
|
||||
return 'retry';
|
||||
}
|
||||
|
||||
export async function recoverExpiredLeases(
|
||||
prisma: PrismaClient,
|
||||
now = new Date(),
|
||||
limit = 100,
|
||||
): Promise<number> {
|
||||
const expired = await prisma.attachmentJob.findMany({
|
||||
where: { status: 'processing', leaseUntil: { lt: now } },
|
||||
orderBy: { leaseUntil: 'asc' },
|
||||
take: limit,
|
||||
});
|
||||
let recovered = 0;
|
||||
for (const candidate of expired) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: {
|
||||
id: candidate.id,
|
||||
status: 'processing',
|
||||
leaseOwner: candidate.leaseOwner,
|
||||
leaseUntil: { lt: now },
|
||||
},
|
||||
data: { leaseOwner: null, leaseUntil: null, status: 'queued' },
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return;
|
||||
}
|
||||
await retryOrFail(tx, candidate, 'lease expired', now, 'lease_expired');
|
||||
recovered++;
|
||||
});
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
export type ReplayResult = 'replayed' | 'already_ready' | 'not_failed' | 'not_found';
|
||||
|
||||
export async function replayJob(
|
||||
prisma: PrismaClient,
|
||||
jobId: string,
|
||||
actor: string,
|
||||
now = new Date(),
|
||||
): Promise<ReplayResult> {
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const job = await tx.attachmentJob.findUnique({ where: { id: jobId } });
|
||||
if (!job) {
|
||||
return 'not_found';
|
||||
}
|
||||
if (job.status === 'ready') {
|
||||
return 'already_ready';
|
||||
}
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: { id: jobId, status: 'failed' },
|
||||
data: {
|
||||
status: 'queued',
|
||||
attempt: 1,
|
||||
errorCode: null,
|
||||
nextAttemptAt: now,
|
||||
finishedAt: null,
|
||||
},
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return 'not_failed';
|
||||
}
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'queued', errorCode: null },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId,
|
||||
attempt: 1,
|
||||
event: 'replayed',
|
||||
detail: `by ${actor}; previous ${job.errorCode} after attempt ${job.attempt}`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
await publishLater(tx, { ...job, attempt: 1 }, now);
|
||||
return 'replayed';
|
||||
});
|
||||
}
|
||||
|
||||
async function publishLater(tx: Tx, job: AttachmentJob, availableAt: Date): Promise<void> {
|
||||
await tx.outboxEntry.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
payload: jobMessage(job, availableAt) as unknown as Prisma.InputJsonObject,
|
||||
availableAt,
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import {
|
||||
ObjectMissingError,
|
||||
ObjectTooLargeError,
|
||||
S3Service,
|
||||
} from '../storage/s3.service';
|
||||
import { JobMessage, PIPELINE_VERSION } from './job-message';
|
||||
import { markFailed, retryOrFail } from './job-store';
|
||||
import { InvalidImageError, RenderedVariant, renderVariants } from './variants';
|
||||
|
||||
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
|
||||
|
||||
export interface ProcessorOptions {
|
||||
timeoutMs: number;
|
||||
leaseMs: number;
|
||||
maxBytes: number;
|
||||
maxPixels: number;
|
||||
}
|
||||
|
||||
export function processorOptions(): ProcessorOptions {
|
||||
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
|
||||
return {
|
||||
timeoutMs,
|
||||
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
|
||||
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
|
||||
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
|
||||
};
|
||||
}
|
||||
|
||||
class PermanentError extends Error {
|
||||
constructor(
|
||||
readonly code: string,
|
||||
reason: string,
|
||||
) {
|
||||
super(reason);
|
||||
}
|
||||
}
|
||||
|
||||
class TimeoutError extends Error {}
|
||||
|
||||
export class JobProcessor {
|
||||
private readonly logger = new Logger('JobProcessor');
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaClient,
|
||||
private readonly s3: S3Service,
|
||||
private readonly options: ProcessorOptions = processorOptions(),
|
||||
private readonly hooks: { afterStore?: () => Promise<void> } = {},
|
||||
) {}
|
||||
|
||||
async handle(message: JobMessage): Promise<Outcome> {
|
||||
const job = await this.prisma.attachmentJob.findUnique({
|
||||
where: { id: message.job_id },
|
||||
include: { attachment: true },
|
||||
});
|
||||
if (
|
||||
!job ||
|
||||
job.attachmentId !== message.attachment_id ||
|
||||
job.pipelineVersion !== message.pipeline_version ||
|
||||
job.pipelineVersion !== PIPELINE_VERSION
|
||||
) {
|
||||
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
|
||||
return 'skipped';
|
||||
}
|
||||
const lease = await this.claim(job);
|
||||
if (!lease) {
|
||||
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
|
||||
return 'skipped';
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await this.withTimeout(this.produce(job.attachment));
|
||||
} catch (error) {
|
||||
return this.recordFailure(lease, error);
|
||||
}
|
||||
await this.hooks.afterStore?.();
|
||||
return this.commit(lease, job.attachment, variants);
|
||||
}
|
||||
|
||||
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
|
||||
const now = new Date();
|
||||
const owner = randomUUID();
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
|
||||
data: {
|
||||
status: 'processing',
|
||||
leaseOwner: owner,
|
||||
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
|
||||
},
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return null;
|
||||
}
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'processing' },
|
||||
});
|
||||
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
|
||||
});
|
||||
return leased;
|
||||
});
|
||||
}
|
||||
|
||||
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
|
||||
if (!attachment.stagingKey) {
|
||||
throw new PermanentError('staging_missing', 'staging object was already removed');
|
||||
}
|
||||
let original: Buffer;
|
||||
try {
|
||||
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectMissingError) {
|
||||
throw new PermanentError('staging_missing', 'staging object not found');
|
||||
}
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const sha256 = createHash('sha256').update(original).digest('hex');
|
||||
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
|
||||
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await renderVariants(original, this.options.maxPixels);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw new PermanentError('invalid_image', 'image cannot be decoded');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
|
||||
for (const variant of variants) {
|
||||
await this.s3.putBuffer(
|
||||
variantKey(attachment.id, variant),
|
||||
variant.body,
|
||||
variant.contentType,
|
||||
variant.sha256,
|
||||
);
|
||||
}
|
||||
return variants;
|
||||
}
|
||||
|
||||
private async commit(
|
||||
lease: AttachmentJob,
|
||||
attachment: Attachment,
|
||||
variants: RenderedVariant[],
|
||||
): Promise<Outcome> {
|
||||
const now = new Date();
|
||||
const committed = await this.prisma.$transaction(async (tx) => {
|
||||
const owned = await tx.attachmentJob.updateMany({
|
||||
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
||||
data: {
|
||||
status: 'ready',
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: null,
|
||||
finishedAt: now,
|
||||
},
|
||||
});
|
||||
if (owned.count === 0) {
|
||||
return false;
|
||||
}
|
||||
await tx.attachmentVariant.createMany({
|
||||
data: variants.map((variant) => ({
|
||||
attachmentId: attachment.id,
|
||||
pipelineVersion: PIPELINE_VERSION,
|
||||
name: variant.name,
|
||||
contentType: variant.contentType,
|
||||
objectKey: variantKey(attachment.id, variant),
|
||||
sizeBytes: variant.body.length,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
createdAt: now,
|
||||
})),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: attachment.id },
|
||||
data: { status: 'ready', errorCode: null },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
|
||||
});
|
||||
return true;
|
||||
});
|
||||
if (!committed) {
|
||||
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
|
||||
return 'skipped';
|
||||
}
|
||||
await this.removeStaging(attachment);
|
||||
return 'ready';
|
||||
}
|
||||
|
||||
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
|
||||
const now = new Date();
|
||||
const reason =
|
||||
error instanceof PermanentError
|
||||
? error.message
|
||||
: error instanceof TimeoutError
|
||||
? 'processing timed out'
|
||||
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
|
||||
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const owned = await tx.attachmentJob.count({
|
||||
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
||||
});
|
||||
if (owned === 0) {
|
||||
return 'skipped' as const;
|
||||
}
|
||||
if (error instanceof PermanentError) {
|
||||
await markFailed(tx, lease, error.code, reason, now);
|
||||
return 'failed' as const;
|
||||
}
|
||||
return retryOrFail(tx, lease, reason, now);
|
||||
});
|
||||
}
|
||||
|
||||
private async removeStaging(attachment: Attachment): Promise<void> {
|
||||
if (!attachment.stagingKey) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await this.s3.delete(attachment.stagingKey);
|
||||
await this.prisma.attachment.update({
|
||||
where: { id: attachment.id },
|
||||
data: { stagingKey: null },
|
||||
});
|
||||
} catch {
|
||||
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
private withTimeout<T>(work: Promise<T>): Promise<T> {
|
||||
let timer: NodeJS.Timeout;
|
||||
const deadline = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
|
||||
});
|
||||
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
}
|
||||
|
||||
export function variantKey(
|
||||
attachmentId: string,
|
||||
variant: Pick<RenderedVariant, 'name' | 'extension'>,
|
||||
): string {
|
||||
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { createHash } from 'crypto';
|
||||
import sharp, { Sharp } from 'sharp';
|
||||
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
const WHITE = { r: 255, g: 255, b: 255 };
|
||||
const JPEG_QUALITY = 82;
|
||||
const WEBP_QUALITY = 80;
|
||||
|
||||
export interface RenderedVariant {
|
||||
name: 'thumbnail' | 'crop' | 'optimized';
|
||||
contentType: string;
|
||||
extension: string;
|
||||
body: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export class InvalidImageError extends Error {
|
||||
constructor(reason: string) {
|
||||
super(reason);
|
||||
this.name = 'InvalidImageError';
|
||||
}
|
||||
}
|
||||
|
||||
export function fitInside(
|
||||
width: number,
|
||||
height: number,
|
||||
box: number,
|
||||
): { width: number; height: number } {
|
||||
const scale = Math.min(box / width, box / height, 1);
|
||||
return {
|
||||
width: Math.max(1, Math.floor(width * scale)),
|
||||
height: Math.max(1, Math.floor(height * scale)),
|
||||
};
|
||||
}
|
||||
|
||||
export async function renderVariants(
|
||||
original: Buffer,
|
||||
maxPixels: number,
|
||||
): Promise<RenderedVariant[]> {
|
||||
const open = (): Sharp =>
|
||||
sharp(original, { failOn: 'error', limitInputPixels: maxPixels }).rotate();
|
||||
try {
|
||||
const meta = await sharp(original, { limitInputPixels: maxPixels }).metadata();
|
||||
if (!meta.width || !meta.height) {
|
||||
throw new InvalidImageError('image has no dimensions');
|
||||
}
|
||||
const swapped = (meta.orientation ?? 1) >= 5;
|
||||
const width = swapped ? meta.height : meta.width;
|
||||
const height = swapped ? meta.width : meta.height;
|
||||
const thumb = fitInside(width, height, 256);
|
||||
const large = fitInside(width, height, 1280);
|
||||
const thumbnail = open()
|
||||
.resize(thumb.width, thumb.height, { fit: 'fill' })
|
||||
.flatten({ background: WHITE })
|
||||
.jpeg({ quality: JPEG_QUALITY });
|
||||
const crop = open()
|
||||
.resize(128, 128, { fit: 'cover', position: 'centre' })
|
||||
.flatten({ background: WHITE })
|
||||
.jpeg({ quality: JPEG_QUALITY });
|
||||
const optimized = open()
|
||||
.resize(large.width, large.height, { fit: 'fill' })
|
||||
.webp({ quality: WEBP_QUALITY, effort: 4 });
|
||||
return [
|
||||
await finish('thumbnail', 'image/jpeg', 'jpg', thumbnail),
|
||||
await finish('crop', 'image/jpeg', 'jpg', crop),
|
||||
await finish('optimized', 'image/webp', 'webp', optimized),
|
||||
];
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw error;
|
||||
}
|
||||
throw new InvalidImageError(error instanceof Error ? error.message : 'decode failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function finish(
|
||||
name: RenderedVariant['name'],
|
||||
contentType: string,
|
||||
extension: string,
|
||||
pipeline: Sharp,
|
||||
): Promise<RenderedVariant> {
|
||||
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
|
||||
return {
|
||||
name,
|
||||
contentType,
|
||||
extension,
|
||||
body: data,
|
||||
width: info.width,
|
||||
height: info.height,
|
||||
sha256: createHash('sha256').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
@@ -166,12 +166,16 @@ export class MessagesService {
|
||||
if (attachmentIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
const found = await this.prisma.attachment.count({
|
||||
const found = await this.prisma.attachment.findMany({
|
||||
where: { id: { in: attachmentIds }, chatId },
|
||||
select: { status: true },
|
||||
});
|
||||
if (found !== attachmentIds.length) {
|
||||
if (found.length !== attachmentIds.length) {
|
||||
throw new ApiError('not_found', 'attachment not found');
|
||||
}
|
||||
if (found.some((row) => row.status === 'failed')) {
|
||||
throw new ApiError('conflict', 'failed attachment cannot be attached');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import { S3Service } from './storage/s3.service';
|
||||
|
||||
const DEFAULT_GRACE_SECONDS = 3600;
|
||||
const DEFAULT_INTERVAL_SECONDS = 300;
|
||||
const DEFAULT_STAGING_RETENTION_SECONDS = 86400;
|
||||
|
||||
function option(name: string, fallback: number): number {
|
||||
const prefix = `--${name}=`;
|
||||
@@ -23,6 +24,10 @@ async function main(): Promise<void> {
|
||||
const loop = process.argv.includes('--loop');
|
||||
const graceSeconds = option('grace-seconds', DEFAULT_GRACE_SECONDS);
|
||||
const intervalSeconds = option('interval-seconds', DEFAULT_INTERVAL_SECONDS);
|
||||
const stagingRetentionSeconds = option(
|
||||
'staging-retention-seconds',
|
||||
Number(process.env.STAGING_RETENTION_SECONDS ?? DEFAULT_STAGING_RETENTION_SECONDS),
|
||||
);
|
||||
const prisma = new PrismaService();
|
||||
const s3 = new S3Service();
|
||||
do {
|
||||
@@ -30,6 +35,7 @@ async function main(): Promise<void> {
|
||||
const report = await reconcile(prisma, s3, {
|
||||
apply,
|
||||
graceMs: graceSeconds * 1000,
|
||||
stagingRetentionMs: stagingRetentionSeconds * 1000,
|
||||
});
|
||||
process.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
} catch (error) {
|
||||
|
||||
@@ -76,6 +76,69 @@ export class S3Service implements OnModuleDestroy {
|
||||
);
|
||||
}
|
||||
|
||||
async putBuffer(
|
||||
key: string,
|
||||
body: Buffer,
|
||||
contentType: string,
|
||||
sha256: string,
|
||||
): Promise<void> {
|
||||
await this.guard(`put ${key}`, () =>
|
||||
this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentLength: body.length,
|
||||
ContentType: contentType,
|
||||
ChecksumSHA256: Buffer.from(sha256, 'hex').toString('base64'),
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async getBuffer(key: string, maxBytes: number): Promise<Buffer> {
|
||||
let body: Readable;
|
||||
try {
|
||||
const response = await this.client.send(
|
||||
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
|
||||
{ abortSignal: AbortSignal.timeout(OPERATION_TIMEOUT_MS) },
|
||||
);
|
||||
if ((response.ContentLength ?? 0) > maxBytes) {
|
||||
throw new ObjectTooLargeError(key);
|
||||
}
|
||||
body = response.Body as Readable;
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw error;
|
||||
}
|
||||
const name = error instanceof Error ? error.name : 'unknown error';
|
||||
this.logger.error(`get ${key} failed: ${name}`);
|
||||
if (name === 'NoSuchKey') {
|
||||
throw new ObjectMissingError(key);
|
||||
}
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for await (const chunk of body) {
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
body.destroy();
|
||||
throw new ObjectTooLargeError(key);
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw error;
|
||||
}
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async getStream(key: string): Promise<Readable> {
|
||||
const deadline = new AbortController();
|
||||
const timer = setTimeout(() => deadline.abort(), IDLE_TIMEOUT_MS);
|
||||
@@ -158,6 +221,20 @@ export class S3Service implements OnModuleDestroy {
|
||||
}
|
||||
}
|
||||
|
||||
export class ObjectMissingError extends Error {
|
||||
constructor(key: string) {
|
||||
super(`object ${key} is missing from storage`);
|
||||
this.name = 'ObjectMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ObjectTooLargeError extends Error {
|
||||
constructor(key: string) {
|
||||
super(`object ${key} exceeds size limit`);
|
||||
this.name = 'ObjectTooLargeError';
|
||||
}
|
||||
}
|
||||
|
||||
function readSecret(name: string): string {
|
||||
const file = process.env[`${name}_FILE`];
|
||||
const value = file ? readFileSync(file, 'utf8').trim() : process.env[name];
|
||||
|
||||
@@ -3,7 +3,13 @@ import { createHash, randomUUID } from 'crypto';
|
||||
import request from 'supertest';
|
||||
import { PrismaService } from '../src/db/prisma.service';
|
||||
import { S3Service } from '../src/storage/s3.service';
|
||||
import { bearer, createTestApp, registerUser, TestUser } from './helpers';
|
||||
import {
|
||||
bearer,
|
||||
createTestApp,
|
||||
processAttachment,
|
||||
registerUser,
|
||||
TestUser,
|
||||
} from './helpers';
|
||||
import { jpeg, SAMPLE_PNG } from './images';
|
||||
|
||||
describe('Скачивание и привязка вложений (e2e)', () => {
|
||||
@@ -35,8 +41,13 @@ describe('Скачивание и привязка вложений (e2e)', () =
|
||||
.set('Authorization', bearer(user))
|
||||
.set('Content-Type', type)
|
||||
.send(body)
|
||||
.expect(201);
|
||||
return res.body;
|
||||
.expect(202);
|
||||
expect(await processAttachment(app, res.body.id)).toBe('ready');
|
||||
const ready = await request(server)
|
||||
.get(`/api/v1/attachments/${res.body.id}`)
|
||||
.set('Authorization', bearer(user))
|
||||
.expect(200);
|
||||
return ready.body;
|
||||
};
|
||||
|
||||
const get = (path: string, user: TestUser | null = bob) => {
|
||||
@@ -134,7 +145,11 @@ describe('Скачивание и привязка вложений (e2e)', () =
|
||||
it('variant: original явно, отсутствующий 404, неизвестный и повторённый 400', async () => {
|
||||
const { id } = await upload();
|
||||
await get(`/api/v1/attachments/${id}/content?variant=original`).expect(200);
|
||||
await get(`/api/v1/attachments/${id}/content?variant=thumbnail`).expect(404);
|
||||
await get(`/api/v1/attachments/${id}/content?variant=thumbnail`).expect(200);
|
||||
await app.get(PrismaService).attachmentVariant.deleteMany({
|
||||
where: { attachmentId: id, name: 'crop' },
|
||||
});
|
||||
await get(`/api/v1/attachments/${id}/content?variant=crop`).expect(404);
|
||||
await get(`/api/v1/attachments/${id}/content?variant=nonsense`).expect(400);
|
||||
await get(`/api/v1/attachments/${id}/content?variant=`).expect(400);
|
||||
await get(`/api/v1/attachments/${id}/content?variant=original&variant=original`).expect(400);
|
||||
|
||||
@@ -2,9 +2,10 @@ import { INestApplication } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import Ajv2020 from 'ajv/dist/2020';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import { readdirSync } from 'fs';
|
||||
import { mkdtempSync, readdirSync, rmSync } from 'fs';
|
||||
import { connect } from 'net';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import request from 'supertest';
|
||||
import { PrismaService } from '../src/db/prisma.service';
|
||||
import { loadDocument } from '../src/docs/swagger';
|
||||
@@ -61,8 +62,13 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
expect(res.body.error.request_id).toBe(res.headers['x-request-id']);
|
||||
};
|
||||
|
||||
const uploadTmp = mkdtempSync(join(tmpdir(), 'attachments-test-'));
|
||||
|
||||
const activeTempDirs = (): string[] =>
|
||||
readdirSync(uploadTmp).filter((name) => name.startsWith('messenger-upload-'));
|
||||
|
||||
beforeAll(async () => {
|
||||
({ app, server } = await createTestApp());
|
||||
({ app, server } = await createTestApp({ UPLOAD_TMP_DIR: uploadTmp }));
|
||||
alice = await registerUser(server, 'att_alice');
|
||||
eve = await registerUser(server, 'att_eve');
|
||||
const chat = await request(server)
|
||||
@@ -75,11 +81,12 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
rmSync(uploadTmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('успешная загрузка', () => {
|
||||
it('возвращает 201 Attachment по контракту и сохраняет метаданные', async () => {
|
||||
const res = await upload(SAMPLE_PNG).expect(201);
|
||||
it('возвращает 202 Attachment queued по контракту, staging и задание сохранены до ответа', async () => {
|
||||
const res = await upload(SAMPLE_PNG).expect(202);
|
||||
const document: any = loadDocument();
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
ajv.addSchema(document, 'contract');
|
||||
@@ -95,22 +102,42 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
content_type: 'image/png',
|
||||
size_bytes: SAMPLE_PNG.length,
|
||||
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
|
||||
status: 'ready',
|
||||
status: 'queued',
|
||||
variants: [],
|
||||
error_code: null,
|
||||
});
|
||||
const row = await app.get(PrismaService).attachment.findUnique({
|
||||
const prisma = app.get(PrismaService);
|
||||
const row = await prisma.attachment.findUnique({
|
||||
where: { id: res.body.id },
|
||||
include: { jobs: { include: { outbox: true } } },
|
||||
});
|
||||
expect(row?.objectKey).toBe(`attachments/${res.body.id}`);
|
||||
expect(JSON.stringify(res.body)).not.toContain('attachments/');
|
||||
expect(row).toMatchObject({
|
||||
status: 'queued',
|
||||
objectKey: `attachments/${res.body.id}/original`,
|
||||
stagingKey: `staging/${res.body.id}`,
|
||||
});
|
||||
expect(row!.jobs).toHaveLength(1);
|
||||
expect(row!.jobs[0]).toMatchObject({ status: 'queued', attempt: 1, pipelineVersion: 1 });
|
||||
expect(row!.jobs[0].outbox).toHaveLength(1);
|
||||
expect(row!.jobs[0].outbox[0].payload).toEqual({
|
||||
schema_version: 1,
|
||||
job_id: row!.jobs[0].id,
|
||||
type: 'attachment.process.v1',
|
||||
attachment_id: res.body.id,
|
||||
pipeline_version: 1,
|
||||
attempt: 1,
|
||||
created_at: expect.any(String),
|
||||
});
|
||||
const staged = await app.get(S3Service).getBuffer(`staging/${res.body.id}`, MAX_BYTES);
|
||||
expect(Buffer.compare(staged, SAMPLE_PNG)).toBe(0);
|
||||
expect(JSON.stringify(res.body)).not.toMatch(/attachments\/|staging\//);
|
||||
});
|
||||
|
||||
it('принимает реальный JPEG', async () => {
|
||||
const res = await upload(await jpeg(), {
|
||||
type: 'image/jpeg',
|
||||
filename: 'фото 1.jpg',
|
||||
}).expect(201);
|
||||
}).expect(202);
|
||||
expect(res.body).toMatchObject({
|
||||
content_type: 'image/jpeg',
|
||||
filename: 'фото 1.jpg',
|
||||
@@ -118,21 +145,21 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
});
|
||||
|
||||
it('игнорирует параметры и регистр Content-Type', async () => {
|
||||
await upload(SAMPLE_PNG, { type: 'IMAGE/PNG; charset=binary' }).expect(201);
|
||||
await upload(SAMPLE_PNG, { type: 'IMAGE/PNG; charset=binary' }).expect(202);
|
||||
});
|
||||
|
||||
it('принимает файл ровно 10 MiB, filename ровно 255 символов и PNG с байтами после IEND', async () => {
|
||||
const exact = pngOfExactSize(MAX_BYTES);
|
||||
expect(exact.length).toBe(MAX_BYTES);
|
||||
const res = await upload(exact).expect(201);
|
||||
const res = await upload(exact).expect(202);
|
||||
expect(res.body.size_bytes).toBe(MAX_BYTES);
|
||||
await upload(SAMPLE_PNG, { filename: `${'ф'.repeat(251)}.png` }).expect(201);
|
||||
await upload(Buffer.concat([png(4, 4), Buffer.from('tail')])).expect(201);
|
||||
await upload(SAMPLE_PNG, { filename: `${'ф'.repeat(251)}.png` }).expect(202);
|
||||
await upload(Buffer.concat([png(4, 4), Buffer.from('tail')])).expect(202);
|
||||
});
|
||||
|
||||
it('принимает ровно 40 млн пикселей', async () => {
|
||||
const res = await upload(await bigPng(8000, 5000));
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.status).toBe(202);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -154,10 +181,10 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
|
||||
it('посторонний и несуществующий чат — одинаковый 404, чужой файл не сохраняется', async () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const before = await prisma.attachment.count();
|
||||
const before = await prisma.attachment.count({ where: { chatId } });
|
||||
expectError(await upload(SAMPLE_PNG, { user: eve }), 404, 'not_found');
|
||||
expectError(await upload(SAMPLE_PNG, { chat: randomUUID() }), 404, 'not_found');
|
||||
expect(await prisma.attachment.count()).toBe(before);
|
||||
expect(await prisma.attachment.count({ where: { chatId } })).toBe(before);
|
||||
});
|
||||
|
||||
it('неверный UUID чата — 400', async () => {
|
||||
@@ -260,7 +287,7 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
|
||||
it('оборванная загрузка не оставляет временных файлов и строк в БД', async () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const before = await prisma.attachment.count();
|
||||
const before = await prisma.attachment.count({ where: { chatId } });
|
||||
const { port } = server.address();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const socket = connect(port, '127.0.0.1', () => {
|
||||
@@ -278,7 +305,7 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
socket.on('error', reject);
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
expect(await prisma.attachment.count()).toBe(before);
|
||||
expect(await prisma.attachment.count({ where: { chatId } })).toBe(before);
|
||||
expect(activeTempDirs()).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -290,7 +317,7 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
});
|
||||
try {
|
||||
const prisma = app.get(PrismaService);
|
||||
const before = await prisma.attachment.count();
|
||||
const before = await prisma.attachment.count({ where: { chatId } });
|
||||
const started = Date.now();
|
||||
const res = await request(downServer)
|
||||
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
||||
@@ -299,17 +326,17 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
.send(SAMPLE_PNG);
|
||||
expectError(res, 503, 'unavailable');
|
||||
expect(Date.now() - started).toBeLessThan(3000);
|
||||
expect(await prisma.attachment.count()).toBe(before);
|
||||
expect(await prisma.attachment.count({ where: { chatId } })).toBe(before);
|
||||
} finally {
|
||||
await down.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('явный отказ INSERT после PUT удаляет объект', async () => {
|
||||
it('явный отказ транзакции после PUT удаляет объект staging', async () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const s3 = app.get(S3Service);
|
||||
const remove = jest.spyOn(s3, 'delete');
|
||||
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
|
||||
const create = jest.spyOn(prisma, '$transaction').mockRejectedValueOnce(
|
||||
new Prisma.PrismaClientKnownRequestError('rejected', {
|
||||
code: 'P2003',
|
||||
clientVersion: 'test',
|
||||
@@ -319,6 +346,7 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
const res = await upload(SAMPLE_PNG);
|
||||
expect(res.status).toBe(500);
|
||||
expect(remove).toHaveBeenCalledTimes(1);
|
||||
expect(remove.mock.calls[0][0]).toMatch(/^staging\//);
|
||||
} finally {
|
||||
create.mockRestore();
|
||||
remove.mockRestore();
|
||||
@@ -329,7 +357,7 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const s3 = app.get(S3Service);
|
||||
const remove = jest.spyOn(s3, 'delete');
|
||||
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
|
||||
const create = jest.spyOn(prisma, '$transaction').mockRejectedValueOnce(
|
||||
new Prisma.PrismaClientUnknownRequestError('server closed the connection', {
|
||||
clientVersion: 'test',
|
||||
}),
|
||||
@@ -347,7 +375,7 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
const prisma = app.get(PrismaService);
|
||||
const s3 = app.get(S3Service);
|
||||
const remove = jest.spyOn(s3, 'delete');
|
||||
const create = jest.spyOn(prisma.attachment, 'create').mockRejectedValueOnce(
|
||||
const create = jest.spyOn(prisma, '$transaction').mockRejectedValueOnce(
|
||||
new Prisma.PrismaClientKnownRequestError('connection lost', {
|
||||
code: 'P1017',
|
||||
clientVersion: 'test',
|
||||
@@ -364,9 +392,6 @@ describe('Загрузка вложений (e2e)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
function activeTempDirs(): string[] {
|
||||
return readdirSync(tmpdir()).filter((name) => name.startsWith('messenger-upload-'));
|
||||
}
|
||||
|
||||
async function bigPng(width: number, height: number): Promise<Buffer> {
|
||||
const { default: sharp } = await import('sharp');
|
||||
|
||||
@@ -8,6 +8,15 @@ import { randomUUID } from 'crypto';
|
||||
import request from 'supertest';
|
||||
import { AppModule } from '../src/app.module';
|
||||
import { configureApp } from '../src/configure-app';
|
||||
import { PrismaService } from '../src/db/prisma.service';
|
||||
import { parseJobMessage } from '../src/jobs/job-message';
|
||||
import {
|
||||
JobProcessor,
|
||||
Outcome,
|
||||
ProcessorOptions,
|
||||
processorOptions,
|
||||
} from '../src/jobs/processor';
|
||||
import { S3Service } from '../src/storage/s3.service';
|
||||
|
||||
export const PASSWORD = 'Correct-Horse-Battery-42';
|
||||
|
||||
@@ -43,6 +52,30 @@ export async function createTestApp(
|
||||
return { app, server: app.getHttpServer() };
|
||||
}
|
||||
|
||||
export async function processAttachment(
|
||||
app: INestApplication,
|
||||
attachmentId: string,
|
||||
hooks: { afterStore?: () => Promise<void> } = {},
|
||||
options: Partial<ProcessorOptions> = {},
|
||||
): Promise<Outcome> {
|
||||
const prisma = app.get(PrismaService);
|
||||
const entry = await prisma.outboxEntry.findFirstOrThrow({
|
||||
where: { job: { attachmentId } },
|
||||
orderBy: { id: 'desc' },
|
||||
});
|
||||
const message = parseJobMessage(Buffer.from(JSON.stringify(entry.payload)));
|
||||
if (!message) {
|
||||
throw new Error('outbox payload does not match jobs.schema.json');
|
||||
}
|
||||
const processor = new JobProcessor(
|
||||
prisma,
|
||||
app.get(S3Service),
|
||||
{ ...processorOptions(), ...options },
|
||||
hooks,
|
||||
);
|
||||
return processor.handle(message);
|
||||
}
|
||||
|
||||
export function basic(username: string, password: string): string {
|
||||
return `Basic ${Buffer.from(`${username}:${password}`, 'utf8').toString('base64')}`;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import sharp from 'sharp';
|
||||
import request from 'supertest';
|
||||
import { ApiError } from '../src/common/filters/api-error';
|
||||
import { PrismaService } from '../src/db/prisma.service';
|
||||
import { jobMessage, parseJobMessage } from '../src/jobs/job-message';
|
||||
import { backoffMs, recoverExpiredLeases, replayJob } from '../src/jobs/job-store';
|
||||
import { JobProcessor, processorOptions } from '../src/jobs/processor';
|
||||
import { fitInside } from '../src/jobs/variants';
|
||||
import { S3Service } from '../src/storage/s3.service';
|
||||
import {
|
||||
bearer,
|
||||
createTestApp,
|
||||
processAttachment,
|
||||
registerUser,
|
||||
TestUser,
|
||||
} from './helpers';
|
||||
import { SAMPLE_PNG } from './images';
|
||||
|
||||
const MAX_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
describe('Фоновая обработка вложений (e2e)', () => {
|
||||
let app: INestApplication;
|
||||
let server: any;
|
||||
let prisma: PrismaService;
|
||||
let s3: S3Service;
|
||||
let alice: TestUser;
|
||||
let eve: TestUser;
|
||||
let chatId: string;
|
||||
|
||||
const upload = async (body: Buffer = SAMPLE_PNG, type = 'image/png') => {
|
||||
const res = await request(server)
|
||||
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.set('Content-Type', type)
|
||||
.send(body)
|
||||
.expect(202);
|
||||
return res.body.id as string;
|
||||
};
|
||||
|
||||
const metadata = async (id: string) =>
|
||||
(await request(server).get(`/api/v1/attachments/${id}`).set('Authorization', bearer(alice)).expect(200)).body;
|
||||
|
||||
const content = (id: string, variant = 'original', user = alice) =>
|
||||
request(server)
|
||||
.get(`/api/v1/attachments/${id}/content?variant=${variant}`)
|
||||
.set('Authorization', bearer(user))
|
||||
.buffer(true)
|
||||
.parse(binaryParser);
|
||||
|
||||
const job = (attachmentId: string) =>
|
||||
prisma.attachmentJob.findFirstOrThrow({ where: { attachmentId } });
|
||||
|
||||
const makeDue = (attachmentId: string) =>
|
||||
prisma.attachmentJob.updateMany({ where: { attachmentId }, data: { nextAttemptAt: new Date() } });
|
||||
|
||||
const events = async (attachmentId: string) =>
|
||||
(
|
||||
await prisma.jobEvent.findMany({
|
||||
where: { job: { attachmentId } },
|
||||
orderBy: { id: 'asc' },
|
||||
})
|
||||
).map((event) => event.event);
|
||||
|
||||
const sendMessage = (attachmentIds: string[]) =>
|
||||
request(server)
|
||||
.post(`/api/v1/chats/${chatId}/messages`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.set('Idempotency-Key', randomUUID())
|
||||
.send({ text: '', attachment_ids: attachmentIds });
|
||||
|
||||
beforeAll(async () => {
|
||||
({ app, server } = await createTestApp());
|
||||
prisma = app.get(PrismaService);
|
||||
s3 = app.get(S3Service);
|
||||
alice = await registerUser(server, 'job_alice');
|
||||
eve = await registerUser(server, 'job_eve');
|
||||
const chat = await request(server)
|
||||
.post('/api/v1/chats')
|
||||
.set('Authorization', bearer(alice))
|
||||
.send({ title: 'Jobs', member_ids: [alice.id] })
|
||||
.expect(201);
|
||||
chatId = chat.body.id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
describe('до обработки', () => {
|
||||
it('queued: метаданные доступны, content любого варианта — 409, сообщение с вложением создаётся', async () => {
|
||||
const id = await upload();
|
||||
expect(await metadata(id)).toMatchObject({ status: 'queued', variants: [], error_code: null });
|
||||
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
|
||||
const res = await content(id, variant).expect(409);
|
||||
expect(JSON.parse(res.body.toString()).error.code).toBe('conflict');
|
||||
}
|
||||
await content(id, 'original', eve).expect(404);
|
||||
await content(id, 'nonsense').expect(400);
|
||||
const message = await sendMessage([id]).expect(201);
|
||||
expect(message.body.attachment_ids).toEqual([id]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('результат worker', () => {
|
||||
it('ready: оригинал неизменен, thumbnail 256×160 JPEG, crop 128×128 JPEG, optimized WebP без увеличения', async () => {
|
||||
const id = await upload();
|
||||
expect(await processAttachment(app, id)).toBe('ready');
|
||||
const meta = await metadata(id);
|
||||
expect(meta.status).toBe('ready');
|
||||
expect(meta.error_code).toBeNull();
|
||||
expect(meta.variants.map((v: any) => [v.name, v.content_type, v.width, v.height])).toEqual([
|
||||
['thumbnail', 'image/jpeg', 256, 160],
|
||||
['crop', 'image/jpeg', 128, 128],
|
||||
['optimized', 'image/webp', 320, 200],
|
||||
]);
|
||||
const original = await content(id).expect(200);
|
||||
expect(Buffer.compare(original.body, SAMPLE_PNG)).toBe(0);
|
||||
for (const variant of meta.variants) {
|
||||
const res = await content(id, variant.name).expect(200);
|
||||
expect(res.headers['content-type']).toBe(variant.content_type);
|
||||
expect(res.body.length).toBe(variant.size_bytes);
|
||||
expect(createHash('sha256').update(res.body).digest('hex')).toBe(variant.sha256);
|
||||
const actual = await sharp(res.body).metadata();
|
||||
expect([actual.format, actual.width, actual.height, actual.exif]).toEqual([
|
||||
variant.content_type.split('/')[1],
|
||||
variant.width,
|
||||
variant.height,
|
||||
undefined,
|
||||
]);
|
||||
}
|
||||
const row = await prisma.attachment.findUniqueOrThrow({ where: { id } });
|
||||
expect(row.stagingKey).toBeNull();
|
||||
await expect(s3.getBuffer(`staging/${id}`, MAX_BYTES)).rejects.toThrow('missing');
|
||||
expect(await events(id)).toEqual(['enqueued', 'processing', 'ready']);
|
||||
});
|
||||
|
||||
it('маленький исходник не увеличивается в thumbnail/optimized, crop — увеличивается', async () => {
|
||||
const small = await sharp({ create: { width: 100, height: 50, channels: 3, background: '#123456' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
const id = await upload(small);
|
||||
expect(await processAttachment(app, id)).toBe('ready');
|
||||
const sizes = (await metadata(id)).variants.map((v: any) => [v.name, v.width, v.height]);
|
||||
expect(sizes).toEqual([
|
||||
['thumbnail', 100, 50],
|
||||
['crop', 128, 128],
|
||||
['optimized', 100, 50],
|
||||
]);
|
||||
});
|
||||
|
||||
it('прозрачность компонуется на белый, EXIF orientation применяется', async () => {
|
||||
const transparent = await sharp({
|
||||
create: { width: 40, height: 20, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } },
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
const alphaId = await upload(transparent);
|
||||
expect(await processAttachment(app, alphaId)).toBe('ready');
|
||||
const thumb = await content(alphaId, 'thumbnail').expect(200);
|
||||
const { data } = await sharp(thumb.body).raw().toBuffer({ resolveWithObject: true });
|
||||
expect(Math.min(...data)).toBeGreaterThanOrEqual(250);
|
||||
|
||||
const rotated = await sharp({ create: { width: 320, height: 200, channels: 3, background: '#808080' } })
|
||||
.jpeg()
|
||||
.withMetadata({ orientation: 6 })
|
||||
.toBuffer();
|
||||
const rotatedId = await upload(rotated, 'image/jpeg');
|
||||
expect(await processAttachment(app, rotatedId)).toBe('ready');
|
||||
const thumbnail = (await metadata(rotatedId)).variants.find((v: any) => v.name === 'thumbnail');
|
||||
expect([thumbnail.width, thumbnail.height]).toEqual([160, 256]);
|
||||
});
|
||||
|
||||
it('размеры округляются вниз, минимум 1 px', () => {
|
||||
expect(fitInside(320, 200, 256)).toEqual({ width: 256, height: 160 });
|
||||
expect(fitInside(1000, 3, 256)).toEqual({ width: 256, height: 1 });
|
||||
expect(fitInside(333, 1000, 256)).toEqual({ width: 85, height: 256 });
|
||||
expect(fitInside(10, 10, 256)).toEqual({ width: 10, height: 10 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('идемпотентность и повторная доставка', () => {
|
||||
it('повтор уже ready-задания — no-op: второго набора вариантов нет, объекты не меняются', async () => {
|
||||
const id = await upload();
|
||||
expect(await processAttachment(app, id)).toBe('ready');
|
||||
const before = await prisma.attachmentVariant.findMany({ where: { attachmentId: id } });
|
||||
const put = jest.spyOn(s3, 'putBuffer');
|
||||
try {
|
||||
expect(await processAttachment(app, id)).toBe('skipped');
|
||||
expect(put).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
put.mockRestore();
|
||||
}
|
||||
const after = await prisma.attachmentVariant.findMany({ where: { attachmentId: id } });
|
||||
expect(after).toEqual(before);
|
||||
expect(after).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('пока lease действует, второй обработчик того же job не запускается', async () => {
|
||||
const id = await upload();
|
||||
let concurrent: string | undefined;
|
||||
const outcome = await processAttachment(app, id, {
|
||||
afterStore: async () => {
|
||||
concurrent = await processAttachment(app, id);
|
||||
},
|
||||
});
|
||||
expect([outcome, concurrent]).toEqual(['ready', 'skipped']);
|
||||
});
|
||||
|
||||
it('worker упал после записи объектов, до COMMIT/ack: lease истекает, повтор завершает тот же результат', async () => {
|
||||
const id = await upload();
|
||||
await expect(
|
||||
processAttachment(app, id, {
|
||||
afterStore: async () => {
|
||||
throw new Error('worker crashed');
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('worker crashed');
|
||||
expect(await job(id)).toMatchObject({ status: 'processing', attempt: 1 });
|
||||
expect((await metadata(id)).status).toBe('processing');
|
||||
const stored = await s3.getBuffer(`attachments/${id}/v1/thumbnail.jpg`, MAX_BYTES);
|
||||
|
||||
expect(await recoverExpiredLeases(prisma)).toBe(0);
|
||||
const future = new Date(Date.now() + 10 * 60 * 1000);
|
||||
expect(await recoverExpiredLeases(prisma, future)).toBeGreaterThanOrEqual(1);
|
||||
expect(await job(id)).toMatchObject({ status: 'queued', attempt: 2 });
|
||||
await makeDue(id);
|
||||
expect(await processAttachment(app, id)).toBe('ready');
|
||||
|
||||
const variants = await prisma.attachmentVariant.findMany({ where: { attachmentId: id } });
|
||||
expect(variants).toHaveLength(3);
|
||||
const thumbnail = variants.find((v) => v.name === 'thumbnail')!;
|
||||
expect(thumbnail.sha256).toBe(createHash('sha256').update(stored).digest('hex'));
|
||||
expect(await events(id)).toEqual(['enqueued', 'processing', 'lease_expired', 'processing', 'ready']);
|
||||
});
|
||||
|
||||
it('устаревшее сообщение, чужой attachment_id и другая pipeline_version игнорируются', async () => {
|
||||
const id = await upload();
|
||||
const row = await job(id);
|
||||
const processor = new JobProcessor(prisma, s3, processorOptions());
|
||||
const base = jobMessage(row, new Date());
|
||||
expect(await processor.handle({ ...base, attachment_id: randomUUID() })).toBe('skipped');
|
||||
expect(await processor.handle({ ...base, pipeline_version: 2 })).toBe('skipped');
|
||||
expect(await processor.handle({ ...base, job_id: randomUUID() })).toBe('skipped');
|
||||
expect((await job(id)).status).toBe('queued');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ошибки, повторы и replay', () => {
|
||||
it('временная ошибка S3: повтор с backoff, до срока задание не берётся, затем ready', async () => {
|
||||
const id = await upload();
|
||||
const put = jest
|
||||
.spyOn(s3, 'putBuffer')
|
||||
.mockRejectedValueOnce(new ApiError('unavailable', 'object storage unavailable'));
|
||||
try {
|
||||
expect(await processAttachment(app, id)).toBe('retry');
|
||||
} finally {
|
||||
put.mockRestore();
|
||||
}
|
||||
const retried = await job(id);
|
||||
expect(retried).toMatchObject({ status: 'queued', attempt: 2, lastError: 'temporary failure: ApiError' });
|
||||
expect(retried.nextAttemptAt.getTime()).toBeGreaterThan(Date.now());
|
||||
expect((await metadata(id)).status).toBe('queued');
|
||||
const outbox = await prisma.outboxEntry.findMany({ where: { jobId: retried.id }, orderBy: { id: 'asc' } });
|
||||
expect(outbox).toHaveLength(2);
|
||||
expect(outbox[1].availableAt).toEqual(retried.nextAttemptAt);
|
||||
expect((outbox[1].payload as any).attempt).toBe(2);
|
||||
expect((outbox[1].payload as any).job_id).toBe(retried.id);
|
||||
|
||||
expect(await processAttachment(app, id)).toBe('skipped');
|
||||
await makeDue(id);
|
||||
expect(await processAttachment(app, id)).toBe('ready');
|
||||
expect(await events(id)).toEqual(['enqueued', 'processing', 'retry_scheduled', 'processing', 'ready']);
|
||||
});
|
||||
|
||||
it('попытки исчерпаны — failed retry_exhausted; replay явно возвращает в очередь и доводит до ready', async () => {
|
||||
const id = await upload();
|
||||
await prisma.attachmentJob.updateMany({ where: { attachmentId: id }, data: { maxAttempts: 2 } });
|
||||
const put = jest
|
||||
.spyOn(s3, 'putBuffer')
|
||||
.mockRejectedValue(new ApiError('unavailable', 'object storage unavailable'));
|
||||
try {
|
||||
expect(await processAttachment(app, id)).toBe('retry');
|
||||
await makeDue(id);
|
||||
expect(await processAttachment(app, id)).toBe('failed');
|
||||
} finally {
|
||||
put.mockRestore();
|
||||
}
|
||||
expect(await metadata(id)).toMatchObject({ status: 'failed', error_code: 'retry_exhausted', variants: [] });
|
||||
const failed = await job(id);
|
||||
expect(failed).toMatchObject({ status: 'failed', errorCode: 'retry_exhausted', attempt: 2 });
|
||||
await sendMessage([id]).expect(409);
|
||||
|
||||
expect(await replayJob(prisma, failed.id, 'test-operator')).toBe('replayed');
|
||||
expect(await job(id)).toMatchObject({ status: 'queued', attempt: 1, errorCode: null });
|
||||
expect(await metadata(id)).toMatchObject({ status: 'queued', error_code: null });
|
||||
expect(await processAttachment(app, id)).toBe('ready');
|
||||
|
||||
const outboxBefore = await prisma.outboxEntry.count({ where: { jobId: failed.id } });
|
||||
expect(await replayJob(prisma, failed.id, 'test-operator')).toBe('already_ready');
|
||||
expect(await prisma.outboxEntry.count({ where: { jobId: failed.id } })).toBe(outboxBefore);
|
||||
expect(await replayJob(prisma, randomUUID(), 'test-operator')).toBe('not_found');
|
||||
const replayed = await prisma.jobEvent.findFirstOrThrow({ where: { jobId: failed.id, event: 'replayed' } });
|
||||
expect(replayed.detail).toBe('by test-operator; previous retry_exhausted after attempt 2');
|
||||
});
|
||||
|
||||
it('replay задания в очереди не создаёт дубль', async () => {
|
||||
const id = await upload();
|
||||
expect(await replayJob(prisma, (await job(id)).id, 'test-operator')).toBe('not_failed');
|
||||
expect(await prisma.outboxEntry.count({ where: { job: { attachmentId: id } } })).toBe(1);
|
||||
});
|
||||
|
||||
it('нераспаковываемое изображение — постоянная ошибка: сразу failed invalid_image без повторов', async () => {
|
||||
const id = await upload();
|
||||
const garbage = Buffer.concat([SAMPLE_PNG.subarray(0, 64), Buffer.alloc(512, 7)]);
|
||||
const sha256 = createHash('sha256').update(garbage).digest('hex');
|
||||
await s3.putBuffer(`staging/${id}`, garbage, 'image/png', sha256);
|
||||
await prisma.attachment.update({ where: { id }, data: { sizeBytes: garbage.length, sha256 } });
|
||||
expect(await processAttachment(app, id)).toBe('failed');
|
||||
expect(await metadata(id)).toMatchObject({ status: 'failed', error_code: 'invalid_image', variants: [] });
|
||||
expect(await job(id)).toMatchObject({ status: 'failed', attempt: 1, errorCode: 'invalid_image' });
|
||||
await content(id).expect(409);
|
||||
await sendMessage([id]).expect(409);
|
||||
expect(await prisma.attachment.findUniqueOrThrow({ where: { id } })).toMatchObject({
|
||||
stagingKey: `staging/${id}`,
|
||||
});
|
||||
});
|
||||
|
||||
it('staging подменён — failed checksum_mismatch; staging пропал — failed staging_missing', async () => {
|
||||
const changed = await upload();
|
||||
const other = await sharp({ create: { width: 8, height: 8, channels: 3, background: '#ff0000' } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
await s3.putBuffer(`staging/${changed}`, other, 'image/png', createHash('sha256').update(other).digest('hex'));
|
||||
expect(await processAttachment(app, changed)).toBe('failed');
|
||||
expect((await metadata(changed)).error_code).toBe('checksum_mismatch');
|
||||
|
||||
const missing = await upload();
|
||||
await s3.delete(`staging/${missing}`);
|
||||
expect(await processAttachment(app, missing)).toBe('failed');
|
||||
expect((await metadata(missing)).error_code).toBe('staging_missing');
|
||||
});
|
||||
|
||||
it('превышение времени обработки — временная ошибка с повтором', async () => {
|
||||
const id = await upload();
|
||||
const slow = jest
|
||||
.spyOn(s3, 'getBuffer')
|
||||
.mockImplementationOnce(() => new Promise((resolve) => setTimeout(() => resolve(SAMPLE_PNG), 500)));
|
||||
try {
|
||||
expect(await processAttachment(app, id, {}, { timeoutMs: 50 })).toBe('retry');
|
||||
} finally {
|
||||
slow.mockRestore();
|
||||
}
|
||||
expect((await job(id)).lastError).toBe('processing timed out');
|
||||
});
|
||||
});
|
||||
|
||||
describe('формат сообщения и backoff', () => {
|
||||
it('сообщение соответствует jobs.schema.json, лишние/неверные поля отклоняются', () => {
|
||||
const valid = jobMessage(
|
||||
{ id: randomUUID(), attachmentId: randomUUID(), pipelineVersion: 1, attempt: 1 },
|
||||
new Date(),
|
||||
);
|
||||
const encode = (value: unknown) => Buffer.from(JSON.stringify(value));
|
||||
expect(parseJobMessage(encode(valid))).toEqual(valid);
|
||||
for (const bad of [
|
||||
{ ...valid, url: 'http://evil' },
|
||||
{ ...valid, job_id: 'nope' },
|
||||
{ ...valid, type: 'other' },
|
||||
{ ...valid, attempt: 0 },
|
||||
{ ...valid, schema_version: 2 },
|
||||
{ ...valid, created_at: 'yesterday' },
|
||||
[valid],
|
||||
]) {
|
||||
expect(parseJobMessage(encode(bad))).toBeNull();
|
||||
}
|
||||
expect(parseJobMessage(Buffer.from('{bad'))).toBeNull();
|
||||
});
|
||||
|
||||
it('backoff растёт экспоненциально с jitter и ограничен сверху', () => {
|
||||
expect(backoffMs(1, () => 0)).toBe(500);
|
||||
expect(backoffMs(1, () => 1)).toBe(1000);
|
||||
expect(backoffMs(3, () => 0)).toBe(2000);
|
||||
expect(backoffMs(3, () => 1)).toBe(4000);
|
||||
expect(backoffMs(20, () => 1)).toBe(30000);
|
||||
expect(backoffMs(20, () => 0)).toBe(15000);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function binaryParser(res: any, done: (error: Error | null, body: Buffer) => void): void {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
res.on('end', () => done(null, Buffer.concat(chunks)));
|
||||
}
|
||||
@@ -7,7 +7,13 @@ import request from 'supertest';
|
||||
import { reconcile } from '../src/attachments/reconciler';
|
||||
import { PrismaService } from '../src/db/prisma.service';
|
||||
import { S3Service } from '../src/storage/s3.service';
|
||||
import { bearer, createTestApp, registerUser, TestUser } from './helpers';
|
||||
import {
|
||||
bearer,
|
||||
createTestApp,
|
||||
processAttachment,
|
||||
registerUser,
|
||||
TestUser,
|
||||
} from './helpers';
|
||||
import { SAMPLE_PNG } from './images';
|
||||
|
||||
const HOUR_MS = 60 * 60 * 1000;
|
||||
@@ -100,6 +106,7 @@ describe('Сверка объектов S3 с метаданными (e2e)', ()
|
||||
sizeBytes: SAMPLE_PNG.length,
|
||||
sha256: createHash('sha256').update(SAMPLE_PNG).digest('hex'),
|
||||
objectKey: key,
|
||||
status: 'ready',
|
||||
},
|
||||
});
|
||||
const report = await reconcile(prisma, s3, {
|
||||
@@ -164,13 +171,75 @@ describe('Сверка объектов S3 с метаданными (e2e)', ()
|
||||
.set('Authorization', bearer(alice))
|
||||
.set('Content-Type', 'image/png')
|
||||
.send(SAMPLE_PNG)
|
||||
.expect(201);
|
||||
.expect(202);
|
||||
expect(await processAttachment(app, upload.body.id)).toBe('ready');
|
||||
const report = await reconcile(prisma, s3, { apply: true, graceMs: HOUR_MS });
|
||||
expect(report.fresh_skipped).toBeGreaterThanOrEqual(1);
|
||||
await request(server)
|
||||
.get(`/api/v1/attachments/${upload.body.id}/content`)
|
||||
expect(report.fresh_skipped).toBeGreaterThanOrEqual(4);
|
||||
for (const variant of ['original', 'thumbnail', 'crop', 'optimized']) {
|
||||
await request(server)
|
||||
.get(`/api/v1/attachments/${upload.body.id}/content?variant=${variant}`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.expect(200);
|
||||
}
|
||||
});
|
||||
|
||||
it('после grace period объекты вложения и staging в очереди не считаются сиротами', async () => {
|
||||
const queued = await request(server)
|
||||
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.expect(200);
|
||||
.set('Content-Type', 'image/png')
|
||||
.send(SAMPLE_PNG)
|
||||
.expect(202);
|
||||
const ready = await request(server)
|
||||
.post(`/api/v1/chats/${chatId}/attachments?filename=b.png`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.set('Content-Type', 'image/png')
|
||||
.send(SAMPLE_PNG)
|
||||
.expect(202);
|
||||
expect(await processAttachment(app, ready.body.id)).toBe('ready');
|
||||
for (const id of [queued.body.id, ready.body.id]) {
|
||||
for (const prefix of [`attachments/${id}/`, `staging/${id}`]) {
|
||||
const report = await reconcile(prisma, s3, {
|
||||
apply: true,
|
||||
graceMs: HOUR_MS,
|
||||
prefix,
|
||||
now: later(),
|
||||
});
|
||||
expect(report).toMatchObject({ orphans: 0, deleted: 0 });
|
||||
}
|
||||
}
|
||||
expect(await exists(`staging/${queued.body.id}`)).toBe(true);
|
||||
expect(await exists(`staging/${ready.body.id}`)).toBe(false);
|
||||
});
|
||||
|
||||
it('staging failed-вложения удаляется после срока хранения', async () => {
|
||||
const res = await request(server)
|
||||
.post(`/api/v1/chats/${chatId}/attachments?filename=a.png`)
|
||||
.set('Authorization', bearer(alice))
|
||||
.set('Content-Type', 'image/png')
|
||||
.send(SAMPLE_PNG)
|
||||
.expect(202);
|
||||
await prisma.attachment.update({
|
||||
where: { id: res.body.id },
|
||||
data: { status: 'failed', errorCode: 'invalid_image' },
|
||||
});
|
||||
const options = { graceMs: HOUR_MS, prefix: `staging/${res.body.id}`, now: later() };
|
||||
const kept = await reconcile(prisma, s3, {
|
||||
...options,
|
||||
apply: true,
|
||||
stagingRetentionMs: 3 * HOUR_MS,
|
||||
});
|
||||
expect(kept.staging_expired).toBe(0);
|
||||
expect(await exists(`staging/${res.body.id}`)).toBe(true);
|
||||
const expired = await reconcile(prisma, s3, {
|
||||
...options,
|
||||
apply: true,
|
||||
stagingRetentionMs: HOUR_MS,
|
||||
});
|
||||
expect(expired.staging_expired).toBeGreaterThanOrEqual(1);
|
||||
expect(await exists(`staging/${res.body.id}`)).toBe(false);
|
||||
const row = await prisma.attachment.findUniqueOrThrow({ where: { id: res.body.id } });
|
||||
expect(row.stagingKey).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user