forked from mtusi2026/mtusi2026lab6
299 lines
9.0 KiB
TypeScript
299 lines
9.0 KiB
TypeScript
import { Logger } from '@nestjs/common';
|
|
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';
|
|
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
|
|
import { createHash, randomUUID } from 'crypto';
|
|
import {
|
|
ObjectMissingError,
|
|
ObjectTooLargeError,
|
|
S3Service,
|
|
} from '../storage/s3.service';
|
|
import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message';
|
|
import { markFailed, retryOrFail } from './job-store';
|
|
import {
|
|
InvalidImageError,
|
|
RenderedVariant,
|
|
RenderTimeoutError,
|
|
renderVariants,
|
|
} from './variants';
|
|
|
|
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
|
|
|
|
export interface ProcessorOptions {
|
|
timeoutMs: number;
|
|
leaseMs: number;
|
|
maxBytes: number;
|
|
maxPixels: number;
|
|
}
|
|
|
|
export function processorOptions(): ProcessorOptions {
|
|
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
|
|
return {
|
|
timeoutMs,
|
|
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
|
|
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
|
|
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
|
|
};
|
|
}
|
|
|
|
export interface ProcessorHooks {
|
|
afterStore?: () => Promise<void>;
|
|
afterCommit?: () => Promise<void>;
|
|
}
|
|
|
|
class PermanentError extends Error {
|
|
constructor(
|
|
readonly code: string,
|
|
reason: string,
|
|
) {
|
|
super(reason);
|
|
}
|
|
}
|
|
|
|
class TimeoutError extends Error {}
|
|
|
|
const tracer = trace.getTracer('messenger-worker');
|
|
const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', {
|
|
description: 'From durable upload acceptance to the committed ready state',
|
|
unit: 's',
|
|
advice: {
|
|
explicitBucketBoundaries: [
|
|
0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 45, 60, 90, 120, 150, 180, 240, 300, 450, 600,
|
|
],
|
|
},
|
|
});
|
|
|
|
export class JobProcessor {
|
|
private readonly logger = new Logger('JobProcessor');
|
|
|
|
constructor(
|
|
private readonly prisma: PrismaClient,
|
|
private readonly s3: S3Service,
|
|
private readonly options: ProcessorOptions = processorOptions(),
|
|
private readonly hooks: ProcessorHooks = {},
|
|
) {}
|
|
|
|
async handle(message: JobMessage): Promise<Outcome> {
|
|
const job = await this.prisma.attachmentJob.findUnique({
|
|
where: { id: message.job_id },
|
|
include: { attachment: true },
|
|
});
|
|
if (
|
|
!job ||
|
|
job.attachmentId !== message.attachment_id ||
|
|
job.pipelineVersion !== message.pipeline_version ||
|
|
job.pipelineVersion !== PIPELINE_VERSION
|
|
) {
|
|
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
|
|
return 'skipped';
|
|
}
|
|
const lease = await this.claim(job);
|
|
if (!lease) {
|
|
if (job.status === 'ready') {
|
|
await this.removeStaging(job.attachment);
|
|
}
|
|
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
|
|
return 'skipped';
|
|
}
|
|
let variants: RenderedVariant[];
|
|
try {
|
|
variants = await this.withTimeout(this.produce(job.attachment));
|
|
} catch (error) {
|
|
return this.recordFailure(lease, error);
|
|
}
|
|
await this.hooks.afterStore?.();
|
|
return this.commit(lease, job.attachment, variants);
|
|
}
|
|
|
|
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
|
|
const now = new Date();
|
|
const owner = randomUUID();
|
|
return this.prisma.$transaction(async (tx) => {
|
|
const taken = await tx.attachmentJob.updateMany({
|
|
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
|
|
data: {
|
|
status: 'processing',
|
|
leaseOwner: owner,
|
|
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
|
|
},
|
|
});
|
|
if (taken.count === 0) {
|
|
return null;
|
|
}
|
|
await tx.attachment.update({
|
|
where: { id: job.attachmentId },
|
|
data: { status: 'processing' },
|
|
});
|
|
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
|
|
await tx.jobEvent.create({
|
|
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
|
|
});
|
|
return leased;
|
|
});
|
|
}
|
|
|
|
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
|
|
if (!attachment.stagingKey) {
|
|
throw new PermanentError('staging_missing', 'staging object was already removed');
|
|
}
|
|
let original: Buffer;
|
|
try {
|
|
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
|
|
} catch (error) {
|
|
if (error instanceof ObjectMissingError) {
|
|
throw new PermanentError('staging_missing', 'staging object not found');
|
|
}
|
|
if (error instanceof ObjectTooLargeError) {
|
|
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
|
|
}
|
|
throw error;
|
|
}
|
|
const sha256 = createHash('sha256').update(original).digest('hex');
|
|
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
|
|
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
|
|
}
|
|
let variants: RenderedVariant[];
|
|
try {
|
|
variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => {
|
|
span.setAttribute('messenger.image.bytes', original.length);
|
|
try {
|
|
return await renderVariants(
|
|
original,
|
|
this.options.maxPixels,
|
|
Math.ceil(this.options.timeoutMs / 1000),
|
|
);
|
|
} catch (error) {
|
|
span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' });
|
|
throw error;
|
|
} finally {
|
|
span.end();
|
|
}
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof InvalidImageError) {
|
|
throw new PermanentError('invalid_image', 'image cannot be decoded');
|
|
}
|
|
throw error;
|
|
}
|
|
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
|
|
for (const variant of variants) {
|
|
await this.s3.putBuffer(
|
|
variantKey(attachment.id, variant),
|
|
variant.body,
|
|
variant.contentType,
|
|
variant.sha256,
|
|
);
|
|
}
|
|
return variants;
|
|
}
|
|
|
|
private async commit(
|
|
lease: AttachmentJob,
|
|
attachment: Attachment,
|
|
variants: RenderedVariant[],
|
|
): Promise<Outcome> {
|
|
const now = new Date();
|
|
const committed = await this.prisma.$transaction(async (tx) => {
|
|
const owned = await tx.attachmentJob.updateMany({
|
|
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
|
data: {
|
|
status: 'ready',
|
|
leaseOwner: null,
|
|
leaseUntil: null,
|
|
lastError: null,
|
|
finishedAt: now,
|
|
},
|
|
});
|
|
if (owned.count === 0) {
|
|
return false;
|
|
}
|
|
await tx.attachmentVariant.createMany({
|
|
data: variants.map((variant) => ({
|
|
attachmentId: attachment.id,
|
|
pipelineVersion: PIPELINE_VERSION,
|
|
name: variant.name,
|
|
contentType: variant.contentType,
|
|
objectKey: variantKey(attachment.id, variant),
|
|
sizeBytes: variant.body.length,
|
|
width: variant.width,
|
|
height: variant.height,
|
|
sha256: variant.sha256,
|
|
createdAt: now,
|
|
})),
|
|
skipDuplicates: true,
|
|
});
|
|
await tx.attachment.update({
|
|
where: { id: attachment.id },
|
|
data: { status: 'ready', errorCode: null },
|
|
});
|
|
await tx.jobEvent.create({
|
|
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
|
|
});
|
|
return true;
|
|
});
|
|
if (!committed) {
|
|
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
|
|
return 'skipped';
|
|
}
|
|
timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, {
|
|
'job.type': JOB_TYPE,
|
|
});
|
|
await this.hooks.afterCommit?.();
|
|
await this.removeStaging(attachment);
|
|
return 'ready';
|
|
}
|
|
|
|
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
|
|
const now = new Date();
|
|
const reason =
|
|
error instanceof PermanentError
|
|
? error.message
|
|
: error instanceof TimeoutError || error instanceof RenderTimeoutError
|
|
? 'processing timed out'
|
|
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
|
|
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
|
|
return this.prisma.$transaction(async (tx) => {
|
|
const owned = await tx.attachmentJob.count({
|
|
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
|
});
|
|
if (owned === 0) {
|
|
return 'skipped' as const;
|
|
}
|
|
if (error instanceof PermanentError) {
|
|
await markFailed(tx, lease, error.code, reason, now);
|
|
return 'failed' as const;
|
|
}
|
|
return retryOrFail(tx, lease, reason, now);
|
|
});
|
|
}
|
|
|
|
private async removeStaging(attachment: Attachment): Promise<void> {
|
|
if (!attachment.stagingKey) {
|
|
return;
|
|
}
|
|
try {
|
|
await this.s3.delete(attachment.stagingKey);
|
|
await this.prisma.attachment.update({
|
|
where: { id: attachment.id },
|
|
data: { stagingKey: null },
|
|
});
|
|
} catch {
|
|
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
|
|
}
|
|
}
|
|
|
|
private withTimeout<T>(work: Promise<T>): Promise<T> {
|
|
let timer: NodeJS.Timeout;
|
|
const deadline = new Promise<never>((_, reject) => {
|
|
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
|
|
});
|
|
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
|
|
}
|
|
}
|
|
|
|
export function variantKey(
|
|
attachmentId: string,
|
|
variant: Pick<RenderedVariant, 'name' | 'extension'>,
|
|
): string {
|
|
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
|
|
}
|