Files
mtusi2026lab6/src/jobs/processor.ts
T

299 lines
9.0 KiB
TypeScript

import { Logger } from '@nestjs/common';
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import {
ObjectMissingError,
ObjectTooLargeError,
S3Service,
} from '../storage/s3.service';
import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message';
import { markFailed, retryOrFail } from './job-store';
import {
InvalidImageError,
RenderedVariant,
RenderTimeoutError,
renderVariants,
} from './variants';
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
export interface ProcessorOptions {
timeoutMs: number;
leaseMs: number;
maxBytes: number;
maxPixels: number;
}
export function processorOptions(): ProcessorOptions {
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
return {
timeoutMs,
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
};
}
export interface ProcessorHooks {
afterStore?: () => Promise<void>;
afterCommit?: () => Promise<void>;
}
class PermanentError extends Error {
constructor(
readonly code: string,
reason: string,
) {
super(reason);
}
}
class TimeoutError extends Error {}
const tracer = trace.getTracer('messenger-worker');
const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', {
description: 'From durable upload acceptance to the committed ready state',
unit: 's',
advice: {
explicitBucketBoundaries: [
0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 45, 60, 90, 120, 150, 180, 240, 300, 450, 600,
],
},
});
export class JobProcessor {
private readonly logger = new Logger('JobProcessor');
constructor(
private readonly prisma: PrismaClient,
private readonly s3: S3Service,
private readonly options: ProcessorOptions = processorOptions(),
private readonly hooks: ProcessorHooks = {},
) {}
async handle(message: JobMessage): Promise<Outcome> {
const job = await this.prisma.attachmentJob.findUnique({
where: { id: message.job_id },
include: { attachment: true },
});
if (
!job ||
job.attachmentId !== message.attachment_id ||
job.pipelineVersion !== message.pipeline_version ||
job.pipelineVersion !== PIPELINE_VERSION
) {
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
return 'skipped';
}
const lease = await this.claim(job);
if (!lease) {
if (job.status === 'ready') {
await this.removeStaging(job.attachment);
}
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
return 'skipped';
}
let variants: RenderedVariant[];
try {
variants = await this.withTimeout(this.produce(job.attachment));
} catch (error) {
return this.recordFailure(lease, error);
}
await this.hooks.afterStore?.();
return this.commit(lease, job.attachment, variants);
}
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
const now = new Date();
const owner = randomUUID();
return this.prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
data: {
status: 'processing',
leaseOwner: owner,
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
},
});
if (taken.count === 0) {
return null;
}
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'processing' },
});
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
await tx.jobEvent.create({
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
});
return leased;
});
}
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
if (!attachment.stagingKey) {
throw new PermanentError('staging_missing', 'staging object was already removed');
}
let original: Buffer;
try {
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
} catch (error) {
if (error instanceof ObjectMissingError) {
throw new PermanentError('staging_missing', 'staging object not found');
}
if (error instanceof ObjectTooLargeError) {
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
}
throw error;
}
const sha256 = createHash('sha256').update(original).digest('hex');
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
}
let variants: RenderedVariant[];
try {
variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => {
span.setAttribute('messenger.image.bytes', original.length);
try {
return await renderVariants(
original,
this.options.maxPixels,
Math.ceil(this.options.timeoutMs / 1000),
);
} catch (error) {
span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' });
throw error;
} finally {
span.end();
}
});
} catch (error) {
if (error instanceof InvalidImageError) {
throw new PermanentError('invalid_image', 'image cannot be decoded');
}
throw error;
}
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
for (const variant of variants) {
await this.s3.putBuffer(
variantKey(attachment.id, variant),
variant.body,
variant.contentType,
variant.sha256,
);
}
return variants;
}
private async commit(
lease: AttachmentJob,
attachment: Attachment,
variants: RenderedVariant[],
): Promise<Outcome> {
const now = new Date();
const committed = await this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.updateMany({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
data: {
status: 'ready',
leaseOwner: null,
leaseUntil: null,
lastError: null,
finishedAt: now,
},
});
if (owned.count === 0) {
return false;
}
await tx.attachmentVariant.createMany({
data: variants.map((variant) => ({
attachmentId: attachment.id,
pipelineVersion: PIPELINE_VERSION,
name: variant.name,
contentType: variant.contentType,
objectKey: variantKey(attachment.id, variant),
sizeBytes: variant.body.length,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
createdAt: now,
})),
skipDuplicates: true,
});
await tx.attachment.update({
where: { id: attachment.id },
data: { status: 'ready', errorCode: null },
});
await tx.jobEvent.create({
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
});
return true;
});
if (!committed) {
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
return 'skipped';
}
timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, {
'job.type': JOB_TYPE,
});
await this.hooks.afterCommit?.();
await this.removeStaging(attachment);
return 'ready';
}
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
const now = new Date();
const reason =
error instanceof PermanentError
? error.message
: error instanceof TimeoutError || error instanceof RenderTimeoutError
? 'processing timed out'
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
return this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.count({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
});
if (owned === 0) {
return 'skipped' as const;
}
if (error instanceof PermanentError) {
await markFailed(tx, lease, error.code, reason, now);
return 'failed' as const;
}
return retryOrFail(tx, lease, reason, now);
});
}
private async removeStaging(attachment: Attachment): Promise<void> {
if (!attachment.stagingKey) {
return;
}
try {
await this.s3.delete(attachment.stagingKey);
await this.prisma.attachment.update({
where: { id: attachment.id },
data: { stagingKey: null },
});
} catch {
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
}
}
private withTimeout<T>(work: Promise<T>): Promise<T> {
let timer: NodeJS.Timeout;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
});
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
}
}
export function variantKey(
attachmentId: string,
variant: Pick<RenderedVariant, 'name' | 'extension'>,
): string {
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
}