import { BadRequestException, HttpException, Injectable, InternalServerErrorException, Logger, NotFoundException, PayloadTooLargeException, ServiceUnavailableException, UnsupportedMediaTypeException, } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { createHash, randomUUID } from 'node:crypto'; import { pipeline, Readable, Transform } from 'node:stream'; import type { AppConfig } from '../config/app-config.schema.js'; import { ChatsService } from '../chats/chats.service.js'; import { MetricsService } from '../metrics/metrics.service.js'; import { ObjectStorage } from '../storage/object-storage.js'; import type { User } from '../users/user.js'; import { toAttachment, type Attachment, type AttachmentRow, } from './attachment.js'; import { validateFilename } from './filename.js'; import { IMAGE_HEADER_MAX_BYTES, ImageHeader, ImageInspector, ImageRejectedError, } from './image/image-inspector.js'; import { AttachmentsRepository } from './attachments.repository.js'; const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']); export function mediaType(header: unknown): string | undefined { if (typeof header !== 'string') return undefined; return header.split(';', 1)[0].trim().toLowerCase(); } export function imageRejection(error: ImageRejectedError): HttpException { return error.reason === 'too_many_pixels' ? new PayloadTooLargeException(error.message) : new UnsupportedMediaTypeException(error.message); } class UploadInterruptedError extends Error {} @Injectable() export class AttachmentsService { private readonly logger = new Logger(AttachmentsService.name); private inFlight = 0; private readonly maxUploadBytes: number; private readonly maxPixels: number; private readonly uploadTimeoutMs: number; private readonly maxInFlight: number; constructor( config: ConfigService, private readonly repository: AttachmentsRepository, private readonly storage: ObjectStorage, private readonly chats: ChatsService, private readonly metrics: MetricsService, ) { this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES'); this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS'); this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS'); this.maxInFlight = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT'); } async uploadProxy( chatId: string, user: User, rawFilename: unknown, contentTypeHeader: unknown, contentLengthHeader: unknown, body: Readable | undefined, ): Promise { const filename = validateFilename(rawFilename); const declared = mediaType(contentTypeHeader); if (declared === undefined || !MEDIA_TYPES.has(declared)) throw new UnsupportedMediaTypeException( 'Content-Type must be image/png or image/jpeg', ); if ( typeof contentLengthHeader !== 'string' || !/^\d{1,15}$/u.test(contentLengthHeader) ) throw new BadRequestException('Content-Length is required'); const contentLength = Number(contentLengthHeader); if (contentLength === 0 || body === undefined) throw new UnsupportedMediaTypeException('Image body is empty'); if (contentLength > this.maxUploadBytes) throw new PayloadTooLargeException('Image is too large'); if (this.inFlight >= this.maxInFlight) throw new HttpException( { message: 'Too many concurrent uploads', retryAfter: 1 }, 429, ); this.inFlight += 1; try { await this.chats.getForUser(chatId, user); return await this.receive( chatId, user, filename, declared, contentLength, body, ); } finally { this.inFlight -= 1; } } private async receive( chatId: string, user: User, filename: string, declared: string, contentLength: number, body: Readable, ): Promise { const controller = new AbortController(); let timedOut = false; const timer = setTimeout(() => { timedOut = true; body.destroy(new Error('upload timeout')); controller.abort(); }, this.uploadTimeoutMs); const iterator = body[Symbol.asyncIterator]() as AsyncIterator; const next = async (): Promise> => { try { return await iterator.next(); } catch { throw new UploadInterruptedError('Upload was interrupted'); } }; const inspector = new ImageInspector(this.maxPixels); const head: Buffer[] = []; let id: string | undefined; let key = ''; let rejection: unknown; try { // Header phase: nothing exists in the database or S3 yet. let headBytes = 0; try { while (inspector.header === undefined) { const step = await next(); if (step.done) { inspector.finish(); throw new ImageRejectedError( 'unsupported_media_type', 'Image header is missing', ); } head.push(step.value); headBytes += step.value.length; inspector.push(step.value); if ( inspector.header === undefined && headBytes > IMAGE_HEADER_MAX_BYTES ) throw new ImageRejectedError( 'unsupported_media_type', 'Image header is missing', ); } } catch (error) { throw this.mapUploadError(error, timedOut); } const header = inspector.header as ImageHeader; if (header.contentType !== declared) throw new UnsupportedMediaTypeException( 'Image content does not match Content-Type', ); id = randomUUID(); key = this.storage.objectKey(id); await this.repository.insertPending({ id, chatId, ownerId: user.id, filename, contentType: declared, uploadMode: 'proxy', objectKey: key, sizeBytes: null, sha256: null, ttlMs: this.uploadTimeoutMs, }); const sha256 = createHash('sha256'); let total = 0; const account = (chunk: Buffer, inspect: boolean): void => { total += chunk.length; if (total > contentLength) throw new UploadInterruptedError( 'Body exceeds Content-Length', ); if (inspect) inspector.push(chunk); sha256.update(chunk); this.metrics.add('attachments.proxy.bytes_in', chunk.length); // Validate before the last bytes reach S3: it cannot commit an // object whose body is incomplete. if (total === contentLength) inspector.finish(); }; async function* generate(): AsyncGenerator { for (const chunk of head) { account(chunk, false); yield chunk; } for (;;) { const step = await next(); if (step.done) break; account(step.value, true); yield step.value; } if (total < contentLength) throw new UploadInterruptedError('Upload was interrupted'); } const readable = Readable.from( (async function* () { try { yield* generate(); } catch (error) { rejection = error; throw error; } })(), ); // A failing body must abort the PUT at once: the SDK does not end a // request whose Content-Length can no longer be satisfied, it would // hang until the request timeout. The error itself is reported via // `rejection`; the listener also keeps a late error from crashing us. readable.on('error', () => controller.abort()); let claimed = false; try { await this.storage.putStream( key, readable, contentLength, declared, controller.signal, ); this.metrics.add( 'attachments.proxy.bytes_to_s3', contentLength, ); const row = await this.repository.markReady(id, { sizeBytes: total, sha256: sha256.digest('hex'), width: header.width, height: header.height, }); if (row === undefined) { claimed = true; throw new ServiceUnavailableException(); } this.metrics.increment('attachments.upload.created'); return toAttachment(row); } catch (error) { readable.destroy(); if (!claimed) await this.compensate(id, key); throw this.mapUploadError(rejection ?? error, timedOut); } } finally { clearTimeout(timer); } } private mapUploadError(error: unknown, timedOut: boolean): unknown { if (error instanceof ImageRejectedError) return imageRejection(error); if (error instanceof UploadInterruptedError || timedOut) return new BadRequestException('Upload was interrupted'); return error; } async compensate(id: string, key: string): Promise { try { await this.storage.delete(key); await this.repository.deletePending(id); this.metrics.increment('attachments.compensated'); } catch (error) { this.metrics.increment('attachments.compensation_deferred'); this.logger.warn( `Compensation of attachment ${id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`, ); } } async getMetadata(id: string, user: User): Promise { const row = await this.repository.findVisible(id, user.id); if (!row) throw new NotFoundException(); return toAttachment(row); } async openContent( id: string, user: User, variant: string | undefined, ): Promise<{ row: AttachmentRow; stream: Readable }> { if (variant !== undefined && variant !== 'original') { if (['thumbnail', 'crop', 'optimized'].includes(variant)) throw new NotFoundException(); throw new BadRequestException('variant is invalid'); } const row = await this.repository.findVisible(id, user.id); if (!row) throw new NotFoundException(); const source = await this.storage.getStream(row.object_key); if (source === null) { this.metrics.increment('attachments.object_missing'); this.logger.error( `Object for ready attachment ${row.id} is missing in storage`, ); throw new InternalServerErrorException(); } const metrics = this.metrics; const counter = new Transform({ transform(chunk: Buffer, _encoding, callback) { metrics.add('attachments.download.bytes_out', chunk.length); callback(null, chunk); }, }); pipeline(source, counter, () => undefined); return { row, stream: counter }; } }