Files
mtusi2026lab4/app/src/attachments/attachments.service.ts
T

332 lines
9.7 KiB
TypeScript

import {
BadRequestException,
HttpException,
Injectable,
InternalServerErrorException,
Logger,
NotFoundException,
PayloadTooLargeException,
ServiceUnavailableException,
UnsupportedMediaTypeException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createHash, randomUUID } from 'node:crypto';
import { pipeline, Readable, Transform } from 'node:stream';
import type { AppConfig } from '../config/app-config.schema.js';
import { ChatsService } from '../chats/chats.service.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { ObjectStorage } from '../storage/object-storage.js';
import type { User } from '../users/user.js';
import {
toAttachment,
type Attachment,
type AttachmentRow,
} from './attachment.js';
import { validateFilename } from './filename.js';
import {
IMAGE_HEADER_MAX_BYTES,
ImageHeader,
ImageInspector,
ImageRejectedError,
} from './image/image-inspector.js';
import { AttachmentsRepository } from './attachments.repository.js';
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
export function mediaType(header: unknown): string | undefined {
if (typeof header !== 'string') return undefined;
return header.split(';', 1)[0].trim().toLowerCase();
}
export function imageRejection(error: ImageRejectedError): HttpException {
return error.reason === 'too_many_pixels'
? new PayloadTooLargeException(error.message)
: new UnsupportedMediaTypeException(error.message);
}
class UploadInterruptedError extends Error {}
@Injectable()
export class AttachmentsService {
private readonly logger = new Logger(AttachmentsService.name);
private inFlight = 0;
private readonly maxUploadBytes: number;
private readonly maxPixels: number;
private readonly uploadTimeoutMs: number;
private readonly maxInFlight: number;
constructor(
config: ConfigService<AppConfig>,
private readonly repository: AttachmentsRepository,
private readonly storage: ObjectStorage,
private readonly chats: ChatsService,
private readonly metrics: MetricsService,
) {
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
this.maxInFlight = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT');
}
async uploadProxy(
chatId: string,
user: User,
rawFilename: unknown,
contentTypeHeader: unknown,
contentLengthHeader: unknown,
body: Readable | undefined,
): Promise<Attachment> {
const filename = validateFilename(rawFilename);
const declared = mediaType(contentTypeHeader);
if (declared === undefined || !MEDIA_TYPES.has(declared))
throw new UnsupportedMediaTypeException(
'Content-Type must be image/png or image/jpeg',
);
if (
typeof contentLengthHeader !== 'string' ||
!/^\d{1,15}$/u.test(contentLengthHeader)
)
throw new BadRequestException('Content-Length is required');
const contentLength = Number(contentLengthHeader);
if (contentLength === 0 || body === undefined)
throw new UnsupportedMediaTypeException('Image body is empty');
if (contentLength > this.maxUploadBytes)
throw new PayloadTooLargeException('Image is too large');
if (this.inFlight >= this.maxInFlight)
throw new HttpException(
{ message: 'Too many concurrent uploads', retryAfter: 1 },
429,
);
this.inFlight += 1;
try {
await this.chats.getForUser(chatId, user);
return await this.receive(
chatId,
user,
filename,
declared,
contentLength,
body,
);
} finally {
this.inFlight -= 1;
}
}
private async receive(
chatId: string,
user: User,
filename: string,
declared: string,
contentLength: number,
body: Readable,
): Promise<Attachment> {
const controller = new AbortController();
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
body.destroy(new Error('upload timeout'));
controller.abort();
}, this.uploadTimeoutMs);
const iterator = body[Symbol.asyncIterator]() as AsyncIterator<Buffer>;
const next = async (): Promise<IteratorResult<Buffer>> => {
try {
return await iterator.next();
} catch {
throw new UploadInterruptedError('Upload was interrupted');
}
};
const inspector = new ImageInspector(this.maxPixels);
const head: Buffer[] = [];
let id: string | undefined;
let key = '';
let rejection: unknown;
try {
// Header phase: nothing exists in the database or S3 yet.
let headBytes = 0;
try {
while (inspector.header === undefined) {
const step = await next();
if (step.done) {
inspector.finish();
throw new ImageRejectedError(
'unsupported_media_type',
'Image header is missing',
);
}
head.push(step.value);
headBytes += step.value.length;
inspector.push(step.value);
if (
inspector.header === undefined &&
headBytes > IMAGE_HEADER_MAX_BYTES
)
throw new ImageRejectedError(
'unsupported_media_type',
'Image header is missing',
);
}
} catch (error) {
throw this.mapUploadError(error, timedOut);
}
const header = inspector.header as ImageHeader;
if (header.contentType !== declared)
throw new UnsupportedMediaTypeException(
'Image content does not match Content-Type',
);
id = randomUUID();
key = this.storage.objectKey(id);
await this.repository.insertPending({
id,
chatId,
ownerId: user.id,
filename,
contentType: declared,
uploadMode: 'proxy',
objectKey: key,
sizeBytes: null,
sha256: null,
ttlMs: this.uploadTimeoutMs,
});
const sha256 = createHash('sha256');
let total = 0;
const account = (chunk: Buffer, inspect: boolean): void => {
total += chunk.length;
if (total > contentLength)
throw new UploadInterruptedError(
'Body exceeds Content-Length',
);
if (inspect) inspector.push(chunk);
sha256.update(chunk);
this.metrics.add('attachments.proxy.bytes_in', chunk.length);
// Validate before the last bytes reach S3: it cannot commit an
// object whose body is incomplete.
if (total === contentLength) inspector.finish();
};
async function* generate(): AsyncGenerator<Buffer> {
for (const chunk of head) {
account(chunk, false);
yield chunk;
}
for (;;) {
const step = await next();
if (step.done) break;
account(step.value, true);
yield step.value;
}
if (total < contentLength)
throw new UploadInterruptedError('Upload was interrupted');
}
const readable = Readable.from(
(async function* () {
try {
yield* generate();
} catch (error) {
rejection = error;
throw error;
}
})(),
);
// A failing body must abort the PUT at once: the SDK does not end a
// request whose Content-Length can no longer be satisfied, it would
// hang until the request timeout. The error itself is reported via
// `rejection`; the listener also keeps a late error from crashing us.
readable.on('error', () => controller.abort());
let claimed = false;
try {
await this.storage.putStream(
key,
readable,
contentLength,
declared,
controller.signal,
);
this.metrics.add(
'attachments.proxy.bytes_to_s3',
contentLength,
);
const row = await this.repository.markReady(id, {
sizeBytes: total,
sha256: sha256.digest('hex'),
width: header.width,
height: header.height,
});
if (row === undefined) {
claimed = true;
throw new ServiceUnavailableException();
}
this.metrics.increment('attachments.upload.created');
return toAttachment(row);
} catch (error) {
readable.destroy();
if (!claimed) await this.compensate(id, key);
throw this.mapUploadError(rejection ?? error, timedOut);
}
} finally {
clearTimeout(timer);
}
}
private mapUploadError(error: unknown, timedOut: boolean): unknown {
if (error instanceof ImageRejectedError) return imageRejection(error);
if (error instanceof UploadInterruptedError || timedOut)
return new BadRequestException('Upload was interrupted');
return error;
}
async compensate(id: string, key: string): Promise<void> {
try {
await this.storage.delete(key);
await this.repository.deletePending(id);
this.metrics.increment('attachments.compensated');
} catch (error) {
this.metrics.increment('attachments.compensation_deferred');
this.logger.warn(
`Compensation of attachment ${id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`,
);
}
}
async getMetadata(id: string, user: User): Promise<Attachment> {
const row = await this.repository.findVisible(id, user.id);
if (!row) throw new NotFoundException();
return toAttachment(row);
}
async openContent(
id: string,
user: User,
variant: string | undefined,
): Promise<{ row: AttachmentRow; stream: Readable }> {
if (variant !== undefined && variant !== 'original') {
if (['thumbnail', 'crop', 'optimized'].includes(variant))
throw new NotFoundException();
throw new BadRequestException('variant is invalid');
}
const row = await this.repository.findVisible(id, user.id);
if (!row) throw new NotFoundException();
const source = await this.storage.getStream(row.object_key);
if (source === null) {
this.metrics.increment('attachments.object_missing');
this.logger.error(
`Object for ready attachment ${row.id} is missing in storage`,
);
throw new InternalServerErrorException();
}
const metrics = this.metrics;
const counter = new Transform({
transform(chunk: Buffer, _encoding, callback) {
metrics.add('attachments.download.bytes_out', chunk.length);
callback(null, chunk);
},
});
pipeline(source, counter, () => undefined);
return { row, stream: counter };
}
}