forked from mtusi2026/mtusi2026lab4
332 lines
9.7 KiB
TypeScript
332 lines
9.7 KiB
TypeScript
import {
|
|
BadRequestException,
|
|
HttpException,
|
|
Injectable,
|
|
InternalServerErrorException,
|
|
Logger,
|
|
NotFoundException,
|
|
PayloadTooLargeException,
|
|
ServiceUnavailableException,
|
|
UnsupportedMediaTypeException,
|
|
} from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { pipeline, Readable, Transform } from 'node:stream';
|
|
import type { AppConfig } from '../config/app-config.schema.js';
|
|
import { ChatsService } from '../chats/chats.service.js';
|
|
import { MetricsService } from '../metrics/metrics.service.js';
|
|
import { ObjectStorage } from '../storage/object-storage.js';
|
|
import type { User } from '../users/user.js';
|
|
import {
|
|
toAttachment,
|
|
type Attachment,
|
|
type AttachmentRow,
|
|
} from './attachment.js';
|
|
import { validateFilename } from './filename.js';
|
|
import {
|
|
IMAGE_HEADER_MAX_BYTES,
|
|
ImageHeader,
|
|
ImageInspector,
|
|
ImageRejectedError,
|
|
} from './image/image-inspector.js';
|
|
import { AttachmentsRepository } from './attachments.repository.js';
|
|
|
|
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
|
|
|
|
export function mediaType(header: unknown): string | undefined {
|
|
if (typeof header !== 'string') return undefined;
|
|
return header.split(';', 1)[0].trim().toLowerCase();
|
|
}
|
|
|
|
export function imageRejection(error: ImageRejectedError): HttpException {
|
|
return error.reason === 'too_many_pixels'
|
|
? new PayloadTooLargeException(error.message)
|
|
: new UnsupportedMediaTypeException(error.message);
|
|
}
|
|
|
|
class UploadInterruptedError extends Error {}
|
|
|
|
@Injectable()
|
|
export class AttachmentsService {
|
|
private readonly logger = new Logger(AttachmentsService.name);
|
|
private inFlight = 0;
|
|
private readonly maxUploadBytes: number;
|
|
private readonly maxPixels: number;
|
|
private readonly uploadTimeoutMs: number;
|
|
private readonly maxInFlight: number;
|
|
|
|
constructor(
|
|
config: ConfigService<AppConfig>,
|
|
private readonly repository: AttachmentsRepository,
|
|
private readonly storage: ObjectStorage,
|
|
private readonly chats: ChatsService,
|
|
private readonly metrics: MetricsService,
|
|
) {
|
|
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
|
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
|
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
|
|
this.maxInFlight = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT');
|
|
}
|
|
|
|
async uploadProxy(
|
|
chatId: string,
|
|
user: User,
|
|
rawFilename: unknown,
|
|
contentTypeHeader: unknown,
|
|
contentLengthHeader: unknown,
|
|
body: Readable | undefined,
|
|
): Promise<Attachment> {
|
|
const filename = validateFilename(rawFilename);
|
|
const declared = mediaType(contentTypeHeader);
|
|
if (declared === undefined || !MEDIA_TYPES.has(declared))
|
|
throw new UnsupportedMediaTypeException(
|
|
'Content-Type must be image/png or image/jpeg',
|
|
);
|
|
if (
|
|
typeof contentLengthHeader !== 'string' ||
|
|
!/^\d{1,15}$/u.test(contentLengthHeader)
|
|
)
|
|
throw new BadRequestException('Content-Length is required');
|
|
const contentLength = Number(contentLengthHeader);
|
|
if (contentLength === 0 || body === undefined)
|
|
throw new UnsupportedMediaTypeException('Image body is empty');
|
|
if (contentLength > this.maxUploadBytes)
|
|
throw new PayloadTooLargeException('Image is too large');
|
|
|
|
if (this.inFlight >= this.maxInFlight)
|
|
throw new HttpException(
|
|
{ message: 'Too many concurrent uploads', retryAfter: 1 },
|
|
429,
|
|
);
|
|
this.inFlight += 1;
|
|
try {
|
|
await this.chats.getForUser(chatId, user);
|
|
return await this.receive(
|
|
chatId,
|
|
user,
|
|
filename,
|
|
declared,
|
|
contentLength,
|
|
body,
|
|
);
|
|
} finally {
|
|
this.inFlight -= 1;
|
|
}
|
|
}
|
|
|
|
private async receive(
|
|
chatId: string,
|
|
user: User,
|
|
filename: string,
|
|
declared: string,
|
|
contentLength: number,
|
|
body: Readable,
|
|
): Promise<Attachment> {
|
|
const controller = new AbortController();
|
|
let timedOut = false;
|
|
const timer = setTimeout(() => {
|
|
timedOut = true;
|
|
body.destroy(new Error('upload timeout'));
|
|
controller.abort();
|
|
}, this.uploadTimeoutMs);
|
|
const iterator = body[Symbol.asyncIterator]() as AsyncIterator<Buffer>;
|
|
const next = async (): Promise<IteratorResult<Buffer>> => {
|
|
try {
|
|
return await iterator.next();
|
|
} catch {
|
|
throw new UploadInterruptedError('Upload was interrupted');
|
|
}
|
|
};
|
|
|
|
const inspector = new ImageInspector(this.maxPixels);
|
|
const head: Buffer[] = [];
|
|
let id: string | undefined;
|
|
let key = '';
|
|
let rejection: unknown;
|
|
try {
|
|
// Header phase: nothing exists in the database or S3 yet.
|
|
let headBytes = 0;
|
|
try {
|
|
while (inspector.header === undefined) {
|
|
const step = await next();
|
|
if (step.done) {
|
|
inspector.finish();
|
|
throw new ImageRejectedError(
|
|
'unsupported_media_type',
|
|
'Image header is missing',
|
|
);
|
|
}
|
|
head.push(step.value);
|
|
headBytes += step.value.length;
|
|
inspector.push(step.value);
|
|
if (
|
|
inspector.header === undefined &&
|
|
headBytes > IMAGE_HEADER_MAX_BYTES
|
|
)
|
|
throw new ImageRejectedError(
|
|
'unsupported_media_type',
|
|
'Image header is missing',
|
|
);
|
|
}
|
|
} catch (error) {
|
|
throw this.mapUploadError(error, timedOut);
|
|
}
|
|
const header = inspector.header as ImageHeader;
|
|
if (header.contentType !== declared)
|
|
throw new UnsupportedMediaTypeException(
|
|
'Image content does not match Content-Type',
|
|
);
|
|
|
|
id = randomUUID();
|
|
key = this.storage.objectKey(id);
|
|
await this.repository.insertPending({
|
|
id,
|
|
chatId,
|
|
ownerId: user.id,
|
|
filename,
|
|
contentType: declared,
|
|
uploadMode: 'proxy',
|
|
objectKey: key,
|
|
sizeBytes: null,
|
|
sha256: null,
|
|
ttlMs: this.uploadTimeoutMs,
|
|
});
|
|
|
|
const sha256 = createHash('sha256');
|
|
let total = 0;
|
|
const account = (chunk: Buffer, inspect: boolean): void => {
|
|
total += chunk.length;
|
|
if (total > contentLength)
|
|
throw new UploadInterruptedError(
|
|
'Body exceeds Content-Length',
|
|
);
|
|
if (inspect) inspector.push(chunk);
|
|
sha256.update(chunk);
|
|
this.metrics.add('attachments.proxy.bytes_in', chunk.length);
|
|
// Validate before the last bytes reach S3: it cannot commit an
|
|
// object whose body is incomplete.
|
|
if (total === contentLength) inspector.finish();
|
|
};
|
|
async function* generate(): AsyncGenerator<Buffer> {
|
|
for (const chunk of head) {
|
|
account(chunk, false);
|
|
yield chunk;
|
|
}
|
|
for (;;) {
|
|
const step = await next();
|
|
if (step.done) break;
|
|
account(step.value, true);
|
|
yield step.value;
|
|
}
|
|
if (total < contentLength)
|
|
throw new UploadInterruptedError('Upload was interrupted');
|
|
}
|
|
const readable = Readable.from(
|
|
(async function* () {
|
|
try {
|
|
yield* generate();
|
|
} catch (error) {
|
|
rejection = error;
|
|
throw error;
|
|
}
|
|
})(),
|
|
);
|
|
|
|
// A failing body must abort the PUT at once: the SDK does not end a
|
|
// request whose Content-Length can no longer be satisfied, it would
|
|
// hang until the request timeout. The error itself is reported via
|
|
// `rejection`; the listener also keeps a late error from crashing us.
|
|
readable.on('error', () => controller.abort());
|
|
let claimed = false;
|
|
try {
|
|
await this.storage.putStream(
|
|
key,
|
|
readable,
|
|
contentLength,
|
|
declared,
|
|
controller.signal,
|
|
);
|
|
this.metrics.add(
|
|
'attachments.proxy.bytes_to_s3',
|
|
contentLength,
|
|
);
|
|
const row = await this.repository.markReady(id, {
|
|
sizeBytes: total,
|
|
sha256: sha256.digest('hex'),
|
|
width: header.width,
|
|
height: header.height,
|
|
});
|
|
if (row === undefined) {
|
|
claimed = true;
|
|
throw new ServiceUnavailableException();
|
|
}
|
|
this.metrics.increment('attachments.upload.created');
|
|
return toAttachment(row);
|
|
} catch (error) {
|
|
readable.destroy();
|
|
if (!claimed) await this.compensate(id, key);
|
|
throw this.mapUploadError(rejection ?? error, timedOut);
|
|
}
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
private mapUploadError(error: unknown, timedOut: boolean): unknown {
|
|
if (error instanceof ImageRejectedError) return imageRejection(error);
|
|
if (error instanceof UploadInterruptedError || timedOut)
|
|
return new BadRequestException('Upload was interrupted');
|
|
return error;
|
|
}
|
|
|
|
async compensate(id: string, key: string): Promise<void> {
|
|
try {
|
|
await this.storage.delete(key);
|
|
await this.repository.deletePending(id);
|
|
this.metrics.increment('attachments.compensated');
|
|
} catch (error) {
|
|
this.metrics.increment('attachments.compensation_deferred');
|
|
this.logger.warn(
|
|
`Compensation of attachment ${id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async getMetadata(id: string, user: User): Promise<Attachment> {
|
|
const row = await this.repository.findVisible(id, user.id);
|
|
if (!row) throw new NotFoundException();
|
|
return toAttachment(row);
|
|
}
|
|
|
|
async openContent(
|
|
id: string,
|
|
user: User,
|
|
variant: string | undefined,
|
|
): Promise<{ row: AttachmentRow; stream: Readable }> {
|
|
if (variant !== undefined && variant !== 'original') {
|
|
if (['thumbnail', 'crop', 'optimized'].includes(variant))
|
|
throw new NotFoundException();
|
|
throw new BadRequestException('variant is invalid');
|
|
}
|
|
const row = await this.repository.findVisible(id, user.id);
|
|
if (!row) throw new NotFoundException();
|
|
const source = await this.storage.getStream(row.object_key);
|
|
if (source === null) {
|
|
this.metrics.increment('attachments.object_missing');
|
|
this.logger.error(
|
|
`Object for ready attachment ${row.id} is missing in storage`,
|
|
);
|
|
throw new InternalServerErrorException();
|
|
}
|
|
const metrics = this.metrics;
|
|
const counter = new Transform({
|
|
transform(chunk: Buffer, _encoding, callback) {
|
|
metrics.add('attachments.download.bytes_out', chunk.length);
|
|
callback(null, chunk);
|
|
},
|
|
});
|
|
pipeline(source, counter, () => undefined);
|
|
return { row, stream: counter };
|
|
}
|
|
}
|