From 4b53e03fd66d77bfd995068725ebd9e12792b786 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 23 Oct 2026 17:09:49 +0300 Subject: [PATCH] =?UTF-8?q?lab4:=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B2=D1=82=D0=BE=D1=80=D0=BE?= =?UTF-8?q?=D0=B3=D0=BE=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E=20(40=20=D0=9C?= =?UTF-8?q?=D0=BF=20=D0=B2=20tmpfs,=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83?= =?UTF-8?q?=D1=82=D1=8B=20S3,=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BB=D0=BB=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9=20complete,=20auth=20=D0=B4?= =?UTF-8?q?=D0=BE=20Content-Type,=20=D0=BB=D0=B8=D0=BC=D0=B8=D1=82=20?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BF=D0=BE=D0=BB=D1=8C=D0=B7=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D1=82=D0=B5=D0=BB=D1=8F,=20=D0=B1=D1=8E=D0=B4=D0=B6=D0=B5?= =?UTF-8?q?=D1=82=20=D0=BF=D0=B0=D0=BC=D1=8F=D1=82=D0=B8=20=D0=B4=D0=B5?= =?UTF-8?q?=D0=BA=D0=BE=D0=B4=D0=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 12 +- app/src/attachments/attachments.controller.ts | 6 +- .../attachments/attachments.service.spec.ts | 4 +- app/src/attachments/attachments.service.ts | 35 +++--- app/src/attachments/direct-uploads.service.ts | 30 ++++- .../attachments/image/image-decoder.spec.ts | 21 +++- app/src/attachments/image/image-decoder.ts | 77 ++++++++++--- app/src/attachments/image/image-inspector.ts | 2 - app/src/attachments/upload-request.ts | 9 ++ app/src/attachments/upload-slots.spec.ts | 54 +++++++++ app/src/attachments/upload-slots.ts | 16 ++- app/src/config/app-config.schema.ts | 8 +- app/src/main.ts | 70 +++++------- app/src/storage/object-storage.ts | 22 ++-- app/test/attachments.e2e-spec.ts | 104 +++++++++++++++++- app/test/dependency-failures.e2e-spec.ts | 60 ++++++++++ app/test/direct-upload.e2e-spec.ts | 15 +++ app/test/support/jpeg.ts | 14 +++ app/test/support/png.ts | 36 +++++- app/tsconfig.json | 2 +- compose.yaml | 5 +- contracts/extensions.openapi.json | 43 +++++++- evidence/lab4/decode-memory.txt | 11 +- scripts/lab4/attachments_bench.py | 64 +++++++---- scripts/lab4/decode_memory.mjs | 23 +++- scripts/lab4/pngkit.py | 28 +++++ scripts/lab4/run_matrix.sh | 12 ++ 27 files changed, 642 insertions(+), 141 deletions(-) create mode 100644 app/src/attachments/upload-request.ts create mode 100644 app/src/attachments/upload-slots.spec.ts diff --git a/.env.example b/.env.example index fd34091..c8470c6 100644 --- a/.env.example +++ b/.env.example @@ -33,7 +33,7 @@ IDEMPOTENCY_TTL_SECONDS=86400 # Стенд (compose.yaml и scripts/): API_REPLICAS=2 API_CPUS_PER_REPLICA=1.5 -API_MEMORY_PER_REPLICA=768m +API_MEMORY_PER_REPLICA=1024m PG_CPUS=1.5 PG_MEMORY=512m REDIS_CPUS=0.25 @@ -68,10 +68,14 @@ S3_CONNECT_TIMEOUT_MS=1000 S3_REQUEST_TIMEOUT_MS=15000 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -# Каждая загрузка держит копию файла (≤ MAX_UPLOAD_BYTES) до полного декода: -# память реплики ≈ UPLOAD_MAX_IN_FLIGHT × 10 MiB + IMAGE_DECODE_MAX_CONCURRENT декодов. +# Каждая загрузка держит копию файла (≤ MAX_UPLOAD_BYTES) до полного декода; +# худший декод (40 Мп RGBA16 interlaced PNG) ≈ 340 MiB. Граница памяти реплики: +# базовый RSS (~250 MiB) + UPLOAD_MAX_IN_FLIGHT × 10 MiB + IMAGE_DECODE_MAX_CONCURRENT × 340 MiB +# = 250 + 160 + 340 ≈ 750 MiB при API_MEMORY_PER_REPLICA=1024m. UPLOAD_MAX_IN_FLIGHT=16 -IMAGE_DECODE_MAX_CONCURRENT=2 +# Не больше стольких одновременных загрузок одного пользователя на реплику. +UPLOAD_MAX_IN_FLIGHT_PER_USER=4 +IMAGE_DECODE_MAX_CONCURRENT=1 DIRECT_UPLOAD_MAX_PENDING_PER_USER=10 # UPLOAD_TIMEOUT_MS, DIRECT_UPLOAD_URL_TTL_SECONDS, ATTACHMENT_SWEEP_GRACE_SECONDS, # ATTACHMENT_SWEEP_INTERVAL_MS задаются в compose.yaml (environment) с умолчаниями diff --git a/app/src/attachments/attachments.controller.ts b/app/src/attachments/attachments.controller.ts index 45d02dd..6435849 100644 --- a/app/src/attachments/attachments.controller.ts +++ b/app/src/attachments/attachments.controller.ts @@ -23,6 +23,7 @@ import type { User } from '../users/user.js'; import { AttachmentsService } from './attachments.service.js'; import { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js'; import { DirectUploadsService } from './direct-uploads.service.js'; +import type { UploadRequest } from './upload-request.js'; import { contentDisposition } from './filename.js'; @Controller('api/v1') @@ -44,9 +45,8 @@ export class AttachmentsController { @Param('chatId', new ParseUUIDPipe()) chatId: string, @CurrentUser() user: User, @Query('filename') filename: unknown, - @Headers('content-type') contentType: string | undefined, @Headers('content-length') contentLength: string | undefined, - @Req() request: FastifyRequest, + @Req() request: FastifyRequest & UploadRequest, @Res({ passthrough: true }) reply: FastifyReply, ) { // A gigantic body is not worth draining: close the connection after the answer. @@ -56,7 +56,7 @@ export class AttachmentsController { chatId, user, filename, - contentType, + request.uploadContentType, contentLength, request.body as Readable | undefined, ); diff --git a/app/src/attachments/attachments.service.spec.ts b/app/src/attachments/attachments.service.spec.ts index 77b0478..b79287a 100644 --- a/app/src/attachments/attachments.service.spec.ts +++ b/app/src/attachments/attachments.service.spec.ts @@ -38,7 +38,9 @@ function setup(repository: Record) { { getForUser: vi.fn() } as never, { increment: vi.fn(), add: vi.fn() } as never, { verify: vi.fn() } as never, - { run: (work: () => Promise) => work() } as never, + { + run: (_user: string, work: () => Promise) => work(), + } as never, ); const upload = () => service.uploadProxy( diff --git a/app/src/attachments/attachments.service.ts b/app/src/attachments/attachments.service.ts index 7a7fa4e..b346b6e 100644 --- a/app/src/attachments/attachments.service.ts +++ b/app/src/attachments/attachments.service.ts @@ -24,7 +24,6 @@ import { } from './attachment.js'; import { validateFilename } from './filename.js'; import { - IMAGE_HEADER_MAX_BYTES, ImageHeader, ImageInspector, ImageRejectedError, @@ -94,7 +93,7 @@ export class AttachmentsService { if (contentLength > this.maxUploadBytes) throw new PayloadTooLargeException('Image is too large'); - return this.slots.run(async () => { + return this.slots.run(user.id, async () => { await this.chats.getForUser(chatId, user); return this.receive( chatId, @@ -137,8 +136,9 @@ export class AttachmentsService { let key = ''; let rejection: unknown; try { - // Header phase: nothing exists in the database or S3 yet. - let headBytes = 0; + // Header phase: nothing exists in the database or S3 yet. It is + // bounded by Content-Length (<= MAX_UPLOAD_BYTES): a JPEG may carry + // megabytes of APPn metadata before its frame header. try { while (inspector.header === undefined) { const step = await next(); @@ -150,16 +150,7 @@ export class AttachmentsService { ); } head.push(step.value); - headBytes += step.value.length; inspector.push(step.value); - if ( - inspector.header === undefined && - headBytes > IMAGE_HEADER_MAX_BYTES - ) - throw new ImageRejectedError( - 'unsupported_media_type', - 'Image header is missing', - ); } } catch (error) { throw this.mapUploadError(error, timedOut); @@ -191,6 +182,7 @@ export class AttachmentsService { // UPLOAD_MAX_IN_FLIGHT, released as soon as the decode starts. const copy: Buffer[] = []; let total = 0; + let allSent = false; const account = (chunk: Buffer, inspect: boolean): boolean => { total += chunk.length; if (total > contentLength) @@ -209,7 +201,10 @@ export class AttachmentsService { inspector.finish(); const data = Buffer.concat(copy, total); copy.length = 0; - await decoder.verify(data); + await decoder.verify(data, controller.signal); + // From here S3 may receive the whole body and commit the object + // even if this request fails or times out afterwards. + allSent = true; }; async function* generate(): AsyncGenerator { for (const chunk of head) { @@ -271,7 +266,12 @@ export class AttachmentsService { } catch (error) { readable.destroy(); if (!claimed) { - const outcome = await this.compensate(id, key); + // After the last byte was handed to S3, a PUT aborted on our + // side may still be committed by S3 after our DELETE: keep the + // row so the sweeper deletes the key again later. + const outcome = await this.compensate(id, key, { + keepRow: allSent && !stored, + }); // markReady failed on our side, but the row is no longer // pending: the commit may have succeeded with its reply lost. // If so, the upload is complete and 201 is the truth. @@ -312,6 +312,7 @@ export class AttachmentsService { async compensate( id: string, key: string, + options: { keepRow?: boolean } = {}, ): Promise<'compensated' | 'not_pending' | 'deferred'> { try { if (!(await this.repository.claimForCompensation(id))) { @@ -319,7 +320,9 @@ export class AttachmentsService { return 'not_pending'; } await this.storage.delete(key); - await this.repository.deleteClaimed(id); + // A kept `deleting` row is re-claimed by the sweeper after a minute, + // which deletes the key once more and only then the row. + if (!options.keepRow) await this.repository.deleteClaimed(id); this.metrics.increment('attachments.compensated'); return 'compensated'; } catch (error) { diff --git a/app/src/attachments/direct-uploads.service.ts b/app/src/attachments/direct-uploads.service.ts index c35cdc4..1b738dd 100644 --- a/app/src/attachments/direct-uploads.service.ts +++ b/app/src/attachments/direct-uploads.service.ts @@ -11,7 +11,10 @@ import { randomUUID } from 'node:crypto'; import type { AppConfig } from '../config/app-config.schema.js'; import { ChatsService } from '../chats/chats.service.js'; import { MetricsService } from '../metrics/metrics.service.js'; -import { ObjectStorage } from '../storage/object-storage.js'; +import { + ObjectStorage, + ObjectStorageUnavailableError, +} from '../storage/object-storage.js'; import type { User } from '../users/user.js'; import { toAttachment, @@ -44,6 +47,7 @@ const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']); export class DirectUploadsService { private readonly maxUploadBytes: number; private readonly maxPixels: number; + private readonly uploadTimeoutMs: number; private readonly ttlSeconds: number; private readonly maxPending: number; @@ -58,6 +62,7 @@ export class DirectUploadsService { ) { this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES'); this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS'); + this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS'); this.ttlSeconds = config.getOrThrow('DIRECT_UPLOAD_URL_TTL_SECONDS'); this.maxPending = config.getOrThrow( 'DIRECT_UPLOAD_MAX_PENDING_PER_USER', @@ -138,7 +143,7 @@ export class DirectUploadsService { ); if (!row) throw new NotFoundException(); if (row.status === 'ready') return toAttachment(row); - return this.slots.run(() => this.verifyAndMarkReady(row)); + return this.slots.run(user.id, () => this.verifyAndMarkReady(row)); } private async verifyAndMarkReady(row: AttachmentRow): Promise { @@ -179,7 +184,15 @@ export class DirectUploadsService { width: header.width, height: header.height, }); - if (!ready) throw new ConflictException('Upload is no longer pending'); + if (!ready) { + // A parallel complete of the same upload won: answer like a repeat. + const current = await this.repository.findForComplete( + row.id, + row.owner_id, + ); + if (current?.status === 'ready') return toAttachment(current); + throw new ConflictException('Upload is no longer pending'); + } this.metrics.increment('attachments.direct.completed'); return toAttachment(ready); } @@ -194,7 +207,9 @@ export class DirectUploadsService { key: string, size: number, ): Promise { - const stream = await this.storage.getStream(key); + // Same deadline as a proxy upload: a slow or hung read releases its slot. + const signal = AbortSignal.timeout(this.uploadTimeoutMs); + const stream = await this.storage.getStream(key, signal); if (stream === null) throw new ConflictException('Upload has not been received'); const inspector = new ImageInspector(this.maxPixels); @@ -211,6 +226,11 @@ export class DirectUploadsService { inspector.push(chunk); chunks.push(chunk); } + } catch (error) { + if (error instanceof ImageRejectedError) throw error; + // The object stream broke mid-read (S3 outage, timeout): the image + // was not judged, so this is an outage, not a client error. + throw new ObjectStorageUnavailableError(); } finally { stream.destroy(); this.metrics.add('attachments.direct.verify_bytes', read); @@ -218,7 +238,7 @@ export class DirectUploadsService { const header = inspector.finish(); const data = Buffer.concat(chunks, read); chunks.length = 0; - await this.decoder.verify(data); + await this.decoder.verify(data, signal); return header; } diff --git a/app/src/attachments/image/image-decoder.spec.ts b/app/src/attachments/image/image-decoder.spec.ts index 9b36dbe..e5d82cf 100644 --- a/app/src/attachments/image/image-decoder.spec.ts +++ b/app/src/attachments/image/image-decoder.spec.ts @@ -3,7 +3,11 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { jpegWithoutScanData } from '../../../test/support/jpeg.js'; -import { noisePng, pngWithGarbageIdat } from '../../../test/support/png.js'; +import { + maxPixelsRgba16Png, + noisePng, + pngWithGarbageIdat, +} from '../../../test/support/png.js'; import { ImageDecoder } from './image-decoder.js'; import { ImageInspector, ImageRejectedError } from './image-inspector.js'; @@ -48,4 +52,19 @@ describe('ImageDecoder', () => { ImageRejectedError, ); }); + + it('accepts the heaviest valid input: 40 MP RGBA 16-bit', async () => { + await decoder.verify(await maxPixelsRgba16Png()); + }); + + it('drops an aborted waiter from the queue without running its decode', async () => { + const busy = decoder.verify(noisePng(4_000_000)); + const controller = new AbortController(); + const waiting = decoder.verify(png, controller.signal); + controller.abort(); + await expect(waiting).rejects.toMatchObject({ status: 503 }); + await busy; + // The slot was not leaked: the next decode runs. + await decoder.verify(png); + }); }); diff --git a/app/src/attachments/image/image-decoder.ts b/app/src/attachments/image/image-decoder.ts index 8dfe35f..fc1ee06 100644 --- a/app/src/attachments/image/image-decoder.ts +++ b/app/src/attachments/image/image-decoder.ts @@ -1,4 +1,9 @@ -import { Injectable } from '@nestjs/common'; +import { + Injectable, + InternalServerErrorException, + Logger, + ServiceUnavailableException, +} from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import sharp from 'sharp'; import type { AppConfig } from '../../config/app-config.schema.js'; @@ -10,19 +15,31 @@ import { ImageRejectedError } from './image-inspector.js'; sharp.cache(false); sharp.concurrency(1); +// Failures of the replica itself, not of the image: they must not turn into a +// client 415. libvips spills large images to a temp file unless +// VIPS_DISC_THRESHOLD (compose.yaml) keeps them in memory; a full /tmp or an +// allocation failure ends up here. +const SERVER_FAULT = + /no space left|out of memory|unable to allocate|cannot allocate|write failed|unable to open|tmp/iu; + /** * Full decode check. ImageInspector only proves the container structure * (signature, chunk CRCs, markers, dimensions); a PNG with a garbage zlib stream - * or a JPEG without scan data passes it. Decoding every pixel with - * failOn=warning rejects those, including truncated or corrupted entropy data. - * Decodes are bounded by IMAGE_DECODE_MAX_CONCURRENT; callers wait for a slot. + * or a JPEG without scan data passes it. Decoding every scanline with + * failOn=warning rejects those, including truncated or corrupted compressed + * data. A downscale to a thumbnail is the sink: it decodes all input but keeps + * far less in memory than a full-size pipeline (measured in + * evidence/lab4/decode-memory.txt). Decodes are bounded by + * IMAGE_DECODE_MAX_CONCURRENT; a waiter whose request was aborted leaves the + * queue and releases its buffer. */ @Injectable() export class ImageDecoder { + private readonly logger = new Logger(ImageDecoder.name); private readonly maxPixels: number; private readonly maxConcurrent: number; private active = 0; - private readonly waiting: (() => void)[] = []; + private readonly waiting = new Set<() => void>(); constructor( config: ConfigService, @@ -32,16 +49,27 @@ export class ImageDecoder { this.maxConcurrent = config.getOrThrow('IMAGE_DECODE_MAX_CONCURRENT'); } - async verify(data: Buffer): Promise { - await this.acquire(); + async verify(data: Buffer, signal?: AbortSignal): Promise { + await this.acquire(signal); const started = performance.now(); try { await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: this.maxPixels, - }).stats(); - } catch { + }) + .resize(64, 64, { fit: 'inside' }) + .toBuffer(); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + if (SERVER_FAULT.test(message)) { + this.metrics.increment('attachments.decode.server_error'); + this.logger.error( + `Image decode failed on the server: ${message}`, + ); + throw new InternalServerErrorException(); + } this.metrics.increment('attachments.decode.rejected'); throw new ImageRejectedError( 'unsupported_media_type', @@ -56,17 +84,36 @@ export class ImageDecoder { } } - private async acquire(): Promise { + private acquire(signal?: AbortSignal): Promise { + const abandoned = new ServiceUnavailableException( + 'Image check was abandoned', + ); + if (signal?.aborted) return Promise.reject(abandoned); if (this.active < this.maxConcurrent) { this.active += 1; - return; + return Promise.resolve(); } - await new Promise((resolve) => this.waiting.push(resolve)); + const { promise, resolve, reject } = Promise.withResolvers(); + const cancel = () => { + this.waiting.delete(grant); + reject(abandoned); + }; + const grant = () => { + signal?.removeEventListener('abort', cancel); + resolve(); + }; + this.waiting.add(grant); + signal?.addEventListener('abort', cancel, { once: true }); + return promise; } private release(): void { - const next = this.waiting.shift(); - if (next) next(); - else this.active -= 1; + const next = this.waiting.values().next(); + if (next.done) { + this.active -= 1; + return; + } + this.waiting.delete(next.value); + next.value(); } } diff --git a/app/src/attachments/image/image-inspector.ts b/app/src/attachments/image/image-inspector.ts index 64a2f0e..aeb88fe 100644 --- a/app/src/attachments/image/image-inspector.ts +++ b/app/src/attachments/image/image-inspector.ts @@ -1,7 +1,5 @@ import { crc32 } from 'node:zlib'; -export const IMAGE_HEADER_MAX_BYTES = 1_048_576; - export type ImageHeader = { contentType: 'image/png' | 'image/jpeg'; width: number; diff --git a/app/src/attachments/upload-request.ts b/app/src/attachments/upload-request.ts new file mode 100644 index 0000000..3479cc3 --- /dev/null +++ b/app/src/attachments/upload-request.ts @@ -0,0 +1,9 @@ +/** Request of the upload route; see the onRequest hook in main.ts. */ +export type UploadRequest = { + method: string; + routeOptions?: { url?: string }; + headers: Record; + /** Content-Type as sent by the client (the header itself is rewritten). */ + uploadContentType?: string | string[]; + body?: unknown; +}; diff --git a/app/src/attachments/upload-slots.spec.ts b/app/src/attachments/upload-slots.spec.ts new file mode 100644 index 0000000..9910b9a --- /dev/null +++ b/app/src/attachments/upload-slots.spec.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { UploadSlots } from './upload-slots.js'; + +const slots = (max: number, perUser: number) => + new UploadSlots({ + getOrThrow: (key: string) => + key === 'UPLOAD_MAX_IN_FLIGHT' ? max : perUser, + } as never); + +function hold() { + const { promise, resolve } = Promise.withResolvers(); + return { work: () => promise, release: resolve }; +} + +describe('UploadSlots', () => { + it('caps one user without starving others, and frees slots after work', async () => { + const limiter = slots(10, 2); + const held = [hold(), hold()]; + const running = held.map((h) => limiter.run('a', h.work)); + await expect(limiter.run('a', async () => 'x')).rejects.toMatchObject({ + status: 429, + }); + await expect(limiter.run('b', async () => 'ok')).resolves.toBe('ok'); + held.forEach((h) => h.release()); + await Promise.all(running); + await expect(limiter.run('a', async () => 'again')).resolves.toBe( + 'again', + ); + }); + + it('applies the replica-wide cap across users', async () => { + const limiter = slots(2, 2); + const held = [hold(), hold()]; + const running = [ + limiter.run('a', held[0].work), + limiter.run('b', held[1].work), + ]; + await expect(limiter.run('c', async () => 'x')).rejects.toMatchObject({ + status: 429, + }); + held.forEach((h) => h.release()); + await Promise.all(running); + }); + + it('releases a slot when the work fails', async () => { + const limiter = slots(1, 1); + await expect( + limiter.run('a', async () => { + throw new Error('boom'); + }), + ).rejects.toThrow('boom'); + await expect(limiter.run('a', async () => 'ok')).resolves.toBe('ok'); + }); +}); diff --git a/app/src/attachments/upload-slots.ts b/app/src/attachments/upload-slots.ts index 528c483..bc4129f 100644 --- a/app/src/attachments/upload-slots.ts +++ b/app/src/attachments/upload-slots.ts @@ -3,29 +3,39 @@ import { ConfigService } from '@nestjs/config'; import type { AppConfig } from '../config/app-config.schema.js'; /** - * Per-replica cap on requests that hold an image in memory (proxy upload, + * Per-replica caps on requests that hold an image in memory (proxy upload, * direct complete): each may keep up to MAX_UPLOAD_BYTES for the decode check. + * The per-user cap keeps one client (e.g. slow senders holding slots until + * UPLOAD_TIMEOUT_MS) from starving everyone else. */ @Injectable() export class UploadSlots { private inFlight = 0; + private readonly perUser = new Map(); private readonly max: number; + private readonly maxPerUser: number; constructor(config: ConfigService) { this.max = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT'); + this.maxPerUser = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT_PER_USER'); } - async run(work: () => Promise): Promise { - if (this.inFlight >= this.max) + async run(userId: string, work: () => Promise): Promise { + const mine = this.perUser.get(userId) ?? 0; + if (this.inFlight >= this.max || mine >= this.maxPerUser) throw new HttpException( { message: 'Too many concurrent uploads', retryAfter: 1 }, 429, ); this.inFlight += 1; + this.perUser.set(userId, mine + 1); try { return await work(); } finally { this.inFlight -= 1; + const left = (this.perUser.get(userId) ?? 1) - 1; + if (left === 0) this.perUser.delete(userId); + else this.perUser.set(userId, left); } } } diff --git a/app/src/config/app-config.schema.ts b/app/src/config/app-config.schema.ts index e115501..86fd5a2 100644 --- a/app/src/config/app-config.schema.ts +++ b/app/src/config/app-config.schema.ts @@ -86,11 +86,17 @@ export const appConfigSchema = z MAX_IMAGE_PIXELS: numeric('MAX_IMAGE_PIXELS', 1, 40000000, 40000000), UPLOAD_TIMEOUT_MS: numeric('UPLOAD_TIMEOUT_MS', 1000, undefined, 25000), UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 16), + UPLOAD_MAX_IN_FLIGHT_PER_USER: numeric( + 'UPLOAD_MAX_IN_FLIGHT_PER_USER', + 1, + 1024, + 4, + ), IMAGE_DECODE_MAX_CONCURRENT: numeric( 'IMAGE_DECODE_MAX_CONCURRENT', 1, 64, - 2, + 1, ), // 0 отключает фоновый проход уборщика (остаётся ручной CLI). ATTACHMENT_SWEEP_INTERVAL_MS: numeric( diff --git a/app/src/main.ts b/app/src/main.ts index b799f2e..8c4a615 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -1,12 +1,8 @@ import { NestFactory } from '@nestjs/core'; import { HttpAdapterHost } from '@nestjs/core'; import { ConfigService } from '@nestjs/config'; -import { - PayloadTooLargeException, - UnsupportedMediaTypeException, - ValidationPipe, -} from '@nestjs/common'; -import type { Readable } from 'node:stream'; +import { ValidationPipe } from '@nestjs/common'; +import type { UploadRequest } from './attachments/upload-request.js'; import { FastifyAdapter, NestFastifyApplication, @@ -16,53 +12,41 @@ import { HealthService } from './health/health.service.js'; import { MetricsService } from './metrics/metrics.service.js'; import { RequestIdInterceptor } from './common/http/request-id.interceptor.js'; -const TEXT_BODY_LIMIT = 1_048_576; const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments'; +const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream'; async function bootstrap(): Promise { const app = await NestFactory.create( AppModule, new FastifyAdapter({ logger: true }), ); - // Raw image bodies reach the controller as an unread stream: the upload is - // bounded by our own checks, not by Fastify's bodyLimit. A stream may reach - // only the upload route: on JSON routes it would hit the ValidationPipe and - // overflow the stack there, so other routes keep the pre-upload behavior. + // The upload route receives its body as an unread stream whatever the + // declared Content-Type: the service checks the type itself, after the + // auth guard (401 before 415, contracts/README.md), and bounds the body by + // its own limits, not Fastify's bodyLimit. The declared type is kept on the + // request; all other routes keep Fastify's parsers untouched. const fastify = app.getHttpAdapter().getInstance(); - type ParserRequest = { method: string; routeOptions?: { url?: string } }; - type Done = (error: Error | null, body?: unknown) => void; - const isUpload = (request: ParserRequest) => - request.method === 'POST' && request.routeOptions?.url === UPLOAD_ROUTE; - fastify.addContentTypeParser( - ['image/png', 'image/jpeg'], - (request: ParserRequest, payload: unknown, done: Done) => - isUpload(request) - ? done(null, payload) - : done(new UnsupportedMediaTypeException()), - ); - // Fastify's built-in text/plain parser buffers the body and fails binary - // input with 400; on the upload route the service answers 415 instead. - // Elsewhere text/plain keeps being buffered as a string (1 MiB cap). - fastify.removeContentTypeParser('text/plain'); - fastify.addContentTypeParser( - 'text/plain', - (request: ParserRequest, payload: Readable, done: Done) => { - if (isUpload(request)) return done(null, payload); - const chunks: Buffer[] = []; - let size = 0; - payload.on('data', (chunk: Buffer) => { - size += chunk.length; - if (size > TEXT_BODY_LIMIT) { - payload.destroy(); - done(new PayloadTooLargeException()); - } else chunks.push(chunk); - }); - payload.on('end', () => - done(null, Buffer.concat(chunks).toString('utf8')), - ); - payload.on('error', (error: Error) => done(error)); + fastify.addHook( + 'onRequest', + (request: UploadRequest, _reply: unknown, done: () => void) => { + if ( + request.method === 'POST' && + request.routeOptions?.url === UPLOAD_ROUTE + ) { + request.uploadContentType = request.headers['content-type']; + request.headers['content-type'] = UPLOAD_STREAM_TYPE; + } + done(); }, ); + fastify.addContentTypeParser( + UPLOAD_STREAM_TYPE, + ( + _request: unknown, + payload: unknown, + done: (error: Error | null, body?: unknown) => void, + ) => done(null, payload), + ); app.useGlobalInterceptors( new RequestIdInterceptor( app.get(HttpAdapterHost), diff --git a/app/src/storage/object-storage.ts b/app/src/storage/object-storage.ts index f644874..cddbb5f 100644 --- a/app/src/storage/object-storage.ts +++ b/app/src/storage/object-storage.ts @@ -62,9 +62,15 @@ export class ObjectStorage { maxAttempts: 1, requestChecksumCalculation: 'WHEN_REQUIRED' as const, responseChecksumValidation: 'WHEN_REQUIRED' as const, + // requestTimeout bounds a request until its response arrives and, + // with throwOnRequestTimeout, actually fails it (otherwise smithy + // only logs a warning); socketTimeout bounds a stalled body read. + // A hung S3 therefore costs at most S3_REQUEST_TIMEOUT_MS per call. requestHandler: new NodeHttpHandler({ connectionTimeout: config.getOrThrow('S3_CONNECT_TIMEOUT_MS'), requestTimeout: config.getOrThrow('S3_REQUEST_TIMEOUT_MS'), + throwOnRequestTimeout: true, + socketTimeout: config.getOrThrow('S3_REQUEST_TIMEOUT_MS'), httpAgent: new http.Agent({ keepAlive: true, maxSockets: 64 }), }), }; @@ -99,21 +105,23 @@ export class ObjectStorage { ContentLength: contentLength, ContentType: contentType, }), - { abortSignal: signal }, + // The upload deadline (UPLOAD_TIMEOUT_MS, via `signal`) bounds the + // PUT: it lasts as long as the client sends, not S3_REQUEST_TIMEOUT_MS. + { abortSignal: signal, requestTimeout: 0 }, ); } catch (error) { throw this.fail('put', error); } } - async getStream(key: string, range?: string): Promise { + async getStream( + key: string, + signal?: AbortSignal, + ): Promise { try { const result = await this.internal.send( - new GetObjectCommand({ - Bucket: this.bucket, - Key: key, - Range: range, - }), + new GetObjectCommand({ Bucket: this.bucket, Key: key }), + { abortSignal: signal }, ); return result.Body as Readable; } catch (error) { diff --git a/app/test/attachments.e2e-spec.ts b/app/test/attachments.e2e-spec.ts index 0758ba6..d35bd73 100644 --- a/app/test/attachments.e2e-spec.ts +++ b/app/test/attachments.e2e-spec.ts @@ -21,8 +21,10 @@ import { upload, uploadRaw, } from './support/e2e.js'; -import { jpegWithoutScanData } from './support/jpeg.js'; +import sharp from 'sharp'; +import { jpegWithMetadata, jpegWithoutScanData } from './support/jpeg.js'; import { + maxPixelsRgba16Png, noisePng, padPngTo, pixelBombPng, @@ -146,6 +148,106 @@ describe('attachments (proxy upload)', () => { expect(content.bytes.equals(data)).toBe(true); }); + it('accepts camera-size images up to the pixel limit', async () => { + // 7728x5152 = 39.8 MP, like a 40 MP camera; decoded size far above the + // replica's 16 MiB /tmp, so libvips must not spill to a temp file. + const camera = sharp({ + create: { + width: 7728, + height: 5152, + channels: 3, + background: { r: 120, g: 80, b: 40 }, + }, + }); + const baseline = await camera.clone().jpeg({ quality: 85 }).toBuffer(); + const progressiveCamera = await camera + .clone() + .jpeg({ quality: 85, progressive: true }) + .toBuffer(); + for (const data of [baseline, progressiveCamera]) + expect((await ready(data, 'image/jpeg')).size_bytes).toBe( + data.length, + ); + const rgba16 = await maxPixelsRgba16Png(); + expect((await ready(rgba16)).size_bytes).toBe(rgba16.length); + }, 120_000); + + it('accepts a JPEG with megabytes of metadata before the frame header', async () => { + const data = jpegWithMetadata(jpeg, 20); + expect(data.length).toBeGreaterThan(1_048_576); + expect((await ready(data, 'image/jpeg')).sha256).toBe(sha256(data)); + }); + + it('authenticates before judging the content type', async () => { + const zip = await http( + `/api/v1/chats/${chatId}/attachments?filename=a.zip`, + { + method: 'POST', + headers: { 'content-type': 'application/zip' }, + body: png as unknown as BodyInit, + }, + ); + expectError(zip, 401, 'unauthorized'); + expectError( + await upload(alice.token, leftoverChatId, png, { + contentType: 'application/json', + }), + 415, + 'unsupported_media_type', + ); + }); + + it('answers 413 with the error envelope to an oversized text body on JSON routes', async () => { + const result = await http( + `/api/v1/chats/${chatId}/messages`, + { + method: 'POST', + headers: { + ...bearer(alice.token), + 'content-type': 'text/plain', + 'idempotency-key': randomUUID().replaceAll('-', ''), + }, + body: 'x'.repeat(2_100_000), + }, + ); + expectError(result, 413, 'payload_too_large'); + }); + + it('keeps one slow client from taking every upload slot', async () => { + const slow = await actor(); + const slowChat = (await createChat(slow, [])).id; + const url = new URL(baseUrl); + const sockets: net.Socket[] = []; + try { + for (let index = 0; index < 12; index += 1) { + const socket = net.connect( + Number(url.port || 80), + url.hostname, + ); + socket.on('error', () => {}); + socket.write( + `POST /api/v1/chats/${slowChat}/attachments?filename=slow.png HTTP/1.1\r\n` + + `Host: ${url.host}\r\nAuthorization: Bearer ${slow.token}\r\n` + + `Content-Type: image/png\r\nContent-Length: ${png.length}\r\n\r\n`, + ); + socket.write(png.subarray(0, 4)); + sockets.push(socket); + } + await sleep(1000); + const extra = await upload(slow.token, slowChat, png); + expectError(extra, 429, 'rate_limited'); + expect(extra.response.headers.get('retry-after')).toBe('1'); + // Another user is not affected by the slow client. + for (let index = 0; index < 4; index += 1) + expect( + (await upload(bob.token, chatId, png)).response.status, + ).toBe(201); + } finally { + for (const socket of sockets) socket.destroy(); + } + await settled(slowChat); + }); + it('enforces the 10 MiB limit exactly', async () => { const exact = padPngTo(png, MAX); const created = await ready(exact); diff --git a/app/test/dependency-failures.e2e-spec.ts b/app/test/dependency-failures.e2e-spec.ts index c44822c..7f489cc 100644 --- a/app/test/dependency-failures.e2e-spec.ts +++ b/app/test/dependency-failures.e2e-spec.ts @@ -602,6 +602,10 @@ describe.skipIf(!enabled)('dependency outages', () => { await compose('start', redisService); await compose('start', postgresService); await compose('start', s3Service); + const s3Container = (await compose('ps', '-q', s3Service)).split( + '\n', + )[0]; + await docker('unpause', s3Container).catch(() => undefined); await waitForReadiness(200); await restoreDatabaseAccess(); const expected = (await apiContainers()).length; @@ -986,6 +990,62 @@ describe.skipIf(!enabled)('dependency outages', () => { outagedRequestBudgetMs, ); + it( + 'bounds requests by S3_REQUEST_TIMEOUT_MS when S3 hangs instead of refusing', + async () => { + const created = await request<{ id: string }>( + uploadPath(chat.id, `hang-${runId}.png`), + { + method: 'POST', + headers: { ...bearer(token), 'content-type': 'image/png' }, + body: samplePng as unknown as BodyInit, + }, + ); + expect(created.response.status).toBe(201); + // A paused container keeps its TCP port open but never answers: the + // failure mode a stopped container (connection refused) does not show. + const container = (await compose('ps', '-q', s3Service)).split( + '\n', + )[0]; + await docker('pause', container); + try { + const started = Date.now(); + const content = await request( + `/api/v1/attachments/${created.body.id}/content`, + { headers: bearer(token) }, + ); + const elapsed = Date.now() - started; + expectBoundedOutage(content); + // S3_REQUEST_TIMEOUT_MS is 15 s in the stand; HAProxy gives up at 30 s. + expect(elapsed).toBeLessThan(20_000); + expect( + ( + await request( + `/api/v1/attachments/${created.body.id}`, + { + headers: bearer(token), + }, + ) + ).response.status, + ).toBe(200); + expect((await request('/health/ready')).response.status).toBe( + 200, + ); + } finally { + await docker('unpause', container); + } + const bytes = await fetch( + `${baseUrl}/api/v1/attachments/${created.body.id}/content`, + { headers: bearer(token) }, + ); + expect(bytes.status).toBe(200); + expect( + Buffer.from(await bytes.arrayBuffer()).equals(samplePng), + ).toBe(true); + }, + outagedRequestBudgetMs, + ); + it( 'keeps the object while PostgreSQL is down before ready and sweeps it afterwards', async () => { diff --git a/app/test/direct-upload.e2e-spec.ts b/app/test/direct-upload.e2e-spec.ts index 0411595..7713bdb 100644 --- a/app/test/direct-upload.e2e-spec.ts +++ b/app/test/direct-upload.e2e-spec.ts @@ -160,6 +160,21 @@ describe('direct upload (presigned PUT)', () => { expect(sent.body.attachment_ids).toEqual([ticket.attachment_id]); }); + it('answers 200 to every parallel complete of one upload', async () => { + const ticket = await initOk(png); + expect((await put(ticket, png)).status).toBe(200); + const results = await Promise.all( + Array.from({ length: 4 }, () => + complete(alice.token, ticket.attachment_id), + ), + ); + expect(results.map((result) => result.response.status)).toEqual([ + 200, 200, 200, 200, + ]); + for (const result of results) + expect(result.body.id).toBe(ticket.attachment_id); + }); + it('rejects invalid declarations', async () => { const base = declaration(png); expectError(await init(eve.token, chatId, base), 404, 'not_found'); diff --git a/app/test/support/jpeg.ts b/app/test/support/jpeg.ts index ea60571..b2d1ef9 100644 --- a/app/test/support/jpeg.ts +++ b/app/test/support/jpeg.ts @@ -8,3 +8,17 @@ export function jpegWithoutScanData(jpeg: Buffer): Buffer { } return Buffer.concat([jpeg.subarray(0, offset), Buffer.from([0xff, 0xd9])]); } + +/** Inserts `count` APP13 segments of ~64 KiB right after SOI (Photoshop-style metadata). */ +export function jpegWithMetadata(jpeg: Buffer, count: number): Buffer { + const payload = Buffer.alloc(65_533, 0x20); + const segment = Buffer.concat([ + Buffer.from([0xff, 0xed, 0xff, 0xff]), + payload, + ]); + return Buffer.concat([ + jpeg.subarray(0, 2), + ...Array.from({ length: count }, () => segment), + jpeg.subarray(2), + ]); +} diff --git a/app/test/support/png.ts b/app/test/support/png.ts index 714ce8a..0605b82 100644 --- a/app/test/support/png.ts +++ b/app/test/support/png.ts @@ -1,5 +1,6 @@ import { randomBytes } from 'node:crypto'; -import { crc32, deflateSync } from 'node:zlib'; +import { once } from 'node:events'; +import { crc32, createDeflate, deflateSync } from 'node:zlib'; const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); @@ -104,3 +105,36 @@ export function pngWithGarbageIdat(): Buffer { idat: randomBytes(5000), }); } + +/** + * 8000x5000 RGBA 16-bit (8 bytes/pixel, 40 000 000 px — the pixel limit): + * the heaviest decode a valid upload can ask for. Rows are streamed into the + * compressor, so the 320 MB raw image is never allocated here. + */ +export async function maxPixelsRgba16Png(): Promise { + const width = 8000; + const height = 5000; + // First row: a gradient; every next row uses filter "Up" with zero + // deltas, i.e. repeats it — a valid image that compresses well. + const first = Buffer.alloc(1 + width * 8); + for (let x = 0; x < width; x += 1) + first.writeUInt16BE((x * 8) & 0xffff, 1 + x * 8); + const repeat = Buffer.alloc(1 + width * 8); + repeat[0] = 2; + const deflate = createDeflate({ level: 1 }); + const chunks: Buffer[] = []; + deflate.on('data', (chunk: Buffer) => chunks.push(chunk)); + const ended = once(deflate, 'end'); + for (let y = 0; y < height; y += 1) + if (!deflate.write(y === 0 ? first : repeat)) + await once(deflate, 'drain'); + deflate.end(); + await ended; + return buildPng({ + width, + height, + colorType: 6, + bitDepth: 16, + idat: Buffer.concat(chunks), + }); +} diff --git a/app/tsconfig.json b/app/tsconfig.json index 8109c41..6aa3ca7 100644 --- a/app/tsconfig.json +++ b/app/tsconfig.json @@ -10,7 +10,7 @@ "emitDecoratorMetadata": true, "experimentalDecorators": true, "allowSyntheticDefaultImports": true, - "target": "ES2023", + "target": "ES2024", "sourceMap": true, "outDir": "./dist", "incremental": true, diff --git a/compose.yaml b/compose.yaml index 139d436..ddef2e2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -169,6 +169,9 @@ services: DIRECT_UPLOAD_URL_TTL_SECONDS: ${DIRECT_UPLOAD_URL_TTL_SECONDS:-300} ATTACHMENT_SWEEP_GRACE_SECONDS: ${ATTACHMENT_SWEEP_GRACE_SECONDS:-900} ATTACHMENT_SWEEP_INTERVAL_MS: ${ATTACHMENT_SWEEP_INTERVAL_MS:-60000} + # libvips держит декодируемое изображение в памяти, а не во временном + # файле: /tmp — tmpfs 16 MiB, и изображение ~40 Мп туда не помещается. + VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g} secrets: [s3_api_access_key_id, s3_api_secret_access_key] # Корневая ФС только для чтения: локального файла оригинала на API нет. read_only: true @@ -178,7 +181,7 @@ services: resources: limits: cpus: '${API_CPUS_PER_REPLICA:-1.5}' - memory: ${API_MEMORY_PER_REPLICA:-768m} + memory: ${API_MEMORY_PER_REPLICA:-1024m} stop_grace_period: 20s restart: unless-stopped depends_on: diff --git a/contracts/extensions.openapi.json b/contracts/extensions.openapi.json index d676f35..8dded17 100644 --- a/contracts/extensions.openapi.json +++ b/contracts/extensions.openapi.json @@ -2,8 +2,8 @@ "openapi": "3.1.0", "info": { "title": "MTUSI Messenger — расширение lab4: прямая загрузка", - "version": "1.4.0-ext.1", - "description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments//original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые заявки и объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)." + "version": "1.4.0-ext.2", + "description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments//original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые и отклонённые заявки и их объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)." }, "servers": [ { @@ -264,8 +264,8 @@ "/api/v1/attachments/{attachment_id}/complete": { "post": { "operationId": "completeDirectUpload", - "summary": "Подтвердить прямую загрузку: HEAD объекта и проверка заголовка изображения", - "description": "Только владелец заявки. Проверяет размер и подписанный SHA-256 объекта (HEAD), читает заголовок изображения (до 1 MiB) и сверяет тип и число пикселей. При несоответствии заявка и объект удаляются, повторный complete даёт 404. Повторный complete готового вложения идемпотентен и возвращает 200.", + "summary": "Подтвердить прямую загрузку: HEAD объекта, полное чтение и декод изображения", + "description": "Только владелец заявки. Сверяет размер и подписанный SHA-256 объекта (HEAD), затем читает объект целиком: структурная проверка PNG/JPEG и полный декод (повреждённые и усечённые изображения отвергаются, даже если checksum совпадает). При несоответствии объект удаляется, а заявка получает внутренний статус rejected до истечения URL и grace-периода уборщика (повторный PUT по ещё действующему URL не оставит неучтённый объект); повторный complete отклонённой заявки — 404. Если объект ещё не загружен — 409, заявка не меняется. Повторный и параллельный complete готового вложения идемпотентен (200).", "parameters": [ { "name": "attachment_id", @@ -461,7 +461,7 @@ } }, "503": { - "description": "Хранилище недоступно", + "description": "Хранилище недоступно или не ответило за S3_REQUEST_TIMEOUT_MS / UPLOAD_TIMEOUT_MS", "headers": { "X-Request-Id": { "description": "Идентификатор запроса.", @@ -485,6 +485,39 @@ } } } + }, + "429": { + "description": "Реплика проверяет слишком много изображений одновременно (UPLOAD_MAX_IN_FLIGHT) или у пользователя слишком много одновременных проверок (UPLOAD_MAX_IN_FLIGHT_PER_USER); Retry-After — секунды до повтора.", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "Retry-After": { + "description": "Секунды до повтора.", + "schema": { + "type": "integer", + "minimum": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } } } } diff --git a/evidence/lab4/decode-memory.txt b/evidence/lab4/decode-memory.txt index 09ebb0d..6a6bbf2 100644 --- a/evidence/lab4/decode-memory.txt +++ b/evidence/lab4/decode-memory.txt @@ -1,4 +1,7 @@ -# inputs: magick -size 6320x6320 gradient:red-blue (PNG); same as JPEG q85 baseline and -interlace JPEG -png 6320x6320 bytes=53350 decode_ms=1159 peak_delta_rss_mib=116.8 -jpeg 6320x6320 bytes=4899767 decode_ms=1234 peak_delta_rss_mib=126.5 -jpeg 6320x6320 progressive bytes=4791710 decode_ms=1379 peak_delta_rss_mib=130.5 +# container /mtusi-lab4-api-1, limits: 805306368 bytes, VIPS_DISC_THRESHOLD=1g; commit a06471a+worktree +jpeg 7728x5152 3ch uchar bytes=4921012 decode_ms=58 peak_delta_rss_mib=3.9 ok +jpeg 7728x5152 3ch uchar progressive bytes=4826968 decode_ms=278 peak_delta_rss_mib=117.7 ok +jpeg 7728x5152 4ch uchar bytes=10601868 decode_ms=161 peak_delta_rss_mib=7.7 ok +png 8000x5000 4ch ushort bytes=1420114 decode_ms=496 peak_delta_rss_mib=66.6 ok +png 8000x5000 4ch ushort progressive bytes=364096 decode_ms=1270 peak_delta_rss_mib=338.6 ok +png 8000x5000 3ch uchar progressive palette bytes=61042 decode_ms=341 peak_delta_rss_mib=126.0 ok diff --git a/scripts/lab4/attachments_bench.py b/scripts/lab4/attachments_bench.py index e5b4174..3ed8e62 100755 --- a/scripts/lab4/attachments_bench.py +++ b/scripts/lab4/attachments_bench.py @@ -141,7 +141,14 @@ def one_upload(env, mode, token, chat_id, data, seed): return ok, elapsed_ms -def run_once(env, args, token, chat_id, ids): +def payload(args, seed): + if args.image: + with open(args.image, "rb") as handle: + return handle.read() + return noise_png(args.size_bytes, seed) + + +def run_once(env, args, actors, ids): for container in ids: reset_metrics(container) time.sleep(0.2) @@ -153,7 +160,8 @@ def run_once(env, args, token, chat_id, ids): def worker(index): for step in range(args.uploads_per_worker): seed = index * 1000 + step - data = noise_png(args.size_bytes, seed) + data = payload(args, seed) + token, chat_id = actors[index % len(actors)] ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed) with lock: (latencies if ok else failures).append(elapsed) @@ -203,29 +211,39 @@ def run_once(env, args, token, chat_id, ids): def command_run(args): + if args.image: + args.size_bytes = os.path.getsize(args.image) + if args.size_bytes <= 0: + sys.exit("--size-bytes or --image is required") env = read_env() ids = containers() if not ids: sys.exit("no api containers found") - name = f"bench_{uuid.uuid4().hex[:10]}" - password = "bench:" + uuid.uuid4().hex - status, user = api_json(env, "POST", "/api/v1/users", - body={"username": name, "display_name": "bench", "password": password}) - assert status == 201, status - status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}") - assert status == 201, status - token = session["token"] - status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]}) - assert status == 201, status + # One user per worker: the API caps concurrent uploads per user + # (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users. + actors = [] + for _ in range(args.concurrency): + name = f"bench_{uuid.uuid4().hex[:10]}" + password = "bench:" + uuid.uuid4().hex + status, user = api_json(env, "POST", "/api/v1/users", + body={"username": name, "display_name": "bench", "password": password}) + assert status == 201, status + status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}") + assert status == 201, status + token = session["token"] + status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]}) + assert status == 201, status + actors.append((token, chat["id"])) - # Warm-up: one upload per replica path, not measured. + # Warm-up: two uploads per replica, not measured. for seed in range(len(ids) * 2): - one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed) + token, chat_id = actors[seed % len(actors)] + one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed) limits = [] for c in ids: limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c)) - repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)] + repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)] mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal")) result = { "date": datetime.now(timezone.utc).isoformat(timespec="seconds"), @@ -233,6 +251,7 @@ def command_run(args): "params": { "mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency, "uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat, + "users": args.concurrency, "image": os.path.basename(args.image) if args.image else None, }, "host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total}, "api_container_limits_nanocpus_memory": limits, @@ -260,17 +279,19 @@ def command_summarize(args): p = data["params"] reps = data["repeats"] med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731 - rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data)) - rows.sort(key=lambda r: (r[1], r[0], r[2])) - print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |") - print("|---|---|---|---|---|---|---|---|---|") + label = p["mode"] + (f" ({p['image']})" if p.get("image") else "") + rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data)) + rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2])) + print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |") + print("|---|---|---|---|---|---|---|---|---|---|") for mode, size, conc, reps, med, _data in rows: + peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"]) ok = sum(r["succeeded"] for r in reps) total = sum(r["uploads"] for r in reps) p50 = statistics.median(r["latency_ms"]["p50"] for r in reps) p95 = statistics.median(r["latency_ms"]["p95"] for r in reps) print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | " - f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |") + f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |") def main(): @@ -278,7 +299,8 @@ def main(): sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run") run.add_argument("--mode", choices=["proxy", "direct"], required=True) - run.add_argument("--size-bytes", type=int, required=True) + run.add_argument("--size-bytes", type=int, default=0) + run.add_argument("--image", help="upload this file instead of generated noise PNGs") run.add_argument("--concurrency", type=int, default=10) run.add_argument("--uploads-per-worker", type=int, default=5) run.add_argument("--repeat", type=int, default=3) diff --git a/scripts/lab4/decode_memory.mjs b/scripts/lab4/decode_memory.mjs index 475316d..65bd84b 100644 --- a/scripts/lab4/decode_memory.mjs +++ b/scripts/lab4/decode_memory.mjs @@ -1,22 +1,33 @@ -// Peak RSS of one full decode check (same sharp options as ImageDecoder) on -// worst-case inputs near MAX_IMAGE_PIXELS. Run from app/: node ../scripts/lab4/decode_memory.mjs +// Peak RSS of one full decode check with the same sharp pipeline as +// app/src/attachments/image/image-decoder.ts, on worst-case inputs near +// MAX_IMAGE_PIXELS. Runs where the API runs (same libvips, VIPS_DISC_THRESHOLD): +// docker compose exec -T api node - < scripts/lab4/decode_memory.mjs # reads files from IMAGES (paths in the container) +// or on the host from app/: IMAGES="a.png b.jpg" node ../scripts/lab4/decode_memory.mjs import { readFileSync } from 'node:fs'; import { createRequire } from 'node:module'; + const sharp = createRequire(`${process.cwd()}/`)('sharp'); sharp.cache(false); sharp.concurrency(1); -for (const file of process.argv.slice(2)) { +for (const file of (process.env.IMAGES ?? '').split(/\s+/u).filter(Boolean)) { const data = readFileSync(file); const meta = await sharp(data).metadata(); const base = process.memoryUsage().rss; let peak = base; const timer = setInterval(() => { peak = Math.max(peak, process.memoryUsage().rss); - }, 10); + }, 5); const started = Date.now(); - await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 }).stats(); + let verdict = 'ok'; + try { + await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 }) + .resize(64, 64, { fit: 'inside' }) + .toBuffer(); + } catch (error) { + verdict = `rejected: ${String(error.message).split('\n')[0]}`; + } clearInterval(timer); console.log( - `${meta.format} ${meta.width}x${meta.height}${meta.isProgressive ? ' progressive' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)}`, + `${meta.format} ${meta.width}x${meta.height} ${meta.channels}ch ${meta.depth}${meta.isProgressive ? ' progressive' : ''}${meta.isPalette ? ' palette' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)} ${verdict}`, ); } diff --git a/scripts/lab4/pngkit.py b/scripts/lab4/pngkit.py index a29c886..1ec0b8d 100644 --- a/scripts/lab4/pngkit.py +++ b/scripts/lab4/pngkit.py @@ -48,3 +48,31 @@ def noise_png(size: int, seed: int = 0) -> bytes: raise ValueError("size is too small") continue return pad_png(png, size) + + +ADAM7 = [(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)] + + +def worst_decode_png(size: int = 10_485_760, width: int = 8000, height: int = 5000) -> bytes: + """The heaviest valid upload to decode: 40 MP RGBA 16-bit, Adam7-interlaced + (the decoder must hold the whole image), padded to exactly `size` bytes.""" + compressor = zlib.compressobj(1) + parts = [] + for x0, y0, dx, dy in ADAM7: + pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0 + pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0 + if pass_width == 0 or pass_height == 0: + continue + row = bytes(1 + pass_width * 8) + for _ in range(pass_height): + parts.append(compressor.compress(row)) + parts.append(compressor.flush()) + ihdr = struct.pack(">IIBBBBB", width, height, 16, 6, 0, 0, 1) + png = SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"".join(parts)) + chunk(b"IEND", b"") + return pad_png(png, size) + + +if __name__ == "__main__": + import sys + + sys.stdout.buffer.write(worst_decode_png()) diff --git a/scripts/lab4/run_matrix.sh b/scripts/lab4/run_matrix.sh index c45df4e..7edd275 100755 --- a/scripts/lab4/run_matrix.sh +++ b/scripts/lab4/run_matrix.sh @@ -19,5 +19,17 @@ for size in 131072 1048576 9961472; do done echo "== proxy size=9961472 c=20" bench proxy 9961472 20 +# Worst case for memory: every request carries the heaviest valid image +# (40 MP RGBA16 Adam7 PNG padded to 10 MiB, ~340 MiB per decode) and C=32 +# saturates UPLOAD_MAX_IN_FLIGHT (16) on both replicas. +worst=$(mktemp --suffix=.png) +trap 'rm -f "$worst"' EXIT +python3 scripts/lab4/pngkit.py > "$worst" +for mode in proxy direct; do + echo "== $mode worst-case image c=32" + python3 scripts/lab4/attachments_bench.py run --mode "$mode" --image "$worst" \ + --concurrency 32 --uploads-per-worker 2 --repeat 3 \ + --output "$out/$mode-worst-c32.json" +done python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md cat evidence/lab4/summary.md