forked from mtusi2026/mtusi2026lab4
lab4: исправления второго ревью (40 Мп в tmpfs, таймауты S3, параллельный complete, auth до Content-Type, лимит на пользователя, бюджет памяти декода)
This commit is contained in:
+8
-4
@@ -33,7 +33,7 @@ IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Стенд (compose.yaml и scripts/):
|
||||
API_REPLICAS=2
|
||||
API_CPUS_PER_REPLICA=1.5
|
||||
API_MEMORY_PER_REPLICA=768m
|
||||
API_MEMORY_PER_REPLICA=1024m
|
||||
PG_CPUS=1.5
|
||||
PG_MEMORY=512m
|
||||
REDIS_CPUS=0.25
|
||||
@@ -68,10 +68,14 @@ S3_CONNECT_TIMEOUT_MS=1000
|
||||
S3_REQUEST_TIMEOUT_MS=15000
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Каждая загрузка держит копию файла (≤ MAX_UPLOAD_BYTES) до полного декода:
|
||||
# память реплики ≈ UPLOAD_MAX_IN_FLIGHT × 10 MiB + IMAGE_DECODE_MAX_CONCURRENT декодов.
|
||||
# Каждая загрузка держит копию файла (≤ MAX_UPLOAD_BYTES) до полного декода;
|
||||
# худший декод (40 Мп RGBA16 interlaced PNG) ≈ 340 MiB. Граница памяти реплики:
|
||||
# базовый RSS (~250 MiB) + UPLOAD_MAX_IN_FLIGHT × 10 MiB + IMAGE_DECODE_MAX_CONCURRENT × 340 MiB
|
||||
# = 250 + 160 + 340 ≈ 750 MiB при API_MEMORY_PER_REPLICA=1024m.
|
||||
UPLOAD_MAX_IN_FLIGHT=16
|
||||
IMAGE_DECODE_MAX_CONCURRENT=2
|
||||
# Не больше стольких одновременных загрузок одного пользователя на реплику.
|
||||
UPLOAD_MAX_IN_FLIGHT_PER_USER=4
|
||||
IMAGE_DECODE_MAX_CONCURRENT=1
|
||||
DIRECT_UPLOAD_MAX_PENDING_PER_USER=10
|
||||
# UPLOAD_TIMEOUT_MS, DIRECT_UPLOAD_URL_TTL_SECONDS, ATTACHMENT_SWEEP_GRACE_SECONDS,
|
||||
# ATTACHMENT_SWEEP_INTERVAL_MS задаются в compose.yaml (environment) с умолчаниями
|
||||
|
||||
@@ -23,6 +23,7 @@ import type { User } from '../users/user.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
import { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
|
||||
import { DirectUploadsService } from './direct-uploads.service.js';
|
||||
import type { UploadRequest } from './upload-request.js';
|
||||
import { contentDisposition } from './filename.js';
|
||||
|
||||
@Controller('api/v1')
|
||||
@@ -44,9 +45,8 @@ export class AttachmentsController {
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Query('filename') filename: unknown,
|
||||
@Headers('content-type') contentType: string | undefined,
|
||||
@Headers('content-length') contentLength: string | undefined,
|
||||
@Req() request: FastifyRequest,
|
||||
@Req() request: FastifyRequest & UploadRequest,
|
||||
@Res({ passthrough: true }) reply: FastifyReply,
|
||||
) {
|
||||
// A gigantic body is not worth draining: close the connection after the answer.
|
||||
@@ -56,7 +56,7 @@ export class AttachmentsController {
|
||||
chatId,
|
||||
user,
|
||||
filename,
|
||||
contentType,
|
||||
request.uploadContentType,
|
||||
contentLength,
|
||||
request.body as Readable | undefined,
|
||||
);
|
||||
|
||||
@@ -38,7 +38,9 @@ function setup(repository: Record<string, unknown>) {
|
||||
{ getForUser: vi.fn() } as never,
|
||||
{ increment: vi.fn(), add: vi.fn() } as never,
|
||||
{ verify: vi.fn() } as never,
|
||||
{ run: (work: () => Promise<unknown>) => work() } as never,
|
||||
{
|
||||
run: (_user: string, work: () => Promise<unknown>) => work(),
|
||||
} as never,
|
||||
);
|
||||
const upload = () =>
|
||||
service.uploadProxy(
|
||||
|
||||
@@ -24,7 +24,6 @@ import {
|
||||
} from './attachment.js';
|
||||
import { validateFilename } from './filename.js';
|
||||
import {
|
||||
IMAGE_HEADER_MAX_BYTES,
|
||||
ImageHeader,
|
||||
ImageInspector,
|
||||
ImageRejectedError,
|
||||
@@ -94,7 +93,7 @@ export class AttachmentsService {
|
||||
if (contentLength > this.maxUploadBytes)
|
||||
throw new PayloadTooLargeException('Image is too large');
|
||||
|
||||
return this.slots.run(async () => {
|
||||
return this.slots.run(user.id, async () => {
|
||||
await this.chats.getForUser(chatId, user);
|
||||
return this.receive(
|
||||
chatId,
|
||||
@@ -137,8 +136,9 @@ export class AttachmentsService {
|
||||
let key = '';
|
||||
let rejection: unknown;
|
||||
try {
|
||||
// Header phase: nothing exists in the database or S3 yet.
|
||||
let headBytes = 0;
|
||||
// Header phase: nothing exists in the database or S3 yet. It is
|
||||
// bounded by Content-Length (<= MAX_UPLOAD_BYTES): a JPEG may carry
|
||||
// megabytes of APPn metadata before its frame header.
|
||||
try {
|
||||
while (inspector.header === undefined) {
|
||||
const step = await next();
|
||||
@@ -150,16 +150,7 @@ export class AttachmentsService {
|
||||
);
|
||||
}
|
||||
head.push(step.value);
|
||||
headBytes += step.value.length;
|
||||
inspector.push(step.value);
|
||||
if (
|
||||
inspector.header === undefined &&
|
||||
headBytes > IMAGE_HEADER_MAX_BYTES
|
||||
)
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Image header is missing',
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw this.mapUploadError(error, timedOut);
|
||||
@@ -191,6 +182,7 @@ export class AttachmentsService {
|
||||
// UPLOAD_MAX_IN_FLIGHT, released as soon as the decode starts.
|
||||
const copy: Buffer[] = [];
|
||||
let total = 0;
|
||||
let allSent = false;
|
||||
const account = (chunk: Buffer, inspect: boolean): boolean => {
|
||||
total += chunk.length;
|
||||
if (total > contentLength)
|
||||
@@ -209,7 +201,10 @@ export class AttachmentsService {
|
||||
inspector.finish();
|
||||
const data = Buffer.concat(copy, total);
|
||||
copy.length = 0;
|
||||
await decoder.verify(data);
|
||||
await decoder.verify(data, controller.signal);
|
||||
// From here S3 may receive the whole body and commit the object
|
||||
// even if this request fails or times out afterwards.
|
||||
allSent = true;
|
||||
};
|
||||
async function* generate(): AsyncGenerator<Buffer> {
|
||||
for (const chunk of head) {
|
||||
@@ -271,7 +266,12 @@ export class AttachmentsService {
|
||||
} catch (error) {
|
||||
readable.destroy();
|
||||
if (!claimed) {
|
||||
const outcome = await this.compensate(id, key);
|
||||
// After the last byte was handed to S3, a PUT aborted on our
|
||||
// side may still be committed by S3 after our DELETE: keep the
|
||||
// row so the sweeper deletes the key again later.
|
||||
const outcome = await this.compensate(id, key, {
|
||||
keepRow: allSent && !stored,
|
||||
});
|
||||
// markReady failed on our side, but the row is no longer
|
||||
// pending: the commit may have succeeded with its reply lost.
|
||||
// If so, the upload is complete and 201 is the truth.
|
||||
@@ -312,6 +312,7 @@ export class AttachmentsService {
|
||||
async compensate(
|
||||
id: string,
|
||||
key: string,
|
||||
options: { keepRow?: boolean } = {},
|
||||
): Promise<'compensated' | 'not_pending' | 'deferred'> {
|
||||
try {
|
||||
if (!(await this.repository.claimForCompensation(id))) {
|
||||
@@ -319,7 +320,9 @@ export class AttachmentsService {
|
||||
return 'not_pending';
|
||||
}
|
||||
await this.storage.delete(key);
|
||||
await this.repository.deleteClaimed(id);
|
||||
// A kept `deleting` row is re-claimed by the sweeper after a minute,
|
||||
// which deletes the key once more and only then the row.
|
||||
if (!options.keepRow) await this.repository.deleteClaimed(id);
|
||||
this.metrics.increment('attachments.compensated');
|
||||
return 'compensated';
|
||||
} catch (error) {
|
||||
|
||||
@@ -11,7 +11,10 @@ import { randomUUID } from 'node:crypto';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import {
|
||||
ObjectStorage,
|
||||
ObjectStorageUnavailableError,
|
||||
} from '../storage/object-storage.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import {
|
||||
toAttachment,
|
||||
@@ -44,6 +47,7 @@ const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
|
||||
export class DirectUploadsService {
|
||||
private readonly maxUploadBytes: number;
|
||||
private readonly maxPixels: number;
|
||||
private readonly uploadTimeoutMs: number;
|
||||
private readonly ttlSeconds: number;
|
||||
private readonly maxPending: number;
|
||||
|
||||
@@ -58,6 +62,7 @@ export class DirectUploadsService {
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
|
||||
this.ttlSeconds = config.getOrThrow('DIRECT_UPLOAD_URL_TTL_SECONDS');
|
||||
this.maxPending = config.getOrThrow(
|
||||
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
|
||||
@@ -138,7 +143,7 @@ export class DirectUploadsService {
|
||||
);
|
||||
if (!row) throw new NotFoundException();
|
||||
if (row.status === 'ready') return toAttachment(row);
|
||||
return this.slots.run(() => this.verifyAndMarkReady(row));
|
||||
return this.slots.run(user.id, () => this.verifyAndMarkReady(row));
|
||||
}
|
||||
|
||||
private async verifyAndMarkReady(row: AttachmentRow): Promise<Attachment> {
|
||||
@@ -179,7 +184,15 @@ export class DirectUploadsService {
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
});
|
||||
if (!ready) throw new ConflictException('Upload is no longer pending');
|
||||
if (!ready) {
|
||||
// A parallel complete of the same upload won: answer like a repeat.
|
||||
const current = await this.repository.findForComplete(
|
||||
row.id,
|
||||
row.owner_id,
|
||||
);
|
||||
if (current?.status === 'ready') return toAttachment(current);
|
||||
throw new ConflictException('Upload is no longer pending');
|
||||
}
|
||||
this.metrics.increment('attachments.direct.completed');
|
||||
return toAttachment(ready);
|
||||
}
|
||||
@@ -194,7 +207,9 @@ export class DirectUploadsService {
|
||||
key: string,
|
||||
size: number,
|
||||
): Promise<ImageHeader> {
|
||||
const stream = await this.storage.getStream(key);
|
||||
// Same deadline as a proxy upload: a slow or hung read releases its slot.
|
||||
const signal = AbortSignal.timeout(this.uploadTimeoutMs);
|
||||
const stream = await this.storage.getStream(key, signal);
|
||||
if (stream === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
const inspector = new ImageInspector(this.maxPixels);
|
||||
@@ -211,6 +226,11 @@ export class DirectUploadsService {
|
||||
inspector.push(chunk);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ImageRejectedError) throw error;
|
||||
// The object stream broke mid-read (S3 outage, timeout): the image
|
||||
// was not judged, so this is an outage, not a client error.
|
||||
throw new ObjectStorageUnavailableError();
|
||||
} finally {
|
||||
stream.destroy();
|
||||
this.metrics.add('attachments.direct.verify_bytes', read);
|
||||
@@ -218,7 +238,7 @@ export class DirectUploadsService {
|
||||
const header = inspector.finish();
|
||||
const data = Buffer.concat(chunks, read);
|
||||
chunks.length = 0;
|
||||
await this.decoder.verify(data);
|
||||
await this.decoder.verify(data, signal);
|
||||
return header;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,11 @@ import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { jpegWithoutScanData } from '../../../test/support/jpeg.js';
|
||||
import { noisePng, pngWithGarbageIdat } from '../../../test/support/png.js';
|
||||
import {
|
||||
maxPixelsRgba16Png,
|
||||
noisePng,
|
||||
pngWithGarbageIdat,
|
||||
} from '../../../test/support/png.js';
|
||||
import { ImageDecoder } from './image-decoder.js';
|
||||
import { ImageInspector, ImageRejectedError } from './image-inspector.js';
|
||||
|
||||
@@ -48,4 +52,19 @@ describe('ImageDecoder', () => {
|
||||
ImageRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts the heaviest valid input: 40 MP RGBA 16-bit', async () => {
|
||||
await decoder.verify(await maxPixelsRgba16Png());
|
||||
});
|
||||
|
||||
it('drops an aborted waiter from the queue without running its decode', async () => {
|
||||
const busy = decoder.verify(noisePng(4_000_000));
|
||||
const controller = new AbortController();
|
||||
const waiting = decoder.verify(png, controller.signal);
|
||||
controller.abort();
|
||||
await expect(waiting).rejects.toMatchObject({ status: 503 });
|
||||
await busy;
|
||||
// The slot was not leaked: the next decode runs.
|
||||
await decoder.verify(png);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
Injectable,
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import sharp from 'sharp';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
@@ -10,19 +15,31 @@ import { ImageRejectedError } from './image-inspector.js';
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
// Failures of the replica itself, not of the image: they must not turn into a
|
||||
// client 415. libvips spills large images to a temp file unless
|
||||
// VIPS_DISC_THRESHOLD (compose.yaml) keeps them in memory; a full /tmp or an
|
||||
// allocation failure ends up here.
|
||||
const SERVER_FAULT =
|
||||
/no space left|out of memory|unable to allocate|cannot allocate|write failed|unable to open|tmp/iu;
|
||||
|
||||
/**
|
||||
* Full decode check. ImageInspector only proves the container structure
|
||||
* (signature, chunk CRCs, markers, dimensions); a PNG with a garbage zlib stream
|
||||
* or a JPEG without scan data passes it. Decoding every pixel with
|
||||
* failOn=warning rejects those, including truncated or corrupted entropy data.
|
||||
* Decodes are bounded by IMAGE_DECODE_MAX_CONCURRENT; callers wait for a slot.
|
||||
* or a JPEG without scan data passes it. Decoding every scanline with
|
||||
* failOn=warning rejects those, including truncated or corrupted compressed
|
||||
* data. A downscale to a thumbnail is the sink: it decodes all input but keeps
|
||||
* far less in memory than a full-size pipeline (measured in
|
||||
* evidence/lab4/decode-memory.txt). Decodes are bounded by
|
||||
* IMAGE_DECODE_MAX_CONCURRENT; a waiter whose request was aborted leaves the
|
||||
* queue and releases its buffer.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ImageDecoder {
|
||||
private readonly logger = new Logger(ImageDecoder.name);
|
||||
private readonly maxPixels: number;
|
||||
private readonly maxConcurrent: number;
|
||||
private active = 0;
|
||||
private readonly waiting: (() => void)[] = [];
|
||||
private readonly waiting = new Set<() => void>();
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
@@ -32,16 +49,27 @@ export class ImageDecoder {
|
||||
this.maxConcurrent = config.getOrThrow('IMAGE_DECODE_MAX_CONCURRENT');
|
||||
}
|
||||
|
||||
async verify(data: Buffer): Promise<void> {
|
||||
await this.acquire();
|
||||
async verify(data: Buffer, signal?: AbortSignal): Promise<void> {
|
||||
await this.acquire(signal);
|
||||
const started = performance.now();
|
||||
try {
|
||||
await sharp(data, {
|
||||
failOn: 'warning',
|
||||
sequentialRead: true,
|
||||
limitInputPixels: this.maxPixels,
|
||||
}).stats();
|
||||
} catch {
|
||||
})
|
||||
.resize(64, 64, { fit: 'inside' })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
if (SERVER_FAULT.test(message)) {
|
||||
this.metrics.increment('attachments.decode.server_error');
|
||||
this.logger.error(
|
||||
`Image decode failed on the server: ${message}`,
|
||||
);
|
||||
throw new InternalServerErrorException();
|
||||
}
|
||||
this.metrics.increment('attachments.decode.rejected');
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
@@ -56,17 +84,36 @@ export class ImageDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<void> {
|
||||
private acquire(signal?: AbortSignal): Promise<void> {
|
||||
const abandoned = new ServiceUnavailableException(
|
||||
'Image check was abandoned',
|
||||
);
|
||||
if (signal?.aborted) return Promise.reject(abandoned);
|
||||
if (this.active < this.maxConcurrent) {
|
||||
this.active += 1;
|
||||
return;
|
||||
return Promise.resolve();
|
||||
}
|
||||
await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const cancel = () => {
|
||||
this.waiting.delete(grant);
|
||||
reject(abandoned);
|
||||
};
|
||||
const grant = () => {
|
||||
signal?.removeEventListener('abort', cancel);
|
||||
resolve();
|
||||
};
|
||||
this.waiting.add(grant);
|
||||
signal?.addEventListener('abort', cancel, { once: true });
|
||||
return promise;
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
const next = this.waiting.shift();
|
||||
if (next) next();
|
||||
else this.active -= 1;
|
||||
const next = this.waiting.values().next();
|
||||
if (next.done) {
|
||||
this.active -= 1;
|
||||
return;
|
||||
}
|
||||
this.waiting.delete(next.value);
|
||||
next.value();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import { crc32 } from 'node:zlib';
|
||||
|
||||
export const IMAGE_HEADER_MAX_BYTES = 1_048_576;
|
||||
|
||||
export type ImageHeader = {
|
||||
contentType: 'image/png' | 'image/jpeg';
|
||||
width: number;
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
/** Request of the upload route; see the onRequest hook in main.ts. */
|
||||
export type UploadRequest = {
|
||||
method: string;
|
||||
routeOptions?: { url?: string };
|
||||
headers: Record<string, string | string[] | undefined>;
|
||||
/** Content-Type as sent by the client (the header itself is rewritten). */
|
||||
uploadContentType?: string | string[];
|
||||
body?: unknown;
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { UploadSlots } from './upload-slots.js';
|
||||
|
||||
const slots = (max: number, perUser: number) =>
|
||||
new UploadSlots({
|
||||
getOrThrow: (key: string) =>
|
||||
key === 'UPLOAD_MAX_IN_FLIGHT' ? max : perUser,
|
||||
} as never);
|
||||
|
||||
function hold() {
|
||||
const { promise, resolve } = Promise.withResolvers<void>();
|
||||
return { work: () => promise, release: resolve };
|
||||
}
|
||||
|
||||
describe('UploadSlots', () => {
|
||||
it('caps one user without starving others, and frees slots after work', async () => {
|
||||
const limiter = slots(10, 2);
|
||||
const held = [hold(), hold()];
|
||||
const running = held.map((h) => limiter.run('a', h.work));
|
||||
await expect(limiter.run('a', async () => 'x')).rejects.toMatchObject({
|
||||
status: 429,
|
||||
});
|
||||
await expect(limiter.run('b', async () => 'ok')).resolves.toBe('ok');
|
||||
held.forEach((h) => h.release());
|
||||
await Promise.all(running);
|
||||
await expect(limiter.run('a', async () => 'again')).resolves.toBe(
|
||||
'again',
|
||||
);
|
||||
});
|
||||
|
||||
it('applies the replica-wide cap across users', async () => {
|
||||
const limiter = slots(2, 2);
|
||||
const held = [hold(), hold()];
|
||||
const running = [
|
||||
limiter.run('a', held[0].work),
|
||||
limiter.run('b', held[1].work),
|
||||
];
|
||||
await expect(limiter.run('c', async () => 'x')).rejects.toMatchObject({
|
||||
status: 429,
|
||||
});
|
||||
held.forEach((h) => h.release());
|
||||
await Promise.all(running);
|
||||
});
|
||||
|
||||
it('releases a slot when the work fails', async () => {
|
||||
const limiter = slots(1, 1);
|
||||
await expect(
|
||||
limiter.run('a', async () => {
|
||||
throw new Error('boom');
|
||||
}),
|
||||
).rejects.toThrow('boom');
|
||||
await expect(limiter.run('a', async () => 'ok')).resolves.toBe('ok');
|
||||
});
|
||||
});
|
||||
@@ -3,29 +3,39 @@ import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
|
||||
/**
|
||||
* Per-replica cap on requests that hold an image in memory (proxy upload,
|
||||
* Per-replica caps on requests that hold an image in memory (proxy upload,
|
||||
* direct complete): each may keep up to MAX_UPLOAD_BYTES for the decode check.
|
||||
* The per-user cap keeps one client (e.g. slow senders holding slots until
|
||||
* UPLOAD_TIMEOUT_MS) from starving everyone else.
|
||||
*/
|
||||
@Injectable()
|
||||
export class UploadSlots {
|
||||
private inFlight = 0;
|
||||
private readonly perUser = new Map<string, number>();
|
||||
private readonly max: number;
|
||||
private readonly maxPerUser: number;
|
||||
|
||||
constructor(config: ConfigService<AppConfig>) {
|
||||
this.max = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT');
|
||||
this.maxPerUser = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT_PER_USER');
|
||||
}
|
||||
|
||||
async run<T>(work: () => Promise<T>): Promise<T> {
|
||||
if (this.inFlight >= this.max)
|
||||
async run<T>(userId: string, work: () => Promise<T>): Promise<T> {
|
||||
const mine = this.perUser.get(userId) ?? 0;
|
||||
if (this.inFlight >= this.max || mine >= this.maxPerUser)
|
||||
throw new HttpException(
|
||||
{ message: 'Too many concurrent uploads', retryAfter: 1 },
|
||||
429,
|
||||
);
|
||||
this.inFlight += 1;
|
||||
this.perUser.set(userId, mine + 1);
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
this.inFlight -= 1;
|
||||
const left = (this.perUser.get(userId) ?? 1) - 1;
|
||||
if (left === 0) this.perUser.delete(userId);
|
||||
else this.perUser.set(userId, left);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,11 +86,17 @@ export const appConfigSchema = z
|
||||
MAX_IMAGE_PIXELS: numeric('MAX_IMAGE_PIXELS', 1, 40000000, 40000000),
|
||||
UPLOAD_TIMEOUT_MS: numeric('UPLOAD_TIMEOUT_MS', 1000, undefined, 25000),
|
||||
UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 16),
|
||||
UPLOAD_MAX_IN_FLIGHT_PER_USER: numeric(
|
||||
'UPLOAD_MAX_IN_FLIGHT_PER_USER',
|
||||
1,
|
||||
1024,
|
||||
4,
|
||||
),
|
||||
IMAGE_DECODE_MAX_CONCURRENT: numeric(
|
||||
'IMAGE_DECODE_MAX_CONCURRENT',
|
||||
1,
|
||||
64,
|
||||
2,
|
||||
1,
|
||||
),
|
||||
// 0 отключает фоновый проход уборщика (остаётся ручной CLI).
|
||||
ATTACHMENT_SWEEP_INTERVAL_MS: numeric(
|
||||
|
||||
+27
-43
@@ -1,12 +1,8 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import {
|
||||
PayloadTooLargeException,
|
||||
UnsupportedMediaTypeException,
|
||||
ValidationPipe,
|
||||
} from '@nestjs/common';
|
||||
import type { Readable } from 'node:stream';
|
||||
import { ValidationPipe } from '@nestjs/common';
|
||||
import type { UploadRequest } from './attachments/upload-request.js';
|
||||
import {
|
||||
FastifyAdapter,
|
||||
NestFastifyApplication,
|
||||
@@ -16,53 +12,41 @@ import { HealthService } from './health/health.service.js';
|
||||
import { MetricsService } from './metrics/metrics.service.js';
|
||||
import { RequestIdInterceptor } from './common/http/request-id.interceptor.js';
|
||||
|
||||
const TEXT_BODY_LIMIT = 1_048_576;
|
||||
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
|
||||
const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream';
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
AppModule,
|
||||
new FastifyAdapter({ logger: true }),
|
||||
);
|
||||
// Raw image bodies reach the controller as an unread stream: the upload is
|
||||
// bounded by our own checks, not by Fastify's bodyLimit. A stream may reach
|
||||
// only the upload route: on JSON routes it would hit the ValidationPipe and
|
||||
// overflow the stack there, so other routes keep the pre-upload behavior.
|
||||
// The upload route receives its body as an unread stream whatever the
|
||||
// declared Content-Type: the service checks the type itself, after the
|
||||
// auth guard (401 before 415, contracts/README.md), and bounds the body by
|
||||
// its own limits, not Fastify's bodyLimit. The declared type is kept on the
|
||||
// request; all other routes keep Fastify's parsers untouched.
|
||||
const fastify = app.getHttpAdapter().getInstance();
|
||||
type ParserRequest = { method: string; routeOptions?: { url?: string } };
|
||||
type Done = (error: Error | null, body?: unknown) => void;
|
||||
const isUpload = (request: ParserRequest) =>
|
||||
request.method === 'POST' && request.routeOptions?.url === UPLOAD_ROUTE;
|
||||
fastify.addContentTypeParser(
|
||||
['image/png', 'image/jpeg'],
|
||||
(request: ParserRequest, payload: unknown, done: Done) =>
|
||||
isUpload(request)
|
||||
? done(null, payload)
|
||||
: done(new UnsupportedMediaTypeException()),
|
||||
);
|
||||
// Fastify's built-in text/plain parser buffers the body and fails binary
|
||||
// input with 400; on the upload route the service answers 415 instead.
|
||||
// Elsewhere text/plain keeps being buffered as a string (1 MiB cap).
|
||||
fastify.removeContentTypeParser('text/plain');
|
||||
fastify.addContentTypeParser(
|
||||
'text/plain',
|
||||
(request: ParserRequest, payload: Readable, done: Done) => {
|
||||
if (isUpload(request)) return done(null, payload);
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
payload.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size > TEXT_BODY_LIMIT) {
|
||||
payload.destroy();
|
||||
done(new PayloadTooLargeException());
|
||||
} else chunks.push(chunk);
|
||||
});
|
||||
payload.on('end', () =>
|
||||
done(null, Buffer.concat(chunks).toString('utf8')),
|
||||
);
|
||||
payload.on('error', (error: Error) => done(error));
|
||||
fastify.addHook(
|
||||
'onRequest',
|
||||
(request: UploadRequest, _reply: unknown, done: () => void) => {
|
||||
if (
|
||||
request.method === 'POST' &&
|
||||
request.routeOptions?.url === UPLOAD_ROUTE
|
||||
) {
|
||||
request.uploadContentType = request.headers['content-type'];
|
||||
request.headers['content-type'] = UPLOAD_STREAM_TYPE;
|
||||
}
|
||||
done();
|
||||
},
|
||||
);
|
||||
fastify.addContentTypeParser(
|
||||
UPLOAD_STREAM_TYPE,
|
||||
(
|
||||
_request: unknown,
|
||||
payload: unknown,
|
||||
done: (error: Error | null, body?: unknown) => void,
|
||||
) => done(null, payload),
|
||||
);
|
||||
app.useGlobalInterceptors(
|
||||
new RequestIdInterceptor(
|
||||
app.get(HttpAdapterHost),
|
||||
|
||||
@@ -62,9 +62,15 @@ export class ObjectStorage {
|
||||
maxAttempts: 1,
|
||||
requestChecksumCalculation: 'WHEN_REQUIRED' as const,
|
||||
responseChecksumValidation: 'WHEN_REQUIRED' as const,
|
||||
// requestTimeout bounds a request until its response arrives and,
|
||||
// with throwOnRequestTimeout, actually fails it (otherwise smithy
|
||||
// only logs a warning); socketTimeout bounds a stalled body read.
|
||||
// A hung S3 therefore costs at most S3_REQUEST_TIMEOUT_MS per call.
|
||||
requestHandler: new NodeHttpHandler({
|
||||
connectionTimeout: config.getOrThrow('S3_CONNECT_TIMEOUT_MS'),
|
||||
requestTimeout: config.getOrThrow('S3_REQUEST_TIMEOUT_MS'),
|
||||
throwOnRequestTimeout: true,
|
||||
socketTimeout: config.getOrThrow('S3_REQUEST_TIMEOUT_MS'),
|
||||
httpAgent: new http.Agent({ keepAlive: true, maxSockets: 64 }),
|
||||
}),
|
||||
};
|
||||
@@ -99,21 +105,23 @@ export class ObjectStorage {
|
||||
ContentLength: contentLength,
|
||||
ContentType: contentType,
|
||||
}),
|
||||
{ abortSignal: signal },
|
||||
// The upload deadline (UPLOAD_TIMEOUT_MS, via `signal`) bounds the
|
||||
// PUT: it lasts as long as the client sends, not S3_REQUEST_TIMEOUT_MS.
|
||||
{ abortSignal: signal, requestTimeout: 0 },
|
||||
);
|
||||
} catch (error) {
|
||||
throw this.fail('put', error);
|
||||
}
|
||||
}
|
||||
|
||||
async getStream(key: string, range?: string): Promise<Readable | null> {
|
||||
async getStream(
|
||||
key: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readable | null> {
|
||||
try {
|
||||
const result = await this.internal.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
Range: range,
|
||||
}),
|
||||
new GetObjectCommand({ Bucket: this.bucket, Key: key }),
|
||||
{ abortSignal: signal },
|
||||
);
|
||||
return result.Body as Readable;
|
||||
} catch (error) {
|
||||
|
||||
@@ -21,8 +21,10 @@ import {
|
||||
upload,
|
||||
uploadRaw,
|
||||
} from './support/e2e.js';
|
||||
import { jpegWithoutScanData } from './support/jpeg.js';
|
||||
import sharp from 'sharp';
|
||||
import { jpegWithMetadata, jpegWithoutScanData } from './support/jpeg.js';
|
||||
import {
|
||||
maxPixelsRgba16Png,
|
||||
noisePng,
|
||||
padPngTo,
|
||||
pixelBombPng,
|
||||
@@ -146,6 +148,106 @@ describe('attachments (proxy upload)', () => {
|
||||
expect(content.bytes.equals(data)).toBe(true);
|
||||
});
|
||||
|
||||
it('accepts camera-size images up to the pixel limit', async () => {
|
||||
// 7728x5152 = 39.8 MP, like a 40 MP camera; decoded size far above the
|
||||
// replica's 16 MiB /tmp, so libvips must not spill to a temp file.
|
||||
const camera = sharp({
|
||||
create: {
|
||||
width: 7728,
|
||||
height: 5152,
|
||||
channels: 3,
|
||||
background: { r: 120, g: 80, b: 40 },
|
||||
},
|
||||
});
|
||||
const baseline = await camera.clone().jpeg({ quality: 85 }).toBuffer();
|
||||
const progressiveCamera = await camera
|
||||
.clone()
|
||||
.jpeg({ quality: 85, progressive: true })
|
||||
.toBuffer();
|
||||
for (const data of [baseline, progressiveCamera])
|
||||
expect((await ready(data, 'image/jpeg')).size_bytes).toBe(
|
||||
data.length,
|
||||
);
|
||||
const rgba16 = await maxPixelsRgba16Png();
|
||||
expect((await ready(rgba16)).size_bytes).toBe(rgba16.length);
|
||||
}, 120_000);
|
||||
|
||||
it('accepts a JPEG with megabytes of metadata before the frame header', async () => {
|
||||
const data = jpegWithMetadata(jpeg, 20);
|
||||
expect(data.length).toBeGreaterThan(1_048_576);
|
||||
expect((await ready(data, 'image/jpeg')).sha256).toBe(sha256(data));
|
||||
});
|
||||
|
||||
it('authenticates before judging the content type', async () => {
|
||||
const zip = await http<ApiErrorBody>(
|
||||
`/api/v1/chats/${chatId}/attachments?filename=a.zip`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/zip' },
|
||||
body: png as unknown as BodyInit,
|
||||
},
|
||||
);
|
||||
expectError(zip, 401, 'unauthorized');
|
||||
expectError(
|
||||
await upload(alice.token, leftoverChatId, png, {
|
||||
contentType: 'application/json',
|
||||
}),
|
||||
415,
|
||||
'unsupported_media_type',
|
||||
);
|
||||
});
|
||||
|
||||
it('answers 413 with the error envelope to an oversized text body on JSON routes', async () => {
|
||||
const result = await http<ApiErrorBody>(
|
||||
`/api/v1/chats/${chatId}/messages`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...bearer(alice.token),
|
||||
'content-type': 'text/plain',
|
||||
'idempotency-key': randomUUID().replaceAll('-', ''),
|
||||
},
|
||||
body: 'x'.repeat(2_100_000),
|
||||
},
|
||||
);
|
||||
expectError(result, 413, 'payload_too_large');
|
||||
});
|
||||
|
||||
it('keeps one slow client from taking every upload slot', async () => {
|
||||
const slow = await actor();
|
||||
const slowChat = (await createChat(slow, [])).id;
|
||||
const url = new URL(baseUrl);
|
||||
const sockets: net.Socket[] = [];
|
||||
try {
|
||||
for (let index = 0; index < 12; index += 1) {
|
||||
const socket = net.connect(
|
||||
Number(url.port || 80),
|
||||
url.hostname,
|
||||
);
|
||||
socket.on('error', () => {});
|
||||
socket.write(
|
||||
`POST /api/v1/chats/${slowChat}/attachments?filename=slow.png HTTP/1.1\r\n` +
|
||||
`Host: ${url.host}\r\nAuthorization: Bearer ${slow.token}\r\n` +
|
||||
`Content-Type: image/png\r\nContent-Length: ${png.length}\r\n\r\n`,
|
||||
);
|
||||
socket.write(png.subarray(0, 4));
|
||||
sockets.push(socket);
|
||||
}
|
||||
await sleep(1000);
|
||||
const extra = await upload(slow.token, slowChat, png);
|
||||
expectError(extra, 429, 'rate_limited');
|
||||
expect(extra.response.headers.get('retry-after')).toBe('1');
|
||||
// Another user is not affected by the slow client.
|
||||
for (let index = 0; index < 4; index += 1)
|
||||
expect(
|
||||
(await upload(bob.token, chatId, png)).response.status,
|
||||
).toBe(201);
|
||||
} finally {
|
||||
for (const socket of sockets) socket.destroy();
|
||||
}
|
||||
await settled(slowChat);
|
||||
});
|
||||
|
||||
it('enforces the 10 MiB limit exactly', async () => {
|
||||
const exact = padPngTo(png, MAX);
|
||||
const created = await ready(exact);
|
||||
|
||||
@@ -602,6 +602,10 @@ describe.skipIf(!enabled)('dependency outages', () => {
|
||||
await compose('start', redisService);
|
||||
await compose('start', postgresService);
|
||||
await compose('start', s3Service);
|
||||
const s3Container = (await compose('ps', '-q', s3Service)).split(
|
||||
'\n',
|
||||
)[0];
|
||||
await docker('unpause', s3Container).catch(() => undefined);
|
||||
await waitForReadiness(200);
|
||||
await restoreDatabaseAccess();
|
||||
const expected = (await apiContainers()).length;
|
||||
@@ -986,6 +990,62 @@ describe.skipIf(!enabled)('dependency outages', () => {
|
||||
outagedRequestBudgetMs,
|
||||
);
|
||||
|
||||
it(
|
||||
'bounds requests by S3_REQUEST_TIMEOUT_MS when S3 hangs instead of refusing',
|
||||
async () => {
|
||||
const created = await request<{ id: string }>(
|
||||
uploadPath(chat.id, `hang-${runId}.png`),
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { ...bearer(token), 'content-type': 'image/png' },
|
||||
body: samplePng as unknown as BodyInit,
|
||||
},
|
||||
);
|
||||
expect(created.response.status).toBe(201);
|
||||
// A paused container keeps its TCP port open but never answers: the
|
||||
// failure mode a stopped container (connection refused) does not show.
|
||||
const container = (await compose('ps', '-q', s3Service)).split(
|
||||
'\n',
|
||||
)[0];
|
||||
await docker('pause', container);
|
||||
try {
|
||||
const started = Date.now();
|
||||
const content = await request<ApiErrorBody>(
|
||||
`/api/v1/attachments/${created.body.id}/content`,
|
||||
{ headers: bearer(token) },
|
||||
);
|
||||
const elapsed = Date.now() - started;
|
||||
expectBoundedOutage(content);
|
||||
// S3_REQUEST_TIMEOUT_MS is 15 s in the stand; HAProxy gives up at 30 s.
|
||||
expect(elapsed).toBeLessThan(20_000);
|
||||
expect(
|
||||
(
|
||||
await request(
|
||||
`/api/v1/attachments/${created.body.id}`,
|
||||
{
|
||||
headers: bearer(token),
|
||||
},
|
||||
)
|
||||
).response.status,
|
||||
).toBe(200);
|
||||
expect((await request('/health/ready')).response.status).toBe(
|
||||
200,
|
||||
);
|
||||
} finally {
|
||||
await docker('unpause', container);
|
||||
}
|
||||
const bytes = await fetch(
|
||||
`${baseUrl}/api/v1/attachments/${created.body.id}/content`,
|
||||
{ headers: bearer(token) },
|
||||
);
|
||||
expect(bytes.status).toBe(200);
|
||||
expect(
|
||||
Buffer.from(await bytes.arrayBuffer()).equals(samplePng),
|
||||
).toBe(true);
|
||||
},
|
||||
outagedRequestBudgetMs,
|
||||
);
|
||||
|
||||
it(
|
||||
'keeps the object while PostgreSQL is down before ready and sweeps it afterwards',
|
||||
async () => {
|
||||
|
||||
@@ -160,6 +160,21 @@ describe('direct upload (presigned PUT)', () => {
|
||||
expect(sent.body.attachment_ids).toEqual([ticket.attachment_id]);
|
||||
});
|
||||
|
||||
it('answers 200 to every parallel complete of one upload', async () => {
|
||||
const ticket = await initOk(png);
|
||||
expect((await put(ticket, png)).status).toBe(200);
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 4 }, () =>
|
||||
complete(alice.token, ticket.attachment_id),
|
||||
),
|
||||
);
|
||||
expect(results.map((result) => result.response.status)).toEqual([
|
||||
200, 200, 200, 200,
|
||||
]);
|
||||
for (const result of results)
|
||||
expect(result.body.id).toBe(ticket.attachment_id);
|
||||
});
|
||||
|
||||
it('rejects invalid declarations', async () => {
|
||||
const base = declaration(png);
|
||||
expectError(await init(eve.token, chatId, base), 404, 'not_found');
|
||||
|
||||
@@ -8,3 +8,17 @@ export function jpegWithoutScanData(jpeg: Buffer): Buffer {
|
||||
}
|
||||
return Buffer.concat([jpeg.subarray(0, offset), Buffer.from([0xff, 0xd9])]);
|
||||
}
|
||||
|
||||
/** Inserts `count` APP13 segments of ~64 KiB right after SOI (Photoshop-style metadata). */
|
||||
export function jpegWithMetadata(jpeg: Buffer, count: number): Buffer {
|
||||
const payload = Buffer.alloc(65_533, 0x20);
|
||||
const segment = Buffer.concat([
|
||||
Buffer.from([0xff, 0xed, 0xff, 0xff]),
|
||||
payload,
|
||||
]);
|
||||
return Buffer.concat([
|
||||
jpeg.subarray(0, 2),
|
||||
...Array.from({ length: count }, () => segment),
|
||||
jpeg.subarray(2),
|
||||
]);
|
||||
}
|
||||
|
||||
+35
-1
@@ -1,5 +1,6 @@
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { crc32, deflateSync } from 'node:zlib';
|
||||
import { once } from 'node:events';
|
||||
import { crc32, createDeflate, deflateSync } from 'node:zlib';
|
||||
|
||||
const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
|
||||
@@ -104,3 +105,36 @@ export function pngWithGarbageIdat(): Buffer {
|
||||
idat: randomBytes(5000),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 8000x5000 RGBA 16-bit (8 bytes/pixel, 40 000 000 px — the pixel limit):
|
||||
* the heaviest decode a valid upload can ask for. Rows are streamed into the
|
||||
* compressor, so the 320 MB raw image is never allocated here.
|
||||
*/
|
||||
export async function maxPixelsRgba16Png(): Promise<Buffer> {
|
||||
const width = 8000;
|
||||
const height = 5000;
|
||||
// First row: a gradient; every next row uses filter "Up" with zero
|
||||
// deltas, i.e. repeats it — a valid image that compresses well.
|
||||
const first = Buffer.alloc(1 + width * 8);
|
||||
for (let x = 0; x < width; x += 1)
|
||||
first.writeUInt16BE((x * 8) & 0xffff, 1 + x * 8);
|
||||
const repeat = Buffer.alloc(1 + width * 8);
|
||||
repeat[0] = 2;
|
||||
const deflate = createDeflate({ level: 1 });
|
||||
const chunks: Buffer[] = [];
|
||||
deflate.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
const ended = once(deflate, 'end');
|
||||
for (let y = 0; y < height; y += 1)
|
||||
if (!deflate.write(y === 0 ? first : repeat))
|
||||
await once(deflate, 'drain');
|
||||
deflate.end();
|
||||
await ended;
|
||||
return buildPng({
|
||||
width,
|
||||
height,
|
||||
colorType: 6,
|
||||
bitDepth: 16,
|
||||
idat: Buffer.concat(chunks),
|
||||
});
|
||||
}
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
"emitDecoratorMetadata": true,
|
||||
"experimentalDecorators": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"target": "ES2023",
|
||||
"target": "ES2024",
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"incremental": true,
|
||||
|
||||
+4
-1
@@ -169,6 +169,9 @@ services:
|
||||
DIRECT_UPLOAD_URL_TTL_SECONDS: ${DIRECT_UPLOAD_URL_TTL_SECONDS:-300}
|
||||
ATTACHMENT_SWEEP_GRACE_SECONDS: ${ATTACHMENT_SWEEP_GRACE_SECONDS:-900}
|
||||
ATTACHMENT_SWEEP_INTERVAL_MS: ${ATTACHMENT_SWEEP_INTERVAL_MS:-60000}
|
||||
# libvips держит декодируемое изображение в памяти, а не во временном
|
||||
# файле: /tmp — tmpfs 16 MiB, и изображение ~40 Мп туда не помещается.
|
||||
VIPS_DISC_THRESHOLD: ${VIPS_DISC_THRESHOLD:-1g}
|
||||
secrets: [s3_api_access_key_id, s3_api_secret_access_key]
|
||||
# Корневая ФС только для чтения: локального файла оригинала на API нет.
|
||||
read_only: true
|
||||
@@ -178,7 +181,7 @@ services:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '${API_CPUS_PER_REPLICA:-1.5}'
|
||||
memory: ${API_MEMORY_PER_REPLICA:-768m}
|
||||
memory: ${API_MEMORY_PER_REPLICA:-1024m}
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "MTUSI Messenger — расширение lab4: прямая загрузка",
|
||||
"version": "1.4.0-ext.1",
|
||||
"description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments/<attachment_id>/original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые заявки и объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)."
|
||||
"version": "1.4.0-ext.2",
|
||||
"description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments/<attachment_id>/original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые и отклонённые заявки и их объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)."
|
||||
},
|
||||
"servers": [
|
||||
{
|
||||
@@ -264,8 +264,8 @@
|
||||
"/api/v1/attachments/{attachment_id}/complete": {
|
||||
"post": {
|
||||
"operationId": "completeDirectUpload",
|
||||
"summary": "Подтвердить прямую загрузку: HEAD объекта и проверка заголовка изображения",
|
||||
"description": "Только владелец заявки. Проверяет размер и подписанный SHA-256 объекта (HEAD), читает заголовок изображения (до 1 MiB) и сверяет тип и число пикселей. При несоответствии заявка и объект удаляются, повторный complete даёт 404. Повторный complete готового вложения идемпотентен и возвращает 200.",
|
||||
"summary": "Подтвердить прямую загрузку: HEAD объекта, полное чтение и декод изображения",
|
||||
"description": "Только владелец заявки. Сверяет размер и подписанный SHA-256 объекта (HEAD), затем читает объект целиком: структурная проверка PNG/JPEG и полный декод (повреждённые и усечённые изображения отвергаются, даже если checksum совпадает). При несоответствии объект удаляется, а заявка получает внутренний статус rejected до истечения URL и grace-периода уборщика (повторный PUT по ещё действующему URL не оставит неучтённый объект); повторный complete отклонённой заявки — 404. Если объект ещё не загружен — 409, заявка не меняется. Повторный и параллельный complete готового вложения идемпотентен (200).",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "attachment_id",
|
||||
@@ -461,7 +461,7 @@
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Хранилище недоступно",
|
||||
"description": "Хранилище недоступно или не ответило за S3_REQUEST_TIMEOUT_MS / UPLOAD_TIMEOUT_MS",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
@@ -485,6 +485,39 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"429": {
|
||||
"description": "Реплика проверяет слишком много изображений одновременно (UPLOAD_MAX_IN_FLIGHT) или у пользователя слишком много одновременных проверок (UPLOAD_MAX_IN_FLIGHT_PER_USER); Retry-After — секунды до повтора.",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"Retry-After": {
|
||||
"description": "Секунды до повтора.",
|
||||
"schema": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
# inputs: magick -size 6320x6320 gradient:red-blue (PNG); same as JPEG q85 baseline and -interlace JPEG
|
||||
png 6320x6320 bytes=53350 decode_ms=1159 peak_delta_rss_mib=116.8
|
||||
jpeg 6320x6320 bytes=4899767 decode_ms=1234 peak_delta_rss_mib=126.5
|
||||
jpeg 6320x6320 progressive bytes=4791710 decode_ms=1379 peak_delta_rss_mib=130.5
|
||||
# container /mtusi-lab4-api-1, limits: 805306368 bytes, VIPS_DISC_THRESHOLD=1g; commit a06471a+worktree
|
||||
jpeg 7728x5152 3ch uchar bytes=4921012 decode_ms=58 peak_delta_rss_mib=3.9 ok
|
||||
jpeg 7728x5152 3ch uchar progressive bytes=4826968 decode_ms=278 peak_delta_rss_mib=117.7 ok
|
||||
jpeg 7728x5152 4ch uchar bytes=10601868 decode_ms=161 peak_delta_rss_mib=7.7 ok
|
||||
png 8000x5000 4ch ushort bytes=1420114 decode_ms=496 peak_delta_rss_mib=66.6 ok
|
||||
png 8000x5000 4ch ushort progressive bytes=364096 decode_ms=1270 peak_delta_rss_mib=338.6 ok
|
||||
png 8000x5000 3ch uchar progressive palette bytes=61042 decode_ms=341 peak_delta_rss_mib=126.0 ok
|
||||
|
||||
@@ -141,7 +141,14 @@ def one_upload(env, mode, token, chat_id, data, seed):
|
||||
return ok, elapsed_ms
|
||||
|
||||
|
||||
def run_once(env, args, token, chat_id, ids):
|
||||
def payload(args, seed):
|
||||
if args.image:
|
||||
with open(args.image, "rb") as handle:
|
||||
return handle.read()
|
||||
return noise_png(args.size_bytes, seed)
|
||||
|
||||
|
||||
def run_once(env, args, actors, ids):
|
||||
for container in ids:
|
||||
reset_metrics(container)
|
||||
time.sleep(0.2)
|
||||
@@ -153,7 +160,8 @@ def run_once(env, args, token, chat_id, ids):
|
||||
def worker(index):
|
||||
for step in range(args.uploads_per_worker):
|
||||
seed = index * 1000 + step
|
||||
data = noise_png(args.size_bytes, seed)
|
||||
data = payload(args, seed)
|
||||
token, chat_id = actors[index % len(actors)]
|
||||
ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed)
|
||||
with lock:
|
||||
(latencies if ok else failures).append(elapsed)
|
||||
@@ -203,29 +211,39 @@ def run_once(env, args, token, chat_id, ids):
|
||||
|
||||
|
||||
def command_run(args):
|
||||
if args.image:
|
||||
args.size_bytes = os.path.getsize(args.image)
|
||||
if args.size_bytes <= 0:
|
||||
sys.exit("--size-bytes or --image is required")
|
||||
env = read_env()
|
||||
ids = containers()
|
||||
if not ids:
|
||||
sys.exit("no api containers found")
|
||||
name = f"bench_{uuid.uuid4().hex[:10]}"
|
||||
password = "bench:" + uuid.uuid4().hex
|
||||
status, user = api_json(env, "POST", "/api/v1/users",
|
||||
body={"username": name, "display_name": "bench", "password": password})
|
||||
assert status == 201, status
|
||||
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
|
||||
assert status == 201, status
|
||||
token = session["token"]
|
||||
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
|
||||
assert status == 201, status
|
||||
# One user per worker: the API caps concurrent uploads per user
|
||||
# (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users.
|
||||
actors = []
|
||||
for _ in range(args.concurrency):
|
||||
name = f"bench_{uuid.uuid4().hex[:10]}"
|
||||
password = "bench:" + uuid.uuid4().hex
|
||||
status, user = api_json(env, "POST", "/api/v1/users",
|
||||
body={"username": name, "display_name": "bench", "password": password})
|
||||
assert status == 201, status
|
||||
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
|
||||
assert status == 201, status
|
||||
token = session["token"]
|
||||
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
|
||||
assert status == 201, status
|
||||
actors.append((token, chat["id"]))
|
||||
|
||||
# Warm-up: one upload per replica path, not measured.
|
||||
# Warm-up: two uploads per replica, not measured.
|
||||
for seed in range(len(ids) * 2):
|
||||
one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed)
|
||||
token, chat_id = actors[seed % len(actors)]
|
||||
one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed)
|
||||
|
||||
limits = []
|
||||
for c in ids:
|
||||
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
|
||||
repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)]
|
||||
repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)]
|
||||
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
|
||||
result = {
|
||||
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
@@ -233,6 +251,7 @@ def command_run(args):
|
||||
"params": {
|
||||
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
|
||||
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
|
||||
"users": args.concurrency, "image": os.path.basename(args.image) if args.image else None,
|
||||
},
|
||||
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
|
||||
"api_container_limits_nanocpus_memory": limits,
|
||||
@@ -260,17 +279,19 @@ def command_summarize(args):
|
||||
p = data["params"]
|
||||
reps = data["repeats"]
|
||||
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
|
||||
rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data))
|
||||
rows.sort(key=lambda r: (r[1], r[0], r[2]))
|
||||
print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |")
|
||||
print("|---|---|---|---|---|---|---|---|---|")
|
||||
label = p["mode"] + (f" ({p['image']})" if p.get("image") else "")
|
||||
rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data))
|
||||
rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2]))
|
||||
print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |")
|
||||
print("|---|---|---|---|---|---|---|---|---|---|")
|
||||
for mode, size, conc, reps, med, _data in rows:
|
||||
peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"])
|
||||
ok = sum(r["succeeded"] for r in reps)
|
||||
total = sum(r["uploads"] for r in reps)
|
||||
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
|
||||
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
|
||||
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | "
|
||||
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |")
|
||||
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |")
|
||||
|
||||
|
||||
def main():
|
||||
@@ -278,7 +299,8 @@ def main():
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
run = sub.add_parser("run")
|
||||
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
|
||||
run.add_argument("--size-bytes", type=int, required=True)
|
||||
run.add_argument("--size-bytes", type=int, default=0)
|
||||
run.add_argument("--image", help="upload this file instead of generated noise PNGs")
|
||||
run.add_argument("--concurrency", type=int, default=10)
|
||||
run.add_argument("--uploads-per-worker", type=int, default=5)
|
||||
run.add_argument("--repeat", type=int, default=3)
|
||||
|
||||
@@ -1,22 +1,33 @@
|
||||
// Peak RSS of one full decode check (same sharp options as ImageDecoder) on
|
||||
// worst-case inputs near MAX_IMAGE_PIXELS. Run from app/: node ../scripts/lab4/decode_memory.mjs <files...>
|
||||
// Peak RSS of one full decode check with the same sharp pipeline as
|
||||
// app/src/attachments/image/image-decoder.ts, on worst-case inputs near
|
||||
// MAX_IMAGE_PIXELS. Runs where the API runs (same libvips, VIPS_DISC_THRESHOLD):
|
||||
// docker compose exec -T api node - < scripts/lab4/decode_memory.mjs # reads files from IMAGES (paths in the container)
|
||||
// or on the host from app/: IMAGES="a.png b.jpg" node ../scripts/lab4/decode_memory.mjs
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
const sharp = createRequire(`${process.cwd()}/`)('sharp');
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
for (const file of process.argv.slice(2)) {
|
||||
for (const file of (process.env.IMAGES ?? '').split(/\s+/u).filter(Boolean)) {
|
||||
const data = readFileSync(file);
|
||||
const meta = await sharp(data).metadata();
|
||||
const base = process.memoryUsage().rss;
|
||||
let peak = base;
|
||||
const timer = setInterval(() => {
|
||||
peak = Math.max(peak, process.memoryUsage().rss);
|
||||
}, 10);
|
||||
}, 5);
|
||||
const started = Date.now();
|
||||
await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 }).stats();
|
||||
let verdict = 'ok';
|
||||
try {
|
||||
await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 })
|
||||
.resize(64, 64, { fit: 'inside' })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
verdict = `rejected: ${String(error.message).split('\n')[0]}`;
|
||||
}
|
||||
clearInterval(timer);
|
||||
console.log(
|
||||
`${meta.format} ${meta.width}x${meta.height}${meta.isProgressive ? ' progressive' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)}`,
|
||||
`${meta.format} ${meta.width}x${meta.height} ${meta.channels}ch ${meta.depth}${meta.isProgressive ? ' progressive' : ''}${meta.isPalette ? ' palette' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)} ${verdict}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -48,3 +48,31 @@ def noise_png(size: int, seed: int = 0) -> bytes:
|
||||
raise ValueError("size is too small")
|
||||
continue
|
||||
return pad_png(png, size)
|
||||
|
||||
|
||||
ADAM7 = [(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)]
|
||||
|
||||
|
||||
def worst_decode_png(size: int = 10_485_760, width: int = 8000, height: int = 5000) -> bytes:
|
||||
"""The heaviest valid upload to decode: 40 MP RGBA 16-bit, Adam7-interlaced
|
||||
(the decoder must hold the whole image), padded to exactly `size` bytes."""
|
||||
compressor = zlib.compressobj(1)
|
||||
parts = []
|
||||
for x0, y0, dx, dy in ADAM7:
|
||||
pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if pass_width == 0 or pass_height == 0:
|
||||
continue
|
||||
row = bytes(1 + pass_width * 8)
|
||||
for _ in range(pass_height):
|
||||
parts.append(compressor.compress(row))
|
||||
parts.append(compressor.flush())
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 16, 6, 0, 0, 1)
|
||||
png = SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"".join(parts)) + chunk(b"IEND", b"")
|
||||
return pad_png(png, size)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.stdout.buffer.write(worst_decode_png())
|
||||
|
||||
@@ -19,5 +19,17 @@ for size in 131072 1048576 9961472; do
|
||||
done
|
||||
echo "== proxy size=9961472 c=20"
|
||||
bench proxy 9961472 20
|
||||
# Worst case for memory: every request carries the heaviest valid image
|
||||
# (40 MP RGBA16 Adam7 PNG padded to 10 MiB, ~340 MiB per decode) and C=32
|
||||
# saturates UPLOAD_MAX_IN_FLIGHT (16) on both replicas.
|
||||
worst=$(mktemp --suffix=.png)
|
||||
trap 'rm -f "$worst"' EXIT
|
||||
python3 scripts/lab4/pngkit.py > "$worst"
|
||||
for mode in proxy direct; do
|
||||
echo "== $mode worst-case image c=32"
|
||||
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --image "$worst" \
|
||||
--concurrency 32 --uploads-per-worker 2 --repeat 3 \
|
||||
--output "$out/$mode-worst-c32.json"
|
||||
done
|
||||
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
|
||||
cat evidence/lab4/summary.md
|
||||
|
||||
Reference in New Issue
Block a user