forked from mtusi2026/mtusi2026lab4
lab4: прямая загрузка, скрипты экспериментов и проверок
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
Headers,
|
||||
@@ -20,6 +21,8 @@ import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
import { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
|
||||
import { DirectUploadsService } from './direct-uploads.service.js';
|
||||
import { contentDisposition } from './filename.js';
|
||||
|
||||
@Controller('api/v1')
|
||||
@@ -29,6 +32,7 @@ export class AttachmentsController {
|
||||
|
||||
constructor(
|
||||
private readonly attachments: AttachmentsService,
|
||||
private readonly directUploads: DirectUploadsService,
|
||||
config: ConfigService<AppConfig>,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
@@ -58,6 +62,25 @@ export class AttachmentsController {
|
||||
);
|
||||
}
|
||||
|
||||
@Post('chats/:chatId/attachments/direct-uploads')
|
||||
@HttpCode(201)
|
||||
initDirectUpload(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: CreateDirectUploadDto,
|
||||
) {
|
||||
return this.directUploads.init(chatId, user, body);
|
||||
}
|
||||
|
||||
@Post('attachments/:attachmentId/complete')
|
||||
@HttpCode(200)
|
||||
completeDirectUpload(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
) {
|
||||
return this.directUploads.complete(attachmentId, user);
|
||||
}
|
||||
|
||||
@Get('attachments/:attachmentId')
|
||||
metadata(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
|
||||
@@ -8,6 +8,7 @@ import { StorageModule } from '../storage/storage.module.js';
|
||||
import { AttachmentSweeper } from './attachment-sweeper.service.js';
|
||||
import { AttachmentsController } from './attachments.controller.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
import { DirectUploadsService } from './direct-uploads.service.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
|
||||
@Module({
|
||||
@@ -20,7 +21,12 @@ import { AttachmentsService } from './attachments.service.js';
|
||||
StorageModule,
|
||||
],
|
||||
controllers: [AttachmentsController],
|
||||
providers: [AttachmentsRepository, AttachmentsService, AttachmentSweeper],
|
||||
providers: [
|
||||
AttachmentsRepository,
|
||||
AttachmentsService,
|
||||
DirectUploadsService,
|
||||
AttachmentSweeper,
|
||||
],
|
||||
exports: [AttachmentsRepository],
|
||||
})
|
||||
export class AttachmentsModule {}
|
||||
|
||||
@@ -279,7 +279,7 @@ export class AttachmentsService {
|
||||
return error;
|
||||
}
|
||||
|
||||
private async compensate(id: string, key: string): Promise<void> {
|
||||
async compensate(id: string, key: string): Promise<void> {
|
||||
try {
|
||||
await this.storage.delete(key);
|
||||
await this.repository.deletePending(id);
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
import {
|
||||
ConflictException,
|
||||
HttpException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
PayloadTooLargeException,
|
||||
UnsupportedMediaTypeException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import { toAttachment, type Attachment } from './attachment.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
import { AttachmentsService, imageRejection } from './attachments.service.js';
|
||||
import type { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
|
||||
import { validateFilename } from './filename.js';
|
||||
import {
|
||||
IMAGE_HEADER_MAX_BYTES,
|
||||
ImageInspector,
|
||||
ImageRejectedError,
|
||||
} from './image/image-inspector.js';
|
||||
|
||||
export type DirectUpload = {
|
||||
attachment_id: string;
|
||||
method: 'PUT';
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
expires_at: string;
|
||||
};
|
||||
|
||||
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
|
||||
|
||||
@Injectable()
|
||||
export class DirectUploadsService {
|
||||
private readonly maxUploadBytes: number;
|
||||
private readonly maxPixels: number;
|
||||
private readonly ttlSeconds: number;
|
||||
private readonly maxPending: number;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly chats: ChatsService,
|
||||
private readonly metrics: MetricsService,
|
||||
private readonly attachments: AttachmentsService,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.ttlSeconds = config.getOrThrow('DIRECT_UPLOAD_URL_TTL_SECONDS');
|
||||
this.maxPending = config.getOrThrow(
|
||||
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
|
||||
);
|
||||
}
|
||||
|
||||
async init(
|
||||
chatId: string,
|
||||
user: User,
|
||||
dto: CreateDirectUploadDto,
|
||||
): Promise<DirectUpload> {
|
||||
const filename = validateFilename(dto.filename);
|
||||
if (!MEDIA_TYPES.has(dto.content_type))
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'content_type must be image/png or image/jpeg',
|
||||
);
|
||||
if (dto.size_bytes > this.maxUploadBytes)
|
||||
throw new PayloadTooLargeException('Image is too large');
|
||||
await this.chats.getForUser(chatId, user);
|
||||
if (
|
||||
(await this.repository.countPendingDirect(user.id)) >=
|
||||
this.maxPending
|
||||
)
|
||||
throw new HttpException(
|
||||
{
|
||||
message: 'Too many unfinished direct uploads',
|
||||
retryAfter: this.ttlSeconds,
|
||||
},
|
||||
429,
|
||||
);
|
||||
|
||||
const id = randomUUID();
|
||||
const key = this.storage.objectKey(id);
|
||||
await this.repository.insertPending({
|
||||
id,
|
||||
chatId,
|
||||
ownerId: user.id,
|
||||
filename,
|
||||
contentType: dto.content_type,
|
||||
uploadMode: 'direct',
|
||||
objectKey: key,
|
||||
sizeBytes: dto.size_bytes,
|
||||
sha256: dto.sha256,
|
||||
ttlMs: this.ttlSeconds * 1000,
|
||||
});
|
||||
const checksum = Buffer.from(dto.sha256, 'hex').toString('base64');
|
||||
let url: string;
|
||||
try {
|
||||
url = await this.storage.presignPut(key, {
|
||||
contentType: dto.content_type,
|
||||
contentLength: dto.size_bytes,
|
||||
checksumSha256Base64: checksum,
|
||||
expiresInSeconds: this.ttlSeconds,
|
||||
});
|
||||
} catch (error) {
|
||||
await this.repository.deletePending(id);
|
||||
throw error;
|
||||
}
|
||||
this.metrics.increment('attachments.direct.initiated');
|
||||
return {
|
||||
attachment_id: id,
|
||||
method: 'PUT',
|
||||
url,
|
||||
headers: {
|
||||
'Content-Type': dto.content_type,
|
||||
'x-amz-checksum-sha256': checksum,
|
||||
},
|
||||
expires_at: new Date(
|
||||
Date.now() + this.ttlSeconds * 1000,
|
||||
).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async complete(attachmentId: string, user: User): Promise<Attachment> {
|
||||
const row = await this.repository.findForComplete(
|
||||
attachmentId,
|
||||
user.id,
|
||||
);
|
||||
if (!row) throw new NotFoundException();
|
||||
if (row.status === 'ready') return toAttachment(row);
|
||||
|
||||
const size = Number(row.size_bytes);
|
||||
const expectedChecksum = Buffer.from(
|
||||
row.sha256 as string,
|
||||
'hex',
|
||||
).toString('base64');
|
||||
const head = await this.storage.head(row.object_key);
|
||||
if (head === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
if (head.size !== size || head.checksumSha256 !== expectedChecksum) {
|
||||
await this.attachments.compensate(row.id, row.object_key);
|
||||
throw new ConflictException(
|
||||
'Uploaded object does not match the declaration',
|
||||
);
|
||||
}
|
||||
|
||||
// Only the header is read: the signed checksum already pins every byte.
|
||||
const inspector = new ImageInspector(this.maxPixels);
|
||||
const covered = size <= IMAGE_HEADER_MAX_BYTES;
|
||||
const stream = await this.storage.getStream(
|
||||
row.object_key,
|
||||
`bytes=0-${Math.min(size, IMAGE_HEADER_MAX_BYTES) - 1}`,
|
||||
);
|
||||
if (stream === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
let verified = 0;
|
||||
try {
|
||||
for await (const chunk of stream as AsyncIterable<Buffer>) {
|
||||
verified += chunk.length;
|
||||
inspector.push(chunk);
|
||||
if (!covered && inspector.header !== undefined) break;
|
||||
}
|
||||
if (covered) inspector.finish();
|
||||
else if (inspector.header === undefined)
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Image header is missing',
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ImageRejectedError) {
|
||||
await this.attachments.compensate(row.id, row.object_key);
|
||||
throw imageRejection(error);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
stream.destroy();
|
||||
this.metrics.add('attachments.direct.verify_bytes', verified);
|
||||
}
|
||||
const header = inspector.header;
|
||||
if (header === undefined || header.contentType !== row.content_type) {
|
||||
await this.attachments.compensate(row.id, row.object_key);
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Image content does not match content_type',
|
||||
);
|
||||
}
|
||||
const ready = await this.repository.markReady(row.id, {
|
||||
sizeBytes: size,
|
||||
sha256: row.sha256 as string,
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
});
|
||||
if (!ready) throw new ConflictException('Upload is no longer pending');
|
||||
this.metrics.increment('attachments.direct.completed');
|
||||
return toAttachment(ready);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { IsInt, IsString, Matches, Min } from 'class-validator';
|
||||
|
||||
export class CreateDirectUploadDto {
|
||||
@IsString()
|
||||
filename!: string;
|
||||
|
||||
@IsString()
|
||||
content_type!: string;
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
size_bytes!: number;
|
||||
|
||||
@Matches(/^[a-f0-9]{64}$/u)
|
||||
sha256!: string;
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import type { Pool } from 'pg';
|
||||
import type { Attachment } from '../src/attachments/attachment.js';
|
||||
import type { DirectUpload } from '../src/attachments/direct-uploads.service.js';
|
||||
import type { ApiErrorBody } from '../src/common/errors/api-error.js';
|
||||
import {
|
||||
Actor,
|
||||
actor,
|
||||
bearer,
|
||||
createChat,
|
||||
database,
|
||||
download,
|
||||
expectError,
|
||||
http,
|
||||
json,
|
||||
postMessage,
|
||||
} from './support/e2e.js';
|
||||
import { pixelBombPng } from './support/png.js';
|
||||
|
||||
const png = readFileSync(
|
||||
new URL('../../tests/fixtures/sample.png', import.meta.url),
|
||||
);
|
||||
const jpeg = readFileSync(new URL('./fixtures/sample.jpg', import.meta.url));
|
||||
const MAX = 10_485_760;
|
||||
const shortTtl = process.env.E2E_DIRECT_TTL_SHORT === '1';
|
||||
|
||||
let db: Pool;
|
||||
let alice: Actor;
|
||||
let bob: Actor;
|
||||
let eve: Actor;
|
||||
let chatId: string;
|
||||
|
||||
const sha256 = (data: Uint8Array) =>
|
||||
createHash('sha256').update(data).digest('hex');
|
||||
|
||||
function init(token: string, chat: string, body: Record<string, unknown>) {
|
||||
return http<DirectUpload & ApiErrorBody>(
|
||||
`/api/v1/chats/${chat}/attachments/direct-uploads`,
|
||||
json(body, bearer(token)),
|
||||
);
|
||||
}
|
||||
|
||||
function declaration(data: Buffer, contentType = 'image/png') {
|
||||
return {
|
||||
filename: 'direct.png',
|
||||
content_type: contentType,
|
||||
size_bytes: data.length,
|
||||
sha256: sha256(data),
|
||||
};
|
||||
}
|
||||
|
||||
async function initOk(data: Buffer, contentType = 'image/png') {
|
||||
const result = await init(
|
||||
alice.token,
|
||||
chatId,
|
||||
declaration(data, contentType),
|
||||
);
|
||||
expect(result.response.status).toBe(201);
|
||||
return result.body;
|
||||
}
|
||||
|
||||
// The PUT goes straight to the public S3 endpoint, not through the API.
|
||||
async function put(ticket: DirectUpload, data: Buffer): Promise<Response> {
|
||||
return fetch(ticket.url, {
|
||||
method: ticket.method,
|
||||
headers: ticket.headers,
|
||||
body: data as unknown as BodyInit,
|
||||
});
|
||||
}
|
||||
|
||||
function complete(token: string, id: string) {
|
||||
return http<Attachment & ApiErrorBody>(
|
||||
`/api/v1/attachments/${id}/complete`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: bearer(token),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
describe('direct upload (presigned PUT)', () => {
|
||||
beforeAll(async () => {
|
||||
db = database();
|
||||
alice = await actor();
|
||||
bob = await actor();
|
||||
eve = await actor();
|
||||
chatId = (await createChat(alice, [bob])).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db.end();
|
||||
});
|
||||
|
||||
it('uploads around the API and exposes the attachment after complete', async () => {
|
||||
const ticket = await initOk(png);
|
||||
expect(ticket.method).toBe('PUT');
|
||||
expect(ticket.headers['x-amz-checksum-sha256']).toBe(
|
||||
Buffer.from(sha256(png), 'hex').toString('base64'),
|
||||
);
|
||||
expect(Date.parse(ticket.expires_at)).toBeGreaterThan(Date.now());
|
||||
expect(new URL(ticket.url).pathname).toBe(
|
||||
`/messenger-attachments/attachments/${ticket.attachment_id}/original`,
|
||||
);
|
||||
|
||||
const hidden = await http<ApiErrorBody>(
|
||||
`/api/v1/attachments/${ticket.attachment_id}`,
|
||||
{ headers: bearer(bob.token) },
|
||||
);
|
||||
expectError(hidden, 404, 'not_found');
|
||||
expectError(
|
||||
await postMessage(alice.token, chatId, {
|
||||
text: '',
|
||||
attachment_ids: [ticket.attachment_id],
|
||||
}),
|
||||
404,
|
||||
'not_found',
|
||||
);
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
409,
|
||||
'conflict',
|
||||
);
|
||||
|
||||
expect((await put(ticket, png)).status).toBe(200);
|
||||
expectError(
|
||||
await complete(bob.token, ticket.attachment_id),
|
||||
404,
|
||||
'not_found',
|
||||
);
|
||||
expectError(
|
||||
await complete(eve.token, ticket.attachment_id),
|
||||
404,
|
||||
'not_found',
|
||||
);
|
||||
const done = await complete(alice.token, ticket.attachment_id);
|
||||
expect(done.response.status).toBe(200);
|
||||
expect(done.body).toMatchObject({
|
||||
id: ticket.attachment_id,
|
||||
owner_id: alice.user.id,
|
||||
content_type: 'image/png',
|
||||
size_bytes: png.length,
|
||||
sha256: sha256(png),
|
||||
status: 'ready',
|
||||
});
|
||||
const again = await complete(alice.token, ticket.attachment_id);
|
||||
expect(again.response.status).toBe(200);
|
||||
expect(again.body).toEqual(done.body);
|
||||
|
||||
const content = await download(bob.token, ticket.attachment_id);
|
||||
expect(content.response.status).toBe(200);
|
||||
expect(content.bytes.equals(png)).toBe(true);
|
||||
const sent = await postMessage(alice.token, chatId, {
|
||||
text: '',
|
||||
attachment_ids: [ticket.attachment_id],
|
||||
});
|
||||
expect(sent.response.status).toBe(201);
|
||||
expect(sent.body.attachment_ids).toEqual([ticket.attachment_id]);
|
||||
});
|
||||
|
||||
it('rejects invalid declarations', async () => {
|
||||
const base = declaration(png);
|
||||
expectError(await init(eve.token, chatId, base), 404, 'not_found');
|
||||
expectError(
|
||||
await init(alice.token, chatId, { ...base, size_bytes: MAX + 1 }),
|
||||
413,
|
||||
'payload_too_large',
|
||||
);
|
||||
expectError(
|
||||
await init(alice.token, chatId, {
|
||||
...base,
|
||||
content_type: 'image/gif',
|
||||
}),
|
||||
415,
|
||||
'unsupported_media_type',
|
||||
);
|
||||
for (const bad of [
|
||||
{ ...base, extra: true },
|
||||
{ ...base, sha256: 'ABC' },
|
||||
{ ...base, size_bytes: 0 },
|
||||
{ ...base, filename: 'a/b.png' },
|
||||
])
|
||||
expectError(
|
||||
await init(alice.token, chatId, bad),
|
||||
400,
|
||||
'invalid_request',
|
||||
);
|
||||
});
|
||||
|
||||
it('lets the storage refuse bytes that differ from the declaration', async () => {
|
||||
const ticket = await initOk(png);
|
||||
const tampered = Buffer.from(png);
|
||||
tampered[1000] ^= 1;
|
||||
const refused = await put(ticket, tampered);
|
||||
expect(refused.status).toBe(400);
|
||||
expect(await refused.text()).toContain('XAmzContentChecksumMismatch');
|
||||
// No object was created, so the upload can still be finished.
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
409,
|
||||
'conflict',
|
||||
);
|
||||
expect((await put(ticket, png)).status).toBe(200);
|
||||
expect(
|
||||
(await complete(alice.token, ticket.attachment_id)).response.status,
|
||||
).toBe(200);
|
||||
|
||||
const longer = await initOk(png);
|
||||
const wrongLength = await put(
|
||||
longer,
|
||||
Buffer.concat([png, Buffer.from('x')]),
|
||||
);
|
||||
expect(wrongLength.status).toBe(403);
|
||||
});
|
||||
|
||||
it('rejects a declared PNG that is really a JPEG, then forgets the claim', async () => {
|
||||
const ticket = await initOk(jpeg, 'image/png');
|
||||
expect((await put(ticket, jpeg)).status).toBe(200);
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
415,
|
||||
'unsupported_media_type',
|
||||
);
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
404,
|
||||
'not_found',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a pixel bomb on complete', async () => {
|
||||
const bomb = pixelBombPng();
|
||||
const ticket = await initOk(bomb);
|
||||
expect((await put(ticket, bomb)).status).toBe(200);
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
413,
|
||||
'payload_too_large',
|
||||
);
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
404,
|
||||
'not_found',
|
||||
);
|
||||
});
|
||||
|
||||
it('limits unfinished direct uploads per user', async () => {
|
||||
const spammer = await actor();
|
||||
const chat = (await createChat(spammer, [])).id;
|
||||
for (let index = 0; index < 10; index += 1) {
|
||||
const ok = await init(spammer.token, chat, declaration(png));
|
||||
expect(ok.response.status).toBe(201);
|
||||
}
|
||||
const limited = await init(spammer.token, chat, declaration(png));
|
||||
expectError(limited, 429, 'rate_limited');
|
||||
expect(
|
||||
Number(limited.response.headers.get('retry-after')),
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it.runIf(shortTtl)(
|
||||
'refuses an expired URL and removes the claim',
|
||||
async () => {
|
||||
const ticket = await initOk(png);
|
||||
const expiresAt = Date.parse(ticket.expires_at);
|
||||
await sleep(Math.max(0, expiresAt - Date.now()) + 2000);
|
||||
expect((await put(ticket, png)).status).toBe(403);
|
||||
const deadline = Date.now() + 15_000;
|
||||
for (;;) {
|
||||
const rows = await db.query(
|
||||
'SELECT 1 FROM attachments WHERE id = $1',
|
||||
[ticket.attachment_id],
|
||||
);
|
||||
if (rows.rowCount === 0) break;
|
||||
expect(Date.now()).toBeLessThan(deadline);
|
||||
await sleep(250);
|
||||
}
|
||||
expectError(
|
||||
await complete(alice.token, ticket.attachment_id),
|
||||
404,
|
||||
'not_found',
|
||||
);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,585 @@
|
||||
{
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "MTUSI Messenger — расширение lab4: прямая загрузка",
|
||||
"version": "1.4.0-ext.1",
|
||||
"description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments/<attachment_id>/original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые заявки и объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)."
|
||||
},
|
||||
"servers": [
|
||||
{
|
||||
"url": "http://localhost:8080",
|
||||
"description": "Локальный учебный стенд"
|
||||
}
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"bearerAuth": []
|
||||
}
|
||||
],
|
||||
"paths": {
|
||||
"/api/v1/chats/{chat_id}/attachments/direct-uploads": {
|
||||
"post": {
|
||||
"operationId": "initDirectUpload",
|
||||
"summary": "Заявка на прямую загрузку: вернуть presigned PUT URL",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "chat_id",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"format": "uuid"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/CreateDirectUpload"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Заявка создана; вложение pending и не видно никому, кроме загрузки",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/DirectUpload"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Некорректное тело, filename или sha256",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Нет или неверен токен",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Чат недоступен",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "size_bytes больше 10 MiB",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"415": {
|
||||
"description": "content_type не image/png и не image/jpeg",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"429": {
|
||||
"description": "Слишком много незавершённых прямых загрузок пользователя; Retry-After — секунды до освобождения слота (TTL URL).",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"Retry-After": {
|
||||
"description": "Секунды до повтора.",
|
||||
"schema": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Зависимость недоступна",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/attachments/{attachment_id}/complete": {
|
||||
"post": {
|
||||
"operationId": "completeDirectUpload",
|
||||
"summary": "Подтвердить прямую загрузку: HEAD объекта и проверка заголовка изображения",
|
||||
"description": "Только владелец заявки. Проверяет размер и подписанный SHA-256 объекта (HEAD), читает заголовок изображения (до 1 MiB) и сверяет тип и число пикселей. При несоответствии заявка и объект удаляются, повторный complete даёт 404. Повторный complete готового вложения идемпотентен и возвращает 200.",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "attachment_id",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"format": "uuid"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Вложение готово; тело идентично Attachment из openapi.json",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Attachment"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Некорректный attachment_id",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Нет или неверен токен",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Заявка не найдена, не принадлежит пользователю или уже удалена",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"409": {
|
||||
"description": "Объект не получен или не совпадает с заявленным размером/SHA-256",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "Слишком много пикселей",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"415": {
|
||||
"description": "Содержимое не PNG/JPEG или не совпадает с content_type",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Хранилище недоступно",
|
||||
"headers": {
|
||||
"X-Request-Id": {
|
||||
"description": "Идентификатор запроса.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"X-Instance-Id": {
|
||||
"description": "Идентификатор реплики.",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "openapi.json#/components/schemas/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"components": {
|
||||
"securitySchemes": {
|
||||
"bearerAuth": {
|
||||
"type": "http",
|
||||
"scheme": "bearer",
|
||||
"bearerFormat": "opaque session token"
|
||||
}
|
||||
},
|
||||
"schemas": {
|
||||
"CreateDirectUpload": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256"
|
||||
],
|
||||
"properties": {
|
||||
"filename": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 255
|
||||
},
|
||||
"content_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"image/png",
|
||||
"image/jpeg"
|
||||
]
|
||||
},
|
||||
"size_bytes": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"maximum": 10485760
|
||||
},
|
||||
"sha256": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-f0-9]{64}$"
|
||||
}
|
||||
}
|
||||
},
|
||||
"DirectUpload": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"attachment_id",
|
||||
"method",
|
||||
"url",
|
||||
"headers",
|
||||
"expires_at"
|
||||
],
|
||||
"properties": {
|
||||
"attachment_id": {
|
||||
"type": "string",
|
||||
"format": "uuid"
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"PUT"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"format": "uri",
|
||||
"description": "Presigned URL; содержит подпись, не логировать."
|
||||
},
|
||||
"headers": {
|
||||
"type": "object",
|
||||
"description": "Заголовки, которые клиент обязан отправить как есть. Content-Length клиент ставит сам; тело должно быть ровно size_bytes.",
|
||||
"properties": {
|
||||
"Content-Type": {
|
||||
"type": "string"
|
||||
},
|
||||
"x-amz-checksum-sha256": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"Content-Type",
|
||||
"x-amz-checksum-sha256"
|
||||
]
|
||||
},
|
||||
"expires_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+293
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
|
||||
|
||||
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
|
||||
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
|
||||
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
|
||||
|
||||
Run from the repository root with the stand up. Tokens, signed URLs and secrets
|
||||
are never written to the output.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from pngkit import noise_png # noqa: E402
|
||||
|
||||
|
||||
def sh(*args, check=True):
|
||||
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def read_env(path=".env"):
|
||||
values = {}
|
||||
with open(path) as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
key, _, value = line.partition("=")
|
||||
values[key] = value
|
||||
return values
|
||||
|
||||
|
||||
def request(host, port, method, path, headers=None, body=None):
|
||||
connection = http.client.HTTPConnection(host, port, timeout=120)
|
||||
try:
|
||||
connection.request(method, path, body=body, headers=headers or {})
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
return response.status, payload
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def api_json(env, method, path, token=None, body=None, basic=None):
|
||||
headers = {}
|
||||
data = None
|
||||
if body is not None:
|
||||
headers["content-type"] = "application/json"
|
||||
data = json.dumps(body).encode()
|
||||
if token:
|
||||
headers["authorization"] = f"Bearer {token}"
|
||||
if basic:
|
||||
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
|
||||
port = int(env.get("APP_PUBLIC_PORT", "8080"))
|
||||
status, payload = request("127.0.0.1", port, method, path, headers, data)
|
||||
return status, json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def containers():
|
||||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||||
return sorted(ids)
|
||||
|
||||
|
||||
NODE_METRICS = (
|
||||
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
|
||||
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
|
||||
".then(r=>r.text()).then(console.log)"
|
||||
)
|
||||
|
||||
|
||||
def metrics(container):
|
||||
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
|
||||
|
||||
|
||||
def reset_metrics(container):
|
||||
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
|
||||
|
||||
|
||||
def net_bytes(container):
|
||||
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
|
||||
if line.strip().startswith("eth0:"):
|
||||
fields = line.split(":", 1)[1].split()
|
||||
return int(fields[0]), int(fields[8])
|
||||
raise RuntimeError("eth0 not found")
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
ordered = sorted(values)
|
||||
if not ordered:
|
||||
return 0.0
|
||||
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
|
||||
|
||||
|
||||
def one_upload(env, mode, token, chat_id, data, seed):
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
filename = f"bench-{seed}.png"
|
||||
port = int(env.get("APP_PUBLIC_PORT", "8080"))
|
||||
started = time.perf_counter()
|
||||
if mode == "proxy":
|
||||
status, payload = request(
|
||||
"127.0.0.1", port, "POST",
|
||||
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
|
||||
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
|
||||
)
|
||||
ok = status == 201
|
||||
attachment = json.loads(payload) if ok else None
|
||||
else:
|
||||
status, ticket = api_json(
|
||||
env, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", token,
|
||||
{"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest},
|
||||
)
|
||||
ok = status == 201
|
||||
attachment = None
|
||||
if ok:
|
||||
target = urllib.parse.urlsplit(ticket["url"])
|
||||
put_status, _ = request(
|
||||
target.hostname, target.port, "PUT", target.path + "?" + target.query,
|
||||
{**ticket["headers"]}, data,
|
||||
)
|
||||
ok = put_status == 200
|
||||
if ok:
|
||||
status, attachment = api_json(
|
||||
env, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", token
|
||||
)
|
||||
ok = status == 200
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
if ok and (attachment is None or attachment["sha256"] != digest):
|
||||
ok = False
|
||||
return ok, elapsed_ms
|
||||
|
||||
|
||||
def run_once(env, args, token, chat_id, ids):
|
||||
for container in ids:
|
||||
reset_metrics(container)
|
||||
time.sleep(0.2)
|
||||
baseline = {c: metrics(c) for c in ids}
|
||||
net_before = {c: net_bytes(c) for c in ids}
|
||||
latencies, failures = [], []
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker(index):
|
||||
for step in range(args.uploads_per_worker):
|
||||
seed = index * 1000 + step
|
||||
data = noise_png(args.size_bytes, seed)
|
||||
ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed)
|
||||
with lock:
|
||||
(latencies if ok else failures).append(elapsed)
|
||||
|
||||
started = time.perf_counter()
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
duration = time.perf_counter() - started
|
||||
time.sleep(0.2)
|
||||
after = {c: metrics(c) for c in ids}
|
||||
net_after = {c: net_bytes(c) for c in ids}
|
||||
|
||||
replicas = []
|
||||
for c in ids:
|
||||
base_rss = baseline[c]["memory_bytes"]["rss"]
|
||||
peak = after[c]["memory_peak_bytes"]
|
||||
replicas.append({
|
||||
"instance_id": after[c]["instance_id"],
|
||||
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
|
||||
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
|
||||
"baseline_rss_bytes": base_rss,
|
||||
"peak_memory_bytes": peak,
|
||||
"delta_rss_bytes": peak["rss"] - base_rss,
|
||||
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
|
||||
})
|
||||
total = args.concurrency * args.uploads_per_worker
|
||||
return {
|
||||
"uploads": total,
|
||||
"succeeded": len(latencies),
|
||||
"failed": len(failures),
|
||||
"duration_s": round(duration, 3),
|
||||
"latency_ms": {
|
||||
"p50": round(percentile(latencies, 0.50), 1),
|
||||
"p95": round(percentile(latencies, 0.95), 1),
|
||||
"p99": round(percentile(latencies, 0.99), 1),
|
||||
},
|
||||
"replicas": replicas,
|
||||
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
|
||||
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
|
||||
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
|
||||
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
|
||||
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
|
||||
}
|
||||
|
||||
|
||||
def command_run(args):
|
||||
env = read_env()
|
||||
ids = containers()
|
||||
if not ids:
|
||||
sys.exit("no api containers found")
|
||||
name = f"bench_{uuid.uuid4().hex[:10]}"
|
||||
password = "bench:" + uuid.uuid4().hex
|
||||
status, user = api_json(env, "POST", "/api/v1/users",
|
||||
body={"username": name, "display_name": "bench", "password": password})
|
||||
assert status == 201, status
|
||||
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
|
||||
assert status == 201, status
|
||||
token = session["token"]
|
||||
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
|
||||
assert status == 201, status
|
||||
|
||||
# Warm-up: one upload per replica path, not measured.
|
||||
for seed in range(len(ids) * 2):
|
||||
one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed)
|
||||
|
||||
limits = []
|
||||
for c in ids:
|
||||
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
|
||||
repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)]
|
||||
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
|
||||
result = {
|
||||
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
|
||||
"params": {
|
||||
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
|
||||
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
|
||||
},
|
||||
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
|
||||
"api_container_limits_nanocpus_memory": limits,
|
||||
"repeats": repeats,
|
||||
}
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
|
||||
with open(args.output, "w") as handle:
|
||||
json.dump(result, handle, indent=2)
|
||||
handle.write("\n")
|
||||
for index, item in enumerate(repeats):
|
||||
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
|
||||
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
|
||||
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms")
|
||||
|
||||
|
||||
def mib(value):
|
||||
return f"{value / 1048576:.2f}"
|
||||
|
||||
|
||||
def command_summarize(args):
|
||||
rows = []
|
||||
for path in args.paths:
|
||||
with open(path) as handle:
|
||||
data = json.load(handle)
|
||||
p = data["params"]
|
||||
reps = data["repeats"]
|
||||
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
|
||||
rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data))
|
||||
rows.sort(key=lambda r: (r[1], r[0], r[2]))
|
||||
print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |")
|
||||
print("|---|---|---|---|---|---|---|---|---|")
|
||||
for mode, size, conc, reps, med, _data in rows:
|
||||
ok = sum(r["succeeded"] for r in reps)
|
||||
total = sum(r["uploads"] for r in reps)
|
||||
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
|
||||
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
|
||||
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | "
|
||||
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
run = sub.add_parser("run")
|
||||
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
|
||||
run.add_argument("--size-bytes", type=int, required=True)
|
||||
run.add_argument("--concurrency", type=int, default=10)
|
||||
run.add_argument("--uploads-per-worker", type=int, default=5)
|
||||
run.add_argument("--repeat", type=int, default=3)
|
||||
run.add_argument("--output", required=True)
|
||||
summarize = sub.add_parser("summarize")
|
||||
summarize.add_argument("paths", nargs="+")
|
||||
args = parser.parse_args()
|
||||
{"run": command_run, "summarize": command_summarize}[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/bin/sh
|
||||
# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials
|
||||
# API, закрытая консоль. Секреты не печатаются.
|
||||
set -u
|
||||
cd "$(dirname "$0")/../.."
|
||||
env_file="${COMPOSE_ENV_FILE:-.env}"
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "./$env_file"; set +a
|
||||
endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
compose="docker compose --env-file $env_file"
|
||||
fail=0
|
||||
|
||||
report() { # status name
|
||||
if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi
|
||||
}
|
||||
|
||||
key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
|
||||
-c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
|
||||
if [ -z "$key" ]; then
|
||||
echo "FAIL no ready attachment to check; upload one first (make test)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key")
|
||||
[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)"
|
||||
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/")
|
||||
[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)"
|
||||
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write")
|
||||
[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)"
|
||||
|
||||
# The API credentials only, inside a one-off container (secrets come from files).
|
||||
api_mc() {
|
||||
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
|
||||
IFS= read -r k < /run/secrets/s3_api_access_key_id
|
||||
IFS= read -r s < /run/secrets/s3_api_secret_access_key
|
||||
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000"
|
||||
exec mc "$@"' sh "$@" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets"
|
||||
api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API"
|
||||
api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*"
|
||||
api_mc stat "api/$bucket/outside-prefix" ; rc=$?
|
||||
api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/"
|
||||
: "$rc"
|
||||
|
||||
port=$($compose port s3 9001 2>/dev/null || true)
|
||||
[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)"
|
||||
if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi
|
||||
|
||||
exit "$fail"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Сверка bucket и БД (только чтение): orphans — объект без строки, missing —
|
||||
# ready-строка без объекта. Код выхода 1, если что-то найдено.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
env_file="${COMPOSE_ENV_FILE:-.env}"
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "./$env_file"; set +a
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
compose="docker compose --env-file $env_file"
|
||||
|
||||
objects=$(mktemp)
|
||||
rows=$(mktemp)
|
||||
trap 'rm -f "$objects" "$rows"' EXIT
|
||||
|
||||
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
|
||||
IFS= read -r u < /run/secrets/minio_root_user
|
||||
IFS= read -r p < /run/secrets/minio_root_password
|
||||
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000"
|
||||
exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \
|
||||
| sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects"
|
||||
$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
|
||||
-c "SELECT object_key, status FROM attachments" | sort > "$rows"
|
||||
|
||||
echo "bucket: $bucket"
|
||||
echo "objects: $(wc -l < "$objects" | tr -d ' ')"
|
||||
echo "rows by status:"
|
||||
cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /'
|
||||
orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects")
|
||||
missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects")
|
||||
echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)"
|
||||
[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /'
|
||||
echo "missing: $(printf '%s' "$missing" | grep -c . || true)"
|
||||
[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /'
|
||||
[ -z "$orphans$missing" ]
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts."""
|
||||
import random
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
NOISE_WIDTH = 1024
|
||||
NOISE_ROW = 1 + NOISE_WIDTH * 3
|
||||
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
crc = zlib.crc32(kind + data) & 0xFFFFFFFF
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def build_png(width: int, height: int, idat: bytes) -> bytes:
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def pad_png(data: bytes, size: int) -> bytes:
|
||||
"""Insert a private ancillary chunk before IEND to reach exactly `size` bytes."""
|
||||
extra = size - len(data)
|
||||
if extra == 0:
|
||||
return data
|
||||
if extra < 12:
|
||||
raise ValueError("padding needs at least 12 bytes")
|
||||
return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:]
|
||||
|
||||
|
||||
def _noise_of_height(height: int, rng: random.Random) -> bytes:
|
||||
raw = bytearray(rng.randbytes(NOISE_ROW * height))
|
||||
for row in range(height):
|
||||
raw[row * NOISE_ROW] = 0 # filter type "none"
|
||||
compressor = zlib.compressobj(0)
|
||||
return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush())
|
||||
|
||||
|
||||
def noise_png(size: int, seed: int = 0) -> bytes:
|
||||
"""Incompressible, structurally valid PNG of exactly `size` bytes."""
|
||||
rng = random.Random(seed)
|
||||
height = max(1, size // NOISE_ROW)
|
||||
while True:
|
||||
png = _noise_of_height(height, rng)
|
||||
if len(png) > size or 0 < size - len(png) < 12:
|
||||
height -= 1
|
||||
if height < 1:
|
||||
raise ValueError("size is too small")
|
||||
continue
|
||||
return pad_png(png, size)
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20.
|
||||
# Needs a running stand (make stand-up) and `make init-env` credentials.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
out=evidence/lab4/bench
|
||||
mkdir -p "$out"
|
||||
bench() {
|
||||
mode=$1 size=$2 conc=$3
|
||||
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \
|
||||
--concurrency "$conc" --uploads-per-worker 5 --repeat 3 \
|
||||
--output "$out/$mode-$size-c$conc.json"
|
||||
}
|
||||
for size in 131072 1048576 9961472; do
|
||||
for mode in proxy direct; do
|
||||
echo "== $mode size=$size c=10"
|
||||
bench "$mode" "$size" 10
|
||||
done
|
||||
done
|
||||
echo "== proxy size=9961472 c=20"
|
||||
bench proxy 9961472 20
|
||||
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
|
||||
cat evidence/lab4/summary.md
|
||||
Reference in New Issue
Block a user