lab4: прямая загрузка, скрипты экспериментов и проверок

This commit is contained in:
2026-10-22 22:39:25 +03:00
parent 7de3012b46
commit 39a6df8c09
12 changed files with 1574 additions and 2 deletions
@@ -1,4 +1,5 @@
import {
Body,
Controller,
Get,
Headers,
@@ -20,6 +21,8 @@ import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
import type { AppConfig } from '../config/app-config.schema.js';
import type { User } from '../users/user.js';
import { AttachmentsService } from './attachments.service.js';
import { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
import { DirectUploadsService } from './direct-uploads.service.js';
import { contentDisposition } from './filename.js';
@Controller('api/v1')
@@ -29,6 +32,7 @@ export class AttachmentsController {
constructor(
private readonly attachments: AttachmentsService,
private readonly directUploads: DirectUploadsService,
config: ConfigService<AppConfig>,
) {
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
@@ -58,6 +62,25 @@ export class AttachmentsController {
);
}
@Post('chats/:chatId/attachments/direct-uploads')
@HttpCode(201)
initDirectUpload(
@Param('chatId', new ParseUUIDPipe()) chatId: string,
@CurrentUser() user: User,
@Body() body: CreateDirectUploadDto,
) {
return this.directUploads.init(chatId, user, body);
}
@Post('attachments/:attachmentId/complete')
@HttpCode(200)
completeDirectUpload(
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
@CurrentUser() user: User,
) {
return this.directUploads.complete(attachmentId, user);
}
@Get('attachments/:attachmentId')
metadata(
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
+7 -1
View File
@@ -8,6 +8,7 @@ import { StorageModule } from '../storage/storage.module.js';
import { AttachmentSweeper } from './attachment-sweeper.service.js';
import { AttachmentsController } from './attachments.controller.js';
import { AttachmentsRepository } from './attachments.repository.js';
import { DirectUploadsService } from './direct-uploads.service.js';
import { AttachmentsService } from './attachments.service.js';
@Module({
@@ -20,7 +21,12 @@ import { AttachmentsService } from './attachments.service.js';
StorageModule,
],
controllers: [AttachmentsController],
providers: [AttachmentsRepository, AttachmentsService, AttachmentSweeper],
providers: [
AttachmentsRepository,
AttachmentsService,
DirectUploadsService,
AttachmentSweeper,
],
exports: [AttachmentsRepository],
})
export class AttachmentsModule {}
+1 -1
View File
@@ -279,7 +279,7 @@ export class AttachmentsService {
return error;
}
private async compensate(id: string, key: string): Promise<void> {
async compensate(id: string, key: string): Promise<void> {
try {
await this.storage.delete(key);
await this.repository.deletePending(id);
@@ -0,0 +1,199 @@
import {
ConflictException,
HttpException,
Injectable,
NotFoundException,
PayloadTooLargeException,
UnsupportedMediaTypeException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'node:crypto';
import type { AppConfig } from '../config/app-config.schema.js';
import { ChatsService } from '../chats/chats.service.js';
import { MetricsService } from '../metrics/metrics.service.js';
import { ObjectStorage } from '../storage/object-storage.js';
import type { User } from '../users/user.js';
import { toAttachment, type Attachment } from './attachment.js';
import { AttachmentsRepository } from './attachments.repository.js';
import { AttachmentsService, imageRejection } from './attachments.service.js';
import type { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
import { validateFilename } from './filename.js';
import {
IMAGE_HEADER_MAX_BYTES,
ImageInspector,
ImageRejectedError,
} from './image/image-inspector.js';
export type DirectUpload = {
attachment_id: string;
method: 'PUT';
url: string;
headers: Record<string, string>;
expires_at: string;
};
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
@Injectable()
export class DirectUploadsService {
private readonly maxUploadBytes: number;
private readonly maxPixels: number;
private readonly ttlSeconds: number;
private readonly maxPending: number;
constructor(
config: ConfigService<AppConfig>,
private readonly repository: AttachmentsRepository,
private readonly storage: ObjectStorage,
private readonly chats: ChatsService,
private readonly metrics: MetricsService,
private readonly attachments: AttachmentsService,
) {
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
this.ttlSeconds = config.getOrThrow('DIRECT_UPLOAD_URL_TTL_SECONDS');
this.maxPending = config.getOrThrow(
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
);
}
async init(
chatId: string,
user: User,
dto: CreateDirectUploadDto,
): Promise<DirectUpload> {
const filename = validateFilename(dto.filename);
if (!MEDIA_TYPES.has(dto.content_type))
throw new UnsupportedMediaTypeException(
'content_type must be image/png or image/jpeg',
);
if (dto.size_bytes > this.maxUploadBytes)
throw new PayloadTooLargeException('Image is too large');
await this.chats.getForUser(chatId, user);
if (
(await this.repository.countPendingDirect(user.id)) >=
this.maxPending
)
throw new HttpException(
{
message: 'Too many unfinished direct uploads',
retryAfter: this.ttlSeconds,
},
429,
);
const id = randomUUID();
const key = this.storage.objectKey(id);
await this.repository.insertPending({
id,
chatId,
ownerId: user.id,
filename,
contentType: dto.content_type,
uploadMode: 'direct',
objectKey: key,
sizeBytes: dto.size_bytes,
sha256: dto.sha256,
ttlMs: this.ttlSeconds * 1000,
});
const checksum = Buffer.from(dto.sha256, 'hex').toString('base64');
let url: string;
try {
url = await this.storage.presignPut(key, {
contentType: dto.content_type,
contentLength: dto.size_bytes,
checksumSha256Base64: checksum,
expiresInSeconds: this.ttlSeconds,
});
} catch (error) {
await this.repository.deletePending(id);
throw error;
}
this.metrics.increment('attachments.direct.initiated');
return {
attachment_id: id,
method: 'PUT',
url,
headers: {
'Content-Type': dto.content_type,
'x-amz-checksum-sha256': checksum,
},
expires_at: new Date(
Date.now() + this.ttlSeconds * 1000,
).toISOString(),
};
}
async complete(attachmentId: string, user: User): Promise<Attachment> {
const row = await this.repository.findForComplete(
attachmentId,
user.id,
);
if (!row) throw new NotFoundException();
if (row.status === 'ready') return toAttachment(row);
const size = Number(row.size_bytes);
const expectedChecksum = Buffer.from(
row.sha256 as string,
'hex',
).toString('base64');
const head = await this.storage.head(row.object_key);
if (head === null)
throw new ConflictException('Upload has not been received');
if (head.size !== size || head.checksumSha256 !== expectedChecksum) {
await this.attachments.compensate(row.id, row.object_key);
throw new ConflictException(
'Uploaded object does not match the declaration',
);
}
// Only the header is read: the signed checksum already pins every byte.
const inspector = new ImageInspector(this.maxPixels);
const covered = size <= IMAGE_HEADER_MAX_BYTES;
const stream = await this.storage.getStream(
row.object_key,
`bytes=0-${Math.min(size, IMAGE_HEADER_MAX_BYTES) - 1}`,
);
if (stream === null)
throw new ConflictException('Upload has not been received');
let verified = 0;
try {
for await (const chunk of stream as AsyncIterable<Buffer>) {
verified += chunk.length;
inspector.push(chunk);
if (!covered && inspector.header !== undefined) break;
}
if (covered) inspector.finish();
else if (inspector.header === undefined)
throw new ImageRejectedError(
'unsupported_media_type',
'Image header is missing',
);
} catch (error) {
if (error instanceof ImageRejectedError) {
await this.attachments.compensate(row.id, row.object_key);
throw imageRejection(error);
}
throw error;
} finally {
stream.destroy();
this.metrics.add('attachments.direct.verify_bytes', verified);
}
const header = inspector.header;
if (header === undefined || header.contentType !== row.content_type) {
await this.attachments.compensate(row.id, row.object_key);
throw new UnsupportedMediaTypeException(
'Image content does not match content_type',
);
}
const ready = await this.repository.markReady(row.id, {
sizeBytes: size,
sha256: row.sha256 as string,
width: header.width,
height: header.height,
});
if (!ready) throw new ConflictException('Upload is no longer pending');
this.metrics.increment('attachments.direct.completed');
return toAttachment(ready);
}
}
@@ -0,0 +1,16 @@
import { IsInt, IsString, Matches, Min } from 'class-validator';
export class CreateDirectUploadDto {
@IsString()
filename!: string;
@IsString()
content_type!: string;
@IsInt()
@Min(1)
size_bytes!: number;
@Matches(/^[a-f0-9]{64}$/u)
sha256!: string;
}
+288
View File
@@ -0,0 +1,288 @@
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { setTimeout as sleep } from 'node:timers/promises';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { Pool } from 'pg';
import type { Attachment } from '../src/attachments/attachment.js';
import type { DirectUpload } from '../src/attachments/direct-uploads.service.js';
import type { ApiErrorBody } from '../src/common/errors/api-error.js';
import {
Actor,
actor,
bearer,
createChat,
database,
download,
expectError,
http,
json,
postMessage,
} from './support/e2e.js';
import { pixelBombPng } from './support/png.js';
const png = readFileSync(
new URL('../../tests/fixtures/sample.png', import.meta.url),
);
const jpeg = readFileSync(new URL('./fixtures/sample.jpg', import.meta.url));
const MAX = 10_485_760;
const shortTtl = process.env.E2E_DIRECT_TTL_SHORT === '1';
let db: Pool;
let alice: Actor;
let bob: Actor;
let eve: Actor;
let chatId: string;
const sha256 = (data: Uint8Array) =>
createHash('sha256').update(data).digest('hex');
function init(token: string, chat: string, body: Record<string, unknown>) {
return http<DirectUpload & ApiErrorBody>(
`/api/v1/chats/${chat}/attachments/direct-uploads`,
json(body, bearer(token)),
);
}
function declaration(data: Buffer, contentType = 'image/png') {
return {
filename: 'direct.png',
content_type: contentType,
size_bytes: data.length,
sha256: sha256(data),
};
}
async function initOk(data: Buffer, contentType = 'image/png') {
const result = await init(
alice.token,
chatId,
declaration(data, contentType),
);
expect(result.response.status).toBe(201);
return result.body;
}
// The PUT goes straight to the public S3 endpoint, not through the API.
async function put(ticket: DirectUpload, data: Buffer): Promise<Response> {
return fetch(ticket.url, {
method: ticket.method,
headers: ticket.headers,
body: data as unknown as BodyInit,
});
}
function complete(token: string, id: string) {
return http<Attachment & ApiErrorBody>(
`/api/v1/attachments/${id}/complete`,
{
method: 'POST',
headers: bearer(token),
},
);
}
describe('direct upload (presigned PUT)', () => {
beforeAll(async () => {
db = database();
alice = await actor();
bob = await actor();
eve = await actor();
chatId = (await createChat(alice, [bob])).id;
});
afterAll(async () => {
await db.end();
});
it('uploads around the API and exposes the attachment after complete', async () => {
const ticket = await initOk(png);
expect(ticket.method).toBe('PUT');
expect(ticket.headers['x-amz-checksum-sha256']).toBe(
Buffer.from(sha256(png), 'hex').toString('base64'),
);
expect(Date.parse(ticket.expires_at)).toBeGreaterThan(Date.now());
expect(new URL(ticket.url).pathname).toBe(
`/messenger-attachments/attachments/${ticket.attachment_id}/original`,
);
const hidden = await http<ApiErrorBody>(
`/api/v1/attachments/${ticket.attachment_id}`,
{ headers: bearer(bob.token) },
);
expectError(hidden, 404, 'not_found');
expectError(
await postMessage(alice.token, chatId, {
text: '',
attachment_ids: [ticket.attachment_id],
}),
404,
'not_found',
);
expectError(
await complete(alice.token, ticket.attachment_id),
409,
'conflict',
);
expect((await put(ticket, png)).status).toBe(200);
expectError(
await complete(bob.token, ticket.attachment_id),
404,
'not_found',
);
expectError(
await complete(eve.token, ticket.attachment_id),
404,
'not_found',
);
const done = await complete(alice.token, ticket.attachment_id);
expect(done.response.status).toBe(200);
expect(done.body).toMatchObject({
id: ticket.attachment_id,
owner_id: alice.user.id,
content_type: 'image/png',
size_bytes: png.length,
sha256: sha256(png),
status: 'ready',
});
const again = await complete(alice.token, ticket.attachment_id);
expect(again.response.status).toBe(200);
expect(again.body).toEqual(done.body);
const content = await download(bob.token, ticket.attachment_id);
expect(content.response.status).toBe(200);
expect(content.bytes.equals(png)).toBe(true);
const sent = await postMessage(alice.token, chatId, {
text: '',
attachment_ids: [ticket.attachment_id],
});
expect(sent.response.status).toBe(201);
expect(sent.body.attachment_ids).toEqual([ticket.attachment_id]);
});
it('rejects invalid declarations', async () => {
const base = declaration(png);
expectError(await init(eve.token, chatId, base), 404, 'not_found');
expectError(
await init(alice.token, chatId, { ...base, size_bytes: MAX + 1 }),
413,
'payload_too_large',
);
expectError(
await init(alice.token, chatId, {
...base,
content_type: 'image/gif',
}),
415,
'unsupported_media_type',
);
for (const bad of [
{ ...base, extra: true },
{ ...base, sha256: 'ABC' },
{ ...base, size_bytes: 0 },
{ ...base, filename: 'a/b.png' },
])
expectError(
await init(alice.token, chatId, bad),
400,
'invalid_request',
);
});
it('lets the storage refuse bytes that differ from the declaration', async () => {
const ticket = await initOk(png);
const tampered = Buffer.from(png);
tampered[1000] ^= 1;
const refused = await put(ticket, tampered);
expect(refused.status).toBe(400);
expect(await refused.text()).toContain('XAmzContentChecksumMismatch');
// No object was created, so the upload can still be finished.
expectError(
await complete(alice.token, ticket.attachment_id),
409,
'conflict',
);
expect((await put(ticket, png)).status).toBe(200);
expect(
(await complete(alice.token, ticket.attachment_id)).response.status,
).toBe(200);
const longer = await initOk(png);
const wrongLength = await put(
longer,
Buffer.concat([png, Buffer.from('x')]),
);
expect(wrongLength.status).toBe(403);
});
it('rejects a declared PNG that is really a JPEG, then forgets the claim', async () => {
const ticket = await initOk(jpeg, 'image/png');
expect((await put(ticket, jpeg)).status).toBe(200);
expectError(
await complete(alice.token, ticket.attachment_id),
415,
'unsupported_media_type',
);
expectError(
await complete(alice.token, ticket.attachment_id),
404,
'not_found',
);
});
it('rejects a pixel bomb on complete', async () => {
const bomb = pixelBombPng();
const ticket = await initOk(bomb);
expect((await put(ticket, bomb)).status).toBe(200);
expectError(
await complete(alice.token, ticket.attachment_id),
413,
'payload_too_large',
);
expectError(
await complete(alice.token, ticket.attachment_id),
404,
'not_found',
);
});
it('limits unfinished direct uploads per user', async () => {
const spammer = await actor();
const chat = (await createChat(spammer, [])).id;
for (let index = 0; index < 10; index += 1) {
const ok = await init(spammer.token, chat, declaration(png));
expect(ok.response.status).toBe(201);
}
const limited = await init(spammer.token, chat, declaration(png));
expectError(limited, 429, 'rate_limited');
expect(
Number(limited.response.headers.get('retry-after')),
).toBeGreaterThan(0);
});
it.runIf(shortTtl)(
'refuses an expired URL and removes the claim',
async () => {
const ticket = await initOk(png);
const expiresAt = Date.parse(ticket.expires_at);
await sleep(Math.max(0, expiresAt - Date.now()) + 2000);
expect((await put(ticket, png)).status).toBe(403);
const deadline = Date.now() + 15_000;
for (;;) {
const rows = await db.query(
'SELECT 1 FROM attachments WHERE id = $1',
[ticket.attachment_id],
);
if (rows.rowCount === 0) break;
expect(Date.now()).toBeLessThan(deadline);
await sleep(250);
}
expectError(
await complete(alice.token, ticket.attachment_id),
404,
'not_found',
);
},
60_000,
);
});
+585
View File
@@ -0,0 +1,585 @@
{
"openapi": "3.1.0",
"info": {
"title": "MTUSI Messenger — расширение lab4: прямая загрузка",
"version": "1.4.0-ext.1",
"description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments/<attachment_id>/original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые заявки и объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)."
},
"servers": [
{
"url": "http://localhost:8080",
"description": "Локальный учебный стенд"
}
],
"security": [
{
"bearerAuth": []
}
],
"paths": {
"/api/v1/chats/{chat_id}/attachments/direct-uploads": {
"post": {
"operationId": "initDirectUpload",
"summary": "Заявка на прямую загрузку: вернуть presigned PUT URL",
"parameters": [
{
"name": "chat_id",
"in": "path",
"required": true,
"schema": {
"type": "string",
"format": "uuid"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateDirectUpload"
}
}
}
},
"responses": {
"201": {
"description": "Заявка создана; вложение pending и не видно никому, кроме загрузки",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DirectUpload"
}
}
}
},
"400": {
"description": "Некорректное тело, filename или sha256",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"401": {
"description": "Нет или неверен токен",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"404": {
"description": "Чат недоступен",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"413": {
"description": "size_bytes больше 10 MiB",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"415": {
"description": "content_type не image/png и не image/jpeg",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"429": {
"description": "Слишком много незавершённых прямых загрузок пользователя; Retry-After — секунды до освобождения слота (TTL URL).",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
},
"Retry-After": {
"description": "Секунды до повтора.",
"schema": {
"type": "integer",
"minimum": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"503": {
"description": "Зависимость недоступна",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
}
}
}
},
"/api/v1/attachments/{attachment_id}/complete": {
"post": {
"operationId": "completeDirectUpload",
"summary": "Подтвердить прямую загрузку: HEAD объекта и проверка заголовка изображения",
"description": "Только владелец заявки. Проверяет размер и подписанный SHA-256 объекта (HEAD), читает заголовок изображения (до 1 MiB) и сверяет тип и число пикселей. При несоответствии заявка и объект удаляются, повторный complete даёт 404. Повторный complete готового вложения идемпотентен и возвращает 200.",
"parameters": [
{
"name": "attachment_id",
"in": "path",
"required": true,
"schema": {
"type": "string",
"format": "uuid"
}
}
],
"responses": {
"200": {
"description": "Вложение готово; тело идентично Attachment из openapi.json",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Attachment"
}
}
}
},
"400": {
"description": "Некорректный attachment_id",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"401": {
"description": "Нет или неверен токен",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"404": {
"description": "Заявка не найдена, не принадлежит пользователю или уже удалена",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"409": {
"description": "Объект не получен или не совпадает с заявленным размером/SHA-256",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"413": {
"description": "Слишком много пикселей",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"415": {
"description": "Содержимое не PNG/JPEG или не совпадает с content_type",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
},
"503": {
"description": "Хранилище недоступно",
"headers": {
"X-Request-Id": {
"description": "Идентификатор запроса.",
"schema": {
"type": "string",
"minLength": 1
}
},
"X-Instance-Id": {
"description": "Идентификатор реплики.",
"schema": {
"type": "string",
"minLength": 1
}
}
},
"content": {
"application/json": {
"schema": {
"$ref": "openapi.json#/components/schemas/Error"
}
}
}
}
}
}
}
},
"components": {
"securitySchemes": {
"bearerAuth": {
"type": "http",
"scheme": "bearer",
"bearerFormat": "opaque session token"
}
},
"schemas": {
"CreateDirectUpload": {
"type": "object",
"additionalProperties": false,
"required": [
"filename",
"content_type",
"size_bytes",
"sha256"
],
"properties": {
"filename": {
"type": "string",
"minLength": 1,
"maxLength": 255
},
"content_type": {
"type": "string",
"enum": [
"image/png",
"image/jpeg"
]
},
"size_bytes": {
"type": "integer",
"minimum": 1,
"maximum": 10485760
},
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
}
}
},
"DirectUpload": {
"type": "object",
"additionalProperties": false,
"required": [
"attachment_id",
"method",
"url",
"headers",
"expires_at"
],
"properties": {
"attachment_id": {
"type": "string",
"format": "uuid"
},
"method": {
"type": "string",
"enum": [
"PUT"
]
},
"url": {
"type": "string",
"format": "uri",
"description": "Presigned URL; содержит подпись, не логировать."
},
"headers": {
"type": "object",
"description": "Заголовки, которые клиент обязан отправить как есть. Content-Length клиент ставит сам; тело должно быть ровно size_bytes.",
"properties": {
"Content-Type": {
"type": "string"
},
"x-amz-checksum-sha256": {
"type": "string"
}
},
"required": [
"Content-Type",
"x-amz-checksum-sha256"
]
},
"expires_at": {
"type": "string",
"format": "date-time"
}
}
}
}
}
}
+293
View File
@@ -0,0 +1,293 @@
#!/usr/bin/env python3
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
Run from the repository root with the stand up. Tokens, signed URLs and secrets
are never written to the output.
"""
import argparse
import base64
import hashlib
import http.client
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from pngkit import noise_png # noqa: E402
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def read_env(path=".env"):
values = {}
with open(path) as handle:
for line in handle:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, _, value = line.partition("=")
values[key] = value
return values
def request(host, port, method, path, headers=None, body=None):
connection = http.client.HTTPConnection(host, port, timeout=120)
try:
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
return response.status, payload
finally:
connection.close()
def api_json(env, method, path, token=None, body=None, basic=None):
headers = {}
data = None
if body is not None:
headers["content-type"] = "application/json"
data = json.dumps(body).encode()
if token:
headers["authorization"] = f"Bearer {token}"
if basic:
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
port = int(env.get("APP_PUBLIC_PORT", "8080"))
status, payload = request("127.0.0.1", port, method, path, headers, data)
return status, json.loads(payload) if payload else None
def containers():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
return sorted(ids)
NODE_METRICS = (
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(console.log)"
)
def metrics(container):
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
def reset_metrics(container):
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
def net_bytes(container):
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
if line.strip().startswith("eth0:"):
fields = line.split(":", 1)[1].split()
return int(fields[0]), int(fields[8])
raise RuntimeError("eth0 not found")
def percentile(values, fraction):
ordered = sorted(values)
if not ordered:
return 0.0
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
def one_upload(env, mode, token, chat_id, data, seed):
digest = hashlib.sha256(data).hexdigest()
filename = f"bench-{seed}.png"
port = int(env.get("APP_PUBLIC_PORT", "8080"))
started = time.perf_counter()
if mode == "proxy":
status, payload = request(
"127.0.0.1", port, "POST",
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
)
ok = status == 201
attachment = json.loads(payload) if ok else None
else:
status, ticket = api_json(
env, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", token,
{"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest},
)
ok = status == 201
attachment = None
if ok:
target = urllib.parse.urlsplit(ticket["url"])
put_status, _ = request(
target.hostname, target.port, "PUT", target.path + "?" + target.query,
{**ticket["headers"]}, data,
)
ok = put_status == 200
if ok:
status, attachment = api_json(
env, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", token
)
ok = status == 200
elapsed_ms = (time.perf_counter() - started) * 1000
if ok and (attachment is None or attachment["sha256"] != digest):
ok = False
return ok, elapsed_ms
def run_once(env, args, token, chat_id, ids):
for container in ids:
reset_metrics(container)
time.sleep(0.2)
baseline = {c: metrics(c) for c in ids}
net_before = {c: net_bytes(c) for c in ids}
latencies, failures = [], []
lock = threading.Lock()
def worker(index):
for step in range(args.uploads_per_worker):
seed = index * 1000 + step
data = noise_png(args.size_bytes, seed)
ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed)
with lock:
(latencies if ok else failures).append(elapsed)
started = time.perf_counter()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
duration = time.perf_counter() - started
time.sleep(0.2)
after = {c: metrics(c) for c in ids}
net_after = {c: net_bytes(c) for c in ids}
replicas = []
for c in ids:
base_rss = baseline[c]["memory_bytes"]["rss"]
peak = after[c]["memory_peak_bytes"]
replicas.append({
"instance_id": after[c]["instance_id"],
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
"baseline_rss_bytes": base_rss,
"peak_memory_bytes": peak,
"delta_rss_bytes": peak["rss"] - base_rss,
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
})
total = args.concurrency * args.uploads_per_worker
return {
"uploads": total,
"succeeded": len(latencies),
"failed": len(failures),
"duration_s": round(duration, 3),
"latency_ms": {
"p50": round(percentile(latencies, 0.50), 1),
"p95": round(percentile(latencies, 0.95), 1),
"p99": round(percentile(latencies, 0.99), 1),
},
"replicas": replicas,
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
}
def command_run(args):
env = read_env()
ids = containers()
if not ids:
sys.exit("no api containers found")
name = f"bench_{uuid.uuid4().hex[:10]}"
password = "bench:" + uuid.uuid4().hex
status, user = api_json(env, "POST", "/api/v1/users",
body={"username": name, "display_name": "bench", "password": password})
assert status == 201, status
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
assert status == 201, status
token = session["token"]
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
assert status == 201, status
# Warm-up: one upload per replica path, not measured.
for seed in range(len(ids) * 2):
one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed)
limits = []
for c in ids:
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)]
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
result = {
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
"params": {
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
},
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
"api_container_limits_nanocpus_memory": limits,
"repeats": repeats,
}
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
with open(args.output, "w") as handle:
json.dump(result, handle, indent=2)
handle.write("\n")
for index, item in enumerate(repeats):
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms")
def mib(value):
return f"{value / 1048576:.2f}"
def command_summarize(args):
rows = []
for path in args.paths:
with open(path) as handle:
data = json.load(handle)
p = data["params"]
reps = data["repeats"]
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data))
rows.sort(key=lambda r: (r[1], r[0], r[2]))
print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |")
print("|---|---|---|---|---|---|---|---|---|")
for mode, size, conc, reps, med, _data in rows:
ok = sum(r["succeeded"] for r in reps)
total = sum(r["uploads"] for r in reps)
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | "
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |")
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("run")
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
run.add_argument("--size-bytes", type=int, required=True)
run.add_argument("--concurrency", type=int, default=10)
run.add_argument("--uploads-per-worker", type=int, default=5)
run.add_argument("--repeat", type=int, default=3)
run.add_argument("--output", required=True)
summarize = sub.add_parser("summarize")
summarize.add_argument("paths", nargs="+")
args = parser.parse_args()
{"run": command_run, "summarize": command_summarize}[args.command](args)
if __name__ == "__main__":
main()
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials
# API, закрытая консоль. Секреты не печатаются.
set -u
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
fail=0
report() { # status name
if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi
}
key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
if [ -z "$key" ]; then
echo "FAIL no ready attachment to check; upload one first (make test)"
exit 1
fi
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key")
[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/")
[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write")
[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)"
# The API credentials only, inside a one-off container (secrets come from files).
api_mc() {
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r k < /run/secrets/s3_api_access_key_id
IFS= read -r s < /run/secrets/s3_api_secret_access_key
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000"
exec mc "$@"' sh "$@" >/dev/null 2>&1
}
api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets"
api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API"
api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*"
api_mc stat "api/$bucket/outside-prefix" ; rc=$?
api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/"
: "$rc"
port=$($compose port s3 9001 2>/dev/null || true)
[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)"
if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi
exit "$fail"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Сверка bucket и БД (только чтение): orphans — объект без строки, missing —
# ready-строка без объекта. Код выхода 1, если что-то найдено.
set -eu
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
objects=$(mktemp)
rows=$(mktemp)
trap 'rm -f "$objects" "$rows"' EXIT
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r u < /run/secrets/minio_root_user
IFS= read -r p < /run/secrets/minio_root_password
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000"
exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \
| sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects"
$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key, status FROM attachments" | sort > "$rows"
echo "bucket: $bucket"
echo "objects: $(wc -l < "$objects" | tr -d ' ')"
echo "rows by status:"
cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /'
orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects")
missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects")
echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)"
[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /'
echo "missing: $(printf '%s' "$missing" | grep -c . || true)"
[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /'
[ -z "$orphans$missing" ]
+50
View File
@@ -0,0 +1,50 @@
"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts."""
import random
import struct
import zlib
SIGNATURE = b"\x89PNG\r\n\x1a\n"
NOISE_WIDTH = 1024
NOISE_ROW = 1 + NOISE_WIDTH * 3
def chunk(kind: bytes, data: bytes) -> bytes:
crc = zlib.crc32(kind + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
def build_png(width: int, height: int, idat: bytes) -> bytes:
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
def pad_png(data: bytes, size: int) -> bytes:
"""Insert a private ancillary chunk before IEND to reach exactly `size` bytes."""
extra = size - len(data)
if extra == 0:
return data
if extra < 12:
raise ValueError("padding needs at least 12 bytes")
return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:]
def _noise_of_height(height: int, rng: random.Random) -> bytes:
raw = bytearray(rng.randbytes(NOISE_ROW * height))
for row in range(height):
raw[row * NOISE_ROW] = 0 # filter type "none"
compressor = zlib.compressobj(0)
return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush())
def noise_png(size: int, seed: int = 0) -> bytes:
"""Incompressible, structurally valid PNG of exactly `size` bytes."""
rng = random.Random(seed)
height = max(1, size // NOISE_ROW)
while True:
png = _noise_of_height(height, rng)
if len(png) > size or 0 < size - len(png) < 12:
height -= 1
if height < 1:
raise ValueError("size is too small")
continue
return pad_png(png, size)
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20.
# Needs a running stand (make stand-up) and `make init-env` credentials.
set -eu
cd "$(dirname "$0")/../.."
out=evidence/lab4/bench
mkdir -p "$out"
bench() {
mode=$1 size=$2 conc=$3
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \
--concurrency "$conc" --uploads-per-worker 5 --repeat 3 \
--output "$out/$mode-$size-c$conc.json"
}
for size in 131072 1048576 9961472; do
for mode in proxy direct; do
echo "== $mode size=$size c=10"
bench "$mode" "$size" 10
done
done
echo "== proxy size=9961472 c=20"
bench proxy 9961472 20
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
cat evidence/lab4/summary.md