From 39a6df8c09fa69205b271ae9d628ef0bdd7d6781 Mon Sep 17 00:00:00 2001 From: Ilya Date: Thu, 22 Oct 2026 22:39:25 +0300 Subject: [PATCH] =?UTF-8?q?lab4:=20=D0=BF=D1=80=D1=8F=D0=BC=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B0,=20=D1=81?= =?UTF-8?q?=D0=BA=D1=80=D0=B8=D0=BF=D1=82=D1=8B=20=D1=8D=D0=BA=D1=81=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=BE=D0=B2=20=D0=B8?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/src/attachments/attachments.controller.ts | 23 + app/src/attachments/attachments.module.ts | 8 +- app/src/attachments/attachments.service.ts | 2 +- app/src/attachments/direct-uploads.service.ts | 199 ++++++ .../dto/create-direct-upload.dto.ts | 16 + app/test/direct-upload.e2e-spec.ts | 288 +++++++++ contracts/extensions.openapi.json | 585 ++++++++++++++++++ scripts/lab4/attachments_bench.py | 293 +++++++++ scripts/lab4/check_s3_security.sh | 54 ++ scripts/lab4/consistency_check.sh | 35 ++ scripts/lab4/pngkit.py | 50 ++ scripts/lab4/run_matrix.sh | 23 + 12 files changed, 1574 insertions(+), 2 deletions(-) create mode 100644 app/src/attachments/direct-uploads.service.ts create mode 100644 app/src/attachments/dto/create-direct-upload.dto.ts create mode 100644 app/test/direct-upload.e2e-spec.ts create mode 100644 contracts/extensions.openapi.json create mode 100755 scripts/lab4/attachments_bench.py create mode 100755 scripts/lab4/check_s3_security.sh create mode 100755 scripts/lab4/consistency_check.sh create mode 100644 scripts/lab4/pngkit.py create mode 100755 scripts/lab4/run_matrix.sh diff --git a/app/src/attachments/attachments.controller.ts b/app/src/attachments/attachments.controller.ts index 51db035..45d02dd 100644 --- a/app/src/attachments/attachments.controller.ts +++ b/app/src/attachments/attachments.controller.ts @@ -1,4 +1,5 @@ import { + Body, Controller, Get, Headers, @@ -20,6 +21,8 @@ import { CurrentUserGuard } from '../common/auth/current-user.guard.js'; import type { AppConfig } from '../config/app-config.schema.js'; import type { User } from '../users/user.js'; import { AttachmentsService } from './attachments.service.js'; +import { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js'; +import { DirectUploadsService } from './direct-uploads.service.js'; import { contentDisposition } from './filename.js'; @Controller('api/v1') @@ -29,6 +32,7 @@ export class AttachmentsController { constructor( private readonly attachments: AttachmentsService, + private readonly directUploads: DirectUploadsService, config: ConfigService, ) { this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES'); @@ -58,6 +62,25 @@ export class AttachmentsController { ); } + @Post('chats/:chatId/attachments/direct-uploads') + @HttpCode(201) + initDirectUpload( + @Param('chatId', new ParseUUIDPipe()) chatId: string, + @CurrentUser() user: User, + @Body() body: CreateDirectUploadDto, + ) { + return this.directUploads.init(chatId, user, body); + } + + @Post('attachments/:attachmentId/complete') + @HttpCode(200) + completeDirectUpload( + @Param('attachmentId', new ParseUUIDPipe()) attachmentId: string, + @CurrentUser() user: User, + ) { + return this.directUploads.complete(attachmentId, user); + } + @Get('attachments/:attachmentId') metadata( @Param('attachmentId', new ParseUUIDPipe()) attachmentId: string, diff --git a/app/src/attachments/attachments.module.ts b/app/src/attachments/attachments.module.ts index e1d1c1f..32305df 100644 --- a/app/src/attachments/attachments.module.ts +++ b/app/src/attachments/attachments.module.ts @@ -8,6 +8,7 @@ import { StorageModule } from '../storage/storage.module.js'; import { AttachmentSweeper } from './attachment-sweeper.service.js'; import { AttachmentsController } from './attachments.controller.js'; import { AttachmentsRepository } from './attachments.repository.js'; +import { DirectUploadsService } from './direct-uploads.service.js'; import { AttachmentsService } from './attachments.service.js'; @Module({ @@ -20,7 +21,12 @@ import { AttachmentsService } from './attachments.service.js'; StorageModule, ], controllers: [AttachmentsController], - providers: [AttachmentsRepository, AttachmentsService, AttachmentSweeper], + providers: [ + AttachmentsRepository, + AttachmentsService, + DirectUploadsService, + AttachmentSweeper, + ], exports: [AttachmentsRepository], }) export class AttachmentsModule {} diff --git a/app/src/attachments/attachments.service.ts b/app/src/attachments/attachments.service.ts index a3a330c..e46b77d 100644 --- a/app/src/attachments/attachments.service.ts +++ b/app/src/attachments/attachments.service.ts @@ -279,7 +279,7 @@ export class AttachmentsService { return error; } - private async compensate(id: string, key: string): Promise { + async compensate(id: string, key: string): Promise { try { await this.storage.delete(key); await this.repository.deletePending(id); diff --git a/app/src/attachments/direct-uploads.service.ts b/app/src/attachments/direct-uploads.service.ts new file mode 100644 index 0000000..b902ac3 --- /dev/null +++ b/app/src/attachments/direct-uploads.service.ts @@ -0,0 +1,199 @@ +import { + ConflictException, + HttpException, + Injectable, + NotFoundException, + PayloadTooLargeException, + UnsupportedMediaTypeException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { randomUUID } from 'node:crypto'; +import type { AppConfig } from '../config/app-config.schema.js'; +import { ChatsService } from '../chats/chats.service.js'; +import { MetricsService } from '../metrics/metrics.service.js'; +import { ObjectStorage } from '../storage/object-storage.js'; +import type { User } from '../users/user.js'; +import { toAttachment, type Attachment } from './attachment.js'; +import { AttachmentsRepository } from './attachments.repository.js'; +import { AttachmentsService, imageRejection } from './attachments.service.js'; +import type { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js'; +import { validateFilename } from './filename.js'; +import { + IMAGE_HEADER_MAX_BYTES, + ImageInspector, + ImageRejectedError, +} from './image/image-inspector.js'; + +export type DirectUpload = { + attachment_id: string; + method: 'PUT'; + url: string; + headers: Record; + expires_at: string; +}; + +const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']); + +@Injectable() +export class DirectUploadsService { + private readonly maxUploadBytes: number; + private readonly maxPixels: number; + private readonly ttlSeconds: number; + private readonly maxPending: number; + + constructor( + config: ConfigService, + private readonly repository: AttachmentsRepository, + private readonly storage: ObjectStorage, + private readonly chats: ChatsService, + private readonly metrics: MetricsService, + private readonly attachments: AttachmentsService, + ) { + this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES'); + this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS'); + this.ttlSeconds = config.getOrThrow('DIRECT_UPLOAD_URL_TTL_SECONDS'); + this.maxPending = config.getOrThrow( + 'DIRECT_UPLOAD_MAX_PENDING_PER_USER', + ); + } + + async init( + chatId: string, + user: User, + dto: CreateDirectUploadDto, + ): Promise { + const filename = validateFilename(dto.filename); + if (!MEDIA_TYPES.has(dto.content_type)) + throw new UnsupportedMediaTypeException( + 'content_type must be image/png or image/jpeg', + ); + if (dto.size_bytes > this.maxUploadBytes) + throw new PayloadTooLargeException('Image is too large'); + await this.chats.getForUser(chatId, user); + if ( + (await this.repository.countPendingDirect(user.id)) >= + this.maxPending + ) + throw new HttpException( + { + message: 'Too many unfinished direct uploads', + retryAfter: this.ttlSeconds, + }, + 429, + ); + + const id = randomUUID(); + const key = this.storage.objectKey(id); + await this.repository.insertPending({ + id, + chatId, + ownerId: user.id, + filename, + contentType: dto.content_type, + uploadMode: 'direct', + objectKey: key, + sizeBytes: dto.size_bytes, + sha256: dto.sha256, + ttlMs: this.ttlSeconds * 1000, + }); + const checksum = Buffer.from(dto.sha256, 'hex').toString('base64'); + let url: string; + try { + url = await this.storage.presignPut(key, { + contentType: dto.content_type, + contentLength: dto.size_bytes, + checksumSha256Base64: checksum, + expiresInSeconds: this.ttlSeconds, + }); + } catch (error) { + await this.repository.deletePending(id); + throw error; + } + this.metrics.increment('attachments.direct.initiated'); + return { + attachment_id: id, + method: 'PUT', + url, + headers: { + 'Content-Type': dto.content_type, + 'x-amz-checksum-sha256': checksum, + }, + expires_at: new Date( + Date.now() + this.ttlSeconds * 1000, + ).toISOString(), + }; + } + + async complete(attachmentId: string, user: User): Promise { + const row = await this.repository.findForComplete( + attachmentId, + user.id, + ); + if (!row) throw new NotFoundException(); + if (row.status === 'ready') return toAttachment(row); + + const size = Number(row.size_bytes); + const expectedChecksum = Buffer.from( + row.sha256 as string, + 'hex', + ).toString('base64'); + const head = await this.storage.head(row.object_key); + if (head === null) + throw new ConflictException('Upload has not been received'); + if (head.size !== size || head.checksumSha256 !== expectedChecksum) { + await this.attachments.compensate(row.id, row.object_key); + throw new ConflictException( + 'Uploaded object does not match the declaration', + ); + } + + // Only the header is read: the signed checksum already pins every byte. + const inspector = new ImageInspector(this.maxPixels); + const covered = size <= IMAGE_HEADER_MAX_BYTES; + const stream = await this.storage.getStream( + row.object_key, + `bytes=0-${Math.min(size, IMAGE_HEADER_MAX_BYTES) - 1}`, + ); + if (stream === null) + throw new ConflictException('Upload has not been received'); + let verified = 0; + try { + for await (const chunk of stream as AsyncIterable) { + verified += chunk.length; + inspector.push(chunk); + if (!covered && inspector.header !== undefined) break; + } + if (covered) inspector.finish(); + else if (inspector.header === undefined) + throw new ImageRejectedError( + 'unsupported_media_type', + 'Image header is missing', + ); + } catch (error) { + if (error instanceof ImageRejectedError) { + await this.attachments.compensate(row.id, row.object_key); + throw imageRejection(error); + } + throw error; + } finally { + stream.destroy(); + this.metrics.add('attachments.direct.verify_bytes', verified); + } + const header = inspector.header; + if (header === undefined || header.contentType !== row.content_type) { + await this.attachments.compensate(row.id, row.object_key); + throw new UnsupportedMediaTypeException( + 'Image content does not match content_type', + ); + } + const ready = await this.repository.markReady(row.id, { + sizeBytes: size, + sha256: row.sha256 as string, + width: header.width, + height: header.height, + }); + if (!ready) throw new ConflictException('Upload is no longer pending'); + this.metrics.increment('attachments.direct.completed'); + return toAttachment(ready); + } +} diff --git a/app/src/attachments/dto/create-direct-upload.dto.ts b/app/src/attachments/dto/create-direct-upload.dto.ts new file mode 100644 index 0000000..f883a12 --- /dev/null +++ b/app/src/attachments/dto/create-direct-upload.dto.ts @@ -0,0 +1,16 @@ +import { IsInt, IsString, Matches, Min } from 'class-validator'; + +export class CreateDirectUploadDto { + @IsString() + filename!: string; + + @IsString() + content_type!: string; + + @IsInt() + @Min(1) + size_bytes!: number; + + @Matches(/^[a-f0-9]{64}$/u) + sha256!: string; +} diff --git a/app/test/direct-upload.e2e-spec.ts b/app/test/direct-upload.e2e-spec.ts new file mode 100644 index 0000000..40e3d17 --- /dev/null +++ b/app/test/direct-upload.e2e-spec.ts @@ -0,0 +1,288 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { Pool } from 'pg'; +import type { Attachment } from '../src/attachments/attachment.js'; +import type { DirectUpload } from '../src/attachments/direct-uploads.service.js'; +import type { ApiErrorBody } from '../src/common/errors/api-error.js'; +import { + Actor, + actor, + bearer, + createChat, + database, + download, + expectError, + http, + json, + postMessage, +} from './support/e2e.js'; +import { pixelBombPng } from './support/png.js'; + +const png = readFileSync( + new URL('../../tests/fixtures/sample.png', import.meta.url), +); +const jpeg = readFileSync(new URL('./fixtures/sample.jpg', import.meta.url)); +const MAX = 10_485_760; +const shortTtl = process.env.E2E_DIRECT_TTL_SHORT === '1'; + +let db: Pool; +let alice: Actor; +let bob: Actor; +let eve: Actor; +let chatId: string; + +const sha256 = (data: Uint8Array) => + createHash('sha256').update(data).digest('hex'); + +function init(token: string, chat: string, body: Record) { + return http( + `/api/v1/chats/${chat}/attachments/direct-uploads`, + json(body, bearer(token)), + ); +} + +function declaration(data: Buffer, contentType = 'image/png') { + return { + filename: 'direct.png', + content_type: contentType, + size_bytes: data.length, + sha256: sha256(data), + }; +} + +async function initOk(data: Buffer, contentType = 'image/png') { + const result = await init( + alice.token, + chatId, + declaration(data, contentType), + ); + expect(result.response.status).toBe(201); + return result.body; +} + +// The PUT goes straight to the public S3 endpoint, not through the API. +async function put(ticket: DirectUpload, data: Buffer): Promise { + return fetch(ticket.url, { + method: ticket.method, + headers: ticket.headers, + body: data as unknown as BodyInit, + }); +} + +function complete(token: string, id: string) { + return http( + `/api/v1/attachments/${id}/complete`, + { + method: 'POST', + headers: bearer(token), + }, + ); +} + +describe('direct upload (presigned PUT)', () => { + beforeAll(async () => { + db = database(); + alice = await actor(); + bob = await actor(); + eve = await actor(); + chatId = (await createChat(alice, [bob])).id; + }); + + afterAll(async () => { + await db.end(); + }); + + it('uploads around the API and exposes the attachment after complete', async () => { + const ticket = await initOk(png); + expect(ticket.method).toBe('PUT'); + expect(ticket.headers['x-amz-checksum-sha256']).toBe( + Buffer.from(sha256(png), 'hex').toString('base64'), + ); + expect(Date.parse(ticket.expires_at)).toBeGreaterThan(Date.now()); + expect(new URL(ticket.url).pathname).toBe( + `/messenger-attachments/attachments/${ticket.attachment_id}/original`, + ); + + const hidden = await http( + `/api/v1/attachments/${ticket.attachment_id}`, + { headers: bearer(bob.token) }, + ); + expectError(hidden, 404, 'not_found'); + expectError( + await postMessage(alice.token, chatId, { + text: '', + attachment_ids: [ticket.attachment_id], + }), + 404, + 'not_found', + ); + expectError( + await complete(alice.token, ticket.attachment_id), + 409, + 'conflict', + ); + + expect((await put(ticket, png)).status).toBe(200); + expectError( + await complete(bob.token, ticket.attachment_id), + 404, + 'not_found', + ); + expectError( + await complete(eve.token, ticket.attachment_id), + 404, + 'not_found', + ); + const done = await complete(alice.token, ticket.attachment_id); + expect(done.response.status).toBe(200); + expect(done.body).toMatchObject({ + id: ticket.attachment_id, + owner_id: alice.user.id, + content_type: 'image/png', + size_bytes: png.length, + sha256: sha256(png), + status: 'ready', + }); + const again = await complete(alice.token, ticket.attachment_id); + expect(again.response.status).toBe(200); + expect(again.body).toEqual(done.body); + + const content = await download(bob.token, ticket.attachment_id); + expect(content.response.status).toBe(200); + expect(content.bytes.equals(png)).toBe(true); + const sent = await postMessage(alice.token, chatId, { + text: '', + attachment_ids: [ticket.attachment_id], + }); + expect(sent.response.status).toBe(201); + expect(sent.body.attachment_ids).toEqual([ticket.attachment_id]); + }); + + it('rejects invalid declarations', async () => { + const base = declaration(png); + expectError(await init(eve.token, chatId, base), 404, 'not_found'); + expectError( + await init(alice.token, chatId, { ...base, size_bytes: MAX + 1 }), + 413, + 'payload_too_large', + ); + expectError( + await init(alice.token, chatId, { + ...base, + content_type: 'image/gif', + }), + 415, + 'unsupported_media_type', + ); + for (const bad of [ + { ...base, extra: true }, + { ...base, sha256: 'ABC' }, + { ...base, size_bytes: 0 }, + { ...base, filename: 'a/b.png' }, + ]) + expectError( + await init(alice.token, chatId, bad), + 400, + 'invalid_request', + ); + }); + + it('lets the storage refuse bytes that differ from the declaration', async () => { + const ticket = await initOk(png); + const tampered = Buffer.from(png); + tampered[1000] ^= 1; + const refused = await put(ticket, tampered); + expect(refused.status).toBe(400); + expect(await refused.text()).toContain('XAmzContentChecksumMismatch'); + // No object was created, so the upload can still be finished. + expectError( + await complete(alice.token, ticket.attachment_id), + 409, + 'conflict', + ); + expect((await put(ticket, png)).status).toBe(200); + expect( + (await complete(alice.token, ticket.attachment_id)).response.status, + ).toBe(200); + + const longer = await initOk(png); + const wrongLength = await put( + longer, + Buffer.concat([png, Buffer.from('x')]), + ); + expect(wrongLength.status).toBe(403); + }); + + it('rejects a declared PNG that is really a JPEG, then forgets the claim', async () => { + const ticket = await initOk(jpeg, 'image/png'); + expect((await put(ticket, jpeg)).status).toBe(200); + expectError( + await complete(alice.token, ticket.attachment_id), + 415, + 'unsupported_media_type', + ); + expectError( + await complete(alice.token, ticket.attachment_id), + 404, + 'not_found', + ); + }); + + it('rejects a pixel bomb on complete', async () => { + const bomb = pixelBombPng(); + const ticket = await initOk(bomb); + expect((await put(ticket, bomb)).status).toBe(200); + expectError( + await complete(alice.token, ticket.attachment_id), + 413, + 'payload_too_large', + ); + expectError( + await complete(alice.token, ticket.attachment_id), + 404, + 'not_found', + ); + }); + + it('limits unfinished direct uploads per user', async () => { + const spammer = await actor(); + const chat = (await createChat(spammer, [])).id; + for (let index = 0; index < 10; index += 1) { + const ok = await init(spammer.token, chat, declaration(png)); + expect(ok.response.status).toBe(201); + } + const limited = await init(spammer.token, chat, declaration(png)); + expectError(limited, 429, 'rate_limited'); + expect( + Number(limited.response.headers.get('retry-after')), + ).toBeGreaterThan(0); + }); + + it.runIf(shortTtl)( + 'refuses an expired URL and removes the claim', + async () => { + const ticket = await initOk(png); + const expiresAt = Date.parse(ticket.expires_at); + await sleep(Math.max(0, expiresAt - Date.now()) + 2000); + expect((await put(ticket, png)).status).toBe(403); + const deadline = Date.now() + 15_000; + for (;;) { + const rows = await db.query( + 'SELECT 1 FROM attachments WHERE id = $1', + [ticket.attachment_id], + ); + if (rows.rowCount === 0) break; + expect(Date.now()).toBeLessThan(deadline); + await sleep(250); + } + expectError( + await complete(alice.token, ticket.attachment_id), + 404, + 'not_found', + ); + }, + 60_000, + ); +}); diff --git a/contracts/extensions.openapi.json b/contracts/extensions.openapi.json new file mode 100644 index 0000000..d676f35 --- /dev/null +++ b/contracts/extensions.openapi.json @@ -0,0 +1,585 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "MTUSI Messenger — расширение lab4: прямая загрузка", + "version": "1.4.0-ext.1", + "description": "Трек 1. Не меняет обязательный контракт openapi.json: POST /api/v1/chats/{chat_id}/attachments остаётся основным путём. Клиент получает presigned PUT URL и отправляет байты прямо в приватный bucket MinIO, минуя API. Подписаны заголовки content-type, content-length и x-amz-checksum-sha256: хранилище само отвергает тело с другим размером (403 SignatureDoesNotMatch) или другим SHA-256 (400 XAmzContentChecksumMismatch), объект при этом не создаётся. TTL URL — DIRECT_UPLOAD_URL_TTL_SECONDS (300 с по умолчанию). Ключ объекта attachments//original раскрывает только UUID вложения; bucket приватный, анонимный доступ запрещён. Незавершённые заявки и объекты убирает уборщик через ATTACHMENT_SWEEP_GRACE_SECONDS после истечения. Вложение видно участникам чата только после успешного complete (status=ready)." + }, + "servers": [ + { + "url": "http://localhost:8080", + "description": "Локальный учебный стенд" + } + ], + "security": [ + { + "bearerAuth": [] + } + ], + "paths": { + "/api/v1/chats/{chat_id}/attachments/direct-uploads": { + "post": { + "operationId": "initDirectUpload", + "summary": "Заявка на прямую загрузку: вернуть presigned PUT URL", + "parameters": [ + { + "name": "chat_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateDirectUpload" + } + } + } + }, + "responses": { + "201": { + "description": "Заявка создана; вложение pending и не видно никому, кроме загрузки", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DirectUpload" + } + } + } + }, + "400": { + "description": "Некорректное тело, filename или sha256", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Нет или неверен токен", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "Чат недоступен", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "413": { + "description": "size_bytes больше 10 MiB", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "415": { + "description": "content_type не image/png и не image/jpeg", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Слишком много незавершённых прямых загрузок пользователя; Retry-After — секунды до освобождения слота (TTL URL).", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "Retry-After": { + "description": "Секунды до повтора.", + "schema": { + "type": "integer", + "minimum": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "503": { + "description": "Зависимость недоступна", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + } + } + } + }, + "/api/v1/attachments/{attachment_id}/complete": { + "post": { + "operationId": "completeDirectUpload", + "summary": "Подтвердить прямую загрузку: HEAD объекта и проверка заголовка изображения", + "description": "Только владелец заявки. Проверяет размер и подписанный SHA-256 объекта (HEAD), читает заголовок изображения (до 1 MiB) и сверяет тип и число пикселей. При несоответствии заявка и объект удаляются, повторный complete даёт 404. Повторный complete готового вложения идемпотентен и возвращает 200.", + "parameters": [ + { + "name": "attachment_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "Вложение готово; тело идентично Attachment из openapi.json", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Attachment" + } + } + } + }, + "400": { + "description": "Некорректный attachment_id", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Нет или неверен токен", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "Заявка не найдена, не принадлежит пользователю или уже удалена", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "Объект не получен или не совпадает с заявленным размером/SHA-256", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "413": { + "description": "Слишком много пикселей", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "415": { + "description": "Содержимое не PNG/JPEG или не совпадает с content_type", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + }, + "503": { + "description": "Хранилище недоступно", + "headers": { + "X-Request-Id": { + "description": "Идентификатор запроса.", + "schema": { + "type": "string", + "minLength": 1 + } + }, + "X-Instance-Id": { + "description": "Идентификатор реплики.", + "schema": { + "type": "string", + "minLength": 1 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "openapi.json#/components/schemas/Error" + } + } + } + } + } + } + } + }, + "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer", + "bearerFormat": "opaque session token" + } + }, + "schemas": { + "CreateDirectUpload": { + "type": "object", + "additionalProperties": false, + "required": [ + "filename", + "content_type", + "size_bytes", + "sha256" + ], + "properties": { + "filename": { + "type": "string", + "minLength": 1, + "maxLength": 255 + }, + "content_type": { + "type": "string", + "enum": [ + "image/png", + "image/jpeg" + ] + }, + "size_bytes": { + "type": "integer", + "minimum": 1, + "maximum": 10485760 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } + }, + "DirectUpload": { + "type": "object", + "additionalProperties": false, + "required": [ + "attachment_id", + "method", + "url", + "headers", + "expires_at" + ], + "properties": { + "attachment_id": { + "type": "string", + "format": "uuid" + }, + "method": { + "type": "string", + "enum": [ + "PUT" + ] + }, + "url": { + "type": "string", + "format": "uri", + "description": "Presigned URL; содержит подпись, не логировать." + }, + "headers": { + "type": "object", + "description": "Заголовки, которые клиент обязан отправить как есть. Content-Length клиент ставит сам; тело должно быть ровно size_bytes.", + "properties": { + "Content-Type": { + "type": "string" + }, + "x-amz-checksum-sha256": { + "type": "string" + } + }, + "required": [ + "Content-Type", + "x-amz-checksum-sha256" + ] + }, + "expires_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } +} diff --git a/scripts/lab4/attachments_bench.py b/scripts/lab4/attachments_bench.py new file mode 100755 index 0000000..e5b4174 --- /dev/null +++ b/scripts/lab4/attachments_bench.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode. + + attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \ + --uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json + attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md + +Run from the repository root with the stand up. Tokens, signed URLs and secrets +are never written to the output. +""" +import argparse +import base64 +import hashlib +import http.client +import json +import os +import statistics +import subprocess +import sys +import threading +import time +import urllib.parse +import uuid +from datetime import datetime, timezone + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from pngkit import noise_png # noqa: E402 + + +def sh(*args, check=True): + return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip() + + +def read_env(path=".env"): + values = {} + with open(path) as handle: + for line in handle: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, _, value = line.partition("=") + values[key] = value + return values + + +def request(host, port, method, path, headers=None, body=None): + connection = http.client.HTTPConnection(host, port, timeout=120) + try: + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + payload = response.read() + return response.status, payload + finally: + connection.close() + + +def api_json(env, method, path, token=None, body=None, basic=None): + headers = {} + data = None + if body is not None: + headers["content-type"] = "application/json" + data = json.dumps(body).encode() + if token: + headers["authorization"] = f"Bearer {token}" + if basic: + headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() + port = int(env.get("APP_PUBLIC_PORT", "8080")) + status, payload = request("127.0.0.1", port, method, path, headers, data) + return status, json.loads(payload) if payload else None + + +def containers(): + ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split() + return sorted(ids) + + +NODE_METRICS = ( + "fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1]," + "{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})" + ".then(r=>r.text()).then(console.log)" +) + + +def metrics(container): + return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, "")) + + +def reset_metrics(container): + sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset") + + +def net_bytes(container): + for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines(): + if line.strip().startswith("eth0:"): + fields = line.split(":", 1)[1].split() + return int(fields[0]), int(fields[8]) + raise RuntimeError("eth0 not found") + + +def percentile(values, fraction): + ordered = sorted(values) + if not ordered: + return 0.0 + return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))] + + +def one_upload(env, mode, token, chat_id, data, seed): + digest = hashlib.sha256(data).hexdigest() + filename = f"bench-{seed}.png" + port = int(env.get("APP_PUBLIC_PORT", "8080")) + started = time.perf_counter() + if mode == "proxy": + status, payload = request( + "127.0.0.1", port, "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}", + {"authorization": f"Bearer {token}", "content-type": "image/png"}, data, + ) + ok = status == 201 + attachment = json.loads(payload) if ok else None + else: + status, ticket = api_json( + env, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", token, + {"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest}, + ) + ok = status == 201 + attachment = None + if ok: + target = urllib.parse.urlsplit(ticket["url"]) + put_status, _ = request( + target.hostname, target.port, "PUT", target.path + "?" + target.query, + {**ticket["headers"]}, data, + ) + ok = put_status == 200 + if ok: + status, attachment = api_json( + env, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", token + ) + ok = status == 200 + elapsed_ms = (time.perf_counter() - started) * 1000 + if ok and (attachment is None or attachment["sha256"] != digest): + ok = False + return ok, elapsed_ms + + +def run_once(env, args, token, chat_id, ids): + for container in ids: + reset_metrics(container) + time.sleep(0.2) + baseline = {c: metrics(c) for c in ids} + net_before = {c: net_bytes(c) for c in ids} + latencies, failures = [], [] + lock = threading.Lock() + + def worker(index): + for step in range(args.uploads_per_worker): + seed = index * 1000 + step + data = noise_png(args.size_bytes, seed) + ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed) + with lock: + (latencies if ok else failures).append(elapsed) + + started = time.perf_counter() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + duration = time.perf_counter() - started + time.sleep(0.2) + after = {c: metrics(c) for c in ids} + net_after = {c: net_bytes(c) for c in ids} + + replicas = [] + for c in ids: + base_rss = baseline[c]["memory_bytes"]["rss"] + peak = after[c]["memory_peak_bytes"] + replicas.append({ + "instance_id": after[c]["instance_id"], + "delta_rx_bytes": net_after[c][0] - net_before[c][0], + "delta_tx_bytes": net_after[c][1] - net_before[c][1], + "baseline_rss_bytes": base_rss, + "peak_memory_bytes": peak, + "delta_rss_bytes": peak["rss"] - base_rss, + "counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")}, + }) + total = args.concurrency * args.uploads_per_worker + return { + "uploads": total, + "succeeded": len(latencies), + "failed": len(failures), + "duration_s": round(duration, 3), + "latency_ms": { + "p50": round(percentile(latencies, 0.50), 1), + "p95": round(percentile(latencies, 0.95), 1), + "p99": round(percentile(latencies, 0.99), 1), + }, + "replicas": replicas, + "total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas), + "total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas), + "rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total), + "tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total), + "max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas), + } + + +def command_run(args): + env = read_env() + ids = containers() + if not ids: + sys.exit("no api containers found") + name = f"bench_{uuid.uuid4().hex[:10]}" + password = "bench:" + uuid.uuid4().hex + status, user = api_json(env, "POST", "/api/v1/users", + body={"username": name, "display_name": "bench", "password": password}) + assert status == 201, status + status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}") + assert status == 201, status + token = session["token"] + status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]}) + assert status == 201, status + + # Warm-up: one upload per replica path, not measured. + for seed in range(len(ids) * 2): + one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed) + + limits = [] + for c in ids: + limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c)) + repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)] + mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal")) + result = { + "date": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "git": sh("git", "rev-parse", "--short", "HEAD", check=False), + "params": { + "mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency, + "uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat, + }, + "host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total}, + "api_container_limits_nanocpus_memory": limits, + "repeats": repeats, + } + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, "w") as handle: + json.dump(result, handle, indent=2) + handle.write("\n") + for index, item in enumerate(repeats): + print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} " + f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} " + f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms") + + +def mib(value): + return f"{value / 1048576:.2f}" + + +def command_summarize(args): + rows = [] + for path in args.paths: + with open(path) as handle: + data = json.load(handle) + p = data["params"] + reps = data["repeats"] + med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731 + rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data)) + rows.sort(key=lambda r: (r[1], r[0], r[2])) + print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |") + print("|---|---|---|---|---|---|---|---|---|") + for mode, size, conc, reps, med, _data in rows: + ok = sum(r["succeeded"] for r in reps) + total = sum(r["uploads"] for r in reps) + p50 = statistics.median(r["latency_ms"]["p50"] for r in reps) + p95 = statistics.median(r["latency_ms"]["p95"] for r in reps) + print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | " + f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |") + + +def main(): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run") + run.add_argument("--mode", choices=["proxy", "direct"], required=True) + run.add_argument("--size-bytes", type=int, required=True) + run.add_argument("--concurrency", type=int, default=10) + run.add_argument("--uploads-per-worker", type=int, default=5) + run.add_argument("--repeat", type=int, default=3) + run.add_argument("--output", required=True) + summarize = sub.add_parser("summarize") + summarize.add_argument("paths", nargs="+") + args = parser.parse_args() + {"run": command_run, "summarize": command_summarize}[args.command](args) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab4/check_s3_security.sh b/scripts/lab4/check_s3_security.sh new file mode 100755 index 0000000..497c5ba --- /dev/null +++ b/scripts/lab4/check_s3_security.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials +# API, закрытая консоль. Секреты не печатаются. +set -u +cd "$(dirname "$0")/../.." +env_file="${COMPOSE_ENV_FILE:-.env}" +# shellcheck disable=SC1090 +set -a; . "./$env_file"; set +a +endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}" +bucket="${S3_BUCKET:-messenger-attachments}" +compose="docker compose --env-file $env_file" +fail=0 + +report() { # status name + if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi +} + +key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \ + -c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1") +if [ -z "$key" ]; then + echo "FAIL no ready attachment to check; upload one first (make test)" + exit 1 +fi + +code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key") +[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)" + +code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/") +[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)" + +code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write") +[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)" + +# The API credentials only, inside a one-off container (secrets come from files). +api_mc() { + $compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c ' + IFS= read -r k < /run/secrets/s3_api_access_key_id + IFS= read -r s < /run/secrets/s3_api_secret_access_key + export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000" + exec mc "$@"' sh "$@" >/dev/null 2>&1 +} + +api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets" +api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API" +api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*" +api_mc stat "api/$bucket/outside-prefix" ; rc=$? +api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/" +: "$rc" + +port=$($compose port s3 9001 2>/dev/null || true) +[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)" +if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi + +exit "$fail" diff --git a/scripts/lab4/consistency_check.sh b/scripts/lab4/consistency_check.sh new file mode 100755 index 0000000..fd97eb7 --- /dev/null +++ b/scripts/lab4/consistency_check.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Сверка bucket и БД (только чтение): orphans — объект без строки, missing — +# ready-строка без объекта. Код выхода 1, если что-то найдено. +set -eu +cd "$(dirname "$0")/../.." +env_file="${COMPOSE_ENV_FILE:-.env}" +# shellcheck disable=SC1090 +set -a; . "./$env_file"; set +a +bucket="${S3_BUCKET:-messenger-attachments}" +compose="docker compose --env-file $env_file" + +objects=$(mktemp) +rows=$(mktemp) +trap 'rm -f "$objects" "$rows"' EXIT + +$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c ' + IFS= read -r u < /run/secrets/minio_root_user + IFS= read -r p < /run/secrets/minio_root_password + export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000" + exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \ + | sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects" +$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \ + -c "SELECT object_key, status FROM attachments" | sort > "$rows" + +echo "bucket: $bucket" +echo "objects: $(wc -l < "$objects" | tr -d ' ')" +echo "rows by status:" +cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /' +orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects") +missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects") +echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)" +[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /' +echo "missing: $(printf '%s' "$missing" | grep -c . || true)" +[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /' +[ -z "$orphans$missing" ] diff --git a/scripts/lab4/pngkit.py b/scripts/lab4/pngkit.py new file mode 100644 index 0000000..a29c886 --- /dev/null +++ b/scripts/lab4/pngkit.py @@ -0,0 +1,50 @@ +"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts.""" +import random +import struct +import zlib + +SIGNATURE = b"\x89PNG\r\n\x1a\n" +NOISE_WIDTH = 1024 +NOISE_ROW = 1 + NOISE_WIDTH * 3 + + +def chunk(kind: bytes, data: bytes) -> bytes: + crc = zlib.crc32(kind + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc) + + +def build_png(width: int, height: int, idat: bytes) -> bytes: + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") + + +def pad_png(data: bytes, size: int) -> bytes: + """Insert a private ancillary chunk before IEND to reach exactly `size` bytes.""" + extra = size - len(data) + if extra == 0: + return data + if extra < 12: + raise ValueError("padding needs at least 12 bytes") + return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:] + + +def _noise_of_height(height: int, rng: random.Random) -> bytes: + raw = bytearray(rng.randbytes(NOISE_ROW * height)) + for row in range(height): + raw[row * NOISE_ROW] = 0 # filter type "none" + compressor = zlib.compressobj(0) + return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush()) + + +def noise_png(size: int, seed: int = 0) -> bytes: + """Incompressible, structurally valid PNG of exactly `size` bytes.""" + rng = random.Random(seed) + height = max(1, size // NOISE_ROW) + while True: + png = _noise_of_height(height, rng) + if len(png) > size or 0 < size - len(png) < 12: + height -= 1 + if height < 1: + raise ValueError("size is too small") + continue + return pad_png(png, size) diff --git a/scripts/lab4/run_matrix.sh b/scripts/lab4/run_matrix.sh new file mode 100755 index 0000000..c45df4e --- /dev/null +++ b/scripts/lab4/run_matrix.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20. +# Needs a running stand (make stand-up) and `make init-env` credentials. +set -eu +cd "$(dirname "$0")/../.." +out=evidence/lab4/bench +mkdir -p "$out" +bench() { + mode=$1 size=$2 conc=$3 + python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \ + --concurrency "$conc" --uploads-per-worker 5 --repeat 3 \ + --output "$out/$mode-$size-c$conc.json" +} +for size in 131072 1048576 9961472; do + for mode in proxy direct; do + echo "== $mode size=$size c=10" + bench "$mode" "$size" 10 + done +done +echo "== proxy size=9961472 c=20" +bench proxy 9961472 20 +python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md +cat evidence/lab4/summary.md