diff --git a/.env.example b/.env.example index 4d9bbc4a..8dba1057 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ # These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 +# Host port published by the LB. Default 8080; change if that port is already taken. +LB_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 # compose.yaml gives api/migrate DB_HOST/DB_USER/DB_NAME and @@ -28,3 +30,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Reconciler does not delete originals younger than this (seconds). Overridable per run with --grace-seconds. +RECONCILE_GRACE_SECONDS=900 diff --git a/Makefile b/Makefile index 5d86d171..d4c6b250 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,11 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-attachments check-private-bucket upload-mem help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "test-attachments: student negative cases; check-private-bucket: MinIO ACL; upload-mem: 10 concurrent uploads" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +29,12 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachments: + $(PYTHON) tests/attachments_negative.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +check-private-bucket: + $(PYTHON) scripts/check_private_bucket.py + +upload-mem: + $(PYTHON) scripts/upload_mem.py --base-url "$(BASE_URL)" --output evidence/upload-concurrency.json diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..726bb1f9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,91 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Sunrise443 (указать группу) +- Целевая оценка; выбранный трек 5, если нужен: **4**; трек 5 не заявлен +- Commit/PR: TODO после публикации (рабочее дерево поверх `c594b88`) +- Использованные источники, библиотеки и помощь ИИ: OpenAPI/семантика курса; документация MinIO, boto3, HAProxy 3.2; FastAPI/Starlette; реализация с помощью Cursor (Grok) ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose v2, Python 3.9+ для публичных тестов, свободный порт для LB. + +```sh +cp .env.example .env +# Задайте уникальные POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD, S3_SECRET_ACCESS_KEY. +# Если 8080 занят (другая лабораторная), задайте LB_PORT, например 8084. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождаться healthy у lb/api. Миграции и minio-init — one-shot. +make test BASE_URL=http://localhost:${LB_PORT:-8080} +make test-attachments BASE_URL=http://localhost:${LB_PORT:-8080} +make check-private-bucket +# Остановка с сохранением volumes: +docker compose --env-file .env down +# Учебный сброс данных (уничтожает pgdata и miniodata): +docker compose --env-file .env down -v +``` + +Образы (закреплены тегом+digest в `compose.yaml` / `Dockerfile`): + +| Роль | Service | Образ | Volume | Сеть | Порт на host | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | lb | haproxy:3.2.26-alpine@sha256:96ec… | — | edge | 127.0.0.1 / ::1 `LB_PORT` | нет | +| API | api ×2 | messenger-lab4-app (python:3.14.8-slim) | нет оригиналов | edge+data | нет | DB и S3 из secret files uid 10001 | +| PostgreSQL | postgres | postgres:16.10-alpine@sha256:0296… | pgdata | data (internal) | нет | POSTGRES_PASSWORD_FILE | +| S3 | minio | minio RELEASE.2025-09-07…@sha256:14ce… | miniodata | data, alias `s3` | нет | root только у minio-init | +| Bootstrap | minio-init | тот же MinIO + `infra/minio/bootstrap.sh` | — | data | — | root + API keys | +| Сверка | reconcile (profile `ops`) | тот же app image | — | data | — | те же API S3 keys | +| Private probe | check-private-bucket (profile `ops`) | тот же MinIO | — | data | — | root + API keys | + +Локально на этой машине `LB_PORT=8084`, потому что 8080 занят стендом lab3. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy → API ×2 → PostgreSQL (метаданные) + приватный bucket MinIO (байты). Object key назначает сервер: `originals/{uuid}`. Filename только для отображения. + +Источник истины для «файл принят»: строка в `attachments` **и** объект в S3. `201 ready` возвращается только после PUT и INSERT. Неудачный INSERT компенсируется `delete_object`. Обрыв клиента до PUT не создаёт объект. Crash между PUT и INSERT оставляет orphan; его забирает сверка после grace period (по умолчанию 900 с). + +Upload не держит весь файл в Python heap: чанки считаются, SHA-256 копится на лету, заголовок PNG/JPEG смотрится без распаковки bitmap (`inspect_prefix`), тело уходит в `SpooledTemporaryFile` (64 KiB RAM, остальное ephemeral tempfile) и streaming `put_object`. Лимит 10 MiB применяется по `Content-Length` и по счётчику чанков; HAProxy отсекает `Content-Length > 10485760` до API. PUT S3 ограничен 30 с; в backend HAProxy `set-timeout server 30s` только для `POST .../attachments`. + +Перенос из lab3: сессии, идемпотентность сообщений, две реплики, LB. Новое: S3, Attachment, сверка, негативные тесты файлов. + +Закладки под оценку 5 (не реализованы): prefix только `originals/`; reconciler игнорирует другие ключи; HTTP upload остаётся proxy `201`; dry-run по умолчанию. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py --base-url http://localhost:8084` | PASS lab4 public API | [evidence/smoke.txt](evidence/smoke.txt) | +| S3 в стенде, байты не в БД/диске API | compose: minio без host ports, API без volume оригиналов, ключ `originals/{uuid}` | minio ports `[]`, API не публикует 9000 | [evidence/compose-minio-exposure.json](evidence/compose-minio-exposure.json) | +| PNG/JPEG ≤10 MiB / 40M px, MIME по содержимому | smoke fixture + negative suite | 201 на sample.png; 413/415 на отказ | [evidence/attachments-negative.txt](evidence/attachments-negative.txt) | +| Другая реплика / restart volumes | `scripts/restart.sh`, `scripts/stop-one.sh` (уровень 3, сохранено) | hook перезапускает postgres+minio+api+lb; stop-one гасит одну реплику | scripts/restart.sh, scripts/stop-one.sh | +| Негативные вложения, не 500 | `make test-attachments BASE_URL=http://localhost:8084` | oversized CL и chunked → 413; corrupt/MIME spoof → 415; pixel bomb → 413; чужой id и чужой чат → 404 | [evidence/attachments-negative.txt](evidence/attachments-negative.txt) | +| Компенсация / сверка / grace | `docker compose --profile ops run --rm reconcile --seed-orphan` затем `--apply --grace-seconds 0` | dry-run: сирота в `orphans_protected_by_grace`; apply: ключ удалён; живые originals не тронуты | [evidence/reconcile-dry-run.json](evidence/reconcile-dry-run.json), [evidence/reconcile-apply.json](evidence/reconcile-apply.json) | +| RAM, ≥10 одновременных upload | `python scripts/upload_mem.py --base-url http://localhost:8084` | 10/10 ок, ~1.9 MiB PNG, пик RSS api 93.66 MiB при лимите 256M (N×size ≈ 19 MiB) | [evidence/upload-concurrency.json](evidence/upload-concurrency.json) | +| Идемпотентный bootstrap | два `docker compose --env-file .env run --rm --no-deps minio-init` | оба: `bucket messenger-attachments is private`, exit 0 | [evidence/minio-init-rerun.txt](evidence/minio-init-rerun.txt) | +| Узкие S3 creds, консоль закрыта, private bucket вне API | `python scripts/check_private_bucket.py` | anonymous GET 403; API user не листит чужой bucket; MINIO_BROWSER=off; нет host ports; не в edge | [evidence/private-bucket.txt](evidence/private-bucket.txt) | +| Трек 5 | не заявлен | — | — | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза (RAM): десять одновременных ~1.9 MiB PNG не разгоняют RSS API пропорционально `N * size` и укладываются в 256 MiB. Стенд: Windows AMD64, Intel, compose limits api 256M × 2 реплики, MinIO 512M. Профиль: 10 POST, файл 1 994 104 байт, сгенерирован в рантайме. Результат: 10 успехов, 0 ошибок, p50 ≈ 0.52 с, p95 ≈ 0.55 с, пик RSS контейнера api-2 **93.66 MiB**. Ограничение: `docker stats` — RSS контейнера с шагом десятых долей секунды, не профилировщик кучи; в выборке 3 снимка, пик внутри запроса мог быть выше. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Content-Length > 10 MiB | 413 на LB, тело не читается | 413 `payload_too_large` | нет записи | +| Chunked тело > 10 MiB | 413 на API по счётчику чанков | 413 `payload_too_large` | нет объекта | +| Повреждённый PNG/JPEG, подмена MIME | 415 | 415 `unsupported_media_type` | нет объекта | +| IHDR 10000×5000 (50M px) | 413 до decode | 413 | нет объекта | +| GET чужого attachment_id | 404, без утечки | 404 `not_found` | — | +| attachment_id другого чата в POST message | 404 | 404 | сообщение не создано | +| S3 PUT прошёл, INSERT нет | компенсация DELETE; иначе orphan до сверки | `_discard_object`; сирота удаляется `--apply` после grace | объект без метаданных не вечный | +| Повтор minio-init | no-op, bucket private | два exit 0 | нет | +| Unauthenticated GET объекта | 403 мимо API | 403 | — | + +В сверке на учебном volume есть два `missing_objects` с ключами без prefix `originals/` (старые строки БД с прошлых итераций). Reconciler **не** удаляет строки PostgreSQL — это зона трека 5. Живые `originals/{uuid}` после apply на месте. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Выдерживаются клиентские ошибки вложений (4xx, не 500), обрыв до PUT, неудачный INSERT, stale orphan после grace, недоступность bucket с хоста, повтор bootstrap. Не выдерживается потеря всего Docker-host; одна PostgreSQL и один MinIO не HA. Download оригинала пока буферизуется целиком (критерий 4 про загрузку). Spool — ephemeral tempfile запроса, не источник истины. + +Трек 5 не делался: нет 202/worker/thumbnail, нет presigned, нет research-измерения стоимости обхода. Reconciler уже с dry-run и grace, prefix `originals/` не трогает будущий `staging/` / `variants/`. diff --git a/app/images.py b/app/images.py index eb964851..edd9f1f2 100644 --- a/app/images.py +++ b/app/images.py @@ -3,18 +3,25 @@ Dimensions come from IHDR or a JPEG SOF marker. The compressed pixel data is never unpacked, so a small file cannot expand into an unbounded bitmap before the pixel limit is applied. """ -from typing import NoReturn +from typing import NoReturn, Optional from .errors import payload_too_large, unsupported_media_type MAX_UPLOAD_BYTES = 10_485_760 MAX_IMAGE_PIXELS = 40_000_000 +# SOF/IHDR almost always sit in the first tens of kilobytes. The rest of the +# object is hashed and stored without being held as one Python bytes object. +INSPECT_PREFIX = 256 * 1024 _PNG = b'\x89PNG\r\n\x1a\n' # SOF markers that carry width and height. DHT/DAC/JPG are skipped by the length field. _JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} +class _NeedMore(Exception): + """The prefix does not yet contain a complete PNG IHDR or JPEG SOF.""" + + def _reject() -> NoReturn: raise unsupported_media_type('Файл не является PNG или JPEG') @@ -26,18 +33,29 @@ def _check_pixels(width: int, height: int) -> None: raise payload_too_large('Изображение превышает 40 миллионов пикселей') -def _png_size(data: bytes) -> None: - if len(data) < 24 or not data.startswith(_PNG): +def _png_size(data: bytes) -> str: + if len(data) < 8: + if _PNG.startswith(data): + raise _NeedMore() _reject() + if not data.startswith(_PNG): + _reject() + if len(data) < 24: + raise _NeedMore() if int.from_bytes(data[8:12], 'big') != 13 or data[12:16] != b'IHDR': _reject() width = int.from_bytes(data[16:20], 'big') height = int.from_bytes(data[20:24], 'big') _check_pixels(width, height) + return 'image/png' -def _jpeg_size(data: bytes) -> None: - if len(data) < 4 or data[0:2] != b'\xff\xd8': +def _jpeg_size(data: bytes) -> str: + if len(data) < 2: + if b'\xff\xd8'.startswith(data): + raise _NeedMore() + _reject() + if data[0:2] != b'\xff\xd8': _reject() index = 2 while index < len(data): @@ -47,7 +65,7 @@ def _jpeg_size(data: bytes) -> None: while index < len(data) and data[index] == 0xFF: index += 1 if index >= len(data): - break + raise _NeedMore() marker = data[index] index += 1 if marker in (0x01, 0xD8, 0xD9) or 0xD0 <= marker <= 0xD7: @@ -55,10 +73,12 @@ def _jpeg_size(data: bytes) -> None: break continue if index + 2 > len(data): - break + raise _NeedMore() segment_length = int.from_bytes(data[index:index + 2], 'big') - if segment_length < 2 or index + segment_length > len(data): + if segment_length < 2: _reject() + if index + segment_length > len(data): + raise _NeedMore() # Start of scan: entropy-coded bytes follow. Dimensions must already have been seen. if marker == 0xDA: break @@ -68,17 +88,32 @@ def _jpeg_size(data: bytes) -> None: height = int.from_bytes(data[index + 3:index + 5], 'big') width = int.from_bytes(data[index + 5:index + 7], 'big') _check_pixels(width, height) - return + return 'image/jpeg' index += segment_length _reject() +def inspect_prefix(data: bytes, *, complete: bool) -> Optional[str]: + """Return image/png or image/jpeg, or None if `complete` is false and more bytes are required.""" + if not data: + if complete: + _reject() + return None + try: + if data[0] == 0x89: + return _png_size(data) + if data[0] == 0xFF: + return _jpeg_size(data) + _reject() + except _NeedMore: + if complete: + _reject() + return None + + def inspect_image(data: bytes) -> str: """Return image/png or image/jpeg after a header check. Raises 413 or 415.""" - if data.startswith(_PNG): - _png_size(data) - return 'image/png' - if data.startswith(b'\xff\xd8'): - _jpeg_size(data) - return 'image/jpeg' - _reject() + result = inspect_prefix(data, complete=True) + if result is None: + _reject() + return result diff --git a/app/objects.py b/app/objects.py index 76312c6c..188816fa 100644 --- a/app/objects.py +++ b/app/objects.py @@ -1,6 +1,9 @@ """Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one.""" from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FutureTimeout +from dataclasses import dataclass +from datetime import datetime +from typing import BinaryIO, List import boto3 from botocore.config import Config @@ -8,9 +11,11 @@ from botocore.exceptions import BotoCoreError, ClientError from .config import ObjectSettings +ORIGINAL_PREFIX = 'originals/' + # Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler # must still answer inside HAProxy's one-second health check. -_POOL = ThreadPoolExecutor(max_workers=16, thread_name_prefix='s3') +_POOL = ThreadPoolExecutor(max_workers=32, thread_name_prefix='s3') class MissingObject(Exception): @@ -21,6 +26,20 @@ class StorageError(Exception): """The object store could not complete the call. The message stays internal.""" +@dataclass(frozen=True) +class StoredObject: + key: str + last_modified: datetime + size: int + + +@dataclass(frozen=True) +class MultipartUpload: + key: str + upload_id: str + initiated: datetime + + class ObjectStore: def __init__(self, settings: ObjectSettings) -> None: self._bucket = settings.bucket @@ -34,7 +53,7 @@ class ObjectStore: signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, - read_timeout=2, + read_timeout=30, retries={'max_attempts': 1}, ), ) @@ -43,14 +62,26 @@ class ObjectStore: self._bounded(self._head, 0.7) def put(self, key: str, body: bytes, content_type: str) -> None: - self._bounded(lambda: self._put(key, body, content_type), 8) + self._bounded(lambda: self._put_bytes(key, body, content_type), 30) + + def put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None: + self._bounded(lambda: self._put_stream(key, body, size, content_type), 30) def get(self, key: str) -> bytes: - return self._bounded(lambda: self._get(key), 8) + return self._bounded(lambda: self._get(key), 15) def delete(self, key: str) -> None: self._bounded(lambda: self._delete(key), 8) + def list_originals(self) -> List[StoredObject]: + return self._bounded(self._list_originals, 15) + + def list_multipart(self) -> List[MultipartUpload]: + return self._bounded(self._list_multipart, 15) + + def abort_multipart(self, key: str, upload_id: str) -> None: + self._bounded(lambda: self._abort_multipart(key, upload_id), 8) + def _bounded(self, fn, timeout: float): future = _POOL.submit(fn) try: @@ -64,7 +95,7 @@ class ObjectStore: except (BotoCoreError, ClientError): raise StorageError() from None - def _put(self, key: str, body: bytes, content_type: str) -> None: + def _put_bytes(self, key: str, body: bytes, content_type: str) -> None: try: self._client.put_object( Bucket=self._bucket, @@ -76,6 +107,19 @@ class ObjectStore: except (BotoCoreError, ClientError): raise StorageError() from None + def _put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None: + body.seek(0) + try: + self._client.put_object( + Bucket=self._bucket, + Key=key, + Body=body, + ContentType=content_type, + ContentLength=size, + ) + except (BotoCoreError, ClientError): + raise StorageError() from None + def _get(self, key: str) -> bytes: try: response = self._client.get_object(Bucket=self._bucket, Key=key) @@ -97,3 +141,66 @@ class ObjectStore: self._client.delete_object(Bucket=self._bucket, Key=key) except (BotoCoreError, ClientError): raise StorageError() from None + + def _list_originals(self) -> List[StoredObject]: + objects: List[StoredObject] = [] + token = None + try: + while True: + kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX} + if token: + kwargs['ContinuationToken'] = token + response = self._client.list_objects_v2(**kwargs) + for item in response.get('Contents') or (): + key = item['Key'] + if not key.startswith(ORIGINAL_PREFIX) or key.endswith('/'): + continue + objects.append(StoredObject(key, item['LastModified'], int(item['Size']))) + if not response.get('IsTruncated'): + return objects + token = response.get('NextContinuationToken') + if not token: + return objects + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _list_multipart(self) -> List[MultipartUpload]: + uploads: List[MultipartUpload] = [] + key_marker = '' + upload_marker = '' + try: + while True: + kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX} + if key_marker: + kwargs['KeyMarker'] = key_marker + if upload_marker: + kwargs['UploadIdMarker'] = upload_marker + response = self._client.list_multipart_uploads(**kwargs) + for item in response.get('Uploads') or (): + key = item['Key'] + if not key.startswith(ORIGINAL_PREFIX): + continue + uploads.append(MultipartUpload(key, item['UploadId'], item['Initiated'])) + if not response.get('IsTruncated'): + return uploads + key_marker = response.get('NextKeyMarker') or '' + upload_marker = response.get('NextUploadIdMarker') or '' + if not key_marker and not upload_marker: + return uploads + except ClientError as exc: + code = exc.response.get('Error', {}).get('Code', '') + if code in ('NotImplemented', 'AccessDenied'): + return [] + raise StorageError() from None + except BotoCoreError: + raise StorageError() from None + + def _abort_multipart(self, key: str, upload_id: str) -> None: + try: + self._client.abort_multipart_upload( + Bucket=self._bucket, + Key=key, + UploadId=upload_id, + ) + except (BotoCoreError, ClientError): + raise StorageError() from None diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..a38a6f1b --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,142 @@ +"""Compare originals/ in the private bucket with attachment metadata. + +Dry-run is the default. --apply deletes unreferenced originals and aborts stale +multipart uploads older than the grace period. Keys outside originals/ are ignored +so a later staging/ or variants/ prefix is not swept. Missing S3 objects for a DB +row are reported only; they are not deleted from PostgreSQL. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import uuid +from datetime import datetime, timezone +from typing import Any, Dict + +from .config import load_object_settings, load_settings +from .db import open_pool +from .objects import ORIGINAL_PREFIX, ObjectStore, StorageError +from .store import Store + +logger = logging.getLogger('messenger.reconcile') + +_DEFAULT_GRACE = 900 + + +def _grace_default() -> int: + raw = os.getenv('RECONCILE_GRACE_SECONDS', '').strip() + return int(raw) if raw else _DEFAULT_GRACE + + +def _age_seconds(moment: datetime, now: datetime) -> float: + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return max(0.0, (now - moment).total_seconds()) + + +async def _seed(objects: ObjectStore) -> str: + key = ORIGINAL_PREFIX + str(uuid.uuid4()) + await asyncio.to_thread(objects.put, key, b'orphan-probe', 'application/octet-stream') + return key + + +async def _run(args: argparse.Namespace) -> Dict[str, Any]: + settings = load_settings() + objects = ObjectStore(load_object_settings()) + pool = await open_pool(settings.database_url) + now = datetime.now(timezone.utc) + report: Dict[str, Any] = { + 'apply': args.apply, + 'grace_seconds': args.grace_seconds, + 'seeded': None, + 'scanned_originals': 0, + 'referenced': 0, + 'orphans_protected_by_grace': [], + 'orphans_stale': [], + 'multipart_protected_by_grace': [], + 'multipart_stale': [], + 'deleted': [], + 'aborted_multipart': [], + 'missing_objects': [], + } + try: + store = Store( + pool, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + objects=objects, + ) + if args.seed_orphan: + report['seeded'] = await _seed(objects) + referenced = await store.listed_object_keys() + report['referenced'] = len(referenced) + stored = await asyncio.to_thread(objects.list_originals) + report['scanned_originals'] = len(stored) + stored_keys = {item.key for item in stored} + for key in sorted(referenced - stored_keys): + report['missing_objects'].append({'key': key}) + for item in stored: + if item.key in referenced: + continue + age = _age_seconds(item.last_modified, now) + entry = {'key': item.key, 'age_seconds': int(age), 'size': item.size} + if age < args.grace_seconds: + report['orphans_protected_by_grace'].append(entry) + continue + report['orphans_stale'].append(entry) + if args.apply: + try: + await asyncio.to_thread(objects.delete, item.key) + report['deleted'].append(item.key) + except StorageError: + logger.warning('failed to delete orphan %s', item.key) + uploads = await asyncio.to_thread(objects.list_multipart) + for item in uploads: + age = _age_seconds(item.initiated, now) + entry = {'key': item.key, 'upload_id': item.upload_id, 'age_seconds': int(age)} + if age < args.grace_seconds: + report['multipart_protected_by_grace'].append(entry) + continue + report['multipart_stale'].append(entry) + if args.apply: + try: + await asyncio.to_thread(objects.abort_multipart, item.key, item.upload_id) + report['aborted_multipart'].append(entry) + except StorageError: + logger.warning('failed to abort multipart %s', item.key) + return report + finally: + await pool.close() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s') + parser = argparse.ArgumentParser(description='Reconcile originals/ with attachment metadata') + parser.add_argument( + '--apply', + action='store_true', + help='delete stale orphans; default is dry-run', + ) + parser.add_argument( + '--grace-seconds', + type=int, + default=_grace_default(), + help='do not delete objects younger than this (default 900, or RECONCILE_GRACE_SECONDS)', + ) + parser.add_argument( + '--seed-orphan', + action='store_true', + help='put an unreferenced originals/ object, then scan (evidence only)', + ) + args = parser.parse_args() + if args.grace_seconds < 0: + parser.error('grace-seconds must be >= 0') + report = asyncio.run(_run(args)) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/app/routes.py b/app/routes.py index 30df7bff..b9e28513 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,8 +1,9 @@ import hashlib import json import re +import tempfile import uuid -from typing import Optional, Sequence, Type, TypeVar +from typing import BinaryIO, Optional, Sequence, Tuple, Type, TypeVar from fastapi import APIRouter, Depends, Request from fastapi.responses import JSONResponse @@ -12,11 +13,15 @@ from starlette.responses import Response from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer from .drain import draining from .errors import invalid_request, not_found, payload_too_large, unauthorized, unavailable, unsupported_media_type -from .images import MAX_UPLOAD_BYTES, inspect_image +from .images import INSPECT_PREFIX, MAX_UPLOAD_BYTES, inspect_image, inspect_prefix from .pagination import encode_cursor, parse_page from .schemas import CreateChat, CreateMessage, CreateUser from .store import Chat, Store, User +# Bytes kept in RAM before SpooledTemporaryFile rolls over to an ephemeral file. +# The original is never persisted on the API volume; the spool dies with the request. +_SPOOL_RAM = 64 * 1024 + router = APIRouter() Body = TypeVar('Body', bound=BaseModel) @@ -77,21 +82,50 @@ def _declared_image_type(value: Optional[str]) -> str: return media -async def _read_upload(request: Request) -> bytes: - declared = request.headers.get('content-length') - if declared is not None: +async def _spool_upload(request: Request, declared: str) -> Tuple[BinaryIO, int, str, str]: + """Stream the body into a size-capped spool, hash it, and inspect the header. + + Returns (spool, size, sha256, sniffed_type). The caller must close the spool. + One request never holds more than INSPECT_PREFIX plus _SPOOL_RAM in Python heap; + the rest, if any, is an ephemeral tempfile deleted in finally. + """ + length = request.headers.get('content-length') + if length is not None: try: - size = int(declared) + size = int(length) except ValueError: raise invalid_request('Некорректный Content-Length') from None if size > MAX_UPLOAD_BYTES: raise payload_too_large('Файл больше 10 МиБ') - buf = bytearray() - async for chunk in request.stream(): - if len(buf) + len(chunk) > MAX_UPLOAD_BYTES: - raise payload_too_large('Файл больше 10 МиБ') - buf.extend(chunk) - return bytes(buf) + spool = tempfile.SpooledTemporaryFile(max_size=_SPOOL_RAM) + hasher = hashlib.sha256() + head = bytearray() + total = 0 + sniffed: Optional[str] = None + try: + async for chunk in request.stream(): + if not chunk: + continue + if total + len(chunk) > MAX_UPLOAD_BYTES: + raise payload_too_large('Файл больше 10 МиБ') + hasher.update(chunk) + spool.write(chunk) + total += len(chunk) + if len(head) < INSPECT_PREFIX: + head.extend(chunk[: INSPECT_PREFIX - len(head)]) + if sniffed is None and len(head) >= 24: + sniffed = inspect_prefix(bytes(head), complete=False) + if sniffed is not None and sniffed != declared: + raise unsupported_media_type('Содержимое не соответствует Content-Type') + if sniffed is None: + sniffed = inspect_image(bytes(head)) + if sniffed != declared: + raise unsupported_media_type('Содержимое не соответствует Content-Type') + spool.seek(0) + return spool, total, hasher.hexdigest(), sniffed + except BaseException: + spool.close() + raise async def _member_chat(store: Store, chat_id: str, actor: User) -> Chat: @@ -231,18 +265,21 @@ async def upload_attachment( chat = await _member_chat(store, chat_id, actor) filename = _filename(request.query_params.get('filename')) declared = _declared_image_type(request.headers.get('content-type')) - data = await _read_upload(request) - actual = inspect_image(data) - if actual != declared: - raise unsupported_media_type('Содержимое не соответствует Content-Type') - attachment = await store.create_attachment( - chat.id, - actor.id, - filename, - actual, - data, - hashlib.sha256(data).hexdigest(), - ) + spool = None + try: + spool, size, digest, actual = await _spool_upload(request, declared) + attachment = await store.create_attachment( + chat.id, + actor.id, + filename, + actual, + spool, + size, + digest, + ) + finally: + if spool is not None: + spool.close() return JSONResponse(attachment.to_dict(), status_code=201) diff --git a/app/store.py b/app/store.py index 60c4c6da..ff59bace 100644 --- a/app/store.py +++ b/app/store.py @@ -7,13 +7,13 @@ import threading import uuid from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Dict, List, Optional, Sequence, Tuple +from typing import BinaryIO, Dict, List, Optional, Sequence, Set, Tuple import asyncpg from .errors import conflict, invalid_request, not_found, unavailable from .migrate import expected_versions -from .objects import MissingObject, ObjectStore, StorageError +from .objects import ORIGINAL_PREFIX, MissingObject, ObjectStore, StorageError from .pagination import Key from .passwords import hash_password, password_matches @@ -376,7 +376,8 @@ class Store: owner_id: str, filename: str, content_type: str, - body: bytes, + body: BinaryIO, + size_bytes: int, sha256: str, ) -> Attachment: chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') @@ -384,11 +385,12 @@ class Store: attachment_id = uuid.uuid4() # The key is not derived from filename. A crash after PUT and before INSERT leaves an # unreferenced object; a failed INSERT deletes it. 201 is returned only after both succeed. - object_key = 'originals/%s' % attachment_id + object_key = ORIGINAL_PREFIX + str(attachment_id) created = self._now() try: - await asyncio.to_thread(self._objects.put, object_key, body, content_type) + await asyncio.to_thread(self._objects.put_stream, object_key, body, size_bytes, content_type) except StorageError: + await self._discard_object(object_key) raise unavailable('Хранилище недоступно') from None try: async with self._pool.acquire() as conn: @@ -405,7 +407,7 @@ class Store: owner_uuid, filename, content_type, - len(body), + size_bytes, sha256, object_key, created, @@ -420,12 +422,18 @@ class Store: str(owner_uuid), filename, content_type, - len(body), + size_bytes, sha256, stamp, stamp, ) + async def listed_object_keys(self) -> Set[str]: + """Object keys that have a metadata row. Used by the reconciler, not by the HTTP API.""" + async with self._pool.acquire() as conn: + rows = await conn.fetch('SELECT object_key FROM attachments') + return {row['object_key'] for row in rows} + async def find_attachment(self, attachment_id: str, user_id: str) -> Optional[Attachment]: """Metadata for a chat member. Missing and foreign attachments are both absent.""" attachment_uuid = _as_uuid(attachment_id, 'Некорректный attachment_id') diff --git a/compose.yaml b/compose.yaml index d000899d..a98a080a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -129,6 +129,79 @@ services: depends_on: minio: condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: "no" + + # Operator one-shot: python -m app.reconcile (dry-run). Pass --apply / --seed-orphan as args. + # Not started by `up`; use: docker compose --env-file .env --profile ops run --rm reconcile + reconcile: + image: messenger-lab4-app:${APP_VERSION:-dev} + build: *app-build + profiles: ["ops"] + entrypoint: ["python", "-m", "app.reconcile"] + environment: + <<: *db-client + S3_ENDPOINT: ${S3_ENDPOINT:?set in .env} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:?set in .env} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900} + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_access_key_id + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_secret_access_key + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - data + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: "no" + + # Unauthenticated GET and console probe from the internal data network. + check-private-bucket: + image: ghcr.io/tsouza/cerberus-mirror/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + profiles: ["ops"] + entrypoint: ["/bin/sh", "/bootstrap/check-private-bucket.sh"] + environment: + S3_BUCKET: ${S3_BUCKET:?set in .env} + MINIO_ENDPOINT: http://minio:9000 + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/check-private-bucket.sh:/bootstrap/check-private-bucket.sh:ro + networks: + - data + depends_on: + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully deploy: resources: limits: diff --git a/evidence/attachments-negative.txt b/evidence/attachments-negative.txt new file mode 100644 index 00000000..238a6bbd Binary files /dev/null and b/evidence/attachments-negative.txt differ diff --git a/evidence/compose-minio-exposure.json b/evidence/compose-minio-exposure.json new file mode 100644 index 00000000..331e066f --- /dev/null +++ b/evidence/compose-minio-exposure.json @@ -0,0 +1,23 @@ +{ + "minio_ports": [], + "MINIO_BROWSER": "off", + "networks": [ + "data" + ], + "lb_ports": [ + { + "mode": "ingress", + "host_ip": "127.0.0.1", + "target": 8080, + "published": "8084", + "protocol": "tcp" + }, + { + "mode": "ingress", + "host_ip": "::1", + "target": 8080, + "published": "8084", + "protocol": "tcp" + } + ] +} diff --git a/evidence/minio-init-rerun.txt b/evidence/minio-init-rerun.txt new file mode 100644 index 00000000..eb812aca --- /dev/null +++ b/evidence/minio-init-rerun.txt @@ -0,0 +1,7 @@ +# Idempotent MinIO bootstrap. Secrets are not printed. +# docker compose --env-file .env run --rm --no-deps minio-init +# (two sequential runs after the 256M memory limit on minio-init) + +run 1: bucket messenger-attachments is private +run 2: bucket messenger-attachments is private +exit: 0 both times diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..26018cb4 --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,7 @@ +# python scripts/check_private_bucket.py +# (stack already up; in-network probe via compose profile ops) + +PASS: bucket messenger-attachments is private; unauthenticated GET is 403; API user cannot list another bucket; console is closed +PASS: minio has no host ports, console env is off, in-network GET is 403 + +See also evidence/compose-minio-exposure.json diff --git a/evidence/reconcile-apply.json b/evidence/reconcile-apply.json new file mode 100644 index 00000000..05339834 --- /dev/null +++ b/evidence/reconcile-apply.json @@ -0,0 +1,29 @@ +{ + "apply": true, + "grace_seconds": 0, + "seeded": null, + "scanned_originals": 37, + "referenced": 38, + "orphans_protected_by_grace": [], + "orphans_stale": [ + { + "key": "originals/44d0830b-a013-482c-ae07-83805b6470c1", + "age_seconds": 10, + "size": 12 + } + ], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [ + "originals/44d0830b-a013-482c-ae07-83805b6470c1" + ], + "aborted_multipart": [], + "missing_objects": [ + { + "key": "a93d1b7ab79245ec95f792b569c3ee46" + }, + { + "key": "c96e4c9da2114c9e837c372ef9e75e15" + } + ] +} diff --git a/evidence/reconcile-dry-run.json b/evidence/reconcile-dry-run.json new file mode 100644 index 00000000..fca06a4a --- /dev/null +++ b/evidence/reconcile-dry-run.json @@ -0,0 +1,27 @@ +{ + "apply": false, + "grace_seconds": 900, + "seeded": "originals/44d0830b-a013-482c-ae07-83805b6470c1", + "scanned_originals": 37, + "referenced": 38, + "orphans_protected_by_grace": [ + { + "key": "originals/44d0830b-a013-482c-ae07-83805b6470c1", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_objects": [ + { + "key": "a93d1b7ab79245ec95f792b569c3ee46" + }, + { + "key": "c96e4c9da2114c9e837c372ef9e75e15" + } + ] +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..450bb652 Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/evidence/upload-concurrency.json b/evidence/upload-concurrency.json new file mode 100644 index 00000000..34a72728 --- /dev/null +++ b/evidence/upload-concurrency.json @@ -0,0 +1,37 @@ +{ + "hypothesis": "Ten concurrent ~1.9 MiB PNG uploads stay inside the 256 MiB API limit and peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.", + "when": "2026-11-09T17:15:07Z", + "commit": "c594b88", + "host": { + "system": "Windows", + "machine": "AMD64", + "processor": "Intel64 Family 6 Model 165 Stepping 2, GenuineIntel" + }, + "compose_limits": { + "api_memory": "256M", + "replicas": 2 + }, + "profile": { + "concurrency": 10, + "width": 900, + "height": 700, + "size_bytes": 1994104, + "sha256": "03f35c154abaa690cfc41d6d73715992df84d10f45ff48dc14722bd64e49a263" + }, + "results": { + "ok": 10, + "errors": 0, + "elapsed_s": 1.995, + "p50_s": 0.5158, + "p95_s": 0.5466, + "p99_s": 0.5466, + "peak_api_rss_mib": 93.66, + "peak_rss_by_container_mib": { + "messenger-lab4-api-2": 93.66, + "messenger-lab4-api-1": 85.37 + }, + "samples": 3, + "naive_n_times_size_mib": 19.02 + }, + "limits_of_measurement": "docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client." +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index e28b7fff..c28adc3a 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -36,7 +36,8 @@ defaults timeout connect 1s timeout client 30s timeout server 10s - timeout http-request 10s + # Body arrival for a 10 MiB attachment; JSON posts finish well under this. + timeout http-request 30s timeout http-keep-alive 5s timeout queue 2s timeout check 1s @@ -55,7 +56,7 @@ frontend public acl safe_request_id req.hdr(X-Request-Id) -m reg ^[A-Za-z0-9._:-]{1,128}$ http-request set-var(txn.reqid) req.hdr(X-Request-Id) if upload_too_large safe_request_id http-request set-var(txn.reqid) str(lb) if upload_too_large !safe_request_id - http-request return status 413 content-type application/json lf-string "{\"error\":{\"code\":\"payload_too_large\",\"message\":\"Файл больше 10 МиБ\",\"request_id\":\"%[var(txn.reqid)]\"}}" hdr X-Request-Id %[var(txn.reqid)] if upload_too_large + http-request return status 413 content-type application/json lf-string "{\"error\":{\"code\":\"payload_too_large\",\"message\":\"Файл больше 10 МиБ\",\"request_id\":\"%[var(txn.reqid)]\"}}" hdr X-Request-Id %[var(txn.reqid)] hdr X-Instance-Id %[str(lb)] if upload_too_large # Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe. # Other POSTs (users, chats, sessions) are retried only when the connection was never established. @@ -67,6 +68,10 @@ frontend public backend api # Round robin, no stickiness: any replica serves any session from PostgreSQL. + # Attachment PUT to S3 can outlast the default 10 s server timeout under concurrent load. + acl attachment_upload method POST + acl attachment_upload_path path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 30s if attachment_upload attachment_upload_path balance roundrobin option httpchk http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh index 5dfaac06..a5c8024f 100644 --- a/infra/minio/bootstrap.sh +++ b/infra/minio/bootstrap.sh @@ -70,7 +70,7 @@ cat > "$policy_file" < "$policy_file" </dev/null 2>&1; then - mc admin policy create local "$policy_name" "$policy_file" >/dev/null +# Recreate the canned policy on every run so added actions (multipart list) apply. +if mc admin user info local "$access_key" >/dev/null 2>&1; then + mc admin policy detach local "$policy_name" --user "$access_key" >/dev/null 2>&1 || true fi +mc admin policy remove local "$policy_name" >/dev/null 2>&1 || \ + mc admin policy rm local "$policy_name" >/dev/null 2>&1 || true +mc admin policy create local "$policy_name" "$policy_file" >/dev/null if ! mc admin user info local "$access_key" >/dev/null 2>&1; then mc admin user add local "$access_key" "$secret_key" >/dev/null diff --git a/infra/minio/check-private-bucket.sh b/infra/minio/check-private-bucket.sh new file mode 100644 index 00000000..2bb22f97 --- /dev/null +++ b/infra/minio/check-private-bucket.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# In-network proof that the bucket is private and API credentials cannot see other buckets. +# Run via: docker compose --env-file .env --profile ops run --rm check-private-bucket +set -eu + +bucket=${S3_BUCKET:?S3_BUCKET is required} +endpoint=${MINIO_ENDPOINT:-http://minio:9000} + +read_secret() { + tr -d '\r\n' < "$1" +} + +root_user=$(read_secret /run/secrets/minio_root_user) +root_password=$(read_secret /run/secrets/minio_root_password) +access_key=$(read_secret /run/secrets/s3_access_key_id) +secret_key=$(read_secret /run/secrets/s3_secret_access_key) + +mc alias set local "$endpoint" "$root_user" "$root_password" >/dev/null + +# MINIO_BROWSER=off: nothing should answer on the console port. +if curl -sf --connect-timeout 2 "http://minio:9001/" >/dev/null 2>&1; then + echo "MinIO console answered on :9001" >&2 + exit 1 +fi + +probe_key=".bootstrap/private-check" +printf 'probe' | mc pipe "local/${bucket}/${probe_key}" >/dev/null +code=$(curl -s -o /dev/null -w '%{http_code}' "${endpoint}/${bucket}/${probe_key}") +mc rm --force "local/${bucket}/${probe_key}" >/dev/null +if [ "$code" != "403" ]; then + echo "unauthenticated GET returned ${code}, expected 403" >&2 + exit 1 +fi + +mc alias set api "$endpoint" "$access_key" "$secret_key" >/dev/null +if mc ls "api/not-this-bucket" >/dev/null 2>&1; then + echo "API credentials can list a different bucket" >&2 + exit 1 +fi + +echo "PASS: bucket ${bucket} is private; unauthenticated GET is 403; API user cannot list another bucket; console is closed" diff --git a/scripts/check_private_bucket.py b/scripts/check_private_bucket.py new file mode 100644 index 00000000..dcb0e6c9 --- /dev/null +++ b/scripts/check_private_bucket.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Host-side private-bucket checks plus the in-network probe. + +Confirms MinIO publishes no host ports, the console env is off, then runs the +compose service that performs an unauthenticated GET from the data network. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _compose(args): + env_file = ROOT / '.env' + cmd = ['docker', 'compose'] + if env_file.is_file(): + cmd.extend(['--env-file', str(env_file)]) + cmd.extend(args) + return subprocess.run(cmd, cwd=str(ROOT), check=True, capture_output=True, text=True) + + +def main() -> None: + os.chdir(ROOT) + result = _compose(['config', '--format', 'json']) + config = json.loads(result.stdout) + services = config.get('services') or {} + minio = services.get('minio') + if minio is None: + raise SystemExit('FAIL: compose config has no minio service') + ports = minio.get('ports') or [] + if ports: + raise SystemExit('FAIL: minio publishes host ports: %s' % ports) + env = minio.get('environment') or {} + browser = str(env.get('MINIO_BROWSER', '')).strip().strip('"').lower() + if browser not in ('off', 'false', '0'): + raise SystemExit('FAIL: MINIO_BROWSER is %r, expected off' % env.get('MINIO_BROWSER')) + networks = (minio.get('networks') or {}) + if 'edge' in networks: + raise SystemExit('FAIL: minio is attached to the edge network') + probe = subprocess.run( + [ + 'docker', 'compose', + *(['--env-file', str(ROOT / '.env')] if (ROOT / '.env').is_file() else []), + '--profile', 'ops', + 'run', '--rm', 'check-private-bucket', + ], + cwd=str(ROOT), + check=False, + text=True, + ) + if probe.returncode != 0: + raise SystemExit(probe.returncode) + print('PASS: minio has no host ports, console env is off, in-network GET is 403') + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as exc: + sys.stderr.write(exc.stderr or exc.stdout or str(exc)) + raise SystemExit(exc.returncode) diff --git a/scripts/upload_mem.py b/scripts/upload_mem.py new file mode 100644 index 00000000..e22cc932 --- /dev/null +++ b/scripts/upload_mem.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Concurrent attachment uploads with peak RSS of API containers. + +Generates a valid PNG at runtime (not stored in Git). Measures docker stats while +N workers POST the same image. Writes a sanitized JSON summary. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +import sys +import threading +import time +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure # noqa: E402 + + +def make_png(width: int, height: int) -> bytes: + rows = [] + for _y in range(height): + row = bytearray(1 + width * 3) + row[0] = 0 + row[1:] = os.urandom(width * 3) + rows.append(bytes(row)) + raw = b''.join(rows) + compressed = zlib.compress(raw, 1) + + def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xffffffff + return len(data).to_bytes(4, 'big') + tag + data + crc.to_bytes(4, 'big') + + ihdr = width.to_bytes(4, 'big') + height.to_bytes(4, 'big') + bytes([8, 2, 0, 0, 0]) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', compressed) + chunk(b'IEND', b'') + + +def parse_mib(usage: str) -> float: + # "45.23MiB / 256MiB" or "123.4MB / 256MB" + left = usage.split('/', 1)[0].strip() + number = '' + unit = '' + for char in left: + if char.isdigit() or char in '.': + number += char + else: + unit += char + value = float(number or '0') + unit = unit.strip().lower() + if unit in ('gib', 'gb'): + return value * 1024 + if unit in ('kib', 'kb'): + return value / 1024 + if unit in ('b',): + return value / (1024 * 1024) + return value + + +def git_commit() -> str: + try: + return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=str(ROOT), text=True).strip() + except (OSError, subprocess.CalledProcessError): + return 'unknown' + + +def docker_stats() -> list: + try: + ids = subprocess.check_output( + ['docker', 'ps', '-q', '--filter', 'name=messenger-lab4-api'], + cwd=str(ROOT), + text=True, + timeout=5, + ).split() + if not ids: + return [] + output = subprocess.check_output( + ['docker', 'stats', '--no-stream', '--format', '{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}', *ids], + cwd=str(ROOT), + text=True, + timeout=8, + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return [] + rows = [] + for line in output.splitlines(): + parts = line.split('\t') + if len(parts) < 2: + continue + name, usage = parts[0], parts[1] + rows.append({'name': name, 'mem_usage': usage, 'rss_mib': round(parse_mib(usage), 2)}) + return rows + + +def main() -> None: + parser = argparse.ArgumentParser(description='Concurrent upload memory experiment') + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--width', type=int, default=900) + parser.add_argument('--height', type=int, default=700) + parser.add_argument('--output', default=str(ROOT / 'evidence' / 'upload-concurrency.json')) + args = parser.parse_args() + if args.concurrency < 10: + raise SystemExit('concurrency must be >= 10') + + image = make_png(args.width, args.height) + size = len(image) + digest = hashlib.sha256(image).hexdigest() + client = Client(args.base_url, timeout=60) + alice, _bob, _eve, chat = client.bootstrap() + path = '/api/v1/chats/%s/attachments?filename=load.png' % chat['id'] + + stop = threading.Event() + samples = [] + + def watch(): + while not stop.is_set(): + snapshot = docker_stats() + if snapshot: + samples.append({'ts': time.time(), 'containers': snapshot}) + stop.wait(0.05) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + baseline = docker_stats() + if baseline: + samples.append({'ts': time.time(), 'containers': baseline, 'phase': 'baseline'}) + started = time.monotonic() + latencies = [] + errors = 0 + + def one(_index: int): + response = client.call( + 'POST', + path, + 201, + actor=alice, + raw=image, + headers={'Content-Type': 'image/png'}, + schema='Attachment', + ) + body = response['json'] + if body.get('sha256') != digest or body.get('size_bytes') != size: + raise Failure('checksum mismatch on concurrent upload') + return response['elapsed'] + + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futs = [pool.submit(one, i) for i in range(args.concurrency)] + for fut in as_completed(futs): + try: + latencies.append(fut.result()) + except Exception: + errors += 1 + finally: + stop.set() + watcher.join(timeout=2) + + elapsed = time.monotonic() - started + latencies.sort() + + def pct(p): + if not latencies: + return None + index = min(len(latencies) - 1, max(0, int(round((p / 100) * (len(latencies) - 1))))) + return round(latencies[index], 4) + + peak = 0.0 + per_container = {} + for sample in samples: + for row in sample['containers']: + peak = max(peak, row['rss_mib']) + prev = per_container.get(row['name'], 0.0) + per_container[row['name']] = max(prev, row['rss_mib']) + + naive_bound_mib = round((args.concurrency * size) / (1024 * 1024), 2) + summary = { + 'hypothesis': ( + 'Ten concurrent ~%.1f MiB PNG uploads stay inside the 256 MiB API limit and ' + 'peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.' + % (size / (1024 * 1024)) + ), + 'when': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'commit': git_commit(), + 'host': { + 'system': platform.system(), + 'machine': platform.machine(), + 'processor': platform.processor(), + }, + 'compose_limits': {'api_memory': '256M', 'replicas': 2}, + 'profile': { + 'concurrency': args.concurrency, + 'width': args.width, + 'height': args.height, + 'size_bytes': size, + 'sha256': digest, + }, + 'results': { + 'ok': len(latencies), + 'errors': errors, + 'elapsed_s': round(elapsed, 3), + 'p50_s': pct(50), + 'p95_s': pct(95), + 'p99_s': pct(99), + 'peak_api_rss_mib': round(peak, 2), + 'peak_rss_by_container_mib': per_container, + 'samples': len(samples), + 'naive_n_times_size_mib': naive_bound_mib, + }, + 'limits_of_measurement': ( + 'docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. ' + 'Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client.' + ), + } + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print('ok=%s errors=%s peak_rss_mib=%s naive_N_x_size_mib=%s file=%s' % ( + len(latencies), errors, summary['results']['peak_api_rss_mib'], naive_bound_mib, out, + )) + if errors: + raise SystemExit(1) + + +if __name__ == '__main__': + try: + main() + except Failure as exc: + print('FAIL:', exc) + raise SystemExit(1) diff --git a/tests/attachments_negative.py b/tests/attachments_negative.py new file mode 100644 index 00000000..5f6aaba2 --- /dev/null +++ b/tests/attachments_negative.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Student negative suite for lab4 attachments. Public smoke.py is not modified.""" +import http.client +import json +import os +import struct +import uuid +import zlib +from urllib.parse import urlparse + +from client import Client, Failure, arguments, check, run, shape + +MAX_UPLOAD_BYTES = 10_485_760 +ROOT = os.path.dirname(os.path.abspath(__file__)) + + +def png_header(width, height): + ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + data = ihdr + crc = zlib.crc32(b'IHDR' + data) & 0xffffffff + chunk = struct.pack('>I', len(data)) + b'IHDR' + data + struct.pack('>I', crc) + return b'\x89PNG\r\n\x1a\n' + chunk + + +def error_payload(status, headers, body, where): + check(status != 500, where + ' returned 500 for a client error') + check(bool(headers.get('X-Request-Id')), where + ': missing X-Request-Id') + content_type = headers.get_content_type() if hasattr(headers, 'get_content_type') else (headers.get('Content-Type') or '').split(';', 1)[0].strip().lower() + check(content_type == 'application/json', where + ': expected JSON error') + try: + payload = json.loads(body) + except (ValueError, UnicodeError): + raise Failure(where + ': invalid error JSON') from None + shape(payload, 'Error') + check(payload['error']['request_id'] == headers.get('X-Request-Id'), where + ': request_id mismatch') + return payload + + +def raw_post(base, path, headers, body, extra_headers=None, chunked=False): + parsed = urlparse(base) + conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=60) + try: + hdrs = dict(headers) + if extra_headers: + hdrs.update(extra_headers) + conn.putrequest('POST', path) + if chunked: + hdrs.pop('Content-Length', None) + for name, value in hdrs.items(): + conn.putheader(name, value) + conn.putheader('Transfer-Encoding', 'chunked') + conn.endheaders() + if isinstance(body, (bytes, bytearray)): + pieces = [bytes(body)] + else: + pieces = body + for piece in pieces: + conn.send(('%x\r\n' % len(piece)).encode('ascii') + piece + b'\r\n') + conn.send(b'0\r\n\r\n') + else: + payload = body if isinstance(body, (bytes, bytearray)) else b'' + if not any(name.lower() == 'content-length' for name in hdrs): + hdrs['Content-Length'] = str(len(payload)) + for name, value in hdrs.items(): + conn.putheader(name, value) + conn.endheaders() + if payload: + conn.send(payload) + response = conn.getresponse() + data = response.read(1024 * 1024) + hdr = response.headers + return response.status, hdr, data + finally: + conn.close() + + +def expect_error(c, actor, chat, *, filename='photo.png', content_type='image/png', body=b'', status=None, extra_headers=None, chunked=False, pieces=None, where='upload'): + path = '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], filename) + headers = { + 'Authorization': 'Bearer ' + actor['token'], + 'Content-Type': content_type, + 'Accept': 'application/json', + } + payload_body = pieces if chunked and pieces is not None else body + code, hdr, data = raw_post(c.base, path, headers, payload_body, extra_headers=extra_headers, chunked=chunked) + payload = error_payload(code, hdr, data, where) + if status is not None: + check(code == status, '%s: expected %s, received %s (%s)' % (where, status, code, payload['error']['code'])) + return code, payload + + +def main(): + args = arguments('Lab4 attachment negative cases') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + sample = open(os.path.join(ROOT, 'fixtures', 'sample.png'), 'rb').read() + + code, payload = expect_error( + c, alice, chat, + extra_headers={'Content-Length': str(MAX_UPLOAD_BYTES + 1)}, + body=b'x', + status=413, + where='oversized Content-Length', + ) + check(payload['error']['code'] == 'payload_too_large', 'oversized Content-Length: wrong code') + + header = png_header(32, 32) + remain = MAX_UPLOAD_BYTES + 1 - len(header) + chunk = b'\x00' * 65536 + pieces = [header] + sent = len(header) + while sent < len(header) + remain: + take = min(len(chunk), len(header) + remain - sent) + pieces.append(chunk[:take]) + sent += take + code, payload = expect_error( + c, alice, chat, + chunked=True, + pieces=pieces, + status=413, + where='oversized chunked body', + ) + check(payload['error']['code'] == 'payload_too_large', 'oversized chunked body: wrong code') + + expect_error(c, alice, chat, body=b'\x89PNG\r\n\x1a\n\x00\x00\x00\x0dnotH', status=415, where='corrupt PNG') + expect_error(c, alice, chat, body=b'\xff\xd8\xff\xe0\x00', status=415, where='truncated JPEG') + expect_error(c, alice, chat, body=b'GIF89a-not-an-image', status=415, where='foreign format') + + code, payload = expect_error( + c, alice, chat, + content_type='image/jpeg', + body=sample, + status=415, + where='MIME spoof', + ) + check(payload['error']['code'] == 'unsupported_media_type', 'MIME spoof: wrong code') + + bomb = png_header(10000, 5000) + code, payload = expect_error(c, alice, chat, body=bomb, status=413, where='pixel bomb') + check(payload['error']['code'] == 'payload_too_large', 'pixel bomb: wrong code') + + att, _content = c.attachment(alice, chat) + for suffix in ('', '/content'): + unknown = c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + check(unknown['json']['error']['code'] == 'not_found', 'random id: wrong code') + check(unknown['status'] != 500, 'random id returned 500') + foreign = c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve) + check(foreign['json']['error']['code'] == 'not_found', 'outsider: wrong code') + + other = c.call( + 'POST', + '/api/v1/chats', + 201, + actor=alice, + body={'title': 'Чужой чат', 'member_ids': [alice['id']]}, + schema='Chat', + )['json'] + linked = c.call( + 'POST', + '/api/v1/chats/' + other['id'] + '/messages', + 404, + actor=alice, + body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}, + ) + check(linked['json']['error']['code'] == 'not_found', 'cross-chat attach: wrong code') + check(linked['status'] != 500, 'cross-chat attach returned 500') + + print('PASS: lab4 attachment negative cases (413/415/404, never 500)') + + +if __name__ == '__main__': + run(main)