From 2a88f717f30ad434edab02b758ff8d68ff4fb6d7 Mon Sep 17 00:00:00 2001 From: "yurtaev.va" Date: Fri, 6 Nov 2026 15:43:25 +0300 Subject: [PATCH] yurtaev - lab4 --- .env.example | 23 +- Dockerfile | 10 + Makefile | 14 +- REPORT.md | 76 ++- app/__init__.py | 0 app/main.py | 881 +++++++++++++++++++++++++++++++++ compose.yaml | 163 ++++++ contracts/README.md | 13 + infra/README.md | 8 +- infra/haproxy-compose.cfg | 27 + infra/minio-policy.json | 15 + migrations/001_initial.sql | 38 ++ migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 25 + requirements.txt | 6 + scripts/lint.py | 52 ++ scripts/migrate.py | 16 + scripts/restart.sh | 4 + scripts/stop-one.sh | 12 + tests/direct_upload.py | 84 ++++ tests/level4.py | 49 ++ tests/upload_benchmark.py | 143 ++++++ 22 files changed, 1627 insertions(+), 42 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 compose.yaml create mode 100644 infra/haproxy-compose.cfg create mode 100644 infra/minio-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 requirements.txt create mode 100644 scripts/lint.py create mode 100644 scripts/migrate.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 tests/direct_upload.py create mode 100644 tests/level4.py create mode 100644 tests/upload_benchmark.py diff --git a/.env.example b/.env.example index d99d26a2..8f5171e2 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,11 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +POSTGRES_PASSWORD=replace-with-long-random-password +MINIO_ROOT_USER=replace-with-random-admin-name +MINIO_ROOT_PASSWORD=replace-with-long-random-admin-password +S3_ACCESS_KEY=messenger-api +S3_SECRET_KEY=replace-with-long-random-api-secret +S3_PUBLIC_PORT=9000 +PRESIGN_TTL_SECONDS=120 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..74909261 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.10-slim +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt && useradd -r -u 10001 app +COPY app ./app +COPY migrations ./migrations +COPY scripts/migrate.py ./scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"] diff --git a/Makefile b/Makefile index 5d86d171..87a642d3 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,14 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help lint check validate test test-level4 test-direct benchmark-upload load test-persistence test-failover help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" +lint: + $(PYTHON) scripts/lint.py + check: $(PYTHON) scripts/check_contract.py @@ -18,6 +21,15 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-level4: + $(PYTHON) tests/level4.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-direct: + $(PYTHON) tests/direct_upload.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +benchmark-upload: + $(PYTHON) tests/upload_benchmark.py --base-url "$(BASE_URL)" + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..111fb205 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,71 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: Юртаев (группа не указана). Цель: **5, трек 1 — прямой upload**. Основа API перенесена из лабораторной 3. Использованы Python 3.12, FastAPI 0.115.12, psycopg 3.2.6, Redis 5.2.1, MinIO и Pillow 11.2.1; реализация подготовлена с помощью Codex. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose, Python 3.9+ и свободные локальные порты 8080/9000. + +```sh +cp .env.example .env +# Заменить все replace-with-* на случайные локальные значения. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +make lint check validate test test-level4 test-direct +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make benchmark-upload +``` + +Безопасная остановка: `docker compose --env-file .env down`. Намеренное удаление только учебных данных: `docker compose --env-file .env down -v`. `.env` не попадает в Git. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +localhost:8080 -> HAProxy -> API x2 -> PostgreSQL (метаданные, сессии, связи) + |-> Redis (необязательный кэш истории) + `-> private MinIO bucket (оригиналы) +client ------------------------------> MinIO:9000 (только короткий presigned PUT) +``` -## Доказательства критериев +MinIO и PostgreSQL имеют persistent volumes. Bucket `messenger-attachments` создаётся идемпотентным init-контейнером, anonymous access отключён. Root credentials получает только init; API использует отдельного пользователя с доступом только к этому bucket. Консоль MinIO не запускается, S3 API опубликован только на `127.0.0.1` для presigned PUT. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Обязательный proxy endpoint читает тело порциями во временный файл с жёстким пределом 10 MiB, считает SHA-256, проверяет реальный PNG/JPEG и не более 40 млн пикселей, затем пишет случайный object key в S3. `201 ready` возвращается только после записи объекта и метаданных. При ошибке БД выполняется компенсационное удаление S3. Фоновая сверка удаляет неизвестные объекты только после grace period 10 минут. Filename никогда не используется как object key. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Метаданные `Attachment` и связи сообщения находятся в PostgreSQL. Сообщение атомарно проверяет до 10 уникальных ready-вложений своего чата. Чужой чат/attachment скрывается одинаковым `404`. Download всегда идёт через авторизованный API без redirect и потоково возвращает оригинал из S3. -## Эксперименты и отказы +Readiness требует PostgreSQL и bucket; Redis остаётся необязательным. S3 connect/read timeout — 2/15 с. На API установлен лимит 256 MiB. Одновременные загрузки используют отдельные временные файлы и буферы по 64 KiB, поэтому размер входного файла не становится постоянным RAM-буфером API. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Трек 1: прямой upload -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Гипотеза:** presigned flow уберёт тело клиентской загрузки из внешнего трафика API и примерно вдвое уменьшит бинарный трафик API с учётом обязательного чтения объекта при `complete`; пиковая RAM API не должна расти пропорционально размеру файла. -## Ограничения и следующие шаги +1. `POST /api/v1/chats/{chat_id}/attachments/presign` аутентифицирует участника, валидирует filename/type/size/SHA-256, создаёт уникальный ключ и запись `uploading`, выдаёт PUT URL на 120 с. +2. Клиент отправляет bytes прямо в MinIO. +3. Только создатель вызывает `POST /api/v1/attachments/{id}/complete`. API проверяет TTL, наличие, Content-Type, размер, потоковый SHA-256 и фактическое изображение; только затем атомарно ставит `ready`. +4. Ошибочный объект удаляется. Незавершённая загрузка удаляется после TTL + 5 минут; повторный запуск cleanup безопасен. Обязательный proxy endpoint сохранён. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Отрицательные сценарии `tests/direct_upload.py`: complete до PUT -> 409; complete другим участником -> 404; неверный checksum -> 409 с удалением объекта; anonymous GET без подписи -> 401/403. `tests/level4.py` отдельно проверяет >10 MiB, битое изображение, MIME spoofing, опасный filename, чужой attachment и связь из другого чата. + +## Доказательства + +| Критерий | Команда | Результат 06.11.2026 | +| --- | --- | --- | +| Lint | `make lint` | PASS, 14 Python-файлов | +| Структура/контракт | `make check` | PASS | +| Compose model | `docker compose --env-file .env config --quiet` | PASS | +| OpenAPI | `make validate` | не выполнено: на диске осталось 155 MiB, установка venv завершилась `No space left on device` | +| Публичный smoke | `make test` | не выполнено: Docker Desktop не стартует при заполненном диске | +| Негативные сценарии | `make test-level4 test-direct` | подготовлены, интеграционный прогон заблокирован Docker | +| Persistence/failover | команды выше | интеграционный прогон заблокирован Docker | +| 10 concurrent, два размера | `make benchmark-upload` | методика подготовлена, измерения не сфабрикованы; требуется рабочий Docker | + +## Методика эксперимента + +`tests/upload_benchmark.py` генерирует валидные несжатые PNG примерно 1 и 8 MiB. Для каждого размера выполняются 10 одновременных proxy, затем 10 direct загрузок. Каждые 100 мс снимается суммарная RAM двух API через `docker stats`; raw JSON сохраняется в `evidence/upload_benchmark.json`. + +Для proxy бинарный трафик API равен примерно `2*N`: client -> API и API -> S3. Для direct внешний client -> API body отсутствует, но безопасный `complete` читает `N` из S3 для независимой проверки checksum, поэтому полный бинарный трафик API примерно `N`, а не ноль. Control JSON мал относительно 1/8 MiB и сохраняется отдельно. Это ожидаемая модель, не подмена фактического измерения. + +## Ограничения + +MinIO и PostgreSQL одиночные и остаются SPOF. Временный файл использует ephemeral disk API только во время проверки и не является хранилищем оригинала. Complete повторно читает объект через API ради независимой проверки SHA-256: это безопаснее доверия клиентской metadata, но оставляет половину бинарного API-трафика. Текущий локальный интеграционный прогон заблокирован заполненным диском; перед защитой нужно освободить место и заменить строки `не выполнено` фактическими результатами из указанных команд. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..917a7848 --- /dev/null +++ b/app/main.py @@ -0,0 +1,881 @@ +"""Replicated messenger API with PostgreSQL-backed sessions and idempotency.""" + +import asyncio +import base64 +import binascii +import hashlib +import json +import os +import re +import secrets +import socket +import tempfile +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone + +import psycopg +import redis +import urllib3 +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response, StreamingResponse +from minio import Minio +from minio.error import MinioException, S3Error +from PIL import Image, UnidentifiedImageError +from psycopg.conninfo import make_conninfo +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException + + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger") +database_password = os.environ.get("DATABASE_PASSWORD") +if "DATABASE_PASSWORD_FILE" in os.environ: + with open(os.environ["DATABASE_PASSWORD_FILE"], encoding="utf-8") as secret_file: + database_password = secret_file.read().rstrip("\n") +if database_password is not None: + DATABASE_URL = make_conninfo( + host=os.environ.get("DATABASE_HOST", "postgres"), + dbname=os.environ.get("DATABASE_NAME", "messenger"), + user=os.environ.get("DATABASE_USER", "messenger"), + password=database_password, + ) +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") +SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) +MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384")) +POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10")) +CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1" +CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1" +CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0")))) +INSTANCE_ID = os.environ.get("INSTANCE_ID", "") or socket.gethostname() +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "minio:9000") +S3_PUBLIC_ENDPOINT = os.environ.get("S3_PUBLIC_ENDPOINT", "localhost:9000") +S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "messenger-api") +S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "messenger-api-secret") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_SECURE = os.environ.get("S3_SECURE", "0") == "1" +PRESIGN_TTL = min(900, max(30, int(os.environ.get("PRESIGN_TTL_SECONDS", "120")))) +MAX_ATTACHMENT = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +ready_for_traffic = False +pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"}) +cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2) +s3_http = urllib3.PoolManager( + timeout=urllib3.Timeout(connect=2, read=15), + retries=False, + maxsize=max(10, POOL_SIZE * 2), +) +s3 = Minio( + S3_ENDPOINT, + access_key=S3_ACCESS_KEY, + secret_key=S3_SECRET_KEY, + secure=S3_SECURE, + http_client=s3_http, +) +s3_signer = Minio(S3_PUBLIC_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=S3_SECURE) +metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0} +metrics_lock = threading.Lock() + + +class ApiError(Exception): + def __init__(self, status, code, message, challenge=None): + self.status, self.code, self.message, self.challenge = status, code, message, challenge + + +def invalid(message="Некорректный запрос"): + raise ApiError(400, "invalid_request", message) + + +def unavailable(): + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(scope, date, identifier): + raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def decode_cursor(scope, value): + if value is None: + return None + try: + raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + parts = json.loads(raw) + if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope + or not isinstance(parts[1], str) or not isinstance(parts[2], str)): + invalid("Некорректный cursor") + date = datetime.fromisoformat(parts[1].replace("Z", "+00:00")) + if date.tzinfo is None: + invalid("Некорректный cursor") + return date, str(uuid.UUID(parts[2])) + except (ValueError, TypeError, binascii.Error, IndexError): + invalid("Некорректный cursor") + + +def identifier(value): + try: + result = str(uuid.UUID(value)) + if result != value.lower() or len(value) != 36: + invalid("Некорректный UUID") + return result + except (ValueError, TypeError, AttributeError): + invalid("Некорректный UUID") + + +def text_field(value, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum: + invalid("Некорректное текстовое поле") + return value + + +def exact_body(value, fields): + if not isinstance(value, dict) or set(value) != set(fields): + invalid("Некорректные поля JSON") + + +async def body(request, fields): + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + declared = request.headers.get("content-length") + if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + chunks = bytearray() + async for chunk in request.stream(): + chunks.extend(chunk) + if len(chunks) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + try: + value = json.loads(chunks.decode("utf-8")) + except (UnicodeError, ValueError): + invalid("Некорректный JSON") + exact_body(value, fields) + return value + + +def user(row): + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])} + + +def message(row, db=None): + attachment_ids = [] + if db is not None: + linked = db.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", + (row["id"],), + ).fetchall() + attachment_ids = [str(item["attachment_id"]) for item in linked] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": stamp(row["created_at"]), + "attachment_ids": attachment_ids, + } + + +def attachment(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": stamp(row["created_at"]), + "updated_at": stamp(row["updated_at"]), + } + + +def valid_filename(value): + if (not isinstance(value, str) or not 1 <= len(value) <= 255 + or any(char in value for char in ("/", "\\", "\x00")) + or any(ord(char) < 32 or ord(char) == 127 for char in value)): + invalid("Некорректный filename") + return value + + +def valid_content_type(value): + if value not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Допускаются PNG и JPEG") + return value + + +def validate_image(path, declared_type): + Image.MAX_IMAGE_PIXELS = MAX_PIXELS + try: + with Image.open(path) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type != declared_type: + raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому") + if image.width * image.height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + image.verify() + except Image.DecompressionBombError: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None + except (UnidentifiedImageError, OSError, ValueError): + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None + + +def chat(connection, row): + members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall() + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])} + + +def password_hash(password): + salt = os.urandom(16) + derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1) + return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex() + + +def password_ok(password, stored): + try: + _, n, r, p, salt, expected = stored.split(":") + actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p)) + return secrets.compare_digest(actual, bytes.fromhex(expected)) + except (ValueError, TypeError): + return False + + +def connection(): + try: + return pool.connection() + except psycopg.Error: + unavailable() + + +def actor(request): + auth = request.headers.get("authorization", "") + if not auth.startswith("Bearer ") or not auth[7:]: + raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"') + digest = hashlib.sha256(auth[7:].encode()).digest() + with connection() as db: + row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone() + if row is None: + raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"') + return row["user_id"] + + +def member(db, uid, cid): + row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def pagination(request): + raw = request.query_params.get("limit", "50") + if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100: + invalid("Некорректный limit") + return int(raw) + + +def bump(key): + with metrics_lock: + metrics[key] += 1 + + +def cache_get(key): + try: + raw = cache.get(key) + if raw is not None: + bump("history_cache_hit") + return json.loads(raw) + except (redis.RedisError, ValueError): + pass + return None + + +def cache_set(key, data): + try: + cache.set(key, json.dumps(data), ex=30) + except redis.RedisError: + pass + + +def history(db, cid, limit, cursor, version): + key = f"history:{cid}:{version}:{limit}:{cursor or '-'}" + hit = cache_get(key) if CACHE_ENABLED else None + if hit is not None: + return hit + lock_key = "lock:" + key + token = secrets.token_hex(16) + locked = False + if CACHE_ENABLED and CACHE_LOCK_ENABLED: + try: + locked = bool(cache.set(lock_key, token, nx=True, px=2000)) + if not locked: + bump("history_lock_wait") + for _ in range(20): + time.sleep(0.025) + hit = cache_get(key) + if hit is not None: + return hit + locked = bool(cache.set(lock_key, token, nx=True, px=2000)) + except redis.RedisError: + pass + if CACHE_MISS_DELAY_MS: + time.sleep(CACHE_MISS_DELAY_MS / 1000) + if cursor is None: + rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall() + else: + rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall() + bump("history_sql") + result = {"items": [message(row, db) for row in rows[:limit]], "next_cursor": None} + if len(rows) > limit: + last = rows[limit - 1] + result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"])) + if CACHE_ENABLED: + cache_set(key, result) + if locked: + try: + cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token) + except redis.RedisError: + pass + return result + + +def migrate(): + from pathlib import Path + files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql")) + with pool.connection() as db: + with db.transaction(): + db.execute("SELECT pg_advisory_xact_lock(2172002)") + db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)") + for file in files: + if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None: + db.execute(file.read_text()) + db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,)) + + +def cleanup_expired_uploads(): + with connection() as db: + rows = db.execute( + "SELECT id,object_key FROM attachments " + "WHERE status='uploading' AND upload_expires_at < clock_timestamp()-interval '5 minutes' " + "FOR UPDATE SKIP LOCKED LIMIT 100" + ).fetchall() + for row in rows: + try: + s3.remove_object(S3_BUCKET, row["object_key"]) + except S3Error as error: + if error.code not in ("NoSuchKey", "NoSuchObject"): + continue + db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],)) + cutoff = datetime.now(timezone.utc) - timedelta(minutes=10) + for prefix in ("originals/", "direct/"): + for item in s3.list_objects(S3_BUCKET, prefix=prefix, recursive=True): + if item.last_modified >= cutoff: + continue + known = db.execute( + "SELECT 1 FROM attachments WHERE object_key=%s", (item.object_name,) + ).fetchone() + if known is None: + s3.remove_object(S3_BUCKET, item.object_name) + + +async def cleanup_loop(): + while True: + await asyncio.sleep(60) + try: + await asyncio.to_thread(cleanup_expired_uploads) + except (psycopg.Error, MinioException, urllib3.exceptions.HTTPError, OSError): + pass + + +@asynccontextmanager +async def lifespan(_app): + global ready_for_traffic + pool.open() + pool.wait(timeout=20) + if not s3.bucket_exists(S3_BUCKET): + raise RuntimeError("S3 bucket is not initialized") + ready_for_traffic = True + task = asyncio.create_task(cleanup_loop()) + try: + yield + finally: + task.cancel() + ready_for_traffic = False + pool.close() + + +app = FastAPI(lifespan=lifespan) + + +@app.exception_handler(StarletteHTTPException) +async def http_error(request, error): + request_id = request.state.request_id + code = "not_found" if error.status_code == 404 else "invalid_request" + return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code) + + +@app.middleware("http") +async def envelope(request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except ApiError as exc: + response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status) + if exc.challenge: + response.headers["WWW-Authenticate"] = exc.challenge + except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable, PoolTimeout, MinioException, + urllib3.exceptions.HTTPError, OSError): + response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503) + except Exception: + response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(): + if not ready_for_traffic: + unavailable() + with connection() as db: + db.execute("SELECT 1") + if not s3.bucket_exists(S3_BUCKET): + unavailable() + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await body(request, ("username", "display_name", "password")) + name = data["username"] + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name): + invalid("Некорректный username") + display = text_field(data["display_name"], 100) + password = data["password"] + if not isinstance(password, str) or not password.strip() or not 12 <= len(password) <= 128: + invalid("Некорректный пароль") + try: + return await asyncio.to_thread(insert_user, name, display, password) + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username уже занят") from None + + +def insert_user(name, display, password): + digest = password_hash(password) + with connection() as db: + row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone() + return user(row) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + async for chunk in request.stream(): + if chunk: + invalid("Тело запроса не допускается") + auth = request.headers.get("authorization", "") + try: + if not auth.startswith("Basic "): + raise ValueError() + decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8") + name, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None + return await asyncio.to_thread(create_session, name, password) + + +def create_session(name, password): + with connection() as db: + row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone() + if row is None or not password_ok(password, row["password_hash"]): + raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') + token = secrets.token_urlsafe(32) + expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL) + db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry)) + return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + actor(request) + token = request.headers["authorization"][7:] + with connection() as db: + db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),)) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def me(request: Request): + uid = actor(request) + with connection() as db: + row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone() + return user(row) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + uid = actor(request) + data = await body(request, ("title", "member_ids")) + title = text_field(data["title"], 100) + raw = data["member_ids"] + if not isinstance(raw, list) or not 1 <= len(raw) <= 100: + invalid("Некорректный список участников") + members = [identifier(item) for item in raw] + if len(set(members)) != len(members) or str(uid) not in members: + invalid("Некорректный список участников") + return await asyncio.to_thread(insert_chat, uid, title, members) + + +def insert_chat(uid, title, members): + with connection() as db: + with db.transaction(): + count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"] + if count != len(members): + invalid("Неизвестный участник") + row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone() + for member_id in members: + db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id)) + db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],)) + return chat(db, row) + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + uid = actor(request) + limit = pagination(request) + cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor")) + with connection() as db: + if cursor is None: + rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall() + else: + valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone() + if valid is None: + invalid("Некорректный cursor") + rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall() + items = [chat(db, row) for row in rows[:limit]] + next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + with connection() as db: + return chat(db, member(db, uid, cid)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + key = request.headers.get("idempotency-key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII): + invalid("Некорректный Idempotency-Key") + data = await body(request, ("text",)) if request.headers.get("content-length") == "0" else None + if data is None: + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + declared = request.headers.get("content-length") + if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + raw = bytearray() + async for chunk in request.stream(): + raw.extend(chunk) + if len(raw) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeError, ValueError): + invalid("Некорректный JSON") + if not isinstance(data, dict) or set(data) not in ({"text"}, {"text", "attachment_ids"}): + invalid("Некорректные поля JSON") + content = data["text"] + if not isinstance(content, str) or len(content) > 4000: + invalid("Некорректный текст") + raw_ids = data.get("attachment_ids", []) + if (not isinstance(raw_ids, list) or len(raw_ids) > 10 + or len(raw_ids) != len(set(raw_ids))): + invalid("Некорректные вложения") + attachment_ids = [identifier(item) for item in raw_ids] + if not content.strip() and not attachment_ids: + invalid("Пустое сообщение") + return await asyncio.to_thread(insert_message, uid, cid, content, key, attachment_ids) + + +def insert_message(uid, cid, content, key, attachment_ids): + with connection() as db: + with db.transaction(): + member(db, uid, cid) + if attachment_ids: + rows = db.execute( + "SELECT id FROM attachments WHERE id=ANY(%s::uuid[]) AND chat_id=%s AND status='ready'", + (attachment_ids, cid), + ).fetchall() + if len(rows) != len(attachment_ids): + raise ApiError(404, "not_found", "Объект не найден") + # The unique constraint serializes concurrent requests on any replica. + db.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()", (uid, cid, key)) + normalized = {"text": content, "attachment_ids": attachment_ids} + body_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True).encode()).digest() + claimed = db.execute("INSERT INTO message_idempotency(user_id,chat_id,key,body_hash,expires_at) VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING RETURNING key", (uid, cid, key, body_hash)).fetchone() + if claimed is None: + previous = db.execute("SELECT body_hash,message_id FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", (uid, cid, key)).fetchone() + if previous is None or previous["body_hash"] != body_hash: + raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения") + return message(db.execute("SELECT * FROM messages WHERE id=%s", (previous["message_id"],)).fetchone(), db) + row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone() + for position, attachment_id in enumerate(attachment_ids): + db.execute( + "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", + (row["id"], attachment_id, position), + ) + db.execute("UPDATE message_idempotency SET message_id=%s WHERE user_id=%s AND chat_id=%s AND key=%s", (row["id"], uid, cid, key)) + db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,)) + return message(row, db) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + limit = pagination(request) + cursor = decode_cursor(cid, request.query_params.get("cursor")) + with connection() as db: + member(db, uid, cid) + if cursor is not None: + valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone() + if valid is None: + invalid("Некорректный cursor") + version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"] + return history(db, cid, limit, cursor, version) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + filename = valid_filename(request.query_params.get("filename")) + content_type = valid_content_type( + request.headers.get("content-type", "").split(";")[0].strip().lower() + ) + declared = request.headers.get("content-length") + if declared is not None and declared.isdecimal() and int(declared) > MAX_ATTACHMENT: + raise ApiError(413, "payload_too_large", "Файл слишком велик") + with connection() as db: + member(db, uid, cid) + digest = hashlib.sha256() + size = 0 + with tempfile.NamedTemporaryFile() as temporary: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_ATTACHMENT: + raise ApiError(413, "payload_too_large", "Файл слишком велик") + digest.update(chunk) + temporary.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "Пустой файл") + temporary.flush() + validate_image(temporary.name, content_type) + aid = uuid.uuid4() + object_key = f"originals/{aid.hex}" + temporary.seek(0) + await asyncio.to_thread( + s3.put_object, S3_BUCKET, object_key, temporary, size, content_type + ) + try: + return await asyncio.to_thread( + save_proxy_attachment, + aid, + uid, + cid, + filename, + content_type, + size, + digest.hexdigest(), + object_key, + ) + except Exception: + try: + await asyncio.to_thread(s3.remove_object, S3_BUCKET, object_key) + except (MinioException, urllib3.exceptions.HTTPError, OSError): + pass + raise + + +def save_proxy_attachment(aid, uid, cid, filename, content_type, size, digest, object_key): + with connection() as db: + row = db.execute( + "INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *", + (aid, cid, uid, filename, content_type, size, digest, object_key), + ).fetchone() + return attachment(row) + + +@app.post("/api/v1/chats/{chat_id}/attachments/presign", status_code=201) +async def presign_attachment(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + data = await body(request, ("filename", "content_type", "size_bytes", "sha256")) + filename = valid_filename(data["filename"]) + content_type = valid_content_type(data["content_type"]) + size = data["size_bytes"] + digest = data["sha256"] + if not isinstance(size, int) or isinstance(size, bool) or not 1 <= size <= MAX_ATTACHMENT: + raise ApiError(413, "payload_too_large", "Некорректный размер") + if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): + invalid("Некорректный SHA-256") + with connection() as db: + member(db, uid, cid) + aid = uuid.uuid4() + object_key = f"direct/{aid.hex}/{secrets.token_hex(16)}" + expires = datetime.now(timezone.utc) + timedelta(seconds=PRESIGN_TTL) + db.execute( + "INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,upload_expires_at) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,'uploading',%s,%s)", + (aid, cid, uid, filename, content_type, size, digest, object_key, expires), + ) + url = s3_signer.presigned_put_object( + S3_BUCKET, object_key, expires=timedelta(seconds=PRESIGN_TTL) + ) + return { + "attachment_id": str(aid), + "upload_url": url, + "expires_at": stamp(expires), + "required_headers": {"Content-Type": content_type}, + } + + +@app.post("/api/v1/attachments/{attachment_id}/complete") +async def complete_attachment(attachment_id: str, request: Request): + uid = actor(request) + aid = identifier(attachment_id) + data = await body(request, ()) + if data: + invalid("Некорректные поля JSON") + return await asyncio.to_thread(complete_direct_attachment, uid, aid) + + +def complete_direct_attachment(uid, aid): + with connection() as db: + row = db.execute( + "SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s AND a.owner_id=%s FOR UPDATE OF a", + (aid, uid, uid), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + if row["status"] == "ready": + return attachment(row) + if row["upload_expires_at"] < datetime.now(timezone.utc): + raise ApiError(409, "conflict", "Срок загрузки истёк") + try: + info = s3.stat_object(S3_BUCKET, row["object_key"]) + except S3Error as error: + if error.code in ("NoSuchKey", "NoSuchObject"): + raise ApiError(409, "conflict", "Файл ещё не загружен") from None + raise + if info.size != row["size_bytes"] or info.content_type != row["content_type"]: + reject_direct_upload(db, row) + raise ApiError(409, "conflict", "Размер или Content-Type не совпадает") + response = s3.get_object(S3_BUCKET, row["object_key"]) + digest = hashlib.sha256() + size = 0 + try: + with tempfile.NamedTemporaryFile() as temporary: + while True: + chunk = response.read(64 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_ATTACHMENT: + reject_direct_upload(db, row) + raise ApiError(413, "payload_too_large", "Файл слишком велик") + digest.update(chunk) + temporary.write(chunk) + temporary.flush() + validate_image(temporary.name, row["content_type"]) + finally: + response.close() + response.release_conn() + if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]: + reject_direct_upload(db, row) + raise ApiError(409, "conflict", "Checksum не совпадает") + ready_row = db.execute( + "UPDATE attachments SET status='ready',upload_expires_at=NULL,updated_at=clock_timestamp() " + "WHERE id=%s RETURNING *", + (aid,), + ).fetchone() + return attachment(ready_row) + + +def reject_direct_upload(db, row): + s3.remove_object(S3_BUCKET, row["object_key"]) + db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],)) + + +def accessible_attachment(uid, aid, ready_only=True): + with connection() as db: + row = db.execute( + "SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (aid, uid), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + if ready_only and row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: str, request: Request): + uid = actor(request) + return attachment(accessible_attachment(uid, identifier(attachment_id))) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment(attachment_id: str, request: Request): + uid = actor(request) + if request.query_params.get("variant", "original") != "original": + raise ApiError(404, "not_found", "Объект не найден") + row = accessible_attachment(uid, identifier(attachment_id)) + source = s3.get_object(S3_BUCKET, row["object_key"]) + + def stream(): + try: + while True: + chunk = source.read(64 * 1024) + if not chunk: + break + yield chunk + finally: + source.close() + source.release_conn() + + return StreamingResponse( + stream(), + media_type=row["content_type"], + headers={"Content-Length": str(row["size_bytes"])}, + ) + + +@app.get("/internal/cache-metrics") +def cache_metrics(request: Request): + if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"): + raise ApiError(404, "not_found", "Объект не найден") + with metrics_lock: + return dict(metrics) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..779a7cfd --- /dev/null +++ b/compose.yaml @@ -0,0 +1,163 @@ +services: + postgres: + image: postgres:16.8-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 2s + retries: 20 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"] + networks: [data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 20 + deploy: + resources: + limits: + cpus: "0.5" + memory: 192M + + minio: + image: quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z + command: ["server", "/data"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + volumes: + - minio_data:/data + networks: [data] + ports: + - "127.0.0.1:${S3_PUBLIC_PORT:-9000}:9000" + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + minio-init: + image: quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z + entrypoint: ["/bin/sh", "-c"] + command: + - | + until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"; do sleep 1; done + mc mb --ignore-existing local/messenger-attachments + mc anonymous set none local/messenger-attachments + mc admin user add local "$$S3_ACCESS_KEY" "$$S3_SECRET_KEY" + mc admin policy create local messenger-api-policy /policy.json + mc admin policy attach local messenger-api-policy --user "$$S3_ACCESS_KEY" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} + volumes: + - ./infra/minio-policy.json:/policy.json:ro + networks: [data] + depends_on: + minio: + condition: service_started + restart: "no" + + migrate: + build: . + command: ["python", "-m", "scripts.migrate"] + environment: &database_env + DATABASE_HOST: postgres + DATABASE_NAME: messenger + DATABASE_USER: messenger + DATABASE_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + networks: [data] + depends_on: + postgres: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + api: + build: . + environment: + <<: *database_env + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: minio:9000 + S3_PUBLIC_ENDPOINT: localhost:${S3_PUBLIC_PORT:-9000} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} + S3_BUCKET: messenger-attachments + PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-120} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_SIZE: ${DB_POOL_SIZE:-8} + networks: [front, data] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 15s + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.75" + memory: 256M + + lb: + image: haproxy:3.2.6-alpine + volumes: + - ./infra/haproxy-compose.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [front] + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 3s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + +networks: + front: + data: + internal: true + +volumes: + postgres_data: + minio_data: diff --git a/contracts/README.md b/contracts/README.md index 8c58b32f..8815280f 100644 --- a/contracts/README.md +++ b/contracts/README.md @@ -52,6 +52,19 @@ Basic — это кодирование, не шифрование. HTTP lab1– - В ready-метаданных `error_code=null`. В failed — краткий машинный код (например `invalid_image`, `processing_failed`, `retry_exhausted`), без stack trace. `size_bytes`, `sha256`, `content_type` всегда описывают оригинал; у вариантов отдельные поля. Публичный API не раскрывает ключи бакета, внутренние адреса и credentials. - Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html). +### Расширение этой реализации: presigned upload + +`POST /api/v1/chats/{chat_id}/attachments/presign` принимает JSON +`filename`, `content_type`, `size_bytes`, `sha256` и возвращает `201` с +`attachment_id`, `upload_url`, `expires_at`, `required_headers`. URL действует +30–900 секунд (в стенде 120) и разрешает один PUT только по назначенному +случайному object key. Затем владелец вызывает +`POST /api/v1/attachments/{attachment_id}/complete` с `{}`. Ответ `200 +Attachment` означает, что API независимо проверил S3 object, размер, MIME, +SHA-256 и изображение. До complete объект нельзя читать или прикреплять; +complete другого пользователя скрывается `404`, отсутствующий PUT и истёкший +тикет дают `409`. + ## Фоновая обработка (с lab5) До 202 все байты приняты в **устойчивый staging**, а намерение обработать файл надёжно сохранено. HTTP-передача клиента не исчезает и не становится мгновенной. Staging — приватный S3-prefix либо общий persistent volume, доступный worker; локальная память/API ephemeral disk не подходит. Worker асинхронно переносит/публикует оригинал в конечный S3-prefix и создаёт варианты; если staging уже в S3, опишите эту границу честно. diff --git a/infra/README.md b/infra/README.md index 5a39e862..fd25b670 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,7 @@ -# Инфраструктура студента +# Инфраструктура -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +`compose.yaml` поднимает PostgreSQL, Redis, приватный MinIO, одноразовые `minio-init`/`migrate`, две API-реплики и HAProxy. MinIO console не публикуется. Порт S3 `127.0.0.1:9000` нужен только клиентскому presigned PUT; anonymous GET отключён. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +`infra/minio-policy.json` ограничивает API-пользователя bucket `messenger-attachments`. Root credentials используются только bootstrap-контейнером. Метаданные хранятся в PostgreSQL volume, bytes — в MinIO volume. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Запуск и доказательства описаны в [REPORT.md](../REPORT.md). diff --git a/infra/haproxy-compose.cfg b/infra/haproxy-compose.cfg new file mode 100644 index 00000000..f5e41e92 --- /dev/null +++ b/infra/haproxy-compose.cfg @@ -0,0 +1,27 @@ +global + maxconn 2048 + +defaults + mode http + timeout connect 1s + timeout client 30s + timeout server 30s + timeout check 1s + +resolvers docker + nameserver dns1 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + timeout resolve 1s + timeout retry 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 2 + server-template replica 2 api:8080 check resolvers docker init-addr none diff --git a/infra/minio-policy.json b/infra/minio-policy.json new file mode 100644 index 00000000..6beb6bd2 --- /dev/null +++ b/infra/minio-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..2cdfbf05 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,38 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$') +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); +CREATE INDEX chats_order_idx ON chats (created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE TABLE chat_versions ( + chat_id uuid PRIMARY KEY REFERENCES chats(id), + version bigint NOT NULL DEFAULT 0 +); +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..9775cce7 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key varchar(128) NOT NULL, + body_hash bytea NOT NULL, + message_id uuid UNIQUE REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX message_idempotency_expiry_idx ON message_idempotency (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..80630b24 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,25 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + status varchar(16) NOT NULL CHECK (status IN ('uploading', 'ready')), + object_key text NOT NULL UNIQUE, + upload_expires_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat_idx ON attachments (chat_id, id); +CREATE INDEX attachments_expired_idx ON attachments (upload_expires_at) + WHERE status = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a563ddcf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +psycopg[binary,pool]==3.2.6 +redis==5.2.1 +minio==7.2.15 +Pillow==11.2.1 diff --git a/scripts/lint.py b/scripts/lint.py new file mode 100644 index 00000000..bdd29b93 --- /dev/null +++ b/scripts/lint.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Small dependency-free lint: syntax, whitespace and unused imports.""" + +import ast +from pathlib import Path +import sys + + +ROOT = Path(__file__).resolve().parents[1] +FILES = sorted( + path for folder in ("app", "scripts", "tests") + for path in (ROOT / folder).glob("*.py") +) + + +def check(path): + source = path.read_text(encoding="utf-8") + issues = [] + try: + tree = ast.parse(source, filename=str(path)) + except SyntaxError as error: + return [f"{path}:{error.lineno}: {error.msg}"] + used = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports = [(item.asname or item.name.split(".")[0], node.lineno) for item in node.names] + elif isinstance(node, ast.ImportFrom): + imports = [(item.asname or item.name, node.lineno) for item in node.names] + else: + continue + for name, line in imports: + if name not in used: + issues.append(f"{path}:{line}: unused import {name}") + for number, line in enumerate(source.splitlines(), 1): + if line.rstrip() != line: + issues.append(f"{path}:{number}: trailing whitespace") + if "\t" in line: + issues.append(f"{path}:{number}: tab character") + return issues + + +def main(): + issues = [issue for path in FILES for issue in check(path)] + if issues: + print("\n".join(issues)) + return 1 + print(f"PASS: lint ({len(FILES)} Python files)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..c644e75f --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,16 @@ +"""Run schema changes once before starting API replicas.""" + +from app.main import migrate, pool + + +def main(): + pool.open() + try: + pool.wait(timeout=20) + migrate() + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..b19bb733 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres redis minio api lb >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2b8fe9fa --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +case "$TARGET_INSTANCE_ID" in + *[!a-fA-F0-9]*|'') echo "Invalid instance id" >&2; exit 2 ;; +esac +service=$(docker inspect --format '{{ index .Config.Labels "com.docker.compose.service" }}' "$TARGET_INSTANCE_ID") +if [ "$service" != api ]; then + echo "Target is not a Compose API replica" >&2 + exit 2 +fi +docker stop -t 1 "$TARGET_INSTANCE_ID" >/dev/null diff --git a/tests/direct_upload.py b/tests/direct_upload.py new file mode 100644 index 00000000..dbee2be8 --- /dev/null +++ b/tests/direct_upload.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Acceptance checks for the level-5 presigned upload track.""" + +import hashlib +import json +from pathlib import Path +import urllib.error +import urllib.request +from urllib.parse import urlsplit, urlunsplit + +from client import Client, arguments, check, run + + +def presign(client, actor, chat, content, digest=None): + response = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments/presign", 201, + actor=actor, + body={ + "filename": "direct.png", + "content_type": "image/png", + "size_bytes": len(content), + "sha256": digest or hashlib.sha256(content).hexdigest(), + }, + ) + return json.loads(response["bytes"]) + + +def put(ticket, content): + request = urllib.request.Request( + ticket["upload_url"], data=content, method="PUT", + headers=ticket["required_headers"], + ) + with urllib.request.urlopen(request, timeout=15) as response: + check(response.status == 200, "presigned PUT failed") + + +def main(): + args = arguments("Lab4 direct upload track") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + content = (Path(__file__).parent / "fixtures/sample.png").read_bytes() + + unfinished = presign(client, alice, chat, content) + client.call( + "POST", "/api/v1/attachments/" + unfinished["attachment_id"] + "/complete", 409, + actor=alice, body={}, + ) + client.call( + "POST", "/api/v1/attachments/" + unfinished["attachment_id"] + "/complete", 404, + actor=bob, body={}, + ) + + wrong = presign(client, alice, chat, content, "0" * 64) + put(wrong, content) + client.call( + "POST", "/api/v1/attachments/" + wrong["attachment_id"] + "/complete", 409, + actor=alice, body={}, + ) + + ticket = presign(client, alice, chat, content) + put(ticket, content) + ready = client.call( + "POST", "/api/v1/attachments/" + ticket["attachment_id"] + "/complete", 200, + actor=alice, body={}, schema="Attachment", + )["json"] + check(ready["sha256"] == hashlib.sha256(content).hexdigest(), "checksum changed") + downloaded = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob, + )["bytes"] + check(downloaded == content, "direct-uploaded object changed") + + parts = urlsplit(ticket["upload_url"]) + private_url = urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + try: + urllib.request.urlopen(private_url, timeout=5) + except urllib.error.HTTPError as error: + check(error.code in (401, 403), "private bucket returned unexpected status") + else: + check(False, "private bucket allowed anonymous GET") + print("PASS: presign, ownership, checksum, completion and private bucket") + + +if __name__ == "__main__": + run(main) diff --git a/tests/level4.py b/tests/level4.py new file mode 100644 index 00000000..492788bb --- /dev/null +++ b/tests/level4.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Negative and isolation checks required by level 4.""" + +from pathlib import Path +from urllib.parse import quote + +from client import Client, arguments, run + + +def main(): + args = arguments("Lab4 negative upload scenarios") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + base = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + client.call( + "POST", base + "large.png", 413, actor=alice, + raw=b"x" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"}, + ) + client.call( + "POST", base + "broken.png", 415, actor=alice, + raw=b"not-an-image", headers={"Content-Type": "image/png"}, + ) + content = (Path(__file__).parent / "fixtures/sample.png").read_bytes() + client.call( + "POST", base + "spoofed.jpg", 415, actor=alice, + raw=content, headers={"Content-Type": "image/jpeg"}, + ) + client.call( + "POST", base + quote("../unsafe.png", safe=""), 400, actor=alice, + raw=content, headers={"Content-Type": "image/png"}, + ) + attachment, _ = client.attachment(alice, chat) + for suffix in ("", "/content"): + client.call("GET", "/api/v1/attachments/" + attachment["id"] + suffix, 404, actor=eve) + other = client.call( + "POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Другой чат", "member_ids": [alice["id"], eve["id"]]}, schema="Chat", + )["json"] + alien, _ = client.attachment(alice, other) + client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/messages", 404, actor=bob, + body={"text": "", "attachment_ids": [alien["id"]]}, + headers={"Idempotency-Key": "cross-chat-attachment"}, + ) + print("PASS: lab4 level-4 negative and authorization scenarios") + + +if __name__ == "__main__": + run(main) diff --git a/tests/upload_benchmark.py b/tests/upload_benchmark.py new file mode 100644 index 00000000..d4d6fc1c --- /dev/null +++ b/tests/upload_benchmark.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Compare API traffic and memory for proxy and presigned uploads.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import threading +import time +import urllib.request +import uuid +import zlib + +from client import Client, check + + +def chunk(kind, payload): + return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload)) + + +def png(width, height): + compressor = zlib.compressobj(level=0) + encoded = bytearray() + row = b"\0" + bytes(range(256)) * ((width * 3 + 255) // 256) + for _ in range(height): + encoded.extend(compressor.compress(row[:width * 3 + 1])) + encoded.extend(compressor.flush()) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", bytes(encoded)) + chunk(b"IEND", b"") + + +def memory_mib(): + ids = subprocess.run( + ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], + check=True, capture_output=True, text=True, + ).stdout.split() + if not ids: + return 0.0 + values = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids], + check=True, capture_output=True, text=True, + ).stdout.splitlines() + total = 0.0 + for value in values: + match = re.fullmatch(r"([0-9.]+)(KiB|MiB|GiB)", value.split("/")[0].strip()) + if match is None: + raise ValueError("unexpected docker stats memory format") + number, unit = match.groups() + factor = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit] + total += float(number) * factor + return total + + +def sample_memory(stop, samples): + while not stop.is_set(): + try: + samples.append(memory_mib()) + except (OSError, subprocess.SubprocessError, ValueError, KeyError): + pass + stop.wait(0.1) + + +def proxy(client, actor, chat, content): + client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bench.png", 201, + actor=actor, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + + +def direct(client, actor, chat, content): + ticket = json.loads(client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments/presign", 201, + actor=actor, + body={ + "filename": "bench.png", "content_type": "image/png", + "size_bytes": len(content), "sha256": hashlib.sha256(content).hexdigest(), + }, + )["bytes"]) + request = urllib.request.Request( + ticket["upload_url"], data=content, method="PUT", headers=ticket["required_headers"], + ) + with urllib.request.urlopen(request, timeout=60) as response: + check(response.status == 200, "presigned PUT failed") + client.call( + "POST", "/api/v1/attachments/" + ticket["attachment_id"] + "/complete", 200, + actor=actor, body={}, schema="Attachment", + ) + + +def measure(client, actor, chat, content, concurrency, mode): + stop = threading.Event() + samples = [] + sampler = threading.Thread(target=sample_memory, args=(stop, samples), daemon=True) + sampler.start() + started = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [pool.submit(proxy if mode == "proxy" else direct, client, actor, chat, content) + for _ in range(concurrency)] + for future in futures: + future.result() + elapsed = time.monotonic() - started + stop.set() + sampler.join() + return { + "mode": mode, + "file_bytes": len(content), + "concurrency": concurrency, + "elapsed_seconds": round(elapsed, 3), + "api_upload_body_bytes": len(content) * concurrency if mode == "proxy" else 0, + "api_storage_body_bytes": len(content) * concurrency, + "api_total_binary_bytes": len(content) * concurrency * (2 if mode == "proxy" else 1), + "s3_upload_body_bytes": len(content) * concurrency, + "api_replicas_peak_memory_mib": round(max(samples), 3) if samples else None, + "sample_count": len(samples), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload_benchmark.json") + args = parser.parse_args() + client = Client(args.base_url, timeout=90) + alice, _, _, chat = client.bootstrap() + results = [] + for content in (png(600, 600), png(1600, 1600)): + for mode in ("proxy", "direct"): + results.append(measure(client, alice, chat, content, args.concurrency, mode)) + output = { + "run_id": uuid.uuid4().hex, + "method": "closed-loop, same valid PNG shared by 10 client threads; 100 ms docker-stats sampling", + "results": results, + } + Path(args.output).write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() -- 2.54.0