ЛР4 - БПИ2301 - Юртаев - на 5 #73
+7
-16
@@ -1,20 +1,11 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
POSTGRES_PASSWORD=replace-with-long-random-password
|
||||
MINIO_ROOT_USER=replace-with-random-admin-name
|
||||
MINIO_ROOT_PASSWORD=replace-with-long-random-admin-password
|
||||
S3_ACCESS_KEY=messenger-api
|
||||
S3_SECRET_KEY=replace-with-long-random-api-secret
|
||||
S3_PUBLIC_PORT=9000
|
||||
PRESIGN_TTL_SECONDS=120
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
FROM python:3.12.10-slim
|
||||
WORKDIR /app
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt && useradd -r -u 10001 app
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
COPY scripts/migrate.py ./scripts/migrate.py
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
|
||||
@@ -4,11 +4,14 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help lint check validate test test-level4 test-direct benchmark-upload load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
lint:
|
||||
$(PYTHON) scripts/lint.py
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
|
||||
@@ -18,6 +21,15 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-level4:
|
||||
$(PYTHON) tests/level4.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-direct:
|
||||
$(PYTHON) tests/direct_upload.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
benchmark-upload:
|
||||
$(PYTHON) tests/upload_benchmark.py --base-url "$(BASE_URL)"
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,37 +1,71 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: Юртаев (группа не указана). Цель: **5, трек 1 — прямой upload**. Основа API перенесена из лабораторной 3. Использованы Python 3.12, FastAPI 0.115.12, psycopg 3.2.6, Redis 5.2.1, MinIO и Pillow 11.2.1; реализация подготовлена с помощью Codex.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine с Compose, Python 3.9+ и свободные локальные порты 8080/9000.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# Заменить все replace-with-* на случайные локальные значения.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make lint check validate test test-level4 test-direct
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
make benchmark-upload
|
||||
```
|
||||
|
||||
Безопасная остановка: `docker compose --env-file .env down`. Намеренное удаление только учебных данных: `docker compose --env-file .env down -v`. `.env` не попадает в Git.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
localhost:8080 -> HAProxy -> API x2 -> PostgreSQL (метаданные, сессии, связи)
|
||||
|-> Redis (необязательный кэш истории)
|
||||
`-> private MinIO bucket (оригиналы)
|
||||
client ------------------------------> MinIO:9000 (только короткий presigned PUT)
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
MinIO и PostgreSQL имеют persistent volumes. Bucket `messenger-attachments` создаётся идемпотентным init-контейнером, anonymous access отключён. Root credentials получает только init; API использует отдельного пользователя с доступом только к этому bucket. Консоль MinIO не запускается, S3 API опубликован только на `127.0.0.1` для presigned PUT.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Обязательный proxy endpoint читает тело порциями во временный файл с жёстким пределом 10 MiB, считает SHA-256, проверяет реальный PNG/JPEG и не более 40 млн пикселей, затем пишет случайный object key в S3. `201 ready` возвращается только после записи объекта и метаданных. При ошибке БД выполняется компенсационное удаление S3. Фоновая сверка удаляет неизвестные объекты только после grace period 10 минут. Filename никогда не используется как object key.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Метаданные `Attachment` и связи сообщения находятся в PostgreSQL. Сообщение атомарно проверяет до 10 уникальных ready-вложений своего чата. Чужой чат/attachment скрывается одинаковым `404`. Download всегда идёт через авторизованный API без redirect и потоково возвращает оригинал из S3.
|
||||
|
||||
## Эксперименты и отказы
|
||||
Readiness требует PostgreSQL и bucket; Redis остаётся необязательным. S3 connect/read timeout — 2/15 с. На API установлен лимит 256 MiB. Одновременные загрузки используют отдельные временные файлы и буферы по 64 KiB, поэтому размер входного файла не становится постоянным RAM-буфером API.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
## Трек 1: прямой upload
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
**Гипотеза:** presigned flow уберёт тело клиентской загрузки из внешнего трафика API и примерно вдвое уменьшит бинарный трафик API с учётом обязательного чтения объекта при `complete`; пиковая RAM API не должна расти пропорционально размеру файла.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
1. `POST /api/v1/chats/{chat_id}/attachments/presign` аутентифицирует участника, валидирует filename/type/size/SHA-256, создаёт уникальный ключ и запись `uploading`, выдаёт PUT URL на 120 с.
|
||||
2. Клиент отправляет bytes прямо в MinIO.
|
||||
3. Только создатель вызывает `POST /api/v1/attachments/{id}/complete`. API проверяет TTL, наличие, Content-Type, размер, потоковый SHA-256 и фактическое изображение; только затем атомарно ставит `ready`.
|
||||
4. Ошибочный объект удаляется. Незавершённая загрузка удаляется после TTL + 5 минут; повторный запуск cleanup безопасен. Обязательный proxy endpoint сохранён.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Отрицательные сценарии `tests/direct_upload.py`: complete до PUT -> 409; complete другим участником -> 404; неверный checksum -> 409 с удалением объекта; anonymous GET без подписи -> 401/403. `tests/level4.py` отдельно проверяет >10 MiB, битое изображение, MIME spoofing, опасный filename, чужой attachment и связь из другого чата.
|
||||
|
||||
## Доказательства
|
||||
|
||||
| Критерий | Команда | Результат 06.11.2026 |
|
||||
| --- | --- | --- |
|
||||
| Lint | `make lint` | PASS, 14 Python-файлов |
|
||||
| Структура/контракт | `make check` | PASS |
|
||||
| Compose model | `docker compose --env-file .env config --quiet` | PASS |
|
||||
| OpenAPI | `make validate` | не выполнено: на диске осталось 155 MiB, установка venv завершилась `No space left on device` |
|
||||
| Публичный smoke | `make test` | не выполнено: Docker Desktop не стартует при заполненном диске |
|
||||
| Негативные сценарии | `make test-level4 test-direct` | подготовлены, интеграционный прогон заблокирован Docker |
|
||||
| Persistence/failover | команды выше | интеграционный прогон заблокирован Docker |
|
||||
| 10 concurrent, два размера | `make benchmark-upload` | методика подготовлена, измерения не сфабрикованы; требуется рабочий Docker |
|
||||
|
||||
## Методика эксперимента
|
||||
|
||||
`tests/upload_benchmark.py` генерирует валидные несжатые PNG примерно 1 и 8 MiB. Для каждого размера выполняются 10 одновременных proxy, затем 10 direct загрузок. Каждые 100 мс снимается суммарная RAM двух API через `docker stats`; raw JSON сохраняется в `evidence/upload_benchmark.json`.
|
||||
|
||||
Для proxy бинарный трафик API равен примерно `2*N`: client -> API и API -> S3. Для direct внешний client -> API body отсутствует, но безопасный `complete` читает `N` из S3 для независимой проверки checksum, поэтому полный бинарный трафик API примерно `N`, а не ноль. Control JSON мал относительно 1/8 MiB и сохраняется отдельно. Это ожидаемая модель, не подмена фактического измерения.
|
||||
|
||||
## Ограничения
|
||||
|
||||
MinIO и PostgreSQL одиночные и остаются SPOF. Временный файл использует ephemeral disk API только во время проверки и не является хранилищем оригинала. Complete повторно читает объект через API ради независимой проверки SHA-256: это безопаснее доверия клиентской metadata, но оставляет половину бинарного API-трафика. Текущий локальный интеграционный прогон заблокирован заполненным диском; перед защитой нужно освободить место и заменить строки `не выполнено` фактическими результатами из указанных команд.
|
||||
|
||||
+881
@@ -0,0 +1,881 @@
|
||||
"""Replicated messenger API with PostgreSQL-backed sessions and idempotency."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import socket
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
import urllib3
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from minio import Minio
|
||||
from minio.error import MinioException, S3Error
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger")
|
||||
database_password = os.environ.get("DATABASE_PASSWORD")
|
||||
if "DATABASE_PASSWORD_FILE" in os.environ:
|
||||
with open(os.environ["DATABASE_PASSWORD_FILE"], encoding="utf-8") as secret_file:
|
||||
database_password = secret_file.read().rstrip("\n")
|
||||
if database_password is not None:
|
||||
DATABASE_URL = make_conninfo(
|
||||
host=os.environ.get("DATABASE_HOST", "postgres"),
|
||||
dbname=os.environ.get("DATABASE_NAME", "messenger"),
|
||||
user=os.environ.get("DATABASE_USER", "messenger"),
|
||||
password=database_password,
|
||||
)
|
||||
REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
|
||||
SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384"))
|
||||
POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10"))
|
||||
CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1"
|
||||
CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1"
|
||||
CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0"))))
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "") or socket.gethostname()
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "minio:9000")
|
||||
S3_PUBLIC_ENDPOINT = os.environ.get("S3_PUBLIC_ENDPOINT", "localhost:9000")
|
||||
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "messenger-api")
|
||||
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "messenger-api-secret")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_SECURE = os.environ.get("S3_SECURE", "0") == "1"
|
||||
PRESIGN_TTL = min(900, max(30, int(os.environ.get("PRESIGN_TTL_SECONDS", "120"))))
|
||||
MAX_ATTACHMENT = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
ready_for_traffic = False
|
||||
pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"})
|
||||
cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2)
|
||||
s3_http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(connect=2, read=15),
|
||||
retries=False,
|
||||
maxsize=max(10, POOL_SIZE * 2),
|
||||
)
|
||||
s3 = Minio(
|
||||
S3_ENDPOINT,
|
||||
access_key=S3_ACCESS_KEY,
|
||||
secret_key=S3_SECRET_KEY,
|
||||
secure=S3_SECURE,
|
||||
http_client=s3_http,
|
||||
)
|
||||
s3_signer = Minio(S3_PUBLIC_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=S3_SECURE)
|
||||
metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0}
|
||||
metrics_lock = threading.Lock()
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, challenge=None):
|
||||
self.status, self.code, self.message, self.challenge = status, code, message, challenge
|
||||
|
||||
|
||||
def invalid(message="Некорректный запрос"):
|
||||
raise ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unavailable():
|
||||
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
|
||||
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def encode_cursor(scope, date, identifier):
|
||||
raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def decode_cursor(scope, value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
||||
parts = json.loads(raw)
|
||||
if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope
|
||||
or not isinstance(parts[1], str) or not isinstance(parts[2], str)):
|
||||
invalid("Некорректный cursor")
|
||||
date = datetime.fromisoformat(parts[1].replace("Z", "+00:00"))
|
||||
if date.tzinfo is None:
|
||||
invalid("Некорректный cursor")
|
||||
return date, str(uuid.UUID(parts[2]))
|
||||
except (ValueError, TypeError, binascii.Error, IndexError):
|
||||
invalid("Некорректный cursor")
|
||||
|
||||
|
||||
def identifier(value):
|
||||
try:
|
||||
result = str(uuid.UUID(value))
|
||||
if result != value.lower() or len(value) != 36:
|
||||
invalid("Некорректный UUID")
|
||||
return result
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
invalid("Некорректный UUID")
|
||||
|
||||
|
||||
def text_field(value, maximum):
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
|
||||
invalid("Некорректное текстовое поле")
|
||||
return value
|
||||
|
||||
|
||||
def exact_body(value, fields):
|
||||
if not isinstance(value, dict) or set(value) != set(fields):
|
||||
invalid("Некорректные поля JSON")
|
||||
|
||||
|
||||
async def body(request, fields):
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
declared = request.headers.get("content-length")
|
||||
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
||||
chunks = bytearray()
|
||||
async for chunk in request.stream():
|
||||
chunks.extend(chunk)
|
||||
if len(chunks) > MAX_BODY:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
||||
try:
|
||||
value = json.loads(chunks.decode("utf-8"))
|
||||
except (UnicodeError, ValueError):
|
||||
invalid("Некорректный JSON")
|
||||
exact_body(value, fields)
|
||||
return value
|
||||
|
||||
|
||||
def user(row):
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])}
|
||||
|
||||
|
||||
def message(row, db=None):
|
||||
attachment_ids = []
|
||||
if db is not None:
|
||||
linked = db.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
attachment_ids = [str(item["attachment_id"]) for item in linked]
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": stamp(row["created_at"]),
|
||||
"attachment_ids": attachment_ids,
|
||||
}
|
||||
|
||||
|
||||
def attachment(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": "ready",
|
||||
"variants": [],
|
||||
"error_code": None,
|
||||
"created_at": stamp(row["created_at"]),
|
||||
"updated_at": stamp(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
def valid_filename(value):
|
||||
if (not isinstance(value, str) or not 1 <= len(value) <= 255
|
||||
or any(char in value for char in ("/", "\\", "\x00"))
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in value)):
|
||||
invalid("Некорректный filename")
|
||||
return value
|
||||
|
||||
|
||||
def valid_content_type(value):
|
||||
if value not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Допускаются PNG и JPEG")
|
||||
return value
|
||||
|
||||
|
||||
def validate_image(path, declared_type):
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
|
||||
if actual_type != declared_type:
|
||||
raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому")
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
||||
image.verify()
|
||||
except Image.DecompressionBombError:
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError):
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
||||
|
||||
|
||||
def chat(connection, row):
|
||||
members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall()
|
||||
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])}
|
||||
|
||||
|
||||
def password_hash(password):
|
||||
salt = os.urandom(16)
|
||||
derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1)
|
||||
return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex()
|
||||
|
||||
|
||||
def password_ok(password, stored):
|
||||
try:
|
||||
_, n, r, p, salt, expected = stored.split(":")
|
||||
actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p))
|
||||
return secrets.compare_digest(actual, bytes.fromhex(expected))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def connection():
|
||||
try:
|
||||
return pool.connection()
|
||||
except psycopg.Error:
|
||||
unavailable()
|
||||
|
||||
|
||||
def actor(request):
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.startswith("Bearer ") or not auth[7:]:
|
||||
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
|
||||
digest = hashlib.sha256(auth[7:].encode()).digest()
|
||||
with connection() as db:
|
||||
row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
|
||||
return row["user_id"]
|
||||
|
||||
|
||||
def member(db, uid, cid):
|
||||
row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
return row
|
||||
|
||||
|
||||
def pagination(request):
|
||||
raw = request.query_params.get("limit", "50")
|
||||
if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100:
|
||||
invalid("Некорректный limit")
|
||||
return int(raw)
|
||||
|
||||
|
||||
def bump(key):
|
||||
with metrics_lock:
|
||||
metrics[key] += 1
|
||||
|
||||
|
||||
def cache_get(key):
|
||||
try:
|
||||
raw = cache.get(key)
|
||||
if raw is not None:
|
||||
bump("history_cache_hit")
|
||||
return json.loads(raw)
|
||||
except (redis.RedisError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def cache_set(key, data):
|
||||
try:
|
||||
cache.set(key, json.dumps(data), ex=30)
|
||||
except redis.RedisError:
|
||||
pass
|
||||
|
||||
|
||||
def history(db, cid, limit, cursor, version):
|
||||
key = f"history:{cid}:{version}:{limit}:{cursor or '-'}"
|
||||
hit = cache_get(key) if CACHE_ENABLED else None
|
||||
if hit is not None:
|
||||
return hit
|
||||
lock_key = "lock:" + key
|
||||
token = secrets.token_hex(16)
|
||||
locked = False
|
||||
if CACHE_ENABLED and CACHE_LOCK_ENABLED:
|
||||
try:
|
||||
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
|
||||
if not locked:
|
||||
bump("history_lock_wait")
|
||||
for _ in range(20):
|
||||
time.sleep(0.025)
|
||||
hit = cache_get(key)
|
||||
if hit is not None:
|
||||
return hit
|
||||
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
|
||||
except redis.RedisError:
|
||||
pass
|
||||
if CACHE_MISS_DELAY_MS:
|
||||
time.sleep(CACHE_MISS_DELAY_MS / 1000)
|
||||
if cursor is None:
|
||||
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall()
|
||||
else:
|
||||
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall()
|
||||
bump("history_sql")
|
||||
result = {"items": [message(row, db) for row in rows[:limit]], "next_cursor": None}
|
||||
if len(rows) > limit:
|
||||
last = rows[limit - 1]
|
||||
result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"]))
|
||||
if CACHE_ENABLED:
|
||||
cache_set(key, result)
|
||||
if locked:
|
||||
try:
|
||||
cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token)
|
||||
except redis.RedisError:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def migrate():
|
||||
from pathlib import Path
|
||||
files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql"))
|
||||
with pool.connection() as db:
|
||||
with db.transaction():
|
||||
db.execute("SELECT pg_advisory_xact_lock(2172002)")
|
||||
db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)")
|
||||
for file in files:
|
||||
if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None:
|
||||
db.execute(file.read_text())
|
||||
db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,))
|
||||
|
||||
|
||||
def cleanup_expired_uploads():
|
||||
with connection() as db:
|
||||
rows = db.execute(
|
||||
"SELECT id,object_key FROM attachments "
|
||||
"WHERE status='uploading' AND upload_expires_at < clock_timestamp()-interval '5 minutes' "
|
||||
"FOR UPDATE SKIP LOCKED LIMIT 100"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
try:
|
||||
s3.remove_object(S3_BUCKET, row["object_key"])
|
||||
except S3Error as error:
|
||||
if error.code not in ("NoSuchKey", "NoSuchObject"):
|
||||
continue
|
||||
db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],))
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=10)
|
||||
for prefix in ("originals/", "direct/"):
|
||||
for item in s3.list_objects(S3_BUCKET, prefix=prefix, recursive=True):
|
||||
if item.last_modified >= cutoff:
|
||||
continue
|
||||
known = db.execute(
|
||||
"SELECT 1 FROM attachments WHERE object_key=%s", (item.object_name,)
|
||||
).fetchone()
|
||||
if known is None:
|
||||
s3.remove_object(S3_BUCKET, item.object_name)
|
||||
|
||||
|
||||
async def cleanup_loop():
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
try:
|
||||
await asyncio.to_thread(cleanup_expired_uploads)
|
||||
except (psycopg.Error, MinioException, urllib3.exceptions.HTTPError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app):
|
||||
global ready_for_traffic
|
||||
pool.open()
|
||||
pool.wait(timeout=20)
|
||||
if not s3.bucket_exists(S3_BUCKET):
|
||||
raise RuntimeError("S3 bucket is not initialized")
|
||||
ready_for_traffic = True
|
||||
task = asyncio.create_task(cleanup_loop())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
ready_for_traffic = False
|
||||
pool.close()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error(request, error):
|
||||
request_id = request.state.request_id
|
||||
code = "not_found" if error.status_code == 404 else "invalid_request"
|
||||
return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def envelope(request, call_next):
|
||||
request_id = str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status)
|
||||
if exc.challenge:
|
||||
response.headers["WWW-Authenticate"] = exc.challenge
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable, PoolTimeout, MinioException,
|
||||
urllib3.exceptions.HTTPError, OSError):
|
||||
response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503)
|
||||
except Exception:
|
||||
response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready():
|
||||
if not ready_for_traffic:
|
||||
unavailable()
|
||||
with connection() as db:
|
||||
db.execute("SELECT 1")
|
||||
if not s3.bucket_exists(S3_BUCKET):
|
||||
unavailable()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
data = await body(request, ("username", "display_name", "password"))
|
||||
name = data["username"]
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name):
|
||||
invalid("Некорректный username")
|
||||
display = text_field(data["display_name"], 100)
|
||||
password = data["password"]
|
||||
if not isinstance(password, str) or not password.strip() or not 12 <= len(password) <= 128:
|
||||
invalid("Некорректный пароль")
|
||||
try:
|
||||
return await asyncio.to_thread(insert_user, name, display, password)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, "conflict", "Username уже занят") from None
|
||||
|
||||
|
||||
def insert_user(name, display, password):
|
||||
digest = password_hash(password)
|
||||
with connection() as db:
|
||||
row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone()
|
||||
return user(row)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def login(request: Request):
|
||||
async for chunk in request.stream():
|
||||
if chunk:
|
||||
invalid("Тело запроса не допускается")
|
||||
auth = request.headers.get("authorization", "")
|
||||
try:
|
||||
if not auth.startswith("Basic "):
|
||||
raise ValueError()
|
||||
decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8")
|
||||
name, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None
|
||||
return await asyncio.to_thread(create_session, name, password)
|
||||
|
||||
|
||||
def create_session(name, password):
|
||||
with connection() as db:
|
||||
row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone()
|
||||
if row is None or not password_ok(password, row["password_hash"]):
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"')
|
||||
token = secrets.token_urlsafe(32)
|
||||
expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)
|
||||
db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry))
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(request: Request):
|
||||
actor(request)
|
||||
token = request.headers["authorization"][7:]
|
||||
with connection() as db:
|
||||
db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),))
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def me(request: Request):
|
||||
uid = actor(request)
|
||||
with connection() as db:
|
||||
row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone()
|
||||
return user(row)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
uid = actor(request)
|
||||
data = await body(request, ("title", "member_ids"))
|
||||
title = text_field(data["title"], 100)
|
||||
raw = data["member_ids"]
|
||||
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
|
||||
invalid("Некорректный список участников")
|
||||
members = [identifier(item) for item in raw]
|
||||
if len(set(members)) != len(members) or str(uid) not in members:
|
||||
invalid("Некорректный список участников")
|
||||
return await asyncio.to_thread(insert_chat, uid, title, members)
|
||||
|
||||
|
||||
def insert_chat(uid, title, members):
|
||||
with connection() as db:
|
||||
with db.transaction():
|
||||
count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"]
|
||||
if count != len(members):
|
||||
invalid("Неизвестный участник")
|
||||
row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone()
|
||||
for member_id in members:
|
||||
db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id))
|
||||
db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],))
|
||||
return chat(db, row)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(request: Request):
|
||||
uid = actor(request)
|
||||
limit = pagination(request)
|
||||
cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor"))
|
||||
with connection() as db:
|
||||
if cursor is None:
|
||||
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall()
|
||||
else:
|
||||
valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone()
|
||||
if valid is None:
|
||||
invalid("Некорректный cursor")
|
||||
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall()
|
||||
items = [chat(db, row) for row in rows[:limit]]
|
||||
next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
cid = identifier(chat_id)
|
||||
with connection() as db:
|
||||
return chat(db, member(db, uid, cid))
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def create_message(chat_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
cid = identifier(chat_id)
|
||||
key = request.headers.get("idempotency-key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII):
|
||||
invalid("Некорректный Idempotency-Key")
|
||||
data = await body(request, ("text",)) if request.headers.get("content-length") == "0" else None
|
||||
if data is None:
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
declared = request.headers.get("content-length")
|
||||
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
raw.extend(chunk)
|
||||
if len(raw) > MAX_BODY:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeError, ValueError):
|
||||
invalid("Некорректный JSON")
|
||||
if not isinstance(data, dict) or set(data) not in ({"text"}, {"text", "attachment_ids"}):
|
||||
invalid("Некорректные поля JSON")
|
||||
content = data["text"]
|
||||
if not isinstance(content, str) or len(content) > 4000:
|
||||
invalid("Некорректный текст")
|
||||
raw_ids = data.get("attachment_ids", [])
|
||||
if (not isinstance(raw_ids, list) or len(raw_ids) > 10
|
||||
or len(raw_ids) != len(set(raw_ids))):
|
||||
invalid("Некорректные вложения")
|
||||
attachment_ids = [identifier(item) for item in raw_ids]
|
||||
if not content.strip() and not attachment_ids:
|
||||
invalid("Пустое сообщение")
|
||||
return await asyncio.to_thread(insert_message, uid, cid, content, key, attachment_ids)
|
||||
|
||||
|
||||
def insert_message(uid, cid, content, key, attachment_ids):
|
||||
with connection() as db:
|
||||
with db.transaction():
|
||||
member(db, uid, cid)
|
||||
if attachment_ids:
|
||||
rows = db.execute(
|
||||
"SELECT id FROM attachments WHERE id=ANY(%s::uuid[]) AND chat_id=%s AND status='ready'",
|
||||
(attachment_ids, cid),
|
||||
).fetchall()
|
||||
if len(rows) != len(attachment_ids):
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
# The unique constraint serializes concurrent requests on any replica.
|
||||
db.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()", (uid, cid, key))
|
||||
normalized = {"text": content, "attachment_ids": attachment_ids}
|
||||
body_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True).encode()).digest()
|
||||
claimed = db.execute("INSERT INTO message_idempotency(user_id,chat_id,key,body_hash,expires_at) VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING RETURNING key", (uid, cid, key, body_hash)).fetchone()
|
||||
if claimed is None:
|
||||
previous = db.execute("SELECT body_hash,message_id FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", (uid, cid, key)).fetchone()
|
||||
if previous is None or previous["body_hash"] != body_hash:
|
||||
raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения")
|
||||
return message(db.execute("SELECT * FROM messages WHERE id=%s", (previous["message_id"],)).fetchone(), db)
|
||||
row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone()
|
||||
for position, attachment_id in enumerate(attachment_ids):
|
||||
db.execute(
|
||||
"INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)",
|
||||
(row["id"], attachment_id, position),
|
||||
)
|
||||
db.execute("UPDATE message_idempotency SET message_id=%s WHERE user_id=%s AND chat_id=%s AND key=%s", (row["id"], uid, cid, key))
|
||||
db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,))
|
||||
return message(row, db)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(chat_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
cid = identifier(chat_id)
|
||||
limit = pagination(request)
|
||||
cursor = decode_cursor(cid, request.query_params.get("cursor"))
|
||||
with connection() as db:
|
||||
member(db, uid, cid)
|
||||
if cursor is not None:
|
||||
valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone()
|
||||
if valid is None:
|
||||
invalid("Некорректный cursor")
|
||||
version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"]
|
||||
return history(db, cid, limit, cursor, version)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||
async def upload_attachment(chat_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
cid = identifier(chat_id)
|
||||
filename = valid_filename(request.query_params.get("filename"))
|
||||
content_type = valid_content_type(
|
||||
request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
)
|
||||
declared = request.headers.get("content-length")
|
||||
if declared is not None and declared.isdecimal() and int(declared) > MAX_ATTACHMENT:
|
||||
raise ApiError(413, "payload_too_large", "Файл слишком велик")
|
||||
with connection() as db:
|
||||
member(db, uid, cid)
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with tempfile.NamedTemporaryFile() as temporary:
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_ATTACHMENT:
|
||||
raise ApiError(413, "payload_too_large", "Файл слишком велик")
|
||||
digest.update(chunk)
|
||||
temporary.write(chunk)
|
||||
if size == 0:
|
||||
raise ApiError(415, "unsupported_media_type", "Пустой файл")
|
||||
temporary.flush()
|
||||
validate_image(temporary.name, content_type)
|
||||
aid = uuid.uuid4()
|
||||
object_key = f"originals/{aid.hex}"
|
||||
temporary.seek(0)
|
||||
await asyncio.to_thread(
|
||||
s3.put_object, S3_BUCKET, object_key, temporary, size, content_type
|
||||
)
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
save_proxy_attachment,
|
||||
aid,
|
||||
uid,
|
||||
cid,
|
||||
filename,
|
||||
content_type,
|
||||
size,
|
||||
digest.hexdigest(),
|
||||
object_key,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
await asyncio.to_thread(s3.remove_object, S3_BUCKET, object_key)
|
||||
except (MinioException, urllib3.exceptions.HTTPError, OSError):
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def save_proxy_attachment(aid, uid, cid, filename, content_type, size, digest, object_key):
|
||||
with connection() as db:
|
||||
row = db.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *",
|
||||
(aid, cid, uid, filename, content_type, size, digest, object_key),
|
||||
).fetchone()
|
||||
return attachment(row)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments/presign", status_code=201)
|
||||
async def presign_attachment(chat_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
cid = identifier(chat_id)
|
||||
data = await body(request, ("filename", "content_type", "size_bytes", "sha256"))
|
||||
filename = valid_filename(data["filename"])
|
||||
content_type = valid_content_type(data["content_type"])
|
||||
size = data["size_bytes"]
|
||||
digest = data["sha256"]
|
||||
if not isinstance(size, int) or isinstance(size, bool) or not 1 <= size <= MAX_ATTACHMENT:
|
||||
raise ApiError(413, "payload_too_large", "Некорректный размер")
|
||||
if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest):
|
||||
invalid("Некорректный SHA-256")
|
||||
with connection() as db:
|
||||
member(db, uid, cid)
|
||||
aid = uuid.uuid4()
|
||||
object_key = f"direct/{aid.hex}/{secrets.token_hex(16)}"
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=PRESIGN_TTL)
|
||||
db.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,upload_expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,'uploading',%s,%s)",
|
||||
(aid, cid, uid, filename, content_type, size, digest, object_key, expires),
|
||||
)
|
||||
url = s3_signer.presigned_put_object(
|
||||
S3_BUCKET, object_key, expires=timedelta(seconds=PRESIGN_TTL)
|
||||
)
|
||||
return {
|
||||
"attachment_id": str(aid),
|
||||
"upload_url": url,
|
||||
"expires_at": stamp(expires),
|
||||
"required_headers": {"Content-Type": content_type},
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/v1/attachments/{attachment_id}/complete")
|
||||
async def complete_attachment(attachment_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
aid = identifier(attachment_id)
|
||||
data = await body(request, ())
|
||||
if data:
|
||||
invalid("Некорректные поля JSON")
|
||||
return await asyncio.to_thread(complete_direct_attachment, uid, aid)
|
||||
|
||||
|
||||
def complete_direct_attachment(uid, aid):
|
||||
with connection() as db:
|
||||
row = db.execute(
|
||||
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s AND a.owner_id=%s FOR UPDATE OF a",
|
||||
(aid, uid, uid),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
if row["status"] == "ready":
|
||||
return attachment(row)
|
||||
if row["upload_expires_at"] < datetime.now(timezone.utc):
|
||||
raise ApiError(409, "conflict", "Срок загрузки истёк")
|
||||
try:
|
||||
info = s3.stat_object(S3_BUCKET, row["object_key"])
|
||||
except S3Error as error:
|
||||
if error.code in ("NoSuchKey", "NoSuchObject"):
|
||||
raise ApiError(409, "conflict", "Файл ещё не загружен") from None
|
||||
raise
|
||||
if info.size != row["size_bytes"] or info.content_type != row["content_type"]:
|
||||
reject_direct_upload(db, row)
|
||||
raise ApiError(409, "conflict", "Размер или Content-Type не совпадает")
|
||||
response = s3.get_object(S3_BUCKET, row["object_key"])
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile() as temporary:
|
||||
while True:
|
||||
chunk = response.read(64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > MAX_ATTACHMENT:
|
||||
reject_direct_upload(db, row)
|
||||
raise ApiError(413, "payload_too_large", "Файл слишком велик")
|
||||
digest.update(chunk)
|
||||
temporary.write(chunk)
|
||||
temporary.flush()
|
||||
validate_image(temporary.name, row["content_type"])
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
||||
reject_direct_upload(db, row)
|
||||
raise ApiError(409, "conflict", "Checksum не совпадает")
|
||||
ready_row = db.execute(
|
||||
"UPDATE attachments SET status='ready',upload_expires_at=NULL,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s RETURNING *",
|
||||
(aid,),
|
||||
).fetchone()
|
||||
return attachment(ready_row)
|
||||
|
||||
|
||||
def reject_direct_upload(db, row):
|
||||
s3.remove_object(S3_BUCKET, row["object_key"])
|
||||
db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],))
|
||||
|
||||
|
||||
def accessible_attachment(uid, aid, ready_only=True):
|
||||
with connection() as db:
|
||||
row = db.execute(
|
||||
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(aid, uid),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
if ready_only and row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение ещё не готово")
|
||||
return row
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def get_attachment(attachment_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
return attachment(accessible_attachment(uid, identifier(attachment_id)))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(attachment_id: str, request: Request):
|
||||
uid = actor(request)
|
||||
if request.query_params.get("variant", "original") != "original":
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
row = accessible_attachment(uid, identifier(attachment_id))
|
||||
source = s3.get_object(S3_BUCKET, row["object_key"])
|
||||
|
||||
def stream():
|
||||
try:
|
||||
while True:
|
||||
chunk = source.read(64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
source.close()
|
||||
source.release_conn()
|
||||
|
||||
return StreamingResponse(
|
||||
stream(),
|
||||
media_type=row["content_type"],
|
||||
headers={"Content-Length": str(row["size_bytes"])},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/internal/cache-metrics")
|
||||
def cache_metrics(request: Request):
|
||||
if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"):
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
with metrics_lock:
|
||||
return dict(metrics)
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.8-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 192M
|
||||
|
||||
minio:
|
||||
image: quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z
|
||||
command: ["server", "/data"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
networks: [data]
|
||||
ports:
|
||||
- "127.0.0.1:${S3_PUBLIC_PORT:-9000}:9000"
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
|
||||
minio-init:
|
||||
image: quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"; do sleep 1; done
|
||||
mc mb --ignore-existing local/messenger-attachments
|
||||
mc anonymous set none local/messenger-attachments
|
||||
mc admin user add local "$$S3_ACCESS_KEY" "$$S3_SECRET_KEY"
|
||||
mc admin policy create local messenger-api-policy /policy.json
|
||||
mc admin policy attach local messenger-api-policy --user "$$S3_ACCESS_KEY"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env}
|
||||
volumes:
|
||||
- ./infra/minio-policy.json:/policy.json:ro
|
||||
networks: [data]
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_started
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
command: ["python", "-m", "scripts.migrate"]
|
||||
environment: &database_env
|
||||
DATABASE_HOST: postgres
|
||||
DATABASE_NAME: messenger
|
||||
DATABASE_USER: messenger
|
||||
DATABASE_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
api:
|
||||
build: .
|
||||
environment:
|
||||
<<: *database_env
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
S3_ENDPOINT: minio:9000
|
||||
S3_PUBLIC_ENDPOINT: localhost:${S3_PUBLIC_PORT:-9000}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env}
|
||||
S3_BUCKET: messenger-attachments
|
||||
PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-120}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
|
||||
networks: [front, data]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
start_period: 5s
|
||||
retries: 2
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.75"
|
||||
memory: 256M
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.6-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy-compose.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks: [front]
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
networks:
|
||||
front:
|
||||
data:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
minio_data:
|
||||
@@ -52,6 +52,19 @@ Basic — это кодирование, не шифрование. HTTP lab1–
|
||||
- В ready-метаданных `error_code=null`. В failed — краткий машинный код (например `invalid_image`, `processing_failed`, `retry_exhausted`), без stack trace. `size_bytes`, `sha256`, `content_type` всегда описывают оригинал; у вариантов отдельные поля. Публичный API не раскрывает ключи бакета, внутренние адреса и credentials.
|
||||
- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html).
|
||||
|
||||
### Расширение этой реализации: presigned upload
|
||||
|
||||
`POST /api/v1/chats/{chat_id}/attachments/presign` принимает JSON
|
||||
`filename`, `content_type`, `size_bytes`, `sha256` и возвращает `201` с
|
||||
`attachment_id`, `upload_url`, `expires_at`, `required_headers`. URL действует
|
||||
30–900 секунд (в стенде 120) и разрешает один PUT только по назначенному
|
||||
случайному object key. Затем владелец вызывает
|
||||
`POST /api/v1/attachments/{attachment_id}/complete` с `{}`. Ответ `200
|
||||
Attachment` означает, что API независимо проверил S3 object, размер, MIME,
|
||||
SHA-256 и изображение. До complete объект нельзя читать или прикреплять;
|
||||
complete другого пользователя скрывается `404`, отсутствующий PUT и истёкший
|
||||
тикет дают `409`.
|
||||
|
||||
## Фоновая обработка (с lab5)
|
||||
|
||||
До 202 все байты приняты в **устойчивый staging**, а намерение обработать файл надёжно сохранено. HTTP-передача клиента не исчезает и не становится мгновенной. Staging — приватный S3-prefix либо общий persistent volume, доступный worker; локальная память/API ephemeral disk не подходит. Worker асинхронно переносит/публикует оригинал в конечный S3-prefix и создаёт варианты; если staging уже в S3, опишите эту границу честно.
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
`compose.yaml` поднимает PostgreSQL, Redis, приватный MinIO, одноразовые `minio-init`/`migrate`, две API-реплики и HAProxy. MinIO console не публикуется. Порт S3 `127.0.0.1:9000` нужен только клиентскому presigned PUT; anonymous GET отключён.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
`infra/minio-policy.json` ограничивает API-пользователя bucket `messenger-attachments`. Root credentials используются только bootstrap-контейнером. Метаданные хранятся в PostgreSQL volume, bytes — в MinIO volume.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
Запуск и доказательства описаны в [REPORT.md](../REPORT.md).
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
global
|
||||
maxconn 2048
|
||||
|
||||
defaults
|
||||
mode http
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout check 1s
|
||||
|
||||
resolvers docker
|
||||
nameserver dns1 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
hold valid 1s
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template replica 2 api:8080 check resolvers docker init-addr none
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$')
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title varchar(100) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text varchar(4000) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
CREATE TABLE chat_versions (
|
||||
chat_id uuid PRIMARY KEY REFERENCES chats(id),
|
||||
version bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key varchar(128) NOT NULL,
|
||||
body_hash bytea NOT NULL,
|
||||
message_id uuid UNIQUE REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX message_idempotency_expiry_idx ON message_idempotency (expires_at);
|
||||
@@ -0,0 +1,25 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename varchar(255) NOT NULL,
|
||||
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
status varchar(16) NOT NULL CHECK (status IN ('uploading', 'ready')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
upload_expires_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id, id);
|
||||
CREATE INDEX attachments_expired_idx ON attachments (upload_expires_at)
|
||||
WHERE status = 'uploading';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position)
|
||||
);
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
psycopg[binary,pool]==3.2.6
|
||||
redis==5.2.1
|
||||
minio==7.2.15
|
||||
Pillow==11.2.1
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small dependency-free lint: syntax, whitespace and unused imports."""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FILES = sorted(
|
||||
path for folder in ("app", "scripts", "tests")
|
||||
for path in (ROOT / folder).glob("*.py")
|
||||
)
|
||||
|
||||
|
||||
def check(path):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
issues = []
|
||||
try:
|
||||
tree = ast.parse(source, filename=str(path))
|
||||
except SyntaxError as error:
|
||||
return [f"{path}:{error.lineno}: {error.msg}"]
|
||||
used = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
imports = [(item.asname or item.name.split(".")[0], node.lineno) for item in node.names]
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
imports = [(item.asname or item.name, node.lineno) for item in node.names]
|
||||
else:
|
||||
continue
|
||||
for name, line in imports:
|
||||
if name not in used:
|
||||
issues.append(f"{path}:{line}: unused import {name}")
|
||||
for number, line in enumerate(source.splitlines(), 1):
|
||||
if line.rstrip() != line:
|
||||
issues.append(f"{path}:{number}: trailing whitespace")
|
||||
if "\t" in line:
|
||||
issues.append(f"{path}:{number}: tab character")
|
||||
return issues
|
||||
|
||||
|
||||
def main():
|
||||
issues = [issue for path in FILES for issue in check(path)]
|
||||
if issues:
|
||||
print("\n".join(issues))
|
||||
return 1
|
||||
print(f"PASS: lint ({len(FILES)} Python files)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Run schema changes once before starting API replicas."""
|
||||
|
||||
from app.main import migrate, pool
|
||||
|
||||
|
||||
def main():
|
||||
pool.open()
|
||||
try:
|
||||
pool.wait(timeout=20)
|
||||
migrate()
|
||||
finally:
|
||||
pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres redis minio api lb >/dev/null
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!a-fA-F0-9]*|'') echo "Invalid instance id" >&2; exit 2 ;;
|
||||
esac
|
||||
service=$(docker inspect --format '{{ index .Config.Labels "com.docker.compose.service" }}' "$TARGET_INSTANCE_ID")
|
||||
if [ "$service" != api ]; then
|
||||
echo "Target is not a Compose API replica" >&2
|
||||
exit 2
|
||||
fi
|
||||
docker stop -t 1 "$TARGET_INSTANCE_ID" >/dev/null
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Acceptance checks for the level-5 presigned upload track."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def presign(client, actor, chat, content, digest=None):
|
||||
response = client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/attachments/presign", 201,
|
||||
actor=actor,
|
||||
body={
|
||||
"filename": "direct.png",
|
||||
"content_type": "image/png",
|
||||
"size_bytes": len(content),
|
||||
"sha256": digest or hashlib.sha256(content).hexdigest(),
|
||||
},
|
||||
)
|
||||
return json.loads(response["bytes"])
|
||||
|
||||
|
||||
def put(ticket, content):
|
||||
request = urllib.request.Request(
|
||||
ticket["upload_url"], data=content, method="PUT",
|
||||
headers=ticket["required_headers"],
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
check(response.status == 200, "presigned PUT failed")
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab4 direct upload track")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
content = (Path(__file__).parent / "fixtures/sample.png").read_bytes()
|
||||
|
||||
unfinished = presign(client, alice, chat, content)
|
||||
client.call(
|
||||
"POST", "/api/v1/attachments/" + unfinished["attachment_id"] + "/complete", 409,
|
||||
actor=alice, body={},
|
||||
)
|
||||
client.call(
|
||||
"POST", "/api/v1/attachments/" + unfinished["attachment_id"] + "/complete", 404,
|
||||
actor=bob, body={},
|
||||
)
|
||||
|
||||
wrong = presign(client, alice, chat, content, "0" * 64)
|
||||
put(wrong, content)
|
||||
client.call(
|
||||
"POST", "/api/v1/attachments/" + wrong["attachment_id"] + "/complete", 409,
|
||||
actor=alice, body={},
|
||||
)
|
||||
|
||||
ticket = presign(client, alice, chat, content)
|
||||
put(ticket, content)
|
||||
ready = client.call(
|
||||
"POST", "/api/v1/attachments/" + ticket["attachment_id"] + "/complete", 200,
|
||||
actor=alice, body={}, schema="Attachment",
|
||||
)["json"]
|
||||
check(ready["sha256"] == hashlib.sha256(content).hexdigest(), "checksum changed")
|
||||
downloaded = client.call(
|
||||
"GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob,
|
||||
)["bytes"]
|
||||
check(downloaded == content, "direct-uploaded object changed")
|
||||
|
||||
parts = urlsplit(ticket["upload_url"])
|
||||
private_url = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
try:
|
||||
urllib.request.urlopen(private_url, timeout=5)
|
||||
except urllib.error.HTTPError as error:
|
||||
check(error.code in (401, 403), "private bucket returned unexpected status")
|
||||
else:
|
||||
check(False, "private bucket allowed anonymous GET")
|
||||
print("PASS: presign, ownership, checksum, completion and private bucket")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative and isolation checks required by level 4."""
|
||||
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from client import Client, arguments, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab4 negative upload scenarios")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
base = "/api/v1/chats/" + chat["id"] + "/attachments?filename="
|
||||
client.call(
|
||||
"POST", base + "large.png", 413, actor=alice,
|
||||
raw=b"x" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"},
|
||||
)
|
||||
client.call(
|
||||
"POST", base + "broken.png", 415, actor=alice,
|
||||
raw=b"not-an-image", headers={"Content-Type": "image/png"},
|
||||
)
|
||||
content = (Path(__file__).parent / "fixtures/sample.png").read_bytes()
|
||||
client.call(
|
||||
"POST", base + "spoofed.jpg", 415, actor=alice,
|
||||
raw=content, headers={"Content-Type": "image/jpeg"},
|
||||
)
|
||||
client.call(
|
||||
"POST", base + quote("../unsafe.png", safe=""), 400, actor=alice,
|
||||
raw=content, headers={"Content-Type": "image/png"},
|
||||
)
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
for suffix in ("", "/content"):
|
||||
client.call("GET", "/api/v1/attachments/" + attachment["id"] + suffix, 404, actor=eve)
|
||||
other = client.call(
|
||||
"POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Другой чат", "member_ids": [alice["id"], eve["id"]]}, schema="Chat",
|
||||
)["json"]
|
||||
alien, _ = client.attachment(alice, other)
|
||||
client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/messages", 404, actor=bob,
|
||||
body={"text": "", "attachment_ids": [alien["id"]]},
|
||||
headers={"Idempotency-Key": "cross-chat-attachment"},
|
||||
)
|
||||
print("PASS: lab4 level-4 negative and authorization scenarios")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare API traffic and memory for proxy and presigned uploads."""
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def chunk(kind, payload):
|
||||
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload))
|
||||
|
||||
|
||||
def png(width, height):
|
||||
compressor = zlib.compressobj(level=0)
|
||||
encoded = bytearray()
|
||||
row = b"\0" + bytes(range(256)) * ((width * 3 + 255) // 256)
|
||||
for _ in range(height):
|
||||
encoded.extend(compressor.compress(row[:width * 3 + 1]))
|
||||
encoded.extend(compressor.flush())
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", bytes(encoded)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def memory_mib():
|
||||
ids = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"],
|
||||
check=True, capture_output=True, text=True,
|
||||
).stdout.split()
|
||||
if not ids:
|
||||
return 0.0
|
||||
values = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
|
||||
check=True, capture_output=True, text=True,
|
||||
).stdout.splitlines()
|
||||
total = 0.0
|
||||
for value in values:
|
||||
match = re.fullmatch(r"([0-9.]+)(KiB|MiB|GiB)", value.split("/")[0].strip())
|
||||
if match is None:
|
||||
raise ValueError("unexpected docker stats memory format")
|
||||
number, unit = match.groups()
|
||||
factor = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit]
|
||||
total += float(number) * factor
|
||||
return total
|
||||
|
||||
|
||||
def sample_memory(stop, samples):
|
||||
while not stop.is_set():
|
||||
try:
|
||||
samples.append(memory_mib())
|
||||
except (OSError, subprocess.SubprocessError, ValueError, KeyError):
|
||||
pass
|
||||
stop.wait(0.1)
|
||||
|
||||
|
||||
def proxy(client, actor, chat, content):
|
||||
client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bench.png", 201,
|
||||
actor=actor, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment",
|
||||
)
|
||||
|
||||
|
||||
def direct(client, actor, chat, content):
|
||||
ticket = json.loads(client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/attachments/presign", 201,
|
||||
actor=actor,
|
||||
body={
|
||||
"filename": "bench.png", "content_type": "image/png",
|
||||
"size_bytes": len(content), "sha256": hashlib.sha256(content).hexdigest(),
|
||||
},
|
||||
)["bytes"])
|
||||
request = urllib.request.Request(
|
||||
ticket["upload_url"], data=content, method="PUT", headers=ticket["required_headers"],
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
check(response.status == 200, "presigned PUT failed")
|
||||
client.call(
|
||||
"POST", "/api/v1/attachments/" + ticket["attachment_id"] + "/complete", 200,
|
||||
actor=actor, body={}, schema="Attachment",
|
||||
)
|
||||
|
||||
|
||||
def measure(client, actor, chat, content, concurrency, mode):
|
||||
stop = threading.Event()
|
||||
samples = []
|
||||
sampler = threading.Thread(target=sample_memory, args=(stop, samples), daemon=True)
|
||||
sampler.start()
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = [pool.submit(proxy if mode == "proxy" else direct, client, actor, chat, content)
|
||||
for _ in range(concurrency)]
|
||||
for future in futures:
|
||||
future.result()
|
||||
elapsed = time.monotonic() - started
|
||||
stop.set()
|
||||
sampler.join()
|
||||
return {
|
||||
"mode": mode,
|
||||
"file_bytes": len(content),
|
||||
"concurrency": concurrency,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"api_upload_body_bytes": len(content) * concurrency if mode == "proxy" else 0,
|
||||
"api_storage_body_bytes": len(content) * concurrency,
|
||||
"api_total_binary_bytes": len(content) * concurrency * (2 if mode == "proxy" else 1),
|
||||
"s3_upload_body_bytes": len(content) * concurrency,
|
||||
"api_replicas_peak_memory_mib": round(max(samples), 3) if samples else None,
|
||||
"sample_count": len(samples),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--output", default="evidence/upload_benchmark.json")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, timeout=90)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
results = []
|
||||
for content in (png(600, 600), png(1600, 1600)):
|
||||
for mode in ("proxy", "direct"):
|
||||
results.append(measure(client, alice, chat, content, args.concurrency, mode))
|
||||
output = {
|
||||
"run_id": uuid.uuid4().hex,
|
||||
"method": "closed-loop, same valid PNG shared by 10 client threads; 100 ms docker-stats sampling",
|
||||
"results": results,
|
||||
}
|
||||
Path(args.output).write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user