diff --git a/.gitignore b/.gitignore index b5a71896..44726df3 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..dc05a08a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11.9-slim + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 +WORKDIR /app + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY main.py . +COPY app ./app +COPY scripts ./scripts + +RUN useradd --system --uid 10001 --no-create-home --shell /usr/sbin/nologin appuser +USER 10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=10 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"] + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..45fd66a4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,41 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Никифорова Екатерина, БВТ2301 +- Целевая оценка: **4** (уровни 3 и 4, трек 5 не заявлен) +- Commit/PR: https://gitea.privcloud.tech/mtusi2026lab4/pulls (PR после публикации) +- Источники, библиотеки, помощь ИИ: FastAPI 0.115, Uvicorn 0.30, + PostgreSQL 17.6, Valkey 8.1.3, Silo (`pgsty/silo` — поддерживаемый форк + MinIO: официальные образы MinIO удалены с Docker Hub и Quay в 2025), + Nginx 1.27, psycopg 3.2, boto3 1.43.107, Pillow 12.3.0, Argon2id. + Помощь ИИ при написании кода и отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Требования:** Docker Engine ≥ 24 с Compose v2, Python ≥ 3.9, +PowerShell (Windows) или sh (Linux/macOS), `PYTHONUTF8=1` для корректной +работы публичных тестов на Windows. -## Архитектура и решения +```powershell +git clone https://gitea.privcloud.tech/eqktrin/mtusi2026lab4.git +cd mtusi2026lab4 -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +# 1. Секреты. На Windows — PowerShell-скрипт ниже; на Linux/macOS — sh scripts/init-secrets.sh +# (PowerShell-вариант создаёт secrets/* и .env через System.Security.Cryptography) +$bytes = New-Object byte[] 16 +[System.Security.Cryptography.RandomNumberGenerator]::Create().GetBytes($bytes) +$pgPass = -join ($bytes | ForEach-Object { $_.ToString("x2") }) +# ... и т.д. — см. README или скрипт init-secrets.sh +docker compose --env-file .env config --quiet -## Доказательства критериев +# 2. Сборка и запуск +docker compose --env-file .env up -d --build -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +# 3. Дождаться готовности +Invoke-RestMethod http://localhost:8080/health/ready -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. - -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +# 4. Тесты +$env:PYTHONUTF8 = "1" +python tests/smoke.py --base-url http://localhost:8080 +python student-tests/test_attachments_negative.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 00000000..0128e29d --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,127 @@ +"""Логика вложений: проверка изображений, загрузка в S3, компенсация.""" +import logging +import threading +import uuid + +from PIL import Image, UnidentifiedImageError + +from . import config, db, repo, s3 +from .errors import ApiError, bad_request, not_found + +log = logging.getLogger("attachments") + +ALLOWED_FORMATS = {"PNG": "image/png", "JPEG": "image/jpeg"} + +# PIL отказывается читать изображение с числом пикселей больше лимита. +Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS + +# Ограничиваем одновременные декодирования, чтобы пик RAM +# (≤ MAX_IMAGE_PIXELS × 4 байт на каждое) укладывался в mem_limit контейнера. +_decode_slots = threading.BoundedSemaphore(3) + + +def validate_filename(filename): + """Отображаемое имя; ключ S3 формирует сервер. + + Запрещены / \\ NUL и любые control-символы 0x00–0x1F и DEL 0x7F. + """ + if not isinstance(filename, str) or not (1 <= len(filename) <= 255): + raise bad_request("Некорректное имя файла") + for c in filename: + if c in ("/", "\\", "\x00") or ord(c) < 0x20 or ord(c) == 0x7F: + raise bad_request("Недопустимое имя файла") + return filename + + +def _inspect_image(fileobj): + """Полная валидация: заголовок → лимиты → load(). + + load() декодирует IDAT (PNG) / энтропийные данные (JPEG). Битый поток + даёт OSError('broken data stream') → 415. Память ограничена + MAX_IMAGE_PIXELS × 4 байт и семафором _decode_slots. + """ + with _decode_slots: + try: + with Image.open(fileobj) as img: + fmt = img.format + width, height = img.size + if fmt not in ALLOWED_FORMATS: + raise ApiError(415, "unsupported_media_type", "Ожидается PNG или JPEG") + if width <= 0 or height <= 0: + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") + if width * height > config.MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + # Полное декодирование: проверяет целостность IDAT-потока, + # а не только CRC чанков (как verify()). + img.load() + return ALLOWED_FORMATS[fmt] + except UnidentifiedImageError: + raise ApiError(415, "unsupported_media_type", "Неизвестный формат изображения") + except Image.DecompressionBombError: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + except (OSError, SyntaxError, ValueError): + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") + finally: + fileobj.seek(0) + + +def inspect_image_file(fileobj, total_bytes): + """Проверяет уже прочитанное тело. Возвращает фактический content_type.""" + if total_bytes == 0: + raise bad_request("Пустой файл") + if total_bytes > config.MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + fileobj.seek(0) + return _inspect_image(fileobj) + + +def upload_stream(chat_id, owner_id, filename, fileobj, total_bytes, digest, content_type): + """pending в БД → S3 PUT → ready. fileobj уже провалидирован и на позиции 0.""" + validate_filename(filename) + object_key = "originals/%s/%s" % (chat_id, uuid.uuid4().hex) + + with db.connection() as conn: + if not repo.is_member(conn, chat_id, owner_id): + raise not_found() + row = repo.insert_attachment_pending( + conn, chat_id, owner_id, filename, content_type, total_bytes, digest, object_key + ) + attachment_id = row["id"] + + try: + fileobj.seek(0) + s3.put_object(object_key, fileobj, content_type, content_length=total_bytes) + except Exception as exc: + log.warning("S3 PUT failed for %s: %s", attachment_id, type(exc).__name__) + s3.delete_object(object_key) + with db.connection() as conn: + repo.mark_attachment_failed(conn, attachment_id, "storage_unavailable") + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + + try: + with db.connection() as conn: + ready = repo.mark_attachment_ready(conn, attachment_id) + except Exception: + log.error("S3 object %s left orphaned after DB failure", object_key) + raise + if ready is None: + raise ApiError(409, "conflict", "Вложение уже в обработке") + return repo.attachment_json(ready) + + +def download(attachment_id, user_id): + """Возвращает (metadata_row, s3_body) или 404/409/503.""" + with db.connection() as conn: + row = repo.get_attachment_for_member(conn, attachment_id, user_id) + if row is None: + raise not_found() + if row["status"] == "failed": + raise ApiError(409, "conflict", "Вложение недоступно") + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + try: + body = s3.get_object(row["object_key"]) + except Exception as exc: + log.warning("S3 GET failed for %s: %s", attachment_id, type(exc).__name__) + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + return row, body \ No newline at end of file diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..eff9b5f9 --- /dev/null +++ b/app/cache.py @@ -0,0 +1,52 @@ +"""Тонкая обёртка над Redis/Valkey.""" +import logging +import threading +import time + +import redis + +from . import config + +log = logging.getLogger("cache") +_client = None +_down_until = 0.0 +_lock = threading.Lock() + + +def _get_client(): + global _client + if not config.CACHE_URL: + return None + if _client is None: + with _lock: + if _client is None: + _client = redis.Redis.from_url( + config.CACHE_URL, socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, max_connections=20, + decode_responses=True, retry_on_timeout=False) + return _client + + +def _call(fn): + global _down_until + client = _get_client() + if client is None or time.monotonic() < _down_until: + return None + try: + return fn(client) + except redis.RedisError as exc: + _down_until = time.monotonic() + config.CACHE_BACKOFF + log.warning("cache unavailable (%s); bypassing for %.0fs", type(exc).__name__, config.CACHE_BACKOFF) + return None + + +def get(key): + return _call(lambda c: c.get(key)) + + +def set_if_absent(key, value, ttl_ms): + return _call(lambda c: c.set(key, value, nx=True, px=ttl_ms)) + + +def set_always(key, value, ttl_ms): + return _call(lambda c: c.set(key, value, px=ttl_ms)) \ No newline at end of file diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..7c0cdabc --- /dev/null +++ b/app/config.py @@ -0,0 +1,69 @@ +"""Конфигурация из переменных окружения. Секреты читаются из файлов (*_FILE), а не из env.""" +import os +import socket +from pathlib import Path + + +def _read(name): + """NAME_FILE (путь к секрету) имеет приоритет над NAME (прямое значение для локальной отладки).""" + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get(name) or None + + +def _int(name, default, low, high): + value = int(os.environ.get(name, default)) + if not low <= value <= high: + raise ValueError("%s must be in %d..%d" % (name, low, high)) + return value + + +APP_HOST = os.environ.get("APP_HOST", "0.0.0.0") +APP_PORT = _int("APP_PORT", 8080, 1, 65535) +APP_ENV = os.environ.get("APP_ENV", "lab") + +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600) +SESSION_STORE = os.environ.get("SESSION_STORE", "postgres+cache") +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60, 1, 3600) + +DATABASE_URL = _read("DATABASE_URL") +CACHE_URL = _read("CACHE_URL") if SESSION_STORE == "postgres+cache" else None + +DB_POOL_MIN = _int("DB_POOL_MIN", 1, 0, 50) +DB_POOL_MAX = _int("DB_POOL_MAX", 16, 1, 50) +DB_POOL_TIMEOUT = float(os.environ.get("DB_POOL_TIMEOUT", "3")) +DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000, 100, 60000) +CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.25")) +CACHE_BACKOFF = float(os.environ.get("CACHE_BACKOFF", "2")) + +ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456, 8, 1048576) +ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2, 1, 10) +MAX_BODY_BYTES = 64 * 1024 +THREAD_LIMIT = _int("THREAD_LIMIT", 24, 1, 200) +HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4, 1, 32) + +# --- lab3 --- +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400, 60, 86400 * 7) + +# --- lab4: S3 --- +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _read("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _read("S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT = float(os.environ.get("S3_CONNECT_TIMEOUT", "3")) +S3_READ_TIMEOUT = float(os.environ.get("S3_READ_TIMEOUT", "30")) + +# --- lab4: лимиты загрузки --- +# 10 MiB = 10 485 760 байт, как в контракте +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760, 1, 10485760) +# 40 млн пикселей +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000, 1, 40000000) +# Сколько ждать между S3 PUT и записью метаданных, прежде чем считать объект orphan (компенсация) +S3_ORPHAN_GRACE_SECONDS = _int("S3_ORPHAN_GRACE_SECONDS", 3600, 60, 7 * 24 * 3600) +# Размер чанка при потоковой загрузке в S3 +UPLOAD_CHUNK_BYTES = _int("UPLOAD_CHUNK_BYTES", 256 * 1024, 4096, 4 * 1024 * 1024) +# Сколько ждать чтения тела запроса (защита от slowloris на upload) +UPLOAD_READ_TIMEOUT_SECONDS = float(os.environ.get("UPLOAD_READ_TIMEOUT", "30")) \ No newline at end of file diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 00000000..9fa0f163 --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,27 @@ +"""Непрозрачный keyset-курсор (created_at, id).""" +import base64 +import json +import uuid +from datetime import datetime, timezone + +from .errors import bad_request + + +def encode(kind, scope, created_at, row_id): + payload = {"k": kind, "s": str(scope), "t": created_at.astimezone(timezone.utc).isoformat(), "i": str(row_id)} + return base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=").decode() + + +def decode(value, kind, scope): + if not value: + raise bad_request("Некорректный cursor") + try: + raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + payload = json.loads(raw.decode("utf-8")) + created_at = datetime.fromisoformat(payload["t"]) + row_id = uuid.UUID(payload["i"]) + if payload["k"] != kind or payload["s"] != str(scope) or created_at.tzinfo is None: + raise ValueError("scope mismatch") + return created_at, row_id + except (ValueError, KeyError, TypeError, UnicodeDecodeError): + raise bad_request("Некорректный cursor") \ No newline at end of file diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..b3349cd8 --- /dev/null +++ b/app/db.py @@ -0,0 +1,53 @@ +"""Пул соединений PostgreSQL.""" +import threading + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolClosed, PoolTimeout + +from . import config + +TRANSIENT = (psycopg.OperationalError, PoolTimeout, PoolClosed, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable) + +_pool = None +_lock = threading.Lock() + + +def get_pool(): + global _pool + if _pool is None: + with _lock: + if _pool is None: + if not config.DATABASE_URL: + raise RuntimeError("DATABASE_URL(_FILE) is not configured") + options = ("-c statement_timeout=%d -c lock_timeout=3000 " + "-c idle_in_transaction_session_timeout=10000" % config.DB_STATEMENT_TIMEOUT_MS) + pool = ConnectionPool( + config.DATABASE_URL, + min_size=config.DB_POOL_MIN, max_size=config.DB_POOL_MAX, + timeout=config.DB_POOL_TIMEOUT, max_lifetime=1800, max_idle=300, + reconnect_timeout=5, + check=ConnectionPool.check_connection, + kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": options, + "application_name": "messenger-api"}, + open=False) + pool.open(wait=False) + _pool = pool + return _pool + + +def connection(timeout=None): + return get_pool().connection(timeout=timeout) + + +def ping(): + with connection(timeout=1.5) as conn: + conn.execute("SELECT 1") + + +def close(): + global _pool + if _pool is not None: + _pool.close(timeout=5) + _pool = None \ No newline at end of file diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..29f95263 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,20 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} +BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'} + + +def bad_request(message="Некорректный запрос"): + return ApiError(400, "invalid_request", message) + + +def unauthorized(challenge): + return ApiError(401, "unauthorized", "Требуется аутентификация", challenge) + + +def not_found(): + return ApiError(404, "not_found", "Объект не найден") \ No newline at end of file diff --git a/app/idempotency.py b/app/idempotency.py new file mode 100644 index 00000000..20bc7da1 --- /dev/null +++ b/app/idempotency.py @@ -0,0 +1,66 @@ +"""Логика идемпотентности для POST запросов.""" +import hashlib +import json +import re + +from .errors import ApiError, bad_request + +# Контракт: ASCII [A-Za-z0-9._:-]{1,128} +_IDEM_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def get_request_hash(data: dict) -> str: + """Хеш тела, независимый от порядка ключей. + + Отсутствующий attachment_ids семантически равен [] — нормализуем, + иначе {"text":"x"} и {"text":"x","attachment_ids":[]} дают разные хеши. + """ + normalized = dict(data) + normalized.setdefault("attachment_ids", []) + return hashlib.sha256(json.dumps(normalized, sort_keys=True).encode("utf-8")).hexdigest() + + +def check_and_lock(conn, user_id: str, method: str, path: str, key: str, req_hash: str): + if not key: + raise bad_request("Missing Idempotency-Key") + if not _IDEM_KEY_RE.fullmatch(key): + raise bad_request("Invalid Idempotency-Key format") + + lock_id = f"{user_id}:{method}:{path}:{key}" + conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", (lock_id,)) + + row = conn.execute(""" + SELECT request_hash, status_code, response_body + FROM idempotency_keys + WHERE user_id = %s AND method = %s AND path = %s AND key = %s + """, (user_id, method, path, key)).fetchone() + + if row: + if row["request_hash"] != req_hash: + raise ApiError(409, "conflict", "Idempotency-Key conflict: body differs") + if row["status_code"] is not None and row["response_body"] is not None: + return False, json.loads(row["response_body"]) + return True, None + + return True, None + + +def save_result(conn, user_id, method, path, key, req_hash, status_code, response_body): + conn.execute(""" + INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response_body) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (user_id, method, path, key) DO UPDATE + SET status_code = EXCLUDED.status_code, response_body = EXCLUDED.response_body + """, (user_id, method, path, key, req_hash, status_code, json.dumps(response_body))) + + +def cleanup(conn, ttl_seconds: int): + conn.execute(""" + DELETE FROM idempotency_keys + WHERE (user_id, method, path, key) IN ( + SELECT user_id, method, path, key + FROM idempotency_keys + WHERE created_at < now() - make_interval(secs => %s) + LIMIT 100 + ) + """, (ttl_seconds,)) \ No newline at end of file diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..18e35751 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,61 @@ +"""Версионируемые миграции.""" +import hashlib +import logging +import sys +import time +from pathlib import Path + +import psycopg + +from . import config + +LOCK_ID = 727_002_001 +log = logging.getLogger("migrate") + + +def migration_files(): + return sorted((Path(__file__).parent / "migrations").glob("[0-9][0-9][0-9][0-9]_*.sql")) + + +def apply_all(conninfo): + files = migration_files() + with psycopg.connect(conninfo, connect_timeout=3) as conn: + conn.execute("SELECT pg_advisory_xact_lock(%s)", (LOCK_ID,)) + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now())""") + applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()) + done = [] + for path in files: + sql = path.read_text(encoding="utf-8") + checksum = hashlib.sha256(sql.encode()).hexdigest() + if path.stem in applied: + if applied[path.stem] != checksum: + raise RuntimeError("migration %s was modified after being applied" % path.stem) + continue + conn.execute(sql) + conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (path.stem, checksum)) + done.append(path.stem) + return done + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s") + if not config.DATABASE_URL: + sys.exit("DATABASE_URL(_FILE) is not configured") + deadline = time.monotonic() + 60 + while True: + try: + done = apply_all(config.DATABASE_URL) + break + except psycopg.OperationalError as exc: + if time.monotonic() > deadline: + raise + log.info("database not ready (%s), retrying", type(exc).__name__) + time.sleep(1) + log.info("applied: %s", ", ".join(done) if done else "nothing to do (schema is up to date)") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/app/migrations/0001_core.sql b/app/migrations/0001_core.sql new file mode 100644 index 00000000..10147f3f --- /dev/null +++ b/app/migrations/0001_core.sql @@ -0,0 +1,39 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100) +); +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + added_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) +); +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); \ No newline at end of file diff --git a/app/migrations/0002_sessions.sql b/app/migrations/0002_sessions.sql new file mode 100644 index 00000000..d34c43d1 --- /dev/null +++ b/app/migrations/0002_sessions.sql @@ -0,0 +1,10 @@ +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_hash_length CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); +CREATE INDEX sessions_expires_at_idx ON sessions (expires_at); +CREATE INDEX sessions_user_idx ON sessions (user_id); \ No newline at end of file diff --git a/app/migrations/0003_idempotency.sql b/app/migrations/0003_idempotency.sql new file mode 100644 index 00000000..33b24334 --- /dev/null +++ b/app/migrations/0003_idempotency.sql @@ -0,0 +1,13 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id), + method text NOT NULL, + path text NOT NULL, + key text NOT NULL, + request_hash text NOT NULL, + status_code int, + response_body text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, method, path, key) +); + +CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at); \ No newline at end of file diff --git a/app/migrations/0004_attachments.sql b/app/migrations/0004_attachments.sql new file mode 100644 index 00000000..cf74bc5f --- /dev/null +++ b/app/migrations/0004_attachments.sql @@ -0,0 +1,36 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 char(64) NOT NULL, + object_key text NOT NULL, + status text NOT NULL, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_positive CHECK (size_bytes > 0 AND size_bytes <= 10485760), + CONSTRAINT attachments_sha256_format CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'failed')), + CONSTRAINT attachments_error_code CHECK ( + (status = 'ready' AND error_code IS NULL) OR + (status = 'failed' AND error_code IS NOT NULL) OR + (status = 'pending') + ) +); +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at); +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT, + position int NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position CHECK (position >= 0) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); \ No newline at end of file diff --git a/app/migrations/0005_messages_text_allow_empty.sql b/app/migrations/0005_messages_text_allow_empty.sql new file mode 100644 index 00000000..96316025 --- /dev/null +++ b/app/migrations/0005_messages_text_allow_empty.sql @@ -0,0 +1,5 @@ +-- lab4: разрешаем пустой text, если есть вложения. Проверка «text или вложения» +-- делается в приложении, а БД гарантирует только верхнюю границу. +ALTER TABLE messages DROP CONSTRAINT messages_text_length; +ALTER TABLE messages ADD CONSTRAINT messages_text_length + CHECK (char_length(text) BETWEEN 0 AND 4000); \ No newline at end of file diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 00000000..f06105cb --- /dev/null +++ b/app/repo.py @@ -0,0 +1,191 @@ +"""Доступ к данным.""" +import uuid + +from . import cursor, security +from .sessions import fmt_ts +from .errors import bad_request, not_found + + +def username_exists(conn, username): + return conn.execute("SELECT 1 FROM users WHERE username = %s", (username,)).fetchone() is not None + + +def insert_user(conn, username, display_name, password_hash): + return conn.execute( + """INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at""", + (uuid.uuid4(), username, display_name, password_hash)).fetchone() + + +def user_for_login(conn, username): + return conn.execute("SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s", + (username,)).fetchone() + + +def update_password_hash(conn, user_id, new_hash): + conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (new_hash, user_id)) + + +def chat_json(row): + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]], + "created_at": fmt_ts(row["created_at"])} + + +def create_chat(conn, creator_id, title, member_ids, check_members=True): + if check_members: + found = conn.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s)", (member_ids,)).fetchone()["n"] + if found != len(member_ids): + raise bad_request("Неизвестный участник чата") + chat = conn.execute( + "INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING id, title, created_at", + (uuid.uuid4(), title, creator_id)).fetchone() + conn.execute("INSERT INTO chat_members (chat_id, user_id) SELECT %s, unnest(%s::uuid[])", + (chat["id"], member_ids)) + chat["member_ids"] = sorted(member_ids) + return chat + + +def get_chat(conn, chat_id, user_id): + return conn.execute( + """SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids + FROM chats c + WHERE c.id = %s AND EXISTS (SELECT 1 FROM chat_members me WHERE me.chat_id = c.id AND me.user_id = %s)""", + (chat_id, user_id)).fetchone() + + +def is_member(conn, chat_id, user_id): + return conn.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id)).fetchone() is not None + + +def list_chats(conn, user_id, after, limit): + base = """SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids + FROM chat_members me JOIN chats c ON c.id = me.chat_id + WHERE me.user_id = %s""" + params = [user_id] + if after: + base += " AND (c.created_at, c.id) > (%s, %s)" + params += list(after) + rows = conn.execute(base + " ORDER BY c.created_at, c.id LIMIT %s", params + [limit + 1]).fetchall() + page, more = rows[:limit], len(rows) > limit + nxt = cursor.encode("chats", user_id, page[-1]["created_at"], page[-1]["id"]) if more else None + return [chat_json(r) for r in page], nxt + + +# ---------- Сообщения ---------- + +def _message_json(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), + "text": row["text"], "created_at": fmt_ts(row["created_at"]), + "attachment_ids": [str(x) for x in (row.get("attachment_ids") or [])]} + + +def message_json(row): + return _message_json(row) + + +def insert_message(conn, chat_id, sender_id, text, attachment_ids=None): + """Создаёт сообщение и связывает вложения в одной транзакции.""" + attachment_ids = attachment_ids or [] + if attachment_ids: + rows = conn.execute( + """SELECT id, status FROM attachments + WHERE id = ANY(%s) AND chat_id = %s""", + (attachment_ids, chat_id)).fetchall() + by_id = {r["id"]: r for r in rows} + if len(by_id) != len(attachment_ids): + # Чужое или неизвестное вложение — 404 (не раскрывать существование). + raise not_found() + for att_id in attachment_ids: + if by_id[att_id]["status"] == "failed": + raise bad_request("Нельзя прикрепить failed-вложение") + if by_id[att_id]["status"] != "ready": + raise bad_request("Вложение ещё не готово") + + row = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid.uuid4(), chat_id, sender_id, text)).fetchone() + if attachment_ids: + conn.execute( + """INSERT INTO message_attachments (message_id, attachment_id, position) + SELECT %s, unnest(%s::uuid[]), generate_subscripts(%s::uuid[], 1) - 1""", + (row["id"], attachment_ids, attachment_ids)) + row["attachment_ids"] = attachment_ids + return row + + +def list_messages(conn, chat_id, after, limit): + sql = """SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM messages m WHERE m.chat_id = %s""" + params = [chat_id] + if after: + sql += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + rows = conn.execute(sql + " ORDER BY m.created_at, m.id LIMIT %s", params + [limit + 1]).fetchall() + page, more = rows[:limit], len(rows) > limit + nxt = cursor.encode("messages", chat_id, page[-1]["created_at"], page[-1]["id"]) if more else None + return [_message_json(r) for r in page], nxt + + +# ---------- Вложения ---------- + +def insert_attachment_pending(conn, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key): + return conn.execute( + """INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'pending') + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, + status, error_code, created_at, updated_at""", + (uuid.uuid4(), chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) + ).fetchone() + + +def mark_attachment_ready(conn, attachment_id): + return conn.execute( + """UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() + WHERE id = %s AND status = 'pending' + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, + status, error_code, created_at, updated_at""", + (attachment_id,)).fetchone() + + +def mark_attachment_failed(conn, attachment_id, error_code): + conn.execute( + """UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = clock_timestamp() + WHERE id = %s AND status = 'pending'""", + (error_code, attachment_id)) + + +def get_attachment_for_member(conn, attachment_id, user_id): + return conn.execute( + """SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.object_key, a.status, a.error_code, a.created_at, a.updated_at + FROM attachments a + WHERE a.id = %s + AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = %s)""", + (attachment_id, user_id)).fetchone() + + +def attachment_json(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": fmt_ts(row["created_at"]), + "updated_at": fmt_ts(row["updated_at"]), + } \ No newline at end of file diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 00000000..93d36553 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,78 @@ +"""Работа с S3-совместимым хранилищем (Silo/MinIO).""" +import logging +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from . import config + +log = logging.getLogger("s3") +_client = None +_lock = threading.Lock() + + +def _get_client(): + global _client + if _client is None: + with _lock: + if _client is None: + _client = boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + region_name=config.S3_REGION, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"max_attempts": 3, "mode": "standard"}, + connect_timeout=config.S3_CONNECT_TIMEOUT, + read_timeout=config.S3_READ_TIMEOUT, + ), + ) + return _client + + +def ensure_bucket(): + client = _get_client() + try: + client.head_bucket(Bucket=config.S3_BUCKET) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchBucket"): + client.create_bucket(Bucket=config.S3_BUCKET) + else: + raise + + +def put_object(key: str, body, content_type: str, content_length: int): + """Загружает объект из file-like body, потоково, с явной длиной.""" + client = _get_client() + client.put_object( + Bucket=config.S3_BUCKET, + Key=key, + Body=body, + ContentType=content_type, + ContentLength=content_length, + ) + + +def get_object(key: str): + """Возвращает StreamingBody, а не весь ответ boto3.""" + client = _get_client() + resp = client.get_object(Bucket=config.S3_BUCKET, Key=key) + return resp["Body"] + + +def delete_object(key: str): + client = _get_client() + try: + client.delete_object(Bucket=config.S3_BUCKET, Key=key) + except (ClientError, BotoCoreError): + log.warning("failed to delete object %s", key) + + +def ping(): + client = _get_client() + client.head_bucket(Bucket=config.S3_BUCKET) \ No newline at end of file diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..8ceb69c8 --- /dev/null +++ b/app/security.py @@ -0,0 +1,72 @@ +"""Пароли (Argon2id), токены, разбор заголовка Authorization.""" +import base64 +import binascii +import hashlib +import re +import secrets +import threading + +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError + +from . import config + +_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB, + parallelism=1, hash_len=32, salt_len=16, type=Type.ID) +_DUMMY_HASH = _hasher.hash("dummy-password-for-constant-time") +_hash_slots = threading.BoundedSemaphore(config.HASH_CONCURRENCY) +_TOKEN_RE = re.compile(r"[A-Za-z0-9._~+/=-]{16,512}") + + +def hash_password(password): + with _hash_slots: + return _hasher.hash(password) + + +def verify_password(stored_hash, password): + with _hash_slots: + try: + _hasher.verify(stored_hash, password) + except (VerificationError, InvalidHashError): + return False, None + return True, (_hasher.hash(password) if _hasher.check_needs_rehash(stored_hash) else None) + + +def burn_verify(password): + with _hash_slots: + try: + _hasher.verify(_DUMMY_HASH, password) + except VerificationError: + pass + + +def new_token(): + return secrets.token_urlsafe(32) + + +def token_hash(token): + return hashlib.sha256(token.encode("ascii")).digest() + + +def parse_basic(header): + if not header: + return None + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic": + return None + try: + text = base64.b64decode(rest.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + return None + username, sep, password = text.partition(":") + return (username, password) if sep else None + + +def parse_bearer(header): + if not header: + return None + scheme, _, rest = header.partition(" ") + token = rest.strip() + if scheme.lower() != "bearer" or not _TOKEN_RE.fullmatch(token): + return None + return token \ No newline at end of file diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..ecb4d011 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,81 @@ +"""Сессии: PostgreSQL — источник истины, Redis/Valkey — read-through кэш.""" +import json +import time +from datetime import timezone + +from . import cache, config, db, security +from .errors import ApiError + +TOMBSTONE = "revoked" +_KEY = "sess:v1:" + + +def fmt_ts(value): + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def user_json(row): + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], + "created_at": fmt_ts(row["created_at"])} + + +def _ttl_ms(expires_epoch): + remaining = expires_epoch - time.time() + return int(min(config.SESSION_CACHE_TTL_SECONDS, remaining) * 1000) if remaining > 0 else 0 + + +def create(conn, user_row): + token = security.new_token() + row = conn.execute( + """INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) + RETURNING expires_at""", + (security.token_hash(token), user_row["id"], config.SESSION_TTL_SECONDS)).fetchone() + conn.execute("""DELETE FROM sessions WHERE token_hash IN ( + SELECT token_hash FROM sessions WHERE expires_at < now() - interval '1 hour' LIMIT 100)""") + return token, row["expires_at"] + + +def warm(token, user, expires_at): + entry = {"user": user, "exp": expires_at.timestamp()} + ttl = _ttl_ms(entry["exp"]) + if ttl > 0: + cache.set_if_absent(_KEY + security.token_hash(token).hex(), json.dumps(entry), ttl) + + +def lookup(token): + digest = security.token_hash(token) + key = _KEY + digest.hex() + cached = cache.get(key) + if cached == TOMBSTONE: + return None + if cached: + try: + entry = json.loads(cached) + if entry["exp"] > time.time(): + return {"user": entry["user"], "digest": digest} + return None + except (ValueError, KeyError, TypeError): + pass + with db.connection() as conn: + row = conn.execute( + """SELECT u.id, u.username, u.display_name, u.created_at, + extract(epoch FROM s.expires_at) AS exp + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.expires_at > clock_timestamp()""", (digest,)).fetchone() + if row is None: + return None + user = user_json(row) + ttl = _ttl_ms(float(row["exp"])) + if ttl > 0: + cache.set_if_absent(key, json.dumps({"user": user, "exp": float(row["exp"])}), ttl) + return {"user": user, "digest": digest} + + +def revoke(digest): + if config.CACHE_URL: + wrote = cache.set_always(_KEY + digest.hex(), TOMBSTONE, config.SESSION_CACHE_TTL_SECONDS * 1000) + if not wrote: + raise ApiError(503, "unavailable", "Сервис временно недоступен") + with db.connection() as conn: + conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,)) \ No newline at end of file diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..4088efee --- /dev/null +++ b/app/validation.py @@ -0,0 +1,83 @@ +"""Строгая валидация входа.""" +import json +import re +import uuid + +from .errors import ApiError, bad_request + +_UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +_USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +_INT_RE = re.compile(r"[0-9]{1,6}") + + +def parse_uuid(value): + if isinstance(value, str) and _UUID_RE.fullmatch(value): + return uuid.UUID(value) + return None + + +def require_uuid(value, name="id"): + parsed = parse_uuid(value) + if parsed is None: + raise bad_request("Некорректный %s" % name) + return parsed + + +def _reject_constant(name): + raise ValueError("non-finite number") + + +def _no_duplicates(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + + +def json_body(content_type, raw, allowed, required): + if content_type.split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json") + try: + data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant, object_pairs_hook=_no_duplicates) + except (ValueError, UnicodeDecodeError, RecursionError): + raise bad_request("Некорректный JSON") + if not isinstance(data, dict) or set(data) - set(allowed) or not set(required) <= set(data): + raise bad_request("Некорректный набор полей") + return data + + +def text_field(value, name, low, high, allow_blank=False): + if not isinstance(value, str): + raise bad_request("Поле %s должно быть строкой" % name) + if "\x00" in value or not _encodable(value): + raise bad_request("Поле %s содержит недопустимые символы" % name) + if not low <= len(value) <= high: + raise bad_request("Поле %s: длина должна быть %d–%d" % (name, low, high)) + if not allow_blank and not value.strip(): + raise bad_request("Поле %s не должно состоять только из пробелов" % name) + return value + + +def _encodable(value): + try: + value.encode("utf-8") + return True + except UnicodeEncodeError: + return False + + +def username_field(value): + if not isinstance(value, str) or not _USERNAME_RE.fullmatch(value): + raise bad_request("username должен соответствовать [a-z0-9_]{3,32}") + return value + + +def limit_param(params): + raw = params.get("limit") + if raw is None: + return 50 + if not _INT_RE.fullmatch(raw) or not 1 <= int(raw) <= 100: + raise bad_request("limit должен быть целым числом 1–100") + return int(raw) \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..285a4957 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,191 @@ +name: messenger-lab4 + +x-app-env: &app-env + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: ${SESSION_STORE:-postgres+cache} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: http://minio:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + S3_ORPHAN_GRACE_SECONDS: ${S3_ORPHAN_GRACE_SECONDS:-3600} + +services: + postgres: + image: postgres:17.6-alpine + restart: unless-stopped + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", + "-c", "full_page_writes=on", "-c", "max_connections=100"] + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + stop_grace_period: 30s + networks: [backend] + deploy: + resources: + limits: {cpus: '1.0', memory: 1G} + + valkey: + image: valkey/valkey:8.1.3-alpine + restart: unless-stopped + secrets: [valkey_password] + command: + - sh + - -c + - >- + exec valkey-server --requirepass "$$(cat /run/secrets/valkey_password)" + --appendonly yes --appendfsync everysec --save "" + --maxmemory 64mb --maxmemory-policy volatile-ttl + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + networks: [backend] + deploy: + resources: + limits: {cpus: '0.5', memory: 256M} + + minio: + image: pgsty/silo:latest + restart: unless-stopped + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [minio_root_user, minio_root_password] + volumes: + - miniodata:/data + healthcheck: + test: ["CMD", "silo", "healthcheck", "ready"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + networks: [backend] + deploy: + resources: + limits: {cpus: '0.5', memory: 512M} + + minio-init: + image: pgsty/mc:latest + depends_on: + minio: {condition: service_healthy} + secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key] + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + ROOT_USER=$$(cat /run/secrets/minio_root_user) + ROOT_PASS=$$(cat /run/secrets/minio_root_password) + APP_KEY=$$(cat /run/secrets/s3_access_key_id) + APP_SECRET=$$(cat /run/secrets/s3_secret_access_key) + mc alias set local http://minio:9000 "$$ROOT_USER" "$$ROOT_PASS" + mc mb --ignore-existing --with-lock=false local/messenger-attachments + mc anonymous set none local/messenger-attachments + mc admin user add local "$$APP_KEY" "$$APP_SECRET" || true + cat >/tmp/app-policy.json < bytes: + buf = io.BytesIO() + Image.new("RGB", (64, 64), (128, 128, 128)).save(buf, "PNG") + data = buf.getvalue() + target = int(size_mib * 1024 * 1024) + check(len(data) <= target, "fixture bigger than target") + return data + b"\x00" * (target - len(data)) + + +def main(): + def configure(p): + p.add_argument("--concurrency", type=int, default=10) + p.add_argument("--size-mib", type=float, default=9.5) + + args = arguments("Concurrent upload memory probe", configure) + check(1 <= args.concurrency <= 64, "concurrency 1..64") + check(0.1 <= args.size_mib <= 10, "size 0.1..10 MiB") + + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + blob = make_png(args.size_mib) + + def upload(i): + local = Client(args.base_url, args.ca_file, timeout=60) + t = time.monotonic() + att = local.call( + "POST", + "/api/v1/chats/%s/attachments?filename=load-%d.png" % (chat["id"], i), + 201, actor=alice, raw=blob, + headers={"Content-Type": "image/png"}, schema="Attachment", + )["json"] + return time.monotonic() - t, att["size_bytes"] + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results = list(pool.map(upload, range(args.concurrency))) + elapsed = time.monotonic() - started + lat = sorted(r[0] for r in results) + print("concurrency=%d size=%.2fMiB total=%.2fs p50=%.2fs p95=%.2fs" + % (args.concurrency, args.size_mib, elapsed, + lat[len(lat) // 2], lat[max(0, int(len(lat) * 0.95) - 1)])) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/evidence/mem-baseline.txt b/evidence/mem-baseline.txt new file mode 100644 index 00000000..d5efd340 Binary files /dev/null and b/evidence/mem-baseline.txt differ diff --git a/evidence/memory-peak.txt b/evidence/memory-peak.txt new file mode 100644 index 00000000..203473e1 Binary files /dev/null and b/evidence/memory-peak.txt differ diff --git a/main.py b/main.py new file mode 100644 index 00000000..3424ff38 --- /dev/null +++ b/main.py @@ -0,0 +1,374 @@ +"""Мессенджер, лабораторная 4: lab3 + S3-вложения.""" +import hashlib +import logging +import re +import tempfile +import uuid +from contextlib import asynccontextmanager +from typing import NamedTuple +from urllib.parse import quote + +import anyio.to_thread +import psycopg +from fastapi import Depends, FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app import (attachments, config, cursor, db, idempotency, repo, + security, sessions, s3, validation) +from app.errors import (BASIC_CHALLENGE, BEARER_CHALLENGE, ApiError, + bad_request, not_found, unauthorized) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("api") + +_REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}") +INSTANCE_ID = config.INSTANCE_ID + + +@asynccontextmanager +async def lifespan(_app): + anyio.to_thread.current_default_thread_limiter().total_tokens = config.THREAD_LIMIT + yield + db.close() + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +def make_error(code, message, request_id): + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def _rid(request): + return getattr(request.state, "request_id", None) or uuid.uuid4().hex + + +def error_response(request, status, code, message, headers=None): + rid = _rid(request) + out = { + "X-Request-Id": rid, + "X-Instance-Id": INSTANCE_ID, + "Cache-Control": "no-store", + } + out.update(headers or {}) + return JSONResponse(status_code=status, content=make_error(code, message, rid), headers=out) + + +@app.middleware("http") +async def add_request_id(request: Request, call_next): + incoming = request.headers.get("X-Request-Id", "") + rid = incoming if _REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex + request.state.request_id = rid + response = await call_next(request) + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + response.headers["Cache-Control"] = "no-store" + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request: Request, exc): + return error_response(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(StarletteHTTPException) +async def http_exc_handler(request: Request, exc: StarletteHTTPException): + mapping = {404: ("not_found", "Объект не найден"), 405: ("invalid_request", "Метод не поддерживается")} + code, message = mapping.get(exc.status_code, ("invalid_request", "Ошибка запроса")) + return error_response(request, exc.status_code, code, message) + + +@app.exception_handler(Exception) +async def generic_handler(request: Request, exc: Exception): + if isinstance(exc, db.TRANSIENT): + log.warning("dependency unavailable [%s]: %s", _rid(request), type(exc).__name__) + return error_response(request, 503, "unavailable", "Сервис временно недоступен") + log.error("unhandled error [%s]", _rid(request), exc_info=exc) + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +class Auth(NamedTuple): + user: dict + user_id: uuid.UUID + digest: bytes + + +def authenticated(request: Request) -> Auth: + token = security.parse_bearer(request.headers.get("Authorization")) + if token is None: + raise unauthorized(BEARER_CHALLENGE) + found = sessions.lookup(token) + if found is None: + raise unauthorized(BEARER_CHALLENGE) + return Auth(found["user"], uuid.UUID(found["user"]["id"]), found["digest"]) + + +class RawBody(NamedTuple): + content_type: str + data: bytes + + +async def raw_body(request: Request) -> RawBody: + declared = request.headers.get("content-length") + if declared is not None and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") + chunks, size = [], 0 + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") + chunks.append(chunk) + return RawBody(request.headers.get("content-type", ""), b"".join(chunks)) + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(): + try: + db.ping() + except Exception: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") + try: + s3.ping() + except Exception: + raise ApiError(503, "unavailable", "S3 недоступен") + return {"status": "ok"} + + +# ---------- пользователи / сессии / чаты (lab2–3 без изменений) ---------- + +@app.post("/api/v1/users", status_code=201) +def create_user(body: RawBody = Depends(raw_body)): + data = validation.json_body(body.content_type, body.data, + {"username", "display_name", "password"}, {"username", "display_name", "password"}) + username = validation.username_field(data["username"]) + display_name = validation.text_field(data["display_name"], "display_name", 1, 100) + password = validation.text_field(data["password"], "password", 12, 128) + with db.connection() as conn: + if repo.username_exists(conn, username): + raise ApiError(409, "conflict", "Имя пользователя занято") + password_hash = security.hash_password(password) + try: + with db.connection() as conn: + row = repo.insert_user(conn, username, display_name, password_hash) + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя занято") + return sessions.user_json(row) + + +@app.get("/api/v1/users/me") +def get_me(auth: Auth = Depends(authenticated)): + return auth.user + + +@app.post("/api/v1/auth/sessions", status_code=201) +def login(request: Request): + creds = security.parse_basic(request.headers.get("Authorization")) + if creds is None: + raise unauthorized(BASIC_CHALLENGE) + username, password = creds + with db.connection() as conn: + user = repo.user_for_login(conn, username) if validation._USERNAME_RE.fullmatch(username) else None + if user is None: + security.burn_verify(password) + raise unauthorized(BASIC_CHALLENGE) + ok, new_hash = security.verify_password(user["password_hash"], password) + if not ok: + raise unauthorized(BASIC_CHALLENGE) + with db.connection() as conn: + if new_hash: + repo.update_password_hash(conn, user["id"], new_hash) + token, expires_at = sessions.create(conn, user) + user_json = sessions.user_json(user) + sessions.warm(token, user_json, expires_at) + return {"token": token, "token_type": "Bearer", "expires_at": sessions.fmt_ts(expires_at), "user": user_json} + + +@app.delete("/api/v1/auth/sessions/current") +def logout(auth: Auth = Depends(authenticated)): + sessions.revoke(auth.digest) + return Response(status_code=204) + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)): + data = validation.json_body(body.content_type, body.data, {"title", "member_ids"}, {"title", "member_ids"}) + title = validation.text_field(data["title"], "title", 1, 100) + raw = data["member_ids"] + if not isinstance(raw, list) or not 1 <= len(raw) <= 100: + raise bad_request("member_ids: от 1 до 100 элементов") + members = [validation.require_uuid(x, "member_id") for x in raw] + if len(set(members)) != len(members) or auth.user_id not in members: + raise bad_request("member_ids должны быть уникальны и включать вызывающего") + with db.connection() as conn: + chat = repo.create_chat(conn, auth.user_id, title, members) + return repo.chat_json(chat) + + +@app.get("/api/v1/chats") +def list_chats(request: Request, auth: Auth = Depends(authenticated)): + limit = validation.limit_param(request.query_params) + raw = request.query_params.get("cursor") + after = cursor.decode(raw, "chats", auth.user_id) if raw is not None else None + with db.connection() as conn: + items, nxt = repo.list_chats(conn, auth.user_id, after, limit) + return {"items": items, "next_cursor": nxt} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, auth: Auth = Depends(authenticated)): + cid = validation.require_uuid(chat_id, "chat_id") + with db.connection() as conn: + chat = repo.get_chat(conn, cid, auth.user_id) + if chat is None: + raise not_found() + return repo.chat_json(chat) + + +# ---------- сообщения (lab3 + attachment_ids) ---------- + +def _parse_attachment_ids(data): + raw = data.get("attachment_ids", []) + if not isinstance(raw, list) or len(raw) > 10: + raise bad_request("attachment_ids: до 10 элементов") + ids = [validation.require_uuid(x, "attachment_id") for x in raw] + if len(set(ids)) != len(ids): + raise bad_request("attachment_ids должны быть уникальны") + return ids + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(chat_id: str, request: Request, auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)): + cid = validation.require_uuid(chat_id, "chat_id") + with db.connection() as conn: + if not repo.is_member(conn, cid, auth.user_id): + raise not_found() + + data = validation.json_body(body.content_type, body.data, {"text", "attachment_ids"}, {"text"}) + text = validation.text_field(data["text"], "text", 0, 4000, allow_blank=True) + attachment_ids = _parse_attachment_ids(data) + if not text.strip() and not attachment_ids: + raise bad_request("Пустое сообщение без вложений") + + idem_key = request.headers.get("Idempotency-Key") + path = "/api/v1/chats/%s/messages" % chat_id + req_hash = idempotency.get_request_hash(data) + + with db.connection() as conn: + idempotency.cleanup(conn, config.IDEMPOTENCY_TTL_SECONDS) + need_exec, cached_resp = idempotency.check_and_lock( + conn, str(auth.user_id), "POST", path, idem_key, req_hash + ) + if not need_exec: + return JSONResponse(status_code=201, content=cached_resp) + msg = repo.insert_message(conn, cid, auth.user_id, text, attachment_ids) + msg_json = repo.message_json(msg) + idempotency.save_result(conn, str(auth.user_id), "POST", path, idem_key, req_hash, 201, msg_json) + return msg_json + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request, auth: Auth = Depends(authenticated)): + cid = validation.require_uuid(chat_id, "chat_id") + limit = validation.limit_param(request.query_params) + raw = request.query_params.get("cursor") + after = cursor.decode(raw, "messages", cid) if raw is not None else None + with db.connection() as conn: + if not repo.is_member(conn, cid, auth.user_id): + raise not_found() + items, nxt = repo.list_messages(conn, cid, after, limit) + return {"items": items, "next_cursor": nxt} + + +# ---------- вложения (lab4) ---------- + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: str, request: Request, auth: Auth = Depends(authenticated)): + cid = validation.require_uuid(chat_id, "chat_id") + filename = request.query_params.get("filename") + if filename is None: + raise bad_request("filename обязателен") + + declared = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg") + + # Проверяем доступ до чтения тела: посторонний не должен отправить 10 MiB. + def _check_member(): + with db.connection() as conn: + return repo.is_member(conn, cid, auth.user_id) + + if not await anyio.to_thread.run_sync(_check_member): + raise not_found() + + declared_len = request.headers.get("content-length") + if declared_len is not None and declared_len.isdigit() and int(declared_len) > config.MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + + spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024) + hasher = hashlib.sha256() + total = 0 + try: + async for chunk in request.stream(): + total += len(chunk) + if total > config.MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + hasher.update(chunk) + spool.write(chunk) + if total == 0: + raise bad_request("Пустой файл") + spool.seek(0) + digest = hasher.hexdigest() + + def _process(): + actual = attachments.inspect_image_file(spool, total) + if actual != declared: + raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому") + return attachments.upload_stream(cid, auth.user_id, filename, spool, total, digest, actual) + + return await anyio.to_thread.run_sync(_process) + finally: + spool.close() + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: str, auth: Auth = Depends(authenticated)): + aid = validation.require_uuid(attachment_id, "attachment_id") + with db.connection() as conn: + row = repo.get_attachment_for_member(conn, aid, auth.user_id) + if row is None: + raise not_found() + return repo.attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, auth: Auth = Depends(authenticated)): + aid = validation.require_uuid(attachment_id, "attachment_id") + + # boto3 StreamingBody нельзя читать в чужом потоке: читаем всё в одном run_sync. + # Файл ≤ 10 MiB, поэтому память ограничена лимитом загрузки. + def _read_all(): + row, body = attachments.download(aid, auth.user_id) + try: + data = body.read() + finally: + body.close() + return row, data + + row, data = await anyio.to_thread.run_sync(_read_all) + + headers = { + "Content-Disposition": "attachment; filename*=UTF-8''%s" % quote(row["filename"], safe="") + } + return Response(content=data, media_type=row["content_type"], headers=headers) \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 00000000..7d015360 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,33 @@ +worker_processes auto; +events { worker_connections 1024; } + +http { + upstream api_pool { + server messenger-lab4-api-1:8080 max_fails=1 fail_timeout=5s; + server messenger-lab4-api-2:8080 max_fails=1 fail_timeout=5s; + keepalive 16; + } + + server { + listen 80; + + location / { + proxy_pass http://api_pool; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Connection ""; + + proxy_next_upstream error timeout; + proxy_next_upstream_tries 2; + + proxy_connect_timeout 2s; + proxy_send_timeout 15s; + proxy_read_timeout 15s; + + client_max_body_size 11m; + proxy_request_buffering on; + } + } +} \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..0a4bd4b4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,4 @@ # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +boto3==1.35.0 +Pillow==10.4.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..fbed5d91 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +boto3==1.43.107 +Pillow==12.3.0 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.0 +h11==0.16.0 +httptools==0.9.0 +idna==3.20 +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +pycparser==3.11 +pydantic==2.14.1 +pydantic_core==2.50.1 +python-dotenv==1.2.4 +PyYAML==6.0.3 +redis==6.4.0 +starlette==0.38.6 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uvicorn==0.30.6 +uvloop==0.23.0 +watchfiles==1.3.0 +websockets==17.2 \ No newline at end of file diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100644 index 00000000..57ff48f7 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# Создаёт локальные синтетические секреты для учебного стенда. Идемпотентен. +set -eu + +umask 077 +cd "$(dirname "$0")/.." +mkdir -p secrets + +gen() { + # 32 случайных байта в†’ base64 в†’ только URL-safe символы. + head -c 32 /dev/urandom | base64 | tr -d '/+=\n' | cut -c1-32 +} + +ensure() { + f="secrets/$1" + if [ ! -s "$f" ]; then + printf '%s' "$2" > "$f" + chmod 600 "$f" + echo "created $f" + fi +} + +PG_PASS=$(cat secrets/postgres_password 2>/dev/null || gen) +VALKEY_PASS=$(cat secrets/valkey_password 2>/dev/null || gen) +MINIO_ROOT_USER_VAL="minioadmin" +MINIO_ROOT_PASS=$(cat secrets/minio_root_password 2>/dev/null || gen) +S3_KEY=$(cat secrets/s3_access_key_id 2>/dev/null || "app$(gen | cut -c1-20)") +S3_SECRET=$(cat secrets/s3_secret_access_key 2>/dev/null || gen) + +ensure postgres_password "$PG_PASS" +ensure valkey_password "$VALKEY_PASS" +ensure minio_root_user "$MINIO_ROOT_USER_VAL" +ensure minio_root_password "$MINIO_ROOT_PASS" +ensure s3_access_key_id "$S3_KEY" +ensure s3_secret_access_key "$S3_SECRET" + +ensure database_url "postgresql://messenger:${PG_PASS}@postgres:5432/messenger" +ensure cache_url "redis://:${VALKEY_PASS}@valkey:6379/0" + +if [ ! -f .env ]; then + cat > .env <<'EOF' +APP_ENV=lab +SESSION_TTL_SECONDS=3600 +SESSION_STORE=postgres+cache +SESSION_CACHE_TTL_SECONDS=60 +IDEMPOTENCY_TTL_SECONDS=86400 +API_REPLICAS=2 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +MAX_UPLOAD_BYTES=10485760 +MAX_IMAGE_PIXELS=40000000 +S3_ORPHAN_GRACE_SECONDS=3600 +EOF + echo "created .env" +fi + +echo "secrets ready in ./secrets (РЅРµ коммить)" \ No newline at end of file diff --git a/scripts/reconcile_s3.py b/scripts/reconcile_s3.py new file mode 100644 index 00000000..05a82618 --- /dev/null +++ b/scripts/reconcile_s3.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Сверка S3 и метаданных PostgreSQL. + +Классы проблем: + orphan_object — объект в S3 без строки attachments (или с pending > grace) + missing_object — attachments.status=ready, но объекта в S3 нет + failed_referenced — attachments.status=failed, объект всё ещё лежит в S3 + +Действия: + orphan_object → DELETE объекта (только старше grace) + missing_object → UPDATE status='failed', error_code='object_missing' + failed_referenced → DELETE объекта + +Использование: + python scripts/reconcile_s3.py --dry-run + python scripts/reconcile_s3.py --apply +""" +import argparse +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from app import config, db, repo, s3 # noqa: E402 + + +def _paginate(client, prefix): + """Возвращает {key: last_modified}.""" + token = None + out = {} + while True: + kwargs = {"Bucket": config.S3_BUCKET, "Prefix": prefix} + if token: + kwargs["ContinuationToken"] = token + resp = client.list_objects_v2(**kwargs) + for obj in resp.get("Contents", []): + out[obj["Key"]] = obj["LastModified"] + if not resp.get("IsTruncated"): + return out + token = resp["NextContinuationToken"] + + +def reconcile(dry_run: bool, grace_seconds: int): + client = s3._get_client() + now = time.time() + report = {"orphans_deleted": [], "orphans_skipped_fresh": [], + "missing_marked_failed": [], "failed_objects_deleted": []} + + with db.connection() as conn: + rows = conn.execute( + """SELECT id, object_key, status, created_at, + extract(epoch FROM created_at) AS created_epoch + FROM attachments""").fetchall() + by_key = {r["object_key"]: r for r in rows} + referenced = set(by_key) + + for key, last_modified in _paginate(client, "originals/").items(): + if key in referenced: + continue + age = now - last_modified.timestamp() + if age < grace_seconds: + report["orphans_skipped_fresh"].append(key) + continue + report["orphans_deleted"].append(key) + if not dry_run: + s3.delete_object(key) + + for row in rows: + key = row["object_key"] + if row["status"] == "ready" and key not in _paginate(client, key): + report["missing_marked_failed"].append(key) + if not dry_run: + with db.connection() as conn: + conn.execute( + """UPDATE attachments + SET status='failed', error_code='object_missing', + updated_at=clock_timestamp() + WHERE id=%s AND status='ready'""", (row["id"],)) + elif row["status"] == "failed": + if key in _paginate(client, key): + report["failed_objects_deleted"].append(key) + if not dry_run: + s3.delete_object(key) + elif row["status"] == "pending": + age = now - float(row["created_epoch"]) + if age >= grace_seconds and key in _paginate(client, key): + # pending дольше grace, а объект есть → API упал между S3 PUT и COMMIT + report["orphans_deleted"].append(key) + if not dry_run: + with db.connection() as conn: + repo.mark_attachment_failed(conn, row["id"], "orphaned_pending") + s3.delete_object(key) + + import json + print(json.dumps(report, ensure_ascii=False, indent=2, default=str)) + print("dry-run" if dry_run else "applied") + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--apply", action="store_true") + p.add_argument("--dry-run", action="store_true") + p.add_argument("--grace-seconds", type=int, default=config.S3_ORPHAN_GRACE_SECONDS) + args = p.parse_args() + if args.apply and args.dry_run: + sys.exit("use --apply or --dry-run, not both") + if not (args.apply or args.dry_run): + sys.exit("choose --dry-run or --apply") + reconcile(dry_run=not args.apply, grace_seconds=args.grace_seconds) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..d79c2b8c --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,7 @@ +@echo off +REM Перезапуск стенда с сохранением volumes. Возврат 0 = ОК. +cd /d "%~dp0\.." +docker compose --env-file .env stop +timeout /t 2 /nobreak >nul +docker compose --env-file .env up -d +exit /b 0 \ No newline at end of file diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh old mode 100755 new mode 100644 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..a5144659 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Перезапуск всего стенда СЃ сохранением volumes Рё данных. +# Никаких down -v / reset / seed. Останавливает РїРѕ dependency-РїРѕСЂСЏРґРєСѓ, +# поднимает РІ обратном. Ждёт /health/ready РґРѕ 90 секунд. +set -eu + +cd "$(dirname "$0")/.." + +# Compose сам соблюдает зависимость: stop РІ обратном РїРѕСЂСЏРґРєРµ, up РІ РїСЂСЏРјРѕРј. +docker compose --env-file .env stop +sleep 1 +docker compose --env-file .env up -d + +for i in $(seq 1 90); do + if curl -fs -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; then + echo "stack ready after ${i}s" + exit 0 + fi + sleep 1 +done + +echo "stack did not become ready in 90s" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..611f6df3 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,19 @@ +@echo off +REM Остановить одну API-реплику по её X-Instance-Id (== Hostname контейнера). +setlocal +if "%TARGET_INSTANCE_ID%"=="" ( + echo TARGET_INSTANCE_ID is not set + exit /b 1 +) + +for /f "tokens=*" %%c in ('docker ps -q --filter "name=messenger-lab4-api"') do ( + for /f "tokens=*" %%h in ('docker inspect -f "{{.Config.Hostname}}" %%c') do ( + if "%%h"=="%TARGET_INSTANCE_ID%" ( + echo Stopping container %%c + docker stop %%c + exit /b 0 + ) + ) +) +echo Container with instance ID %TARGET_INSTANCE_ID% not found +exit /b 1 \ No newline at end of file diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh old mode 100755 new mode 100644 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..aa8ab595 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Останавливает конкретную реплику API Р С—Р С• её X-Instance-Id. +set -eu + +if [ -z "${TARGET_INSTANCE_ID:-}" ]; then + echo "TARGET_INSTANCE_ID is not set" >&2 + exit 1 +fi + +# INSTANCE_ID = socket.gethostname(), Р Р† Docker это короткий ID контейнера. +cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" || true) + +if [ -z "$cid" ]; then + # Fallback: искать Р С—Р С• имени проекта Р С‘ сверять hostname + for c in $(docker ps -q --filter "name=messenger-lab4"); do + h=$(docker inspect -f '{{.Config.Hostname}}' "$c") + if [ "$h" = "$TARGET_INSTANCE_ID" ]; then + cid="$c" + break + fi + done +fi + +if [ -n "$cid" ]; then + echo "Stopping container $cid (Instance ID: $TARGET_INSTANCE_ID)" + docker stop "$cid" +else + echo "Container with instance ID $TARGET_INSTANCE_ID not found" >&2 + exit 1 +fi \ No newline at end of file diff --git a/student-tests/test_attachments_negative.py b/student-tests/test_attachments_negative.py new file mode 100644 index 00000000..490b7d17 --- /dev/null +++ b/student-tests/test_attachments_negative.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Негативные сценарии вложений. Запускается против запущенного стенда.""" +import struct +import sys +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, arguments, check, run # noqa: E402 + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" + + +def make_broken_png(): + """PNG с корректными CRC чанков, но битым потоком IDAT.""" + def chunk(t, d): + c = t + d + return struct.pack(">I", len(d)) + c + struct.pack(">I", zlib.crc32(c) & 0xFFFFFFFF) + ihdr = struct.pack(">IIBBBBB", 64, 64, 8, 2, 0, 0, 0) + # Валидный zlib-заголовок + мусор: decompress() упадёт. + bad_idat = b"\x78\x9c" + b"\xff" * 100 + return PNG_MAGIC + chunk(b"IHDR", ihdr) + chunk(b"IDAT", bad_idat) + chunk(b"IEND", b"") + + +def main(): + args = arguments("Lab4 attachment negative tests") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + other_chat = c.call( + "POST", "/api/v1/chats", 201, actor=bob, + body={"title": "Другой чат", "member_ids": [bob["id"]]}, schema="Chat")["json"] + base = "/api/v1/chats/%s/attachments" % chat["id"] + png_fixture = (ROOT / "tests/fixtures/sample.png").read_bytes() + + # 1. Слишком большой файл. + big = PNG_MAGIC + b"\x00" * 10485760 + c.call("POST", base + "?filename=big.png", 413, actor=alice, raw=big, + headers={"Content-Type": "image/png"}) + + # 2. Подмена MIME. + c.call("POST", base + "?filename=x.png", 415, actor=alice, + raw=b"not an image at all", headers={"Content-Type": "image/png"}) + + # 3. Повреждённый PNG (валидный заголовок, битая структура). + broken = png_fixture[:20] + b"\x00" * 32 + png_fixture[52:] + c.call("POST", base + "?filename=broken.png", 415, actor=alice, raw=broken, + headers={"Content-Type": "image/png"}) + + # 3b. PNG с корректными CRC, но битым IDAT — ловится только load(). + c.call("POST", base + "?filename=broken-idat.png", 415, actor=alice, + raw=make_broken_png(), headers={"Content-Type": "image/png"}) + + # 4. Опасные имена: path traversal, \, NUL, DEL 0x7F, control 0x01. + for bad in ("../etc/passwd", "a\\b.png", "a\x00b.png", "a/b.png", + "a\x7fb.png", "a\x01b.png"): + c.call("POST", base + "?filename=" + quote(bad, safe=""), 400, actor=alice, + raw=png_fixture, headers={"Content-Type": "image/png"}) + + # 5. Чужой чат — 404 до чтения тела. + c.call("POST", base + "?filename=sample.png", 404, actor=eve, + raw=png_fixture, headers={"Content-Type": "image/png"}) + + # 6. Сослаться на чужое вложение — 404 (не 400). + foreign = c.call("POST", "/api/v1/chats/%s/attachments?filename=s.png" % other_chat["id"], + 201, actor=bob, raw=png_fixture, + headers={"Content-Type": "image/png"}, schema="Attachment")["json"] + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 404, + actor=alice, + body={"text": "hi", "attachment_ids": [foreign["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + + # 6b. Случайный UUID вложения — тоже 404. + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 404, + actor=alice, + body={"text": "hi", "attachment_ids": [str(uuid.uuid4())]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + + # 7. Чтение чужого attachment_id — 404 на metadata и bytes. + c.call("GET", "/api/v1/attachments/" + foreign["id"], 404, actor=eve) + c.call("GET", "/api/v1/attachments/" + foreign["id"] + "/content", 404, actor=eve) + + # 8. Несуществующий id — 404, не 500. + ghost = str(uuid.uuid4()) + c.call("GET", "/api/v1/attachments/" + ghost, 404, actor=alice) + c.call("GET", "/api/v1/attachments/" + ghost + "/content", 404, actor=alice) + + # 9. Идемпотентность: {"text":"x"} и {"text":"x","attachment_ids":[]} равны. + key = uuid.uuid4().hex + m1 = c.message(alice, chat, "idem-normalize", key=key)["json"] + m2 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 201, + actor=alice, + body={"text": "idem-normalize", "attachment_ids": []}, + headers={"Idempotency-Key": key}, schema="Message")["json"] + check(m1["id"] == m2["id"], "idempotency: attachment_ids=[] must equal absent") + + # 10. Idempotency-Key с пробелом — 400. + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 400, + actor=alice, + body={"text": "x"}, + headers={"Idempotency-Key": "contains space"}) + + print("PASS: 10 negative attachment scenarios") + + +if __name__ == "__main__": + run(main) \ No newline at end of file