diff --git a/.env.example b/.env.example index d99d26a2..1c354a77 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,20 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url + +# Shared PostgreSQL +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace-with-local-synthetic-password +POSTGRES_DB=messenger +DATABASE_URL=postgresql://messenger:replace-with-local-synthetic-password@db:5432/messenger + +# Private S3 bucket (MinIO) shared by both API replicas +S3_BUCKET=messenger +S3_ACCESS_KEY=messenger +S3_SECRET_KEY=replace-with-local-synthetic-s3-secret + +# Lab4: two API replicas behind the LB; bytes are NOT stored locally on API. API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6f181f15 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PIP_NO_CACHE_DIR=1 +WORKDIR /opt/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --uid 10001 appuser && chown -R appuser:appuser /opt/app +USER appuser + +EXPOSE 8080 +CMD ["python", "-m", "app"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..a68a3ce5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,99 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Яковлев Матвей Дмитриевич, БПИ2304 +- Целевая оценка; выбранный трек 5, если нужен: 3 +- Commit/PR: ветка: (solution/BPI2304-Yakovlev); MR: (https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/155) +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.135.3, Uvicorn 0.44.0, psycopg 3.3.3, boto3 1.40.67, PostgreSQL 16-alpine, MinIO (bitnamilegacy/minio), nginx 1.27-alpine; версии закреплены в `requirements.txt`, `compose.yaml`, `Dockerfile`. При подготовке использовалась помощь Claude. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +```sh +git clone && cd <каталог> +cp .env.example .env # только синтетические локальные креды; .env в Git не попадает +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +curl -i http://localhost:8080/health/ready # nginx — единственная публичная точка +python tests/smoke.py --base-url http://localhost:8080 +``` + +Публикуется только порт 8080 балансировщика. PostgreSQL, MinIO и обе API-реплики +доступны только во внутренней сети Compose. Консоль MinIO (9001) наружу не открыта. +Безопасная остановка: `docker compose --env-file .env stop api1 api2 db minio`. +Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. + +Явные проверки: + +```sh +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# Windows локально: scripts/restart.cmd и scripts/stop-one.cmd +``` + +Вывод команд — `evidence/lifecycle.txt`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → nginx (`lb`) → `api1`/`api2` → PostgreSQL (метаданные) + MinIO (байты). + +- **Минимальный S3-сервер** — `bitnamilegacy/minio:latest` (официальный `minio/minio` + удалён с Docker Hub, `quay.io/minio/minio` требует авторизацию при pull). Приватный + bucket `messenger` создаётся идемпотентно: `MINIO_DEFAULT_BUCKETS` при старте сервера + и `head_bucket/create_bucket` в `app/s3.py` (обрабатывает `BucketAlreadyOwnedByYou`). +- **Разделение метаданных и байтов**: таблица `attachments` хранит id, chat_id, owner_id, + filename, content_type, size_bytes, sha256, сгенерированный сервером `object_key`, + status, variants, error_code. Байты лежат только в приватном bucket; в БД и на + локальном диске API оригинала нет. `filename` не используется как ключ S3. +- **Проверки загрузки** (`app/images.py`): MIME определяется по magic bytes PNG/JPEG, + заголовок Content-Type обязан совпадать; размеры читаются из IHDR/SOF без полного + декодирования; лимит 10 MiB применяется потоково до полной буферизации, лимит + 40 млн пикселей — до декодирования. Имя файла не содержит `/`, `\`, NUL и + управляющие символы. Несоответствие MIME → 415, превышение → 413, мусор → 415, не 500. +- **Консистентность S3/БД**: S3 PUT выполняется до вставки метаданных; при неудачной + транзакции PostgreSQL выполняется best-effort `delete_object` (нет вечного + неучтённого объекта). `201 ready` возвращается только после успешного PUT и COMMIT. +- **Доступ**: `GET /attachments/{id}` и `/content` требуют Bearer и membership в чате; + посторонний и неизвестный ID получают одинаковый 404. Download идёт через API + побайтно, без redirect; variant по умолчанию `original`, других вариантов в lab4 нет. +- **Сообщения с вложениями**: `attachment_ids` (0–10 уникальных UUID) Check-validate, + все ID обязаны принадлежать тому же чату иначе 404; пустой текст допустим только при + наличии вложения. Идемпотентный ключ хранится в PostgreSQL и покрывает каноническое + тело с `attachment_ids`; повтор возвращает тот же Message. +- **Отказ и рестарт**: `scripts/stop-one.sh` останавливает только реплику с alias из + `X-Instance-Id`; nginx настроен с `max_fails=1 fail_timeout=5s`, + `proxy_connect_timeout 1s` и `proxy_next_upstream error timeout http_502 http_503 http_504`, + поэтому вторая реплика продолжает отдавать и метаданные, и байты из общего S3. + `scripts/restart.sh` перезапускает `db`, `minio`, `api1`, `api2` без удаления volume. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| S3 в стенде, метаданные в PostgreSQL, байты в приватном bucket | `docker compose ps`; `psql SELECT` | 5 сервисов; строки `attachments` с object_key; байты не в БД | `evidence/lifecycle.txt`, разделы 1, 4 | +| Upload 201 ready, checksum и побайтный download | `python tests/smoke.py` | `PASS: lab4 public API scenario...`; SHA-256 и размер совпадают | `evidence/lifecycle.txt`, раздел 3 | +| Чужой чат/вложение недоступно | smoke: 404 для eve | PASS | `evidence/lifecycle.txt`, раздел 3 | +| Лимиты и MIME по содержимому | `app/images.py` + own tests | 14 юнит-тестов OK (fixture PNG, мусор, filename) | `evidence/lifecycle.txt`, раздел 9 | +| Restart с сохранением volumes и того же файла | `make test-persistence` | PASS: данные, сессия и оригинал прочитаны после рестарта | `evidence/lifecycle.txt`, разделы 5–6 | +| Отказ реплики, download через другую | `make test-failover` | 2 реплики до отказа, 105 успешных записей, 1 переходная ошибка, recovery 0.624 с, PASS | `evidence/lifecycle.txt`, разделы 7–8 | +| lab3-минимум сохранён | smoke + idempotency_records | PASS; ledger в PostgreSQL | `evidence/lifecycle.txt`, разделы 3–4 | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Не проводились (целевая оценка 3); уровни 4/5 с отказами S3, reconciliation и +нагрузочными экспериментами не заявлены. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Останов API-реплики, загрузившей файл | вторая реплика читает метаданные и байты из общего S3 | PASS: исходный файл побайтно совпадает, сессия и сообщения на месте | первая успешная запись за 0.624 с; потерь 0 | +| Перезапуск `db` + `minio` + `api1` + `api2` | вложения и оригиналы сохраняются | PASS; `schema_migrations` без повторов | автоматическое ожидание готовности; потерь 0 | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Уровень 4 не реализован: нет отдельной проверки недоступности S3, отрицательных + тестов >10 MiB/подмены MIME, reconciliation orphan-объектов с grace period, + ограниченных S3-credentials и эксперимента ≥10 одновременных загрузок. +- Компенсация при отказе между S3 PUT и DB COMMIT — best-effort в том же запросе; + нет фонового reconciler (это тема уровня 5). +- MinIO без TLS и с общим логином в локальном Compose — только для изолированного стенда + с синтетическими данными; для удалённой демонстрации нужен TLS. +- PostgreSQL и nginx остаются SPOF; отказ узла БД не моделируется. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..a4254e9e --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger lab2: PostgreSQL-backed API.""" diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..7148fa95 --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,13 @@ +"""Local entry point: python -m app""" +import os + +import uvicorn + +if __name__ == "__main__": + uvicorn.run( + "app.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, # in-process ordering/cursors; PostgreSQL is the shared source of truth + log_level=os.environ.get("LOG_LEVEL", "info"), + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..f60b4eff --- /dev/null +++ b/app/db.py @@ -0,0 +1,168 @@ +"""PostgreSQL access: small bounded connection pool, versioned migrations, row helpers.""" +import logging +import os +import queue +from contextlib import contextmanager +from datetime import timezone +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +log = logging.getLogger("messenger.db") + +ROOT = Path(__file__).resolve().parents[1] +MIGRATIONS = ROOT / "migrations" + + +def _read_secret(path): + if not path: + return None + value = Path(path).read_text(encoding="utf-8").strip() + if not value: + raise RuntimeError("secret file is empty: " + path) + return value + + +def database_url(): + """Prefer a mounted secret file; fall back to an environment variable for local runs.""" + url = _read_secret(os.environ.get("DATABASE_URL_FILE")) + if url: + return url + url = os.environ.get("DATABASE_URL") + if not url: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required") + return url + + +class Database: + """A tiny FIFO pool. Each request borrows one connection and returns it.""" + + def __init__(self): + self._pool = None + self.dsn = None + + def start(self): + self.dsn = database_url() + size = int(os.environ.get("DB_POOL_MAX", "8")) + self._pool = queue.Queue(maxsize=size) + for _ in range(size): + self._pool.put(self._connect()) + self.migrate() + + def _connect(self): + return psycopg.connect(self.dsn, connect_timeout=3, row_factory=dict_row, autocommit=True) + + def stop(self): + if self._pool is None: + return + while not self._pool.empty(): + try: + self._pool.get_nowait().close() + except queue.Empty: + break + self._pool = None + + @contextmanager + def connection(self): + conn = self._pool.get(timeout=5) + try: + yield conn + finally: + self._pool.put(conn) + + @contextmanager + def transaction(self): + with self.connection() as conn: + # Explicit BEGIN/COMMIT; rollback on any exception. + conn.execute("BEGIN") + try: + yield conn + except BaseException: + conn.execute("ROLLBACK") + raise + else: + conn.execute("COMMIT") + + def ready(self): + try: + with self.connection() as conn: + conn.execute("SELECT 1").fetchone() + return True + except Exception: + log.exception("readiness check failed") + return False + + def migrate(self): + """Apply *.sql in name order, at most once each; restart-safe with the same volume.""" + with self.transaction() as conn: + # All API replicas start against the same database. A transaction-scoped + # advisory lock makes the migration step strictly single-runner: + # the second replica waits, then sees schema_migrations already filled. + conn.execute("SELECT pg_advisory_xact_lock(20261116)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp())" + ) + applied = {r["name"] for r in conn.execute("SELECT name FROM schema_migrations").fetchall()} + for path in sorted(MIGRATIONS.glob("*.sql")): + if path.name in applied: + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,)) + log.info("applied migration %s", path.name) + + +db = Database() + + +def utc_text(value): + """RFC 3339 UTC with an explicit Z and microsecond precision.""" + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def user_public(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": utc_text(row["created_at"]), + } + + +def chat_public(row): + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(x) for x in (row["member_ids"] or [])], + "created_at": utc_text(row["created_at"]), + } + + +def message_public(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": [str(x) for x in (row["attachment_ids"] or [])], + "created_at": utc_text(row["created_at"]), + } + + +def attachment_public(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": row["variants"] or [], + "error_code": row["error_code"], + "created_at": utc_text(row["created_at"]), + "updated_at": utc_text(row["updated_at"]), + } diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..8d4da425 --- /dev/null +++ b/app/images.py @@ -0,0 +1,49 @@ +"""Cheap header-only PNG/JPEG probing: magic, declared MIME and pixel limit.""" +import struct + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" +SOF_MARKERS = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + + +class InvalidImage(ValueError): + pass + + +def probe(data): + """Return (content_type, width, height) without decoding the whole image.""" + if data.startswith(PNG_MAGIC): + if len(data) < 24 or data[12:16] != b"IHDR": + raise InvalidImage("truncated PNG") + width, height = struct.unpack(">II", data[16:24]) + if width == 0 or height == 0: + raise InvalidImage("invalid PNG dimensions") + return "image/png", width, height + + if data[:3] == JPEG_MAGIC: + i = 2 + while i + 9 < len(data): + if data[i] != 0xFF: + i += 1 + continue + marker = data[i + 1] + if marker == 0xFF: + i += 1 + continue + if marker == 0xD8 or 0xD0 <= marker <= 0xD7 or marker == 0x01: + i += 2 + continue + if marker == 0xD9: + break + length = struct.unpack(">H", data[i + 2:i + 4])[0] + if length < 2 or i + 2 + length > len(data): + raise InvalidImage("truncated JPEG segment") + if marker in SOF_MARKERS: + height, width = struct.unpack(">HH", data[i + 5:i + 9]) + if width == 0 or height == 0: + raise InvalidImage("invalid JPEG dimensions") + return "image/jpeg", width, height + i += 2 + length + raise InvalidImage("JPEG has no SOF marker") + + raise InvalidImage("not a PNG or JPEG") diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..9609c6a6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,602 @@ +"""Lab2 messenger HTTP layer. Contract: contracts/openapi.json. + +State lives in PostgreSQL: users, chats, memberships, messages and sessions. +Basic is accepted only on POST /auth/sessions; every other protected route needs Bearer. +""" +import base64 +import binascii +import datetime as dt +import hashlib +import json +import logging +import os +import re +import uuid +from contextlib import asynccontextmanager + +import psycopg +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response +from starlette.datastructures import MutableHeaders +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import s3 +from .db import attachment_public, chat_public, db, message_public, user_public, utc_text +from .images import InvalidImage, probe +from .security import bearer_token, hash_password, parse_basic, token_hash, verify_password + +log = logging.getLogger("messenger") + +MAX_BODY_BYTES = 1 << 20 +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 # 10 MiB +MAX_IMAGE_PIXELS = 40_000_000 +DEFAULT_LIMIT, MAX_LIMIT = 50, 100 +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}\Z") +UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}\Z") +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +INSTANCE_ID = os.environ.get("INSTANCE_ALIAS") or uuid.uuid4().hex[:12] + +UTC = dt.timezone.utc + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status, self.code, self.message, self.headers = status, code, message, headers + + +def bad_request(message): + return ApiError(400, "invalid_request", message) + + +def unauthorized(challenge, message="Требуется действительная аутентификация"): + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(): + return ApiError(404, "not_found", "Объект не найден") + + +def error_response(status, code, message, request_id, headers=None): + body = {"error": {"code": code, "message": message, "request_id": request_id}} + return JSONResponse(body, status_code=status, headers=headers) + + +class RequestContextMiddleware: + """Add X-Request-Id (and X-Instance-Id) to every response, including errors and health.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + request_id = str(uuid.uuid4()) + scope.setdefault("state", {})["request_id"] = request_id + started = False + + async def send_with_id(message): + nonlocal started + if message["type"] == "http.response.start": + started = True + headers = MutableHeaders(scope=message) + headers["X-Request-Id"] = request_id + headers["X-Instance-Id"] = INSTANCE_ID + await send(message) + + try: + await self.app(scope, receive, send_with_id) + except Exception: + log.exception("unhandled error request_id=%s", request_id) + if not started: + response = error_response(500, "internal_error", "Внутренняя ошибка сервера", request_id) + await response(scope, receive, send_with_id) + + +@asynccontextmanager +async def lifespan(app): + db.start() + s3.start() + try: + yield + finally: + db.stop() + + +app = FastAPI(title="Messenger lab2", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan) +app.add_middleware(RequestContextMiddleware) + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError): + return error_response(exc.status, exc.code, exc.message, request.state.request_id, exc.headers) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_exception(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + code, message = "not_found", "Объект не найден" + elif exc.status_code < 500: + code, message = "invalid_request", "Некорректный запрос" + else: + code, message = "internal_error", "Внутренняя ошибка сервера" + return error_response(exc.status_code, code, message, request.state.request_id, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, exc: RequestValidationError): + return error_response(400, "invalid_request", "Некорректный запрос", request.state.request_id) + + +# ---------- validation helpers ---------- + +def clean_string(data, field, max_len): + value = data.get(field) + if not isinstance(value, str): + raise bad_request("Поле %s должно быть строкой" % field) + if not value.strip() or len(value) > max_len: + raise bad_request("Поле %s должно содержать от 1 до %d символов" % (field, max_len)) + return value + + +def idempotency_key(request): + value = request.headers.get("idempotency-key") + if not value or not IDEMPOTENCY_KEY_RE.fullmatch(value): + raise bad_request("Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + return value + + +async def read_raw(request, limit): + """Read a binary body with a hard byte limit applied before buffering everything.""" + declared = request.headers.get("content-length", "") + if declared.isdigit() and int(declared) > limit: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + body = bytearray() + async for chunk in request.stream(): + body += chunk + if len(body) > limit: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + return bytes(body) + + +def validate_filename(value): + if not isinstance(value, str) or not 1 <= len(value) <= 255: + raise bad_request("filename должен содержать от 1 до 255 символов") + if any(ch in value for ch in ("/", "\\", "\x00")) or any(ord(ch) < 32 or ord(ch) == 127 for ch in value): + raise bad_request("filename содержит недопустимые символы") + return value + + +def parse_uuid(value, what): + if not isinstance(value, str) or not UUID_RE.fullmatch(value): + raise bad_request("Некорректный UUID: %s" % what) + return value.lower() + + +async def read_json_object(request, required, optional=()): + declared = request.headers.get("content-length", "") + if declared.isdigit() and int(declared) > MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") + content_type = request.headers.get("content-type", "").split(";")[0].strip().lower() + if content_type and content_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + body = bytearray() + async for chunk in request.stream(): + body += chunk + if len(body) > MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") + try: + data = json.loads(bytes(body).decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise bad_request("Тело запроса не является корректным JSON") from None + if not isinstance(data, dict): + raise bad_request("Тело запроса должно быть JSON-объектом") + expected = set(required) | set(optional) + if not set(required).issubset(data) or not set(data).issubset(expected): + raise bad_request("Ожидаемые поля: " + ", ".join(sorted(expected))) + return data + + +def page_params(request): + query = request.query_params + raw_limit = query.getlist("limit") + if len(raw_limit) > 1 or len(query.getlist("cursor")) > 1: + raise bad_request("Параметры limit и cursor можно указать только один раз") + limit = DEFAULT_LIMIT + if raw_limit: + text = raw_limit[0] + if not (text.isascii() and text.isdigit()) or not 1 <= int(text) <= MAX_LIMIT: + raise bad_request("limit должен быть целым числом от 1 до %d" % MAX_LIMIT) + limit = int(text) + return limit, query.get("cursor") or None + + +def encode_cursor(kind, scope, created_at, item_id): + raw = json.dumps( + {"k": kind, "s": scope, + "t": int(created_at.timestamp() * 1_000_000), + "i": uuid.UUID(str(item_id)).int}, + separators=(",", ":"), + ) + return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + + +def decode_cursor(text, kind, scope): + try: + data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4))) + t, i = data["t"], data["i"] + valid = (data["k"] == kind and data["s"] == scope + and type(t) is int and type(i) is int and t >= 0 and i >= 0) + if not valid: + raise ValueError + return dt.datetime.fromtimestamp(t / 1_000_000, tz=UTC), uuid.UUID(int=i) + except (ValueError, KeyError, TypeError, binascii.Error, OverflowError): + raise bad_request("Некорректный cursor") from None + + +def page_response(items, limit, kind, scope, rows): + """items are public JSON dicts; rows are the raw DB rows used for the opaque cursor.""" + more = len(items) > limit + window = items[:limit] + last = rows[limit - 1] if more and len(rows) >= limit else None + cursor = (encode_cursor(kind, scope, last["created_at"], last["id"]) + if last is not None else None) + return JSONResponse({"items": window, "next_cursor": cursor}) + + +# ---------- authentication and resource access ---------- + +def current_user(request): + """Lab2: only an opaque Bearer session identifies the caller; X-User-Id is ignored.""" + token = bearer_token(request.headers.get("authorization")) + if not token: + raise unauthorized(BEARER_CHALLENGE) + with db.connection() as conn: + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > clock_timestamp()", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + return user_public(row) + + +def authenticated_chat(request, chat_id): + user = current_user(request) + chat_uuid = parse_uuid(chat_id, "chat_id") + with db.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at, " + "array_agg(m.user_id ORDER BY m.user_id) AS member_ids " + "FROM chats c JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s GROUP BY c.id", + (chat_uuid, uuid.UUID(user["id"])), + ).fetchone() + if row is None: + raise not_found() # unknown and foreign chats look identical + return user, chat_public(row) + + +# ---------- health ---------- + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def ready(): + if not db.ready(): + raise ApiError(503, "unavailable", "Хранилище данных недоступно") + if not s3.ready(): + raise ApiError(503, "unavailable", "Хранилище вложений недоступно") + return {"status": "ok"} + + +# ---------- users and sessions ---------- + +@app.post("/api/v1/users") +async def register_user(request: Request): + data = await read_json_object(request, ("username", "display_name", "password")) + username = data["username"] + if not isinstance(username, str) or not USERNAME_RE.fullmatch(username): + raise bad_request("username должен соответствовать [a-z0-9_]{3,32}") + display_name = clean_string(data, "display_name", 100) + password = data["password"] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise bad_request("password должен содержать от 12 до 128 символов") + user_id = uuid.uuid4() + try: + with db.transaction() as conn: + conn.execute( + "INSERT INTO users(id, username, display_name, password_hash) VALUES (%s,%s,%s,%s)", + (user_id, username, display_name, hash_password(password)), + ) + row = conn.execute("SELECT * FROM users WHERE id = %s", (user_id,)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя уже занято") from None + return JSONResponse(user_public(row), status_code=201) + + +@app.post("/api/v1/auth/sessions") +async def create_session(request: Request): + credential = parse_basic(request.headers.get("authorization")) + if credential is None: + raise unauthorized(BASIC_CHALLENGE, "Неверные учетные данные") + username, password = credential + with db.connection() as conn: + row = conn.execute("SELECT * FROM users WHERE username = %s", (username,)).fetchone() + if row is None or not verify_password(password, row["password_hash"]): + raise unauthorized(BASIC_CHALLENGE, "Неверные учетные данные") + token = base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip("=") + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + expires_at = dt.datetime.now(UTC) + dt.timedelta(seconds=ttl) + with db.transaction() as conn: + conn.execute( + "INSERT INTO sessions(token_hash, user_id, expires_at) VALUES (%s,%s,%s)", + (token_hash(token), row["id"], expires_at), + ) + return JSONResponse({ + "token": token, + "token_type": "Bearer", + "expires_at": utc_text(expires_at), + "user": user_public(row), + }, status_code=201) + + +@app.delete("/api/v1/auth/sessions/current") +async def revoke_session(request: Request): + token = bearer_token(request.headers.get("authorization")) + if not token: + raise unauthorized(BEARER_CHALLENGE) + with db.transaction() as conn: + result = conn.execute( + "DELETE FROM sessions WHERE token_hash = %s AND expires_at > clock_timestamp()", + (token_hash(token),), + ) + if result.rowcount == 0: + raise unauthorized(BEARER_CHALLENGE) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def get_me(request: Request): + return current_user(request) + + +# ---------- chats ---------- + +@app.post("/api/v1/chats") +async def create_chat(request: Request): + user = current_user(request) + data = await read_json_object(request, ("title", "member_ids")) + title = clean_string(data, "title", 100) + raw_members = data["member_ids"] + if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100: + raise bad_request("member_ids должен быть списком из 1–100 UUID") + members = [parse_uuid(m, "member_ids") for m in raw_members] + if len(set(members)) != len(members): + raise bad_request("member_ids содержит дубликаты") + if user["id"] not in members: + raise bad_request("member_ids должен содержать вызывающего пользователя") + chat_id = uuid.uuid4() + member_uuids = [uuid.UUID(x) for x in members] + try: + with db.transaction() as conn: + found = conn.execute( + "SELECT id FROM users WHERE id = ANY(%s::uuid[])", (member_uuids,) + ).fetchall() + if len(found) != len(member_uuids): + raise bad_request("member_ids содержит неизвестного пользователя") + conn.execute("INSERT INTO chats(id, title) VALUES (%s,%s)", (chat_id, title)) + for member in member_uuids: + conn.execute( + "INSERT INTO chat_members(chat_id, user_id) VALUES (%s,%s)", + (chat_id, member), + ) + row = conn.execute( + "SELECT c.id, c.title, c.created_at, array_agg(m.user_id ORDER BY m.user_id) AS member_ids " + "FROM chats c JOIN chat_members m ON m.chat_id = c.id WHERE c.id = %s GROUP BY c.id", + (chat_id,), + ).fetchone() + except psycopg.errors.ForeignKeyViolation: + raise bad_request("member_ids содержит неизвестного пользователя") from None + return JSONResponse(chat_public(row), status_code=201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = current_user(request) + limit, cursor = page_params(request) + after = decode_cursor(cursor, "chats", user["id"]) if cursor else None + sql = ( + "SELECT c.id, c.title, c.created_at, array_agg(m.user_id ORDER BY m.user_id) AS member_ids " + "FROM chats c JOIN chat_members m ON m.chat_id = c.id WHERE m.user_id = %s" + ) + params = [uuid.UUID(user["id"])] + if after: + sql += " AND (c.created_at, c.id) > (%s, %s::uuid)" + params.extend([after[0], after[1]]) + sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s" + params.append(limit + 1) + with db.connection() as conn: + rows = conn.execute(sql, params).fetchall() + items = [chat_public(r) for r in rows] + return page_response(items, limit, "chats", user["id"], rows) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + _, chat = authenticated_chat(request, chat_id) + return chat + + +# ---------- messages ---------- + +@app.post("/api/v1/chats/{chat_id}/messages") +async def create_message(chat_id: str, request: Request): + """Idempotent POST: the ledger row and the message are committed atomically.""" + user, chat = authenticated_chat(request, chat_id) # replay re-checks session and access + data = await read_json_object(request, ("text",), optional=("attachment_ids",)) + text = data.get("text", "") + if not isinstance(text, str) or len(text) > 4000: + raise bad_request("text должен быть строкой до 4000 символов") + raw_attachments = data.get("attachment_ids", []) + if not isinstance(raw_attachments, list) or len(raw_attachments) > 10: + raise bad_request("attachment_ids должен быть списком до 10 UUID") + attachment_ids = [parse_uuid(x, "attachment_ids") for x in raw_attachments] + if len(set(attachment_ids)) != len(attachment_ids): + raise bad_request("attachment_ids содержит дубликаты") + if not text.strip() and not attachment_ids: + raise bad_request("Нужен непустой текст или хотя бы одно вложение") + key = idempotency_key(request) + # Canonical body hash: key order does not matter; omitted attachment_ids == []. + canonical = json.dumps( + {"text": text, "attachment_ids": sorted(attachment_ids)}, + sort_keys=True, separators=(",", ":"), + ) + request_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + scope_path = request.url.path + actor_id = uuid.UUID(user["id"]) + + for _attempt in range(5): + try: + with db.transaction() as conn: + previous = conn.execute( + "SELECT request_hash, status_code, response_body FROM idempotency_records " + "WHERE user_id=%s AND method=%s AND path=%s AND key=%s FOR UPDATE", + (actor_id, "POST", scope_path, key), + ).fetchone() + if previous is not None: + if previous["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Тот же Idempotency-Key использован с другим телом") + return JSONResponse(previous["response_body"], status_code=previous["status_code"]) + + if attachment_ids: + found = conn.execute( + "SELECT id FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s", + ([uuid.UUID(x) for x in attachment_ids], uuid.UUID(chat["id"])), + ).fetchall() + if len(found) != len(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено в этом чате") + message_id = uuid.uuid4() + conn.execute( + "INSERT INTO messages(id, chat_id, sender_id, text, attachment_ids) " + "VALUES (%s,%s,%s,%s,%s::uuid[])", + (message_id, uuid.UUID(chat["id"]), actor_id, text, + [uuid.UUID(x) for x in attachment_ids]), + ) + row = conn.execute("SELECT * FROM messages WHERE id = %s", (message_id,)).fetchone() + body = message_public(row) + conn.execute( + "INSERT INTO idempotency_records" + "(user_id, method, path, key, request_hash, status_code, response_body) " + "VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb)", + (actor_id, "POST", scope_path, key, request_hash, 201, + json.dumps(body, ensure_ascii=False)), + ) + return JSONResponse(body, status_code=201) + except psycopg.errors.UniqueViolation: + # Another replica won the race for this key. Roll back and read its + # committed response on the next attempt instead of creating a duplicate. + continue + raise ApiError(409, "conflict", "Не удалось зафиксировать Idempotency-Key") + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + _, chat = authenticated_chat(request, chat_id) + limit, cursor = page_params(request) + after = decode_cursor(cursor, "messages", chat["id"]) if cursor else None + sql = "SELECT * FROM messages WHERE chat_id = %s" + params = [uuid.UUID(chat["id"])] + if after: + sql += " AND (created_at, id) > (%s, %s::uuid)" + params.extend([after[0], after[1]]) + sql += " ORDER BY created_at, id LIMIT %s" + params.append(limit + 1) + with db.connection() as conn: + rows = conn.execute(sql, params).fetchall() + items = [message_public(r) for r in rows] + return page_response(items, limit, "messages", chat["id"], rows) + + +# ---------- attachments (lab4) ---------- + +def attachment_for(request, attachment_id): + """An attachment is visible only to members of its chat; outsiders get a plain 404.""" + user = current_user(request) + row = None + try: + aid = uuid.UUID(attachment_id) + except (ValueError, TypeError, AttributeError): + raise bad_request("Некорректный UUID вложения") from None + with db.connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a " + "JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (aid, uuid.UUID(user["id"])), + ).fetchone() + if row is None: + raise not_found() + return row + + +@app.post("/api/v1/chats/{chat_id}/attachments") +async def upload_attachment(chat_id: str, request: Request): + user, chat = authenticated_chat(request, chat_id) + filename = validate_filename(request.query_params.get("filename")) + declared = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg") + data = await read_raw(request, MAX_UPLOAD_BYTES) + try: + content_type, width, height = probe(data) + except InvalidImage: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG или JPEG") from None + if declared != content_type: + raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому") + if width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + digest = hashlib.sha256(data).hexdigest() + object_key = uuid.uuid4().hex + attachment_id = uuid.uuid4() + s3.put_object(object_key, data, content_type) + try: + with db.transaction() as conn: + conn.execute( + "INSERT INTO attachments(id, chat_id, owner_id, filename, content_type, " + "size_bytes, sha256, object_key, status, variants, error_code) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'ready','[]'::jsonb,NULL)", + (attachment_id, uuid.UUID(chat["id"]), uuid.UUID(user["id"]), + filename, content_type, len(data), digest, object_key), + ) + row = conn.execute("SELECT * FROM attachments WHERE id = %s", (attachment_id,)).fetchone() + except Exception: + # Best-effort compensation: no permanently unaccounted S3 object on a DB failure. + s3.delete_object(object_key) + raise + return JSONResponse(attachment_public(row), status_code=201) + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + return attachment_public(attachment_for(request, attachment_id)) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request): + row = attachment_for(request, attachment_id) + variant = request.query_params.get("variant", "original") + if variant != "original": + raise not_found() # lab4 declares no variants + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + body, content_type = s3.get_object(row["object_key"]) + return Response(content=body, media_type=content_type) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 00000000..7c59b2e9 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,69 @@ +"""MinIO/S3 client shared by all API replicas. Bucket bootstrap is idempotent.""" +import logging +import os + +log = logging.getLogger("messenger.s3") + +client = None +bucket = None + + +def start(): + global client, bucket + import boto3 + from botocore.config import Config + from botocore.exceptions import ClientError + endpoint = os.environ.get("S3_ENDPOINT", "http://minio:9000") + bucket = os.environ.get("S3_BUCKET", "messenger") + client = boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=os.environ.get("S3_ACCESS_KEY", "messenger"), + aws_secret_access_key=os.environ.get("S3_SECRET_KEY", "local-only-synthetic-secret"), + region_name="us-east-1", + config=Config(signature_version="s3v4", connect_timeout=3, read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}), + ) + _bootstrap_bucket() + + +def _bootstrap_bucket(): + from botocore.exceptions import ClientError + try: + client.head_bucket(Bucket=bucket) + return + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in ("404", "NoSuchBucket", "NotFound"): + raise + try: + client.create_bucket(Bucket=bucket) + log.info("created bucket %s", bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise + + +def put_object(key, body, content_type): + client.put_object(Bucket=bucket, Key=key, Body=body, ContentType=content_type) + + +def get_object(key): + response = client.get_object(Bucket=bucket, Key=key) + return response["Body"].read(), response.get("ContentType", "application/octet-stream") + + +def delete_object(key): + client.delete_object(Bucket=bucket, Key=key) + + +def ready(): + if client is None: + return False + try: + client.head_bucket(Bucket=bucket) + return True + except Exception: + log.exception("S3 readiness check failed") + return False diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..5fec64a1 --- /dev/null +++ b/app/security.py @@ -0,0 +1,66 @@ +"""Password KDF and HTTP credential parsing (Basic / opaque Bearer).""" +import base64 +import binascii +import hashlib +import hmac +import os +import re + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}\Z") +SCRYPT_N = 2 ** 14 +SCRYPT_R = 8 +SCRYPT_P = 1 +SCRYPT_LEN = 32 + + +def hash_password(password): + salt = os.urandom(16) + derived = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P, dklen=SCRYPT_LEN + ) + return "scrypt$%d$%d$%d$%s$%s" % ( + SCRYPT_N, SCRYPT_R, SCRYPT_P, salt.hex(), derived.hex() + ) + + +def verify_password(password, stored): + try: + scheme, n, r, p, salt_hex, digest_hex = stored.split("$") + if scheme != "scrypt": + return False + derived = hashlib.scrypt( + password.encode("utf-8"), + salt=bytes.fromhex(salt_hex), + n=int(n), r=int(r), p=int(p), dklen=len(bytes.fromhex(digest_hex)), + ) + return hmac.compare_digest(derived, bytes.fromhex(digest_hex)) + except (ValueError, TypeError): + return False + + +def token_hash(token): + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def parse_basic(header): + """Return (username, password) or None. Colon inside the password is allowed.""" + if not header or not header.lower().startswith("basic "): + return None + encoded = header[6:].strip() + try: + raw = base64.b64decode(encoded, validate=True).decode("utf-8") + except (ValueError, UnicodeDecodeError, binascii.Error): + return None + if ":" not in raw: + return None + username, password = raw.split(":", 1) + if not USERNAME_RE.fullmatch(username) or not 12 <= len(password) <= 128: + return None + return username, password + + +def bearer_token(header): + if not header or not header.lower().startswith("bearer "): + return None + token = header[7:].strip() + return token or None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..ea762236 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,102 @@ +# Lab3 level 3: single public entry point (nginx) -> 2 interchangeable API replicas +# -> one shared PostgreSQL database. Sessions and the idempotency ledger live in the DB. +services: + db: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set a synthetic local password in .env} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"] + interval: 3s + timeout: 3s + retries: 20 + + minio: + image: bitnamilegacy/minio:latest + restart: unless-stopped + user: "0:0" # Bitnami setup chowns the named volume, then drops to its own user + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY:-messenger} + MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:?set a synthetic local S3 secret in .env} + MINIO_DATA_DIR: /data + MINIO_DEFAULT_BUCKETS: ${S3_BUCKET:-messenger} + MINIO_CONSOLE_PORT_NUMBER: "9001" + volumes: + - s3data:/data + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/minio/health/live > /dev/null"] + interval: 5s + timeout: 3s + retries: 30 + + api1: + build: . + restart: unless-stopped + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab + INSTANCE_ALIAS: api1 + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-messenger} + S3_SECRET_KEY: ${S3_SECRET_KEY:?set a synthetic local S3 secret in .env} + depends_on: + db: + condition: service_healthy + minio: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2)"] + interval: 5s + timeout: 3s + retries: 20 + + api2: + build: . + restart: unless-stopped + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab + INSTANCE_ALIAS: api2 + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-messenger} + S3_SECRET_KEY: ${S3_SECRET_KEY:?set a synthetic local S3 secret in .env} + depends_on: + db: + condition: service_healthy + minio: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2)"] + interval: 5s + timeout: 3s + retries: 20 + + lb: + image: nginx:1.27-alpine + restart: unless-stopped + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "8080:8080" + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + +volumes: + pgdata: + s3data: diff --git a/evidence/lifecycle.txt b/evidence/lifecycle.txt new file mode 100644 index 00000000..7abd7c8f --- /dev/null +++ b/evidence/lifecycle.txt @@ -0,0 +1,118 @@ + +=== 0. Restore all replicas before the recorded run === +ready after 0 s +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api1-1 mtusi2026lab4-api1 "python -m app" api1 2 minutes ago Up 8 seconds (healthy) 8080/tcp +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "python -m app" api2 2 minutes ago Up About a minute (healthy) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.sтАж" db 4 minutes ago Up About a minute (healthy) 5432/tcp +mtusi2026lab4-lb-1 nginx:1.27-alpine "/docker-entrypoint.тАж" lb 4 minutes ago Up 2 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-minio-1 bitnamilegacy/minio:latest "/opt/bitnami/scriptтАж" minio 3 minutes ago Up About a minute (healthy) 9000-9001/tcp + +=== 1. Stack: LB + 2 API + PostgreSQL + private S3 === +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api1-1 mtusi2026lab4-api1 "python -m app" api1 2 minutes ago Up 8 seconds (healthy) 8080/tcp +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "python -m app" api2 2 minutes ago Up About a minute (healthy) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.sтАж" db 4 minutes ago Up About a minute (healthy) 5432/tcp +mtusi2026lab4-lb-1 nginx:1.27-alpine "/docker-entrypoint.тАж" lb 4 minutes ago Up 2 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-minio-1 bitnamilegacy/minio:latest "/opt/bitnami/scriptтАж" minio 3 minutes ago Up About a minute (healthy) 9000-9001/tcp + +=== 2. X-Instance-Id through the LB === +x-instance-id: api2 +x-instance-id: api1 +x-instance-id: api2 +x-instance-id: api1 +x-instance-id: api2 +x-instance-id: api1 +x-instance-id: api2 +x-instance-id: api1 + +=== 3. Public smoke (upload, checksum, byte download, outsider 404, idempotency) === +ready after 0 s +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. + +=== 4. Metadata in PostgreSQL (bytes are NOT in the DB) === + users | sessions | attachments | messages | idem +-------+----------+-------------+----------+------ + 18 | 16 | 5 | 212 | 212 +(1 row) + + id | filename | content_type | size_bytes | status | variants | error_code +--------------------------------------+------------+--------------+------------+--------+----------+------------ + 63d0a1c8-0a9a-4547-83f5-2c697943ec84 | sample.png | image/png | 111817 | ready | [] | + 33bf0b8d-2ebe-4a16-83e4-9073a264f0a9 | sample.png | image/png | 111817 | ready | [] | + d26993db-4208-4984-8847-292ea8a1542e | sample.png | image/png | 111817 | ready | [] | +(3 rows) + + name +-------------- + 001_init.sql +(1 row) + +=== 5. Persistence hook (restart DB + S3 + both API replicas, volumes preserved) === +cmd : Container mtusi2026lab4-api1-1 Restarting +C:\Users\Tankist\AppData\Local\Temp\opencode\capture_lab4.ps1:48 знак:3 ++ cmd /c 'scripts\restart.cmd' 2>&1 ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : NotSpecified: ( Container mtus...1-1 Restarting :String) [], RemoteException + + FullyQualifiedErrorId : NativeCommandError + + Container mtusi2026lab4-db-1 Restarting + Container mtusi2026lab4-minio-1 Restarting + Container mtusi2026lab4-api2-1 Restarting + Container mtusi2026lab4-minio-1 Started + Container mtusi2026lab4-api1-1 Started + Container mtusi2026lab4-db-1 Started + Container mtusi2026lab4-api2-1 Started +ready after 6 s +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api1-1 mtusi2026lab4-api1 "python -m app" api1 2 minutes ago Up 6 seconds (healthy) 8080/tcp +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "python -m app" api2 2 minutes ago Up 6 seconds (healthy) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.sтАж" db 4 minutes ago Up 6 seconds (healthy) 5432/tcp +mtusi2026lab4-lb-1 nginx:1.27-alpine "/docker-entrypoint.тАж" lb 4 minutes ago Up 2 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-minio-1 bitnamilegacy/minio:latest "/opt/bitnami/scriptтАж" minio 3 minutes ago Up 6 seconds (healthy) 9000-9001/tcp + +=== 6. Persistence test (same token AND same original bytes after restart) === +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +ready after 0 s + +=== 7. Failover hook: stop exactly one API replica === +cmd : Container mtusi2026lab4-api1-1 Stopping +C:\Users\Tankist\AppData\Local\Temp\opencode\capture_lab4.ps1:59 знак:3 ++ cmd /c 'set TARGET_INSTANCE_ID=api1&& scripts\stop-one.cmd' 2>&1 ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : NotSpecified: ( Container mtus...pi1-1 Stopping :String) [], RemoteException + + FullyQualifiedErrorId : NativeCommandError + + Container mtusi2026lab4-api1-1 Stopped +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "python -m app" api2 2 minutes ago Up 5 seconds (health: starting) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.sтАж" db 4 minutes ago Up 5 seconds (healthy) 5432/tcp +mtusi2026lab4-lb-1 nginx:1.27-alpine "/docker-entrypoint.тАж" lb 4 minutes ago Up 2 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-minio-1 bitnamilegacy/minio:latest "/opt/bitnami/scriptтАж" minio 3 minutes ago Up 5 seconds (healthy) 9000-9001/tcp +ready after 0 s +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api1-1 mtusi2026lab4-api1 "python -m app" api1 2 minutes ago Up 8 seconds (healthy) 8080/tcp +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "python -m app" api2 2 minutes ago Up 14 seconds (healthy) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.sтАж" db 4 minutes ago Up 14 seconds (healthy) 5432/tcp +mtusi2026lab4-lb-1 nginx:1.27-alpine "/docker-entrypoint.тАж" lb 4 minutes ago Up 3 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-minio-1 bitnamilegacy/minio:latest "/opt/bitnami/scriptтАж" minio 3 minutes ago Up 14 seconds (healthy) 9000-9001/tcp + +=== 8. Failover test (download via the surviving replica) === +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 105, "failed_attempts": 1, "first_recovery_seconds": 0.624} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +ready after 0 s +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api1-1 mtusi2026lab4-api1 "python -m app" api1 2 minutes ago Up 26 seconds (healthy) 8080/tcp +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "python -m app" api2 2 minutes ago Up Less than a second (health: starting) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.sтАж" db 4 minutes ago Up 33 seconds (healthy) 5432/tcp +mtusi2026lab4-lb-1 nginx:1.27-alpine "/docker-entrypoint.тАж" lb 4 minutes ago Up 3 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-minio-1 bitnamilegacy/minio:latest "/opt/bitnami/scriptтАж" minio 4 minutes ago Up 33 seconds (healthy) 9000-9001/tcp + +=== 9. Own unit tests and offline check === +---------------------------------------------------------------------- +Ran 14 tests in 0.259s + +OK +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..e1e649aa --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,30 @@ +events { + worker_connections 1024; +} + +http { + access_log /dev/stdout; + error_log /dev/stderr warn; + + upstream api_backend { + server api1:8080 max_fails=1 fail_timeout=5s; + server api2:8080 max_fails=1 fail_timeout=5s; + } + + server { + listen 8080; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Connection ""; + # Retry the other replica on a transport failure or a 5xx from a dying backend. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + } + } +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..03acad78 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,77 @@ +-- Lab2 initial schema. Applied once by app/db.py; safe to keep volumes on restart. +-- Source of truth for users, chats, memberships, messages and sessions is PostgreSQL. + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name TEXT NOT NULL CHECK (length(display_name) BETWEEN 1 AND 100), + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL CHECK (length(title) BETWEEN 1 AND 100), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX IF NOT EXISTS ix_chat_members_user ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL CHECK (length(text) <= 4000), + attachment_ids UUID[] NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); +-- Keyset pagination reads strictly one (created_at, id) window; no full-history scan. +CREATE INDEX IF NOT EXISTS ix_messages_history ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS ix_sessions_user ON sessions (user_id); +CREATE INDEX IF NOT EXISTS ix_sessions_expiry ON sessions (expires_at); + +-- Shared idempotency ledger for POST /chats/{id}/messages (lab3+). +-- Uniqueness scope: user + HTTP method + chat path + key. Stored alongside the +-- message insert in the SAME transaction, so a retry cannot create a duplicate. +CREATE TABLE IF NOT EXISTS idempotency_records ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code INTEGER NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, method, path, key) +); +CREATE INDEX IF NOT EXISTS ix_idempotency_created ON idempotency_records (created_at); + +-- Attachment metadata lives in PostgreSQL; bytes live in a private S3 bucket. +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename TEXT NOT NULL CHECK (length(filename) BETWEEN 1 AND 255), + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes INTEGER NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + variants JSONB NOT NULL DEFAULT '[]'::jsonb, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS ix_attachments_chat ON attachments (chat_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6f143d1d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.135.3 +uvicorn==0.44.0 +psycopg[binary]==3.3.3 +boto3==1.40.67 diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..fa8cd2b0 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,5 @@ +@echo off +REM Windows equivalent of scripts/restart.sh for the local persistence hook. +cd /d "%~dp0.." +docker compose --env-file .env restart db minio api1 api2 +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..22f5941d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +# Explicit persistence hook for lab3. +# Restart both API replicas and PostgreSQL. Volumes are preserved: no down -v, +# no flush, no re-seed. The load balancer stays in front of the replicas. +cd "$(dirname "$0")/.." +docker compose restart db minio api1 api2 diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..e8e24ee9 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,10 @@ +@echo off +REM Windows equivalent of scripts/stop-one.sh for the local failover hook. +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api1" goto stop +if "%TARGET_INSTANCE_ID%"=="api2" goto stop +echo unknown target instance: %TARGET_INSTANCE_ID% >&2 +exit /b 1 +:stop +docker compose --env-file .env stop %TARGET_INSTANCE_ID% +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..5d60d149 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +# Explicit failover hook: stop exactly ONE API replica selected by the test. +# TARGET_INSTANCE_ID is the X-Instance-Id served by that replica (api1 or api2). +# DB, volume and load balancer are untouched. Return quickly; leave the replica stopped. +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + api1|api2) docker compose stop "$TARGET_INSTANCE_ID" ;; + *) echo "unknown target instance: $TARGET_INSTANCE_ID" >&2; exit 1 ;; +esac diff --git a/student-tests/test_security.py b/student-tests/test_security.py new file mode 100644 index 00000000..d51620c3 --- /dev/null +++ b/student-tests/test_security.py @@ -0,0 +1,112 @@ +"""Own tests for the security helpers and keyset cursors (stdlib unittest).""" +import base64 +import datetime as dt +import json +import sys +import unittest +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from app import main as api +from app.security import (bearer_token, hash_password, parse_basic, + token_hash, verify_password) + + +class PasswordTests(unittest.TestCase): + def test_kdf_round_trip_and_uniqueness(self): + first = hash_password("T3st-passphrase-123") + second = hash_password("T3st-passphrase-123") + self.assertNotEqual(first, second, "salt must be unique") + self.assertTrue(first.startswith("scrypt$")) + self.assertTrue(verify_password("T3st-passphrase-123", first)) + self.assertFalse(verify_password("wrong-password", first)) + + def test_unicode_and_colon_password(self): + secret = "пароль:с:двоеточием-123" + stored = hash_password(secret) + self.assertTrue(verify_password(secret, stored)) + self.assertFalse(verify_password("пароль", stored)) + + +class CredentialTests(unittest.TestCase): + def test_basic_colon_inside_password(self): + raw = "u_alice_123:T3st-with:colon-12345" + header = "Basic " + base64.b64encode(raw.encode()).decode() + self.assertEqual(parse_basic(header), ("u_alice_123", "T3st-with:colon-12345")) + + def test_basic_rejects_short_password_and_bad_scheme(self): + raw = "u_alice_123:short" + self.assertIsNone(parse_basic("Basic " + base64.b64encode(raw.encode()).decode())) + self.assertIsNone(parse_basic("Bearer abcdef")) + + def test_bearer_only(self): + self.assertEqual(bearer_token("Bearer abc"), "abc") + self.assertIsNone(bearer_token("Basic abc")) + self.assertIsNone(bearer_token(None)) + + def test_token_hash_is_stable_and_not_the_token(self): + token = "s" * 43 + self.assertEqual(token_hash(token), token_hash(token)) + self.assertNotEqual(token_hash(token), token) + + +class IdempotencyTests(unittest.TestCase): + def test_key_pattern(self): + self.assertIsNotNone(api.IDEMPOTENCY_KEY_RE.fullmatch('abc-123.XYZ:9_')) + self.assertIsNone(api.IDEMPOTENCY_KEY_RE.fullmatch('bad key')) + self.assertIsNone(api.IDEMPOTENCY_KEY_RE.fullmatch('')) + + def test_canonical_body_hash_ignores_key_order(self): + import hashlib + a = json.dumps({'text': 'Привет'}, sort_keys=True, separators=(',', ':')) + b = json.dumps({'text': 'Привет'}, separators=(',', ':')) + self.assertEqual(hashlib.sha256(a.encode()).hexdigest(), + hashlib.sha256(b.encode()).hexdigest()) + + +class AttachmentTests(unittest.TestCase): + def test_filename_rejects_path_traversal(self): + for bad in ('../x.png', 'a/b.png', 'a\\b.png', 'a\x00b', 'a\nb'): + with self.assertRaises(api.ApiError): + api.validate_filename(bad) + self.assertEqual(api.validate_filename('photo.png'), 'photo.png') + + def test_probe_real_fixture(self): + from app.images import probe + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + content_type, width, height = probe(data) + self.assertEqual(content_type, 'image/png') + self.assertGreater(width, 0) + self.assertGreater(height, 0) + + def test_probe_rejects_garbage(self): + from app.images import InvalidImage, probe + with self.assertRaises(InvalidImage): + probe(b'not an image at all') + + +class CursorTests(unittest.TestCase): + def test_round_trip(self): + created = dt.datetime(2026, 11, 16, 12, 0, 0, 123456, tzinfo=dt.timezone.utc) + item_id = uuid.uuid4() + encoded = api.encode_cursor("messages", "scope-1", created, item_id) + back_created, back_id = api.decode_cursor(encoded, "messages", "scope-1") + self.assertEqual(back_created, created) + self.assertEqual(back_id, item_id) + + def test_wrong_scope_rejected(self): + created = dt.datetime.now(dt.timezone.utc) + encoded = api.encode_cursor("messages", "chat-A", created, uuid.uuid4()) + with self.assertRaises(api.ApiError): + api.decode_cursor(encoded, "messages", "chat-B") + + def test_garbage_rejected(self): + with self.assertRaises(api.ApiError): + api.decode_cursor("not-a-cursor", "messages", "chat-A") + + +if __name__ == "__main__": + unittest.main(verbosity=2)