From 2e23dbe09593c0fce7fe583557ef01c90befedbd Mon Sep 17 00:00:00 2001 From: "yurtaev.va" Date: Fri, 23 Oct 2026 18:54:43 +0300 Subject: [PATCH] feat: lab3 init --- .env.example | 15 +- Dockerfile | 10 + Makefile | 5 +- app/__init__.py | 0 app/main.py | 500 +++++++++++++++++++++++++++++++++ compose.yaml | 107 +++++++ infra/nginx.conf | 21 ++ infra/swarm-app.yaml | 66 +++++ infra/swarm-data.yaml | 41 +++ migrations/001_initial.sql | 38 +++ migrations/002_idempotency.sql | 10 + requirements.txt | 4 + scripts/lint.py | 49 ++++ scripts/migrate.py | 16 ++ scripts/restart.sh | 4 + scripts/stop-one.sh | 12 + 16 files changed, 886 insertions(+), 12 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 compose.yaml create mode 100644 infra/nginx.conf create mode 100644 infra/swarm-app.yaml create mode 100644 infra/swarm-data.yaml create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.txt create mode 100644 scripts/lint.py create mode 100644 scripts/migrate.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index d32b9be0..55f87a7b 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,6 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Local synthetic test only. Copy to .env and replace the password. APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +SESSION_TTL_SECONDS=3600 +POSTGRES_PASSWORD=replace-me-with-a-local-password +DB_POOL_SIZE=8 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..74909261 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.10-slim +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt && useradd -r -u 10001 app +COPY app ./app +COPY migrations ./migrations +COPY scripts/migrate.py ./scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"] diff --git a/Makefile b/Makefile index 5d86d171..f671ce5e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate lint test load test-persistence test-failover help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -15,6 +15,9 @@ check: validate: $(PYTHON) scripts/validate_openapi.py +lint: + $(PYTHON) scripts/lint.py + test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..fee71a09 --- /dev/null +++ b/app/main.py @@ -0,0 +1,500 @@ +"""Replicated messenger API with PostgreSQL-backed sessions and idempotency.""" + +import asyncio +import base64 +import binascii +import hashlib +import json +import os +import re +import secrets +import socket +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone + +import psycopg +import redis +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from psycopg.conninfo import make_conninfo +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException + + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger") +database_password = os.environ.get("DATABASE_PASSWORD") +if "DATABASE_PASSWORD_FILE" in os.environ: + with open(os.environ["DATABASE_PASSWORD_FILE"], encoding="utf-8") as secret_file: + database_password = secret_file.read().rstrip("\n") +if database_password is not None: + DATABASE_URL = make_conninfo( + host=os.environ.get("DATABASE_HOST", "postgres"), + dbname=os.environ.get("DATABASE_NAME", "messenger"), + user=os.environ.get("DATABASE_USER", "messenger"), + password=database_password, + ) +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") +SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) +MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384")) +POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10")) +CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1" +CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1" +CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0")))) +INSTANCE_ID = os.environ.get("INSTANCE_ID", "") or socket.gethostname() +ready_for_traffic = False +pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"}) +cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2) +metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0} +metrics_lock = threading.Lock() + + +class ApiError(Exception): + def __init__(self, status, code, message, challenge=None): + self.status, self.code, self.message, self.challenge = status, code, message, challenge + + +def invalid(message="Некорректный запрос"): + raise ApiError(400, "invalid_request", message) + + +def unavailable(): + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(scope, date, identifier): + raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def decode_cursor(scope, value): + if value is None: + return None + try: + raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + parts = json.loads(raw) + if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope + or not isinstance(parts[1], str) or not isinstance(parts[2], str)): + invalid("Некорректный cursor") + date = datetime.fromisoformat(parts[1].replace("Z", "+00:00")) + if date.tzinfo is None: + invalid("Некорректный cursor") + return date, str(uuid.UUID(parts[2])) + except (ValueError, TypeError, binascii.Error, IndexError): + invalid("Некорректный cursor") + + +def identifier(value): + try: + result = str(uuid.UUID(value)) + if result != value.lower() or len(value) != 36: + invalid("Некорректный UUID") + return result + except (ValueError, TypeError, AttributeError): + invalid("Некорректный UUID") + + +def text_field(value, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum: + invalid("Некорректное текстовое поле") + return value + + +def exact_body(value, fields): + if not isinstance(value, dict) or set(value) != set(fields): + invalid("Некорректные поля JSON") + + +async def body(request, fields): + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + declared = request.headers.get("content-length") + if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + chunks = bytearray() + async for chunk in request.stream(): + chunks.extend(chunk) + if len(chunks) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + try: + value = json.loads(chunks.decode("utf-8")) + except (UnicodeError, ValueError): + invalid("Некорректный JSON") + exact_body(value, fields) + return value + + +def user(row): + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])} + + +def message(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": stamp(row["created_at"])} + + +def chat(connection, row): + members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall() + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])} + + +def password_hash(password): + salt = os.urandom(16) + derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1) + return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex() + + +def password_ok(password, stored): + try: + _, n, r, p, salt, expected = stored.split(":") + actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p)) + return secrets.compare_digest(actual, bytes.fromhex(expected)) + except (ValueError, TypeError): + return False + + +def connection(): + try: + return pool.connection() + except psycopg.Error: + unavailable() + + +def actor(request): + auth = request.headers.get("authorization", "") + if not auth.startswith("Bearer ") or not auth[7:]: + raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"') + digest = hashlib.sha256(auth[7:].encode()).digest() + with connection() as db: + row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone() + if row is None: + raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"') + return row["user_id"] + + +def member(db, uid, cid): + row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def pagination(request): + raw = request.query_params.get("limit", "50") + if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100: + invalid("Некорректный limit") + return int(raw) + + +def bump(key): + with metrics_lock: + metrics[key] += 1 + + +def cache_get(key): + try: + raw = cache.get(key) + if raw is not None: + bump("history_cache_hit") + return json.loads(raw) + except (redis.RedisError, ValueError): + pass + return None + + +def cache_set(key, data): + try: + cache.set(key, json.dumps(data), ex=30) + except redis.RedisError: + pass + + +def history(db, cid, limit, cursor, version): + key = f"history:{cid}:{version}:{limit}:{cursor or '-'}" + hit = cache_get(key) if CACHE_ENABLED else None + if hit is not None: + return hit + lock_key = "lock:" + key + token = secrets.token_hex(16) + locked = False + if CACHE_ENABLED and CACHE_LOCK_ENABLED: + try: + locked = bool(cache.set(lock_key, token, nx=True, px=2000)) + if not locked: + bump("history_lock_wait") + for _ in range(20): + time.sleep(0.025) + hit = cache_get(key) + if hit is not None: + return hit + locked = bool(cache.set(lock_key, token, nx=True, px=2000)) + except redis.RedisError: + pass + if CACHE_MISS_DELAY_MS: + time.sleep(CACHE_MISS_DELAY_MS / 1000) + if cursor is None: + rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall() + else: + rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall() + bump("history_sql") + result = {"items": [message(row) for row in rows[:limit]], "next_cursor": None} + if len(rows) > limit: + last = rows[limit - 1] + result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"])) + if CACHE_ENABLED: + cache_set(key, result) + if locked: + try: + cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token) + except redis.RedisError: + pass + return result + + +def migrate(): + from pathlib import Path + files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql")) + with pool.connection() as db: + with db.transaction(): + db.execute("SELECT pg_advisory_xact_lock(2172002)") + db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)") + for file in files: + if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None: + db.execute(file.read_text()) + db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,)) + + +@asynccontextmanager +async def lifespan(_app): + global ready_for_traffic + pool.open() + pool.wait(timeout=20) + ready_for_traffic = True + yield + ready_for_traffic = False + pool.close() + + +app = FastAPI(lifespan=lifespan) + + +@app.exception_handler(StarletteHTTPException) +async def http_error(request, error): + request_id = request.state.request_id + code = "not_found" if error.status_code == 404 else "invalid_request" + return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code) + + +@app.middleware("http") +async def envelope(request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except ApiError as exc: + response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status) + if exc.challenge: + response.headers["WWW-Authenticate"] = exc.challenge + except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable, PoolTimeout): + response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503) + except Exception: + response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(): + if not ready_for_traffic: + unavailable() + with connection() as db: + db.execute("SELECT 1") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await body(request, ("username", "display_name", "password")) + name = data["username"] + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name): + invalid("Некорректный username") + display = text_field(data["display_name"], 100) + password = data["password"] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + invalid("Некорректный пароль") + try: + return await asyncio.to_thread(insert_user, name, display, password) + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username уже занят") from None + + +def insert_user(name, display, password): + digest = password_hash(password) + with connection() as db: + row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone() + return user(row) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + async for chunk in request.stream(): + if chunk: + invalid("Тело запроса не допускается") + auth = request.headers.get("authorization", "") + try: + if not auth.startswith("Basic "): + raise ValueError() + decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8") + name, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None + return await asyncio.to_thread(create_session, name, password) + + +def create_session(name, password): + with connection() as db: + row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone() + if row is None or not password_ok(password, row["password_hash"]): + raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') + token = secrets.token_urlsafe(32) + expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL) + db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry)) + return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + actor(request) + token = request.headers["authorization"][7:] + with connection() as db: + db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),)) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def me(request: Request): + uid = actor(request) + with connection() as db: + row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone() + return user(row) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + uid = actor(request) + data = await body(request, ("title", "member_ids")) + title = text_field(data["title"], 100) + raw = data["member_ids"] + if not isinstance(raw, list) or not 1 <= len(raw) <= 100: + invalid("Некорректный список участников") + members = [identifier(item) for item in raw] + if len(set(members)) != len(members) or str(uid) not in members: + invalid("Некорректный список участников") + return await asyncio.to_thread(insert_chat, uid, title, members) + + +def insert_chat(uid, title, members): + with connection() as db: + with db.transaction(): + count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"] + if count != len(members): + invalid("Неизвестный участник") + row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone() + for member_id in members: + db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id)) + db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],)) + return chat(db, row) + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + uid = actor(request) + limit = pagination(request) + cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor")) + with connection() as db: + if cursor is None: + rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall() + else: + valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone() + if valid is None: + invalid("Некорректный cursor") + rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall() + items = [chat(db, row) for row in rows[:limit]] + next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + with connection() as db: + return chat(db, member(db, uid, cid)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + key = request.headers.get("idempotency-key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII): + invalid("Некорректный Idempotency-Key") + data = await body(request, ("text",)) + content = text_field(data["text"], 4000) + return await asyncio.to_thread(insert_message, uid, cid, content, key) + + +def insert_message(uid, cid, content, key): + with connection() as db: + with db.transaction(): + member(db, uid, cid) + # The unique constraint serializes concurrent requests on any replica. + db.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()", (uid, cid, key)) + claimed = db.execute("INSERT INTO message_idempotency(user_id,chat_id,key,body_hash,expires_at) VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING RETURNING key", (uid, cid, key, hashlib.sha256(json.dumps({"text": content}, ensure_ascii=False, sort_keys=True).encode()).digest())).fetchone() + if claimed is None: + previous = db.execute("SELECT body_hash,message_id FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", (uid, cid, key)).fetchone() + expected = hashlib.sha256(json.dumps({"text": content}, ensure_ascii=False, sort_keys=True).encode()).digest() + if previous is None or previous["body_hash"] != expected: + raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения") + return message(db.execute("SELECT * FROM messages WHERE id=%s", (previous["message_id"],)).fetchone()) + row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone() + db.execute("UPDATE message_idempotency SET message_id=%s WHERE user_id=%s AND chat_id=%s AND key=%s", (row["id"], uid, cid, key)) + db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,)) + return message(row) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + limit = pagination(request) + cursor = decode_cursor(cid, request.query_params.get("cursor")) + with connection() as db: + member(db, uid, cid) + if cursor is not None: + valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone() + if valid is None: + invalid("Некорректный cursor") + version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"] + return history(db, cid, limit, cursor, version) + + +@app.get("/internal/cache-metrics") +def cache_metrics(request: Request): + if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"): + raise ApiError(404, "not_found", "Объект не найден") + with metrics_lock: + return dict(metrics) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..261dc824 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,107 @@ +services: + postgres: + image: postgres:16.8-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 2s + retries: 20 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"] + networks: [data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 20 + deploy: + resources: + limits: + cpus: "0.5" + memory: 192M + + migrate: + build: . + command: ["python", "-m", "scripts.migrate"] + environment: &database_env + DATABASE_HOST: postgres + DATABASE_NAME: messenger + DATABASE_USER: messenger + DATABASE_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + networks: [data] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + environment: + <<: *database_env + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_SIZE: ${DB_POOL_SIZE:-8} + networks: [front, data] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + restart: unless-stopped + stop_grace_period: 15s + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.75" + memory: 256M + + lb: + image: nginx:1.27.5-alpine + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + networks: [front] + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 3s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + +networks: + front: + data: + internal: true + +volumes: + postgres_data: diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..9aa22f18 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,21 @@ +events { worker_connections 1024; } +http { + access_log off; + resolver 127.0.0.11 valid=1s ipv6=off; + server { + listen 8080; + client_max_body_size 16k; + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + proxy_next_upstream error timeout http_502 http_503; + proxy_next_upstream_tries 2; + location / { + set $api http://api:8080; + proxy_pass $api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Request-Id $request_id; + } + } +} diff --git a/infra/swarm-app.yaml b/infra/swarm-app.yaml new file mode 100644 index 00000000..4149471c --- /dev/null +++ b/infra/swarm-app.yaml @@ -0,0 +1,66 @@ +version: "3.8" +services: + api: + image: lab3-api:local + environment: + DATABASE_HOST: lab3data_postgres + DATABASE_NAME: messenger + DATABASE_USER: messenger + DATABASE_PASSWORD_FILE: /run/secrets/lab3_db_password + REDIS_URL: redis://lab3data_redis:6379/0 + SESSION_TTL_SECONDS: "3600" + DB_POOL_SIZE: "8" + secrets: [lab3_db_password] + networks: [data] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + retries: 2 + deploy: + replicas: 2 + placement: + constraints: [node.labels.lab3_compute == true] + update_config: + parallelism: 1 + delay: 3s + order: start-first + failure_action: rollback + rollback_config: + parallelism: 1 + order: start-first + restart_policy: + condition: on-failure + resources: + limits: + cpus: "0.75" + memory: 256M + lb: + image: nginx:1.27.5-alpine + configs: + - source: lab3_nginx_conf + target: /etc/nginx/nginx.conf + networks: [data] + ports: + - target: 8080 + published: 8080 + protocol: tcp + mode: host + deploy: + replicas: 1 + placement: + constraints: [node.labels.lab3_storage == true] + resources: + limits: + cpus: "0.5" + memory: 128M +networks: + data: + external: true + name: lab3data_data +secrets: + lab3_db_password: + external: true +configs: + lab3_nginx_conf: + external: true diff --git a/infra/swarm-data.yaml b/infra/swarm-data.yaml new file mode 100644 index 00000000..d9e3979d --- /dev/null +++ b/infra/swarm-data.yaml @@ -0,0 +1,41 @@ +version: "3.8" +services: + postgres: + image: postgres:16.8-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/lab3_db_password + secrets: [lab3_db_password] + volumes: + - postgres_data:/var/lib/postgresql/data + networks: [data] + deploy: + replicas: 1 + placement: + constraints: [node.labels.lab3_storage == true] + resources: + limits: + cpus: "1.0" + memory: 512M + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"] + networks: [data] + deploy: + replicas: 1 + placement: + constraints: [node.labels.lab3_storage == true] + resources: + limits: + cpus: "0.5" + memory: 192M +networks: + data: + driver: overlay + attachable: true +volumes: + postgres_data: +secrets: + lab3_db_password: + external: true diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..2cdfbf05 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,38 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$') +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); +CREATE INDEX chats_order_idx ON chats (created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE TABLE chat_versions ( + chat_id uuid PRIMARY KEY REFERENCES chats(id), + version bigint NOT NULL DEFAULT 0 +); +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..9775cce7 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key varchar(128) NOT NULL, + body_hash bytea NOT NULL, + message_id uuid UNIQUE REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX message_idempotency_expiry_idx ON message_idempotency (expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..1dc75f74 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +psycopg[binary,pool]==3.2.6 +redis==5.2.1 diff --git a/scripts/lint.py b/scripts/lint.py new file mode 100644 index 00000000..89935e33 --- /dev/null +++ b/scripts/lint.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Offline lint for this small Python project: syntax, whitespace, unused imports.""" + +import ast +from pathlib import Path +import sys + + +ROOT = Path(__file__).resolve().parents[1] +FILES = [ROOT / "app/main.py", ROOT / "scripts/migrate.py", ROOT / "scripts/lint.py"] + + +def check(path): + source = path.read_text() + issues = [] + try: + tree = ast.parse(source, filename=str(path)) + except SyntaxError as error: + return [f"{path}:{error.lineno}: {error.msg}"] + used = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports = [(item.asname or item.name.split(".")[0], node.lineno) for item in node.names] + elif isinstance(node, ast.ImportFrom): + imports = [(item.asname or item.name, node.lineno) for item in node.names] + else: + continue + for name, line in imports: + if name not in used: + issues.append(f"{path}:{line}: unused import {name}") + for number, line in enumerate(source.splitlines(), 1): + if line.rstrip() != line: + issues.append(f"{path}:{number}: trailing whitespace") + if "\t" in line: + issues.append(f"{path}:{number}: tab character") + return issues + + +def main(): + issues = [issue for path in FILES for issue in check(path)] + if issues: + print("\n".join(issues)) + return 1 + print(f"PASS: lint ({len(FILES)} Python files)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..c644e75f --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,16 @@ +"""Run schema changes once before starting API replicas.""" + +from app.main import migrate, pool + + +def main(): + pool.open() + try: + pool.wait(timeout=20) + migrate() + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..1522bf0e --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres redis api lb >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2b8fe9fa --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +case "$TARGET_INSTANCE_ID" in + *[!a-fA-F0-9]*|'') echo "Invalid instance id" >&2; exit 2 ;; +esac +service=$(docker inspect --format '{{ index .Config.Labels "com.docker.compose.service" }}' "$TARGET_INSTANCE_ID") +if [ "$service" != api ]; then + echo "Target is not a Compose API replica" >&2 + exit 2 +fi +docker stop -t 1 "$TARGET_INSTANCE_ID" >/dev/null