From a592b519ee11f999ddb8b204f3ec8a595667ee26 Mon Sep 17 00:00:00 2001 From: Getlles Date: Mon, 9 Nov 2026 17:34:11 +0300 Subject: [PATCH 1/4] lab3 --- Dockerfile | 22 ++ app.py | 644 ++++++++++++++++++++++++++++++++++++++ compose.yaml | 130 ++++++++ migrations/init.sql | 41 +++ nginx.conf | 28 ++ requirements.txt | 13 + scripts/restart.bat | 3 + scripts/restart.sh | 3 + scripts/rolling-update.sh | 35 +++ scripts/stop-one.sh | 12 + tests/client.py | 2 +- tests/custom_tests.py | 109 +++++++ tests/resilience.py | 2 +- 13 files changed, 1042 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 app.py create mode 100644 compose.yaml create mode 100644 migrations/init.sql create mode 100644 nginx.conf create mode 100644 requirements.txt create mode 100644 scripts/restart.bat create mode 100644 scripts/restart.sh create mode 100644 scripts/rolling-update.sh create mode 100644 scripts/stop-one.sh create mode 100644 tests/custom_tests.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a33beebc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11.7-slim-bookworm AS builder + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir --prefix=/install -r requirements.txt + +FROM python:3.11.7-slim-bookworm AS runner + +RUN useradd -u 10001 -m appuser +WORKDIR /home/appuser + +COPY --from=builder /install /usr/local +COPY app.py . + +USER appuser +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" + +ENV PYTHONUNBUFFERED=1 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"] \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 00000000..7a9ff3a8 --- /dev/null +++ b/app.py @@ -0,0 +1,644 @@ +import os +import base64 +import secrets +import uuid +import json +import re +from datetime import datetime, timezone, timedelta +from typing import List, Optional +from uuid import UUID + +import psycopg2 +import psycopg2.extras +import bcrypt +from fastapi import FastAPI, Header, HTTPException, Query, Response, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field, field_validator, ConfigDict + +import redis + +import hashlib +import hmac + +app = FastAPI(title="MTUSI Messenger — lab3", version="1.3.0") + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +INSTANCE_ID = os.getenv("INSTANCE_ID", "instance-1") + +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) + +SECRET_HMAC_KEY = os.environ["SECRET_HMAC_KEY"] + +def get_db_connection(): + conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + return conn + +def generate_request_id() -> str: + return uuid.uuid4().hex + +def make_error_response( + status_code: int, + code: str, + message: str, + req_id: str, + extra_headers: Optional[dict] = None, +) -> JSONResponse: + headers = {"X-Request-Id": req_id, "X-Instance-Id": INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status_code, + headers=headers, + content={ + "error": { + "code": code, + "message": message, + "request_id": req_id, + } + }, + ) + +def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str: + payload = f"{chat_id}|{created_at}|{msg_id}" + signature = hmac.new(SECRET_HMAC_KEY.encode(), payload.encode(), hashlib.sha256).hexdigest()[:16] + raw = f"{payload}|{signature}" + return base64.urlsafe_b64encode(raw.encode()).decode() + +def decode_cursor(cursor: str, expected_chat_id: str) -> tuple: + try: + decoded = base64.urlsafe_b64decode(cursor.encode()).decode() + parts = decoded.split("|") + if len(parts) != 4: + raise ValueError() + chat_id, created_at, msg_id, signature = parts + + if chat_id != expected_chat_id: + raise HTTPException(status_code=400, detail="Cursor belongs to another chat") + + expected_sig = hmac.new(SECRET_HMAC_KEY.encode(), f"{chat_id}|{created_at}|{msg_id}".encode(), hashlib.sha256).hexdigest()[:16] + if not hmac.compare_digest(signature, expected_sig): + raise ValueError() + + return created_at, msg_id + except HTTPException: + raise + except Exception: + raise HTTPException(status_code=400, detail="Invalid cursor format") + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + req_id = request.headers.get("X-Request-Id") or generate_request_id() + return make_error_response( + status_code=400, + code="invalid_request", + message="Validation error in request body or parameters", + req_id=req_id + ) + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + req_id = request.headers.get("X-Request-Id") or generate_request_id() + headers = {"X-Request-Id": req_id, "X-Instance-Id": INSTANCE_ID} + if exc.headers: + headers.update(exc.headers) + + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + detail["error"]["request_id"] = req_id + return JSONResponse(status_code=exc.status_code, headers=headers, content=detail) + + code_map = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", + } + return JSONResponse( + status_code=exc.status_code, + headers=headers, + content={ + "error": { + "code": code_map.get(exc.status_code, "internal_error"), + "message": str(detail), + "request_id": req_id, + } + }, + ) + +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): + req_id = request.headers.get("X-Request-Id") or generate_request_id() + return make_error_response( + status_code=500, + code="internal_error", + message="Internal server error", + req_id=req_id, + ) + +class CreateUser(BaseModel): + model_config = ConfigDict(extra='forbid') + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + @field_validator("display_name", "password") + def not_blank(cls, v): + if not v.strip(): + raise ValueError("Must not be blank") + return v + +class CreateChat(BaseModel): + model_config = ConfigDict(extra='forbid') + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[uuid.UUID] = Field(..., min_items=1, max_items=100) + + @field_validator("title") + def not_blank(cls, v): + if not v.strip(): + raise ValueError("Must not be blank") + return v + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra='forbid') + text: str = Field(..., min_length=1, max_length=4000) + + @field_validator("text") + def not_blank(cls, v): + if not v.strip(): + raise ValueError("Must not be blank") + return v + +@app.get("/health/live") +def health_live(response: Response): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return {"status": "ok"} + +@app.get("/health/ready") +def health_ready(response: Response): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + try: + conn = get_db_connection() + conn.close() + redis_client.ping() + except Exception: + raise HTTPException(status_code=503, detail="Database or Redis unavailable") + return {"status": "ok"} + +@app.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser, response: Response): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + + password_hash = bcrypt.hashpw(body.password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8') + user_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES (%s, %s, %s, %s, %s)", + (user_id, body.username, body.display_name, password_hash, created_at) + ) + conn.commit() + except psycopg2.errors.UniqueViolation: + conn.rollback() + raise HTTPException(status_code=409, detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}}) + finally: + conn.close() + + return { + "id": user_id, + "username": body.username, + "display_name": body.display_name, + "created_at": created_at + } + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(authorization: Optional[str] = Header(None), response: Response = None): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + + if not authorization or not authorization.startswith("Basic "): + raise HTTPException( + status_code=401, + detail="Missing or invalid basic auth", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + ) + + try: + encoded = authorization.split(" ")[1] + decoded = base64.b64decode(encoded).decode("utf-8") + username, password = decoded.split(":", 1) + except Exception: + raise HTTPException( + status_code=401, + detail="Invalid credentials format", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + ) + + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE username = %s", (username,)) + user = cur.fetchone() + + if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')): + conn.close() + raise HTTPException( + status_code=401, + detail="Invalid credentials", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + ) + + token = secrets.token_urlsafe(48) + redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, user['id']) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + expires_at_str = expires_at.isoformat().replace("+00:00", "Z") + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at_str, + "user": { + "id": user['id'], + "username": user['username'], + "display_name": user['display_name'], + "created_at": user['created_at'].isoformat().replace("+00:00", "Z") if isinstance(user['created_at'], datetime) else user['created_at'] + } + } + +def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=401, + detail="Missing or invalid bearer token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + token = authorization.split(" ")[1] + + try: + user_id = redis_client.get(f"session:{token}") + except Exception: + raise HTTPException(status_code=503, detail="Session store unavailable") + + if not user_id: + raise HTTPException( + status_code=401, + detail="Invalid or expired session token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + conn.close() + + if not row: + raise HTTPException( + status_code=401, + detail="User not found", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + return { + "id": row['id'], + "username": row['username'], + "display_name": row['display_name'], + "created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at'] + } + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + token = authorization.split(" ")[1] + deleted = redis_client.delete(f"session:{token}") + + if not deleted: + raise HTTPException( + status_code=401, + detail="Invalid session token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + return Response( + status_code=204, + headers={ + "X-Request-Id": req_id, + "X-Instance-Id": INSTANCE_ID + }, + ) + +@app.get("/api/v1/users/me") +def current_user(response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return get_current_user_from_token(authorization) + +@app.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + member_ids_str = [str(mid) for mid in body.member_ids] + + if user["id"] not in member_ids_str: + raise HTTPException(status_code=400, detail="Caller must be in member_ids") + if len(member_ids_str) != len(set(member_ids_str)): + raise HTTPException(status_code=400, detail="Duplicate member IDs") + + conn = get_db_connection() + with conn.cursor() as cur: + for mid_str in member_ids_str: + cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,)) + if not cur.fetchone(): + conn.close() + raise HTTPException(status_code=400, detail=f"User {mid_str} not found") + + chat_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + try: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, body.title, created_at) + ) + for mid_str in member_ids_str: + cur.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (chat_id, mid_str) + ) + conn.commit() + except Exception: + conn.rollback() + conn.close() + raise HTTPException(status_code=500, detail="Internal error creating chat") + conn.close() + + return { + "id": chat_id, + "title": body.title, + "member_ids": member_ids_str, + "created_at": created_at + } + +@app.get("/api/v1/chats") +def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="Invalid limit") + + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute(""" + SELECT c.id, c.title, c.created_at FROM chats c + JOIN chat_members cm ON c.id = cm.chat_id + WHERE cm.user_id = %s + ORDER BY c.created_at ASC, c.id ASC + """, (user["id"],)) + rows = cur.fetchall() + conn.close() + + user_chats = [] + for r in rows: + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],)) + m_rows = cur.fetchall() + conn.close() + user_chats.append({ + "id": r['id'], + "title": r['title'], + "member_ids": [m['user_id'] for m in m_rows], + "created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at'] + }) + + start_idx = 0 + if cursor: + try: + parts = cursor.split("_") + c_time, c_id = parts[0], parts[1] + found = False + for idx, c in enumerate(user_chats): + if (c["created_at"], c["id"]) >= (c_time, c_id): + start_idx = idx if (c["created_at"], c["id"]) > (c_time, c_id) else idx + 1 + found = True + break + if not found: + start_idx = len(user_chats) + except Exception: + raise HTTPException(status_code=400, detail="Invalid cursor format") + + page_items = user_chats[start_idx:start_idx + limit] + next_cursor = None + if start_idx + limit < len(user_chats): + last = page_items[-1] + next_cursor = f"{last['created_at']}_{last['id']}" + + return {"items": page_items, "next_cursor": next_cursor} + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + chat_id_str = str(chat_id) + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute(""" + SELECT c.* FROM chats c + JOIN chat_members cm ON c.id = cm.chat_id + WHERE c.id = %s AND cm.user_id = %s + """, (chat_id_str, user["id"])) + chat = cur.fetchone() + if chat: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,)) + members = cur.fetchall() + conn.close() + + if not chat: + raise HTTPException(status_code=404, detail="Chat not found") + + return { + "id": chat['id'], + "title": chat['title'], + "member_ids": [m['user_id'] for m in members], + "created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at'] + } + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: UUID, + body: CreateMessage, + response: Response, + authorization: Optional[str] = Header(None), + idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key") +): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + if not idempotency_key or not re.match(r"^[A-Za-z0-9._:-]{1,128}$", idempotency_key): + raise HTTPException(status_code=400, detail="Missing or invalid Idempotency-Key") + + chat_id_str = str(chat_id) + method = "POST" + path = f"/api/v1/chats/{chat_id_str}/messages" + + body_json_str = json.dumps(body.model_dump(), sort_keys=True) + request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest() + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id_str, user["id"]) + ) + if not cur.fetchone(): + conn.close() + raise HTTPException(status_code=404, detail="Chat not found") + + msg_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + msg_response = { + "id": msg_id, + "chat_id": chat_id_str, + "sender_id": user["id"], + "text": body.text, + "created_at": created_at + } + + cur.execute(""" + INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, response_status, response_body) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (user_id, method, path, key) DO NOTHING + """, (user["id"], method, path, idempotency_key, request_hash, 201, json.dumps(msg_response))) + + if cur.rowcount == 0: + cur.execute(""" + SELECT request_hash, response_status, response_body + FROM idempotency_keys + WHERE user_id = %s AND method = %s AND path = %s AND key = %s + """, (user["id"], method, path, idempotency_key)) + row = cur.fetchone() + conn.commit() + conn.close() + + if row["request_hash"] != request_hash: + raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload") + + return row["response_body"] + + cur.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", + (msg_id, chat_id_str, user["id"], body.text, created_at) + ) + conn.commit() + except HTTPException: + raise + except Exception as exc: + conn.rollback() + conn.close() + raise HTTPException(status_code=500, detail="Internal error processing message") + finally: + if not conn.closed: + conn.close() + + return msg_response + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + chat_id_str = str(chat_id) + conn = get_db_connection() + + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id_str, user["id"]) + ) + if not cur.fetchone(): + conn.close() + raise HTTPException(status_code=404, detail="Chat not found") + + if limit < 1 or limit > 100: + conn.close() + raise HTTPException(status_code=400, detail="Invalid limit") + + query = """ + SELECT id, chat_id, sender_id, text, created_at FROM messages + WHERE chat_id = %s + """ + params = [chat_id_str] + + if cursor: + c_time, c_id = decode_cursor(cursor, chat_id_str) + query += " AND (created_at, id) > (%s, %s)" + params.extend([c_time, c_id]) + + query += " ORDER BY created_at ASC, id ASC LIMIT %s" + params.append(limit + 1) + + with conn.cursor() as cur: + cur.execute(query, tuple(params)) + rows = cur.fetchall() + conn.close() + + has_next = len(rows) > limit + page_items = rows[:limit] + + formatted_msgs = [] + for m in page_items: + formatted_msgs.append({ + "id": m['id'], + "chat_id": m['chat_id'], + "sender_id": m['sender_id'], + "text": m['text'], + "created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'] + }) + + next_cursor = None + if has_next and formatted_msgs: + last = formatted_msgs[-1] + next_cursor = encode_cursor(chat_id_str, last['created_at'], last['id']) + + return {"items": formatted_msgs, "next_cursor": next_cursor} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..d5599496 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,130 @@ +services: + postgres: + image: postgres:15.5-alpine3.19 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + volumes: + - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + reservations: + cpus: '0.10' + memory: 128M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d messenger"] + interval: 3s + timeout: 3s + retries: 5 + + redis: + image: redis:7.2.4-alpine + command: redis-server --save 60 1 --loglevel warning + volumes: + - redisdata:/data + deploy: + resources: + limits: + cpus: '0.25' + memory: 256M + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 5 + + migrate: + image: postgres:15.5-alpine3.19 + environment: + PGPASSWORD: ${POSTGRES_PASSWORD} + volumes: + - ./migrations:/migrations + command: > + sh -c "until pg_isready -h postgres -U postgres; do sleep 1; done; + psql -h postgres -U postgres -d messenger -f /migrations/init.sql" + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api-1: + build: . + stop_grace_period: 15s + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + - SESSION_TTL_SECONDS=3600 + - INSTANCE_ID=api-replica-1 + - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 5s + timeout: 3s + retries: 3 + + api-2: + build: . + stop_grace_period: 15s + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + - SESSION_TTL_SECONDS=3600 + - INSTANCE_ID=api-replica-2 + - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 5s + timeout: 3s + retries: 3 + + nginx: + image: nginx:1.25.4-alpine + ports: + - "8080:8080" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M + depends_on: + - api-1 + - api-2 + healthcheck: + test: ["CMD", "nginx", "-t"] + interval: 5s + timeout: 3s + retries: 3 + +volumes: + pgdata: + redisdata: \ No newline at end of file diff --git a/migrations/init.sql b/migrations/init.sql new file mode 100644 index 00000000..2725c969 --- /dev/null +++ b/migrations/init.sql @@ -0,0 +1,41 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, + password_hash VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID REFERENCES users(id) ON DELETE CASCADE, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_pagination ON messages(chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL, + method VARCHAR(16) NOT NULL, + path TEXT NOT NULL, + key VARCHAR(128) NOT NULL, + request_hash VARCHAR(64) NOT NULL, + response_status INT NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, method, path, key) +); \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 00000000..bf626811 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,28 @@ +events { worker_connections 1024; } + +http { + upstream messenger_api { + server api-1:8080 max_fails=1 fail_timeout=5s; + server api-2:8080 max_fails=1 fail_timeout=5s; + } + + server { + listen 8080; + + location / { + proxy_pass http://messenger_api; + + proxy_connect_timeout 2s; + proxy_send_timeout 5s; + proxy_read_timeout 5s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..ed45144e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +fastapi==0.110.0 +uvicorn==0.28.0 +pydantic==2.6.2 +starlette==0.36.3 +idna==3.6 +anyio==4.3.0 +sniffio==1.3.1 +click==8.1.7 +h11==0.14.0 +annotated-types==0.6.0 +psycopg2-binary==2.9.9 +bcrypt==4.1.2 +redis==5.0.1 \ No newline at end of file diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..ce7fdbe2 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,3 @@ +@echo off +docker compose restart +exit /b %ERRORLEVEL% \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..e6cfbc5b --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +docker compose restart \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..173f1dba --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu + +wait_ready() { + svc=$1 + timeout=60 + echo "Waiting for $svc readiness (up to ${timeout}s)..." + while [ "$timeout" -gt 0 ]; do + if docker compose exec -T "$svc" python -c \ + "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" 2>/dev/null; then + echo "$svc is ready!" + return 0 + fi + sleep 1 + timeout=$((timeout - 1)) + done + echo "$svc failed to become ready" >&2 + return 1 +} + +echo "Starting Rolling Update" + +echo "1. Rebuilding and replacing api-1..." +docker compose up -d --no-deps --build --force-recreate api-1 +wait_ready api-1 +echo "Reloading nginx to pick up new api-1 IP..." +docker compose exec -T nginx nginx -s reload + +echo "2. Rebuilding and replacing api-2..." +docker compose up -d --no-deps --build --force-recreate api-2 +wait_ready api-2 +echo "Reloading nginx to pick up new api-2 IP..." +docker compose exec -T nginx nginx -s reload + +echo "Rolling Update Completed Successfully" \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..d35a3c03 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +if [ "$TARGET_INSTANCE_ID" = "api-replica-1" ]; then + docker compose stop -t 3 api-1 +elif [ "$TARGET_INSTANCE_ID" = "api-replica-2" ]; then + docker compose stop -t 3 api-2 +else + echo "Unknown target instance: $TARGET_INSTANCE_ID" >&2 + exit 1 +fi \ No newline at end of file diff --git a/tests/client.py b/tests/client.py index f0499a29..38159a0c 100644 --- a/tests/client.py +++ b/tests/client.py @@ -14,7 +14,7 @@ import uuid ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8')) class Failure(AssertionError): pass diff --git a/tests/custom_tests.py b/tests/custom_tests.py new file mode 100644 index 00000000..f5e46e61 --- /dev/null +++ b/tests/custom_tests.py @@ -0,0 +1,109 @@ +import concurrent.futures +import urllib.request +import urllib.error +import json +import unittest +from client import Client + +class TestMessengerLab4(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice = self.client.user("alice") + self.bob = self.client.user("bob") + self.eve = self.client.user("eve") + + res = self.client.call( + 'POST', '/api/v1/chats', 201, + actor=self.alice, + body={"title": "Test Chat", "member_ids": [self.alice["id"], self.bob["id"]]}, + schema='Chat' + ) + self.chat = res['json'] + + def test_unauthorized_access_to_chat(self): + self.client.call('GET', f"/api/v1/chats/{self.chat['id']}", 404, actor=self.eve) + + def test_empty_message(self): + self.client.call('POST', f"/api/v1/chats/{self.chat['id']}/messages", 400, actor=self.alice, body={"text": ""}) + + def test_malformed_json(self): + url = self.client.base + f"/api/v1/chats/{self.chat['id']}/messages" + req = urllib.request.Request( + url, + data=b"{broken json content without quotes", + headers={ + "Content-Type": "application/json", + "X-User-Id": self.alice["id"] + }, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 400 for malformed JSON") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 400) + body = json.loads(e.read().decode()) + self.assertEqual(body["error"]["code"], "invalid_request") + + def test_duplicate_username(self): + self.client.call('POST', '/api/v1/users', 409, body=self.alice["registration"]) + + def test_pagination_multiple_pages(self): + import time + msg_texts = ["msg-page-1", "msg-page-2", "msg-page-3"] + for txt in msg_texts: + self.client.message(self.alice, self.chat, txt) + time.sleep(0.01) + + path = f"/api/v1/chats/{self.chat['id']}/messages" + collected_items = [] + cursor = None + + for _ in range(5): + url_suffix = "?limit=1" + if cursor: + url_suffix += f"&cursor={cursor}" + + page_res = self.client.call('GET', path + url_suffix, actor=self.alice, schema='MessagePage') + page = page_res['json'] + + for item in page['items']: + self.assertNotIn(item['id'], [x['id'] for x in collected_items]) + collected_items.append(item) + + cursor = page.get('next_keyword') or page.get('next_cursor') + if not cursor: + break + + self.assertEqual(len(collected_items), 3) + self.assertEqual(sorted([x['text'] for x in collected_items]), sorted(msg_texts)) + + def test_concurrent_messages_strict(self): + def worker_send(worker_idx): + local_client = Client() + ids = [] + for i in range(10): + res = local_client.message(self.alice, self.chat, f"w-{worker_idx}-msg-{i}") + ids.append(res['json']['id']) + return ids + + all_ids = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(worker_send, w) for w in range(10)] + for f in concurrent.futures.as_completed(futures): + all_ids.extend(f.result()) + + self.assertEqual(len(all_ids), 100) + self.assertEqual(len(set(all_ids)), 100) + + page = self.client.call( + 'GET', f"/api/v1/chats/{self.chat['id']}/messages?limit=100", + actor=self.alice, + schema='MessagePage' + ) + server_ids = [m['id'] for m in page['json']['items']] + self.assertEqual(len(server_ids), 100) + self.assertEqual(len(set(server_ids)), 100) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/resilience.py b/tests/resilience.py index f3fa700a..d5d67fe1 100644 --- a/tests/resilience.py +++ b/tests/resilience.py @@ -55,7 +55,7 @@ def main(): check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') # No shell interpolation and no Docker commands in this test. - proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + proc = subprocess.Popen(['sh', str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) started = time.monotonic() successes,failures,after_instances = [],0,[] try: -- 2.54.0 From 9a09b3838796daee6e0a71a3fe3a573832826ef6 Mon Sep 17 00:00:00 2001 From: Getlles Date: Mon, 9 Nov 2026 20:56:25 +0300 Subject: [PATCH 2/4] grade3 --- app.py | 282 ++++++++++++++++++++++++++++++++++++++++---- compose.yaml | 25 +++- migrations/init.sql | 24 +++- requirements.txt | 3 +- 4 files changed, 310 insertions(+), 24 deletions(-) diff --git a/app.py b/app.py index 7a9ff3a8..7ba13140 100644 --- a/app.py +++ b/app.py @@ -4,24 +4,27 @@ import secrets import uuid import json import re +import struct +import hashlib +import hmac from datetime import datetime, timezone, timedelta from typing import List, Optional from uuid import UUID import psycopg2 import psycopg2.extras +psycopg2.extras.register_uuid() import bcrypt from fastapi import FastAPI, Header, HTTPException, Query, Response, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field, field_validator, ConfigDict +from pydantic import BaseModel, Field, field_validator, model_validator, ConfigDict import redis +import boto3 +from botocore.client import Config -import hashlib -import hmac - -app = FastAPI(title="MTUSI Messenger — lab3", version="1.3.0") +app = FastAPI(title="MTUSI Messenger — lab4", version="1.4.0") DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) @@ -32,6 +35,20 @@ redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) SECRET_HMAC_KEY = os.environ["SECRET_HMAC_KEY"] +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "minioadmin") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadminpassword") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + def get_db_connection(): conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) return conn @@ -168,13 +185,14 @@ class CreateChat(BaseModel): class CreateMessage(BaseModel): model_config = ConfigDict(extra='forbid') - text: str = Field(..., min_length=1, max_length=4000) + text: str = Field(default="", max_length=4000) + attachment_ids: List[uuid.UUID] = Field(default=[], max_items=10) - @field_validator("text") - def not_blank(cls, v): - if not v.strip(): - raise ValueError("Must not be blank") - return v + @model_validator(mode="after") + def check_content(self) -> 'CreateMessage': + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Message must contain either text or attachments") + return self @app.get("/health/live") def health_live(response: Response): @@ -192,8 +210,9 @@ def health_ready(response: Response): conn = get_db_connection() conn.close() redis_client.ping() + s3_client.head_bucket(Bucket=S3_BUCKET) except Exception: - raise HTTPException(status_code=503, detail="Database or Redis unavailable") + raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable") return {"status": "ok"} @app.post("/api/v1/users", status_code=201) @@ -265,7 +284,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon ) token = secrets.token_urlsafe(48) - redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, user['id']) + redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id'])) expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) expires_at_str = expires_at.isoformat().replace("+00:00", "Z") @@ -316,7 +335,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> ) return { - "id": row['id'], + "id": str(row['id']), "username": row['username'], "display_name": row['display_name'], "created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at'] @@ -498,6 +517,205 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H "created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at'] } +def validate_image_and_get_dimensions(data: bytes, content_type: str): + if len(data) > 10 * 1024 * 1024: + raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") + + width, height = 0, 0 + if content_type == "image/png": + if not data.startswith(b'\x89PNG\r\n\x1a\n'): + raise HTTPException(status_code=415, detail="Invalid PNG signature") + if len(data) >= 24: + width, height = struct.unpack('>II', data[16:24]) + elif content_type == "image/jpeg": + if not data.startswith(b'\xff\xd8'): + raise HTTPException(status_code=415, detail="Invalid JPEG signature") + i = 2 + while i < len(data) - 9: + marker = data[i] + if marker == 0xff: + marker_type = data[i+1] + if 0xc0 <= marker_type <= 0xc3: + height, width = struct.unpack('>HH', data[i+5:i+9]) + break + i += 2 + struct.unpack('>H', data[i+2:i+4])[0] + else: + i += 1 + else: + raise HTTPException(status_code=415, detail="Unsupported media type") + + if width * height > 40_000_000: + raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit") + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: UUID, + filename: str = Query(..., min_length=1, max_length=255), + request: Request = None, + response: Response = None, + authorization: Optional[str] = Header(None) +): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + chat_id_str = str(chat_id) + + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id_str, user["id"])) + if not cur.fetchone(): + conn.close() + raise HTTPException(status_code=404, detail="Chat not found or inaccessible") + conn.close() + + if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename): + raise HTTPException(400, "Invalid filename") + + content_type = request.headers.get("content-type") + if content_type not in ["image/png", "image/jpeg"]: + raise HTTPException(status_code=415, detail="Unsupported media type") + + MAX_UPLOAD = 10 * 1024 * 1024 + + cl = request.headers.get("content-length") + if cl is not None: + try: + if int(cl) > MAX_UPLOAD: + raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") + except ValueError: + raise HTTPException(status_code=400, detail="Invalid Content-Length") + + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_UPLOAD: + raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") + chunks.append(chunk) + body_bytes = b"".join(chunks) + + size_bytes = len(body_bytes) + sha256_hash = hashlib.sha256(body_bytes).hexdigest() + attachment_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + s3_key = f"attachments/{chat_id_str}/{attachment_id}/{filename}" + try: + s3_client.put_object( + Bucket=S3_BUCKET, + Key=s3_key, + Body=body_bytes, + ContentType=content_type + ) + except Exception: + raise HTTPException(status_code=503, detail="S3 storage unavailable") + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, 'ready', '[]', NULL, %s, %s) + """, (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now)) + conn.commit() + except Exception: + conn.rollback() + raise HTTPException(status_code=500, detail="Internal error saving attachment metadata") + finally: + conn.close() + + return { + "id": attachment_id, + "chat_id": chat_id_str, + "owner_id": user["id"], + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "sha256": sha256_hash, + "status": "ready", + "variants": [], + "error_code": None, + "created_at": now, + "updated_at": now + } + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + att_id_str = str(attachment_id) + + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, (att_id_str, user["id"])) + row = cur.fetchone() + conn.close() + + if not row: + raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") + + return { + "id": str(row['id']), + "chat_id": str(row['chat_id']), + "owner_id": str(row['owner_id']), + "filename": row['filename'], + "content_type": row['content_type'], + "size_bytes": row['size_bytes'], + "sha256": row['sha256'], + "status": row['status'], + "variants": row['variants'], + "error_code": row['error_code'], + "created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at'], + "updated_at": row['updated_at'].isoformat().replace("+00:00", "Z") if isinstance(row['updated_at'], datetime) else row['updated_at'] + } + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + att_id_str = str(attachment_id) + + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, (att_id_str, user["id"])) + row = cur.fetchone() + conn.close() + + if not row: + raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") + + if row['status'] != 'ready': + raise HTTPException(status_code=409, detail="Attachment not ready") + + s3_key = f"attachments/{row['chat_id']}/{row['id']}/{row['filename']}" + try: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key) + data = obj['Body'].read() + except Exception: + raise HTTPException(status_code=503, detail="S3 storage unavailable or object missing") + + return Response( + content=data, + media_type=row['content_type'], + headers={ + "X-Request-Id": req_id, + "X-Instance-Id": INSTANCE_ID, + }, + ) + @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message( chat_id: UUID, @@ -518,7 +736,7 @@ def create_message( method = "POST" path = f"/api/v1/chats/{chat_id_str}/messages" - body_json_str = json.dumps(body.model_dump(), sort_keys=True) + body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True) request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest() conn = get_db_connection() @@ -532,6 +750,21 @@ def create_message( conn.close() raise HTTPException(status_code=404, detail="Chat not found") + attachment_ids_str = [str(aid) for aid in body.attachment_ids] + if attachment_ids_str: + cur.execute(""" + SELECT id, status FROM attachments + WHERE id = ANY(%s::uuid[]) AND chat_id = %s + """, (attachment_ids_str, chat_id_str)) + found_atts = cur.fetchall() + if len(found_atts) != len(attachment_ids_str): + conn.close() + raise HTTPException(status_code=404, detail="One or more attachments not found or in another chat") + for att in found_atts: + if att['status'] == 'failed': + conn.close() + raise HTTPException(status_code=409, detail="Cannot attach failed attachment") + msg_id = str(uuid.uuid4()) created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") @@ -540,7 +773,8 @@ def create_message( "chat_id": chat_id_str, "sender_id": user["id"], "text": body.text, - "created_at": created_at + "created_at": created_at, + "attachment_ids": attachment_ids_str } cur.execute(""" @@ -565,8 +799,8 @@ def create_message( return row["response_body"] cur.execute( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", - (msg_id, chat_id_str, user["id"], body.text, created_at) + "INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) VALUES (%s, %s, %s, %s, %s, %s::uuid[])", + (msg_id, chat_id_str, user["id"], body.text, created_at, attachment_ids_str) ) conn.commit() except HTTPException: @@ -605,7 +839,7 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, raise HTTPException(status_code=400, detail="Invalid limit") query = """ - SELECT id, chat_id, sender_id, text, created_at FROM messages + SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id = %s """ params = [chat_id_str] @@ -628,12 +862,20 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, formatted_msgs = [] for m in page_items: + raw_atts = m.get('attachment_ids') + atts = [] + if raw_atts: + for aid in raw_atts: + if aid: + atts.append(str(aid)) + formatted_msgs.append({ "id": m['id'], "chat_id": m['chat_id'], "sender_id": m['sender_id'], "text": m['text'], - "created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'] + "created_at": m['created_at'].isoformat().replace("+00:00", "Z") if isinstance(m['created_at'], datetime) else m['created_at'], + "attachment_ids": atts }) next_cursor = None diff --git a/compose.yaml b/compose.yaml index d5599496..d6952547 100644 --- a/compose.yaml +++ b/compose.yaml @@ -125,6 +125,29 @@ services: timeout: 3s retries: 3 + s3: + image: chrislusf/seaweedfs:latest + command: "mini -dir=/data" + environment: + - AWS_ACCESS_KEY_ID=minioadmin + - AWS_SECRET_ACCESS_KEY=minioadminpassword + - S3_BUCKET=messenger-attachments + ports: + - "8333:8333" + volumes: + - s3data:/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + healthcheck: + test: ["CMD", "nc", "-z", "localhost", "8333"] + interval: 5s + timeout: 3s + retries: 3 + volumes: pgdata: - redisdata: \ No newline at end of file + redisdata: + s3data: \ No newline at end of file diff --git a/migrations/init.sql b/migrations/init.sql index 2725c969..978ebac5 100644 --- a/migrations/init.sql +++ b/migrations/init.sql @@ -23,7 +23,8 @@ CREATE TABLE IF NOT EXISTS messages ( chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, sender_id UUID REFERENCES users(id) ON DELETE CASCADE, text VARCHAR(4000) NOT NULL, - created_at TIMESTAMPTZ NOT NULL + created_at TIMESTAMPTZ NOT NULL, + attachment_ids UUID[] DEFAULT '{}' ); CREATE INDEX IF NOT EXISTS idx_messages_chat_pagination ON messages(chat_id, created_at, id); @@ -38,4 +39,23 @@ CREATE TABLE IF NOT EXISTS idempotency_keys ( response_body JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), PRIMARY KEY (user_id, method, path, key) -); \ No newline at end of file +); + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(50) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'ready', + variants JSONB NOT NULL DEFAULT '[]', + error_code VARCHAR(64), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id); + +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}'; \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ed45144e..ab928d42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ h11==0.14.0 annotated-types==0.6.0 psycopg2-binary==2.9.9 bcrypt==4.1.2 -redis==5.0.1 \ No newline at end of file +redis==5.0.1 +boto3==1.34.0 \ No newline at end of file -- 2.54.0 From e1d42e069fd34773f1c3ba38f4e45bb861712b69 Mon Sep 17 00:00:00 2001 From: Getlles Date: Mon, 9 Nov 2026 23:42:13 +0300 Subject: [PATCH 3/4] grade4 --- app.py | 45 ++++++++--- compose.yaml | 36 ++++++++- evidence/bucket_private.txt | 1 + evidence/compose_ps.txt | 7 ++ evidence/failover.txt | 4 + evidence/level4.txt | 17 ++++ evidence/persistence.txt | 3 + evidence/reconcile_dry_run.txt | 4 + evidence/smoke.txt | 2 + evidence/upload_load.json | 7 ++ evidence/upload_load_stats.txt | 40 +++++++++ scripts/bootstrap_s3.py | 29 +++++++ scripts/check_bucket_private.py | 11 +++ scripts/reconcile.py | 81 +++++++++++++++++++ scripts/upload_load_test.py | 54 +++++++++++++ tests/level4_tests.py | 139 ++++++++++++++++++++++++++++++++ 16 files changed, 467 insertions(+), 13 deletions(-) create mode 100644 evidence/bucket_private.txt create mode 100644 evidence/compose_ps.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/level4.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/reconcile_dry_run.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/upload_load.json create mode 100644 evidence/upload_load_stats.txt create mode 100644 scripts/bootstrap_s3.py create mode 100644 scripts/check_bucket_private.py create mode 100644 scripts/reconcile.py create mode 100644 scripts/upload_load_test.py create mode 100644 tests/level4_tests.py diff --git a/app.py b/app.py index 7ba13140..a7d43e34 100644 --- a/app.py +++ b/app.py @@ -525,8 +525,14 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str): if content_type == "image/png": if not data.startswith(b'\x89PNG\r\n\x1a\n'): raise HTTPException(status_code=415, detail="Invalid PNG signature") - if len(data) >= 24: - width, height = struct.unpack('>II', data[16:24]) + if len(data) < 33: + raise HTTPException(status_code=415, detail="Truncated PNG") + if data[12:16] != b'IHDR': + raise HTTPException(status_code=415, detail="PNG missing IHDR chunk") + ihdr_len = struct.unpack('>I', data[8:12])[0] + if ihdr_len != 13: + raise HTTPException(status_code=415, detail="Invalid IHDR length") + width, height = struct.unpack('>II', data[16:24]) elif content_type == "image/jpeg": if not data.startswith(b'\xff\xd8'): raise HTTPException(status_code=415, detail="Invalid JPEG signature") @@ -544,6 +550,9 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str): else: raise HTTPException(status_code=415, detail="Unsupported media type") + if width == 0 or height == 0: + raise HTTPException(status_code=415, detail="Could not read image dimensions") + if width * height > 40_000_000: raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit") @@ -569,7 +578,7 @@ async def upload_attachment( raise HTTPException(status_code=404, detail="Chat not found or inaccessible") conn.close() - if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename): + if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename: raise HTTPException(400, "Invalid filename") content_type = request.headers.get("content-type") @@ -577,7 +586,6 @@ async def upload_attachment( raise HTTPException(status_code=415, detail="Unsupported media type") MAX_UPLOAD = 10 * 1024 * 1024 - cl = request.headers.get("content-length") if cl is not None: try: @@ -595,12 +603,31 @@ async def upload_attachment( chunks.append(chunk) body_bytes = b"".join(chunks) + validate_image_and_get_dimensions(body_bytes, content_type) + size_bytes = len(body_bytes) sha256_hash = hashlib.sha256(body_bytes).hexdigest() attachment_id = str(uuid.uuid4()) now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") s3_key = f"attachments/{chat_id_str}/{attachment_id}/{filename}" + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, 'pending', '[]', NULL, %s, %s) + """, (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now)) + conn.commit() + except Exception: + conn.rollback() + conn.close() + raise HTTPException(status_code=500, detail="Internal error initializing attachment") + finally: + if not conn.closed: + conn.close() + try: s3_client.put_object( Bucket=S3_BUCKET, @@ -615,15 +642,15 @@ async def upload_attachment( try: with conn.cursor() as cur: cur.execute(""" - INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, 'ready', '[]', NULL, %s, %s) - """, (attachment_id, chat_id_str, user["id"], filename, content_type, size_bytes, sha256_hash, now, now)) + UPDATE attachments SET status = 'ready', updated_at = %s WHERE id = %s + """, (now, attachment_id)) conn.commit() except Exception: conn.rollback() - raise HTTPException(status_code=500, detail="Internal error saving attachment metadata") + raise HTTPException(status_code=500, detail="Internal error finalizing attachment status") finally: - conn.close() + if not conn.closed: + conn.close() return { "id": attachment_id, diff --git a/compose.yaml b/compose.yaml index d6952547..c6f67609 100644 --- a/compose.yaml +++ b/compose.yaml @@ -60,6 +60,10 @@ services: - SESSION_TTL_SECONDS=3600 - INSTANCE_ID=api-replica-1 - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ACCESS_KEY_ID=appuser + - S3_SECRET_ACCESS_KEY=appsecretpassword deploy: resources: limits: @@ -72,6 +76,8 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + s3-init: + condition: service_completed_successfully healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] interval: 5s @@ -87,6 +93,10 @@ services: - SESSION_TTL_SECONDS=3600 - INSTANCE_ID=api-replica-2 - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ACCESS_KEY_ID=appuser + - S3_SECRET_ACCESS_KEY=appsecretpassword deploy: resources: limits: @@ -99,6 +109,8 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + s3-init: + condition: service_completed_successfully healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] interval: 5s @@ -129,11 +141,9 @@ services: image: chrislusf/seaweedfs:latest command: "mini -dir=/data" environment: - - AWS_ACCESS_KEY_ID=minioadmin - - AWS_SECRET_ACCESS_KEY=minioadminpassword + - AWS_ACCESS_KEY_ID=appuser + - AWS_SECRET_ACCESS_KEY=appsecretpassword - S3_BUCKET=messenger-attachments - ports: - - "8333:8333" volumes: - s3data:/data deploy: @@ -147,6 +157,24 @@ services: timeout: 3s retries: 3 + s3-init: + image: python:3.11-slim + volumes: + - ./scripts:/scripts + - ./requirements.txt:/requirements.txt + environment: + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ROOT_ACCESS_KEY=appuser + - S3_ROOT_SECRET_KEY=appsecretpassword + command: > + sh -c "pip install --no-cache-dir -r /requirements.txt && + python /scripts/bootstrap_s3.py" + depends_on: + s3: + condition: service_healthy + restart: "no" + volumes: pgdata: redisdata: diff --git a/evidence/bucket_private.txt b/evidence/bucket_private.txt new file mode 100644 index 00000000..01e3dccd --- /dev/null +++ b/evidence/bucket_private.txt @@ -0,0 +1 @@ +PASS: anonymous GET -> 403 diff --git a/evidence/compose_ps.txt b/evidence/compose_ps.txt new file mode 100644 index 00000000..4d1a2278 --- /dev/null +++ b/evidence/compose_ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-1-1 mtusi2026lab4-api-1 "uvicorn app:app --h…" api-1 15 minutes ago Up 12 minutes (healthy) 8080/tcp +mtusi2026lab4-api-2-1 mtusi2026lab4-api-2 "uvicorn app:app --h…" api-2 15 minutes ago Up 23 seconds (healthy) 8080/tcp +mtusi2026lab4-nginx-1 nginx:1.25.4-alpine "/docker-entrypoint.…" nginx 15 minutes ago Up 12 minutes (healthy) 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab4-postgres-1 postgres:15.5-alpine3.19 "docker-entrypoint.s…" postgres 15 minutes ago Up 12 minutes (healthy) 5432/tcp +mtusi2026lab4-redis-1 redis:7.2.4-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 12 minutes (healthy) 6379/tcp +mtusi2026lab4-s3-1 chrislusf/seaweedfs:latest "/entrypoint.sh mini…" s3 15 minutes ago Up 12 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..6cde1399 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 82, "failed_attempts": 2, "first_recovery_seconds": 1.312} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/level4.txt b/evidence/level4.txt new file mode 100644 index 00000000..74291158 --- /dev/null +++ b/evidence/level4.txt @@ -0,0 +1,17 @@ +test_corrupted_image_signature (__main__.TestMessengerLevel4Complete.test_corrupted_image_signature) +Ïðîâåðêà 415: áèòàÿ ñòðóêòóðà / íåâåðíûå ðàçìåðû PNG. ... ok +test_cross_chat_attachment_isolation (__main__.TestMessengerLevel4Complete.test_cross_chat_attachment_isolation) +Ïðîâåðêà 404: ïîïûòêà ïðèêðåïèòü âëîæåíèå èç ÷óæîãî ÷àòà. ... ok +test_invalid_filename_characters (__main__.TestMessengerLevel4Complete.test_invalid_filename_characters) +Ïðîâåðêà 400: íåäîïóñòèìûå ñèìâîëû (path traversal) â èìåíè ôàéëà. ... ok +test_nonexistent_attachment_id (__main__.TestMessengerLevel4Complete.test_nonexistent_attachment_id) +Ïðîâåðêà 404: ïåðåäà÷à íåñóùåñòâóþùåãî ID âëîæåíèÿ ïðè îòïðàâêå ñîîáùåíèÿ. ... ok +test_payload_too_large (__main__.TestMessengerLevel4Complete.test_payload_too_large) +Ïðîâåðêà 413: ôàéë ðàçìåðîì áîëüøå 10 MiB. ... ok +test_unsupported_media_type (__main__.TestMessengerLevel4Complete.test_unsupported_media_type) +Ïðîâåðêà 415: ïåðåäà÷à òåêñòîâîãî ôàéëà ïîä âèäîì êàðòèíêè. ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 19.521s + +OK diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/reconcile_dry_run.txt b/evidence/reconcile_dry_run.txt new file mode 100644 index 00000000..de093e2f --- /dev/null +++ b/evidence/reconcile_dry_run.txt @@ -0,0 +1,4 @@ + Container mtusi2026lab4-api-1-run-9e5b0d92d11b Creating + Container mtusi2026lab4-api-1-run-9e5b0d92d11b Created +Starting reconciliation (dry_run=True)... +Reconciliation completed successfully. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/upload_load.json b/evidence/upload_load.json new file mode 100644 index 00000000..7e96bac0 --- /dev/null +++ b/evidence/upload_load.json @@ -0,0 +1,7 @@ +CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS +de2d9b4cdf10 mtusi2026lab4-nginx-1 2.71% 2.355MiB / 128MiB 1.84% 389kB / 408kB 0B / 12.3kB 2 +bd85245e25be mtusi2026lab4-api-1-1 14.75% 74.22MiB / 256MiB 28.99% 518kB / 647kB 0B / 53.2kB 2 +dd34824ed0c5 mtusi2026lab4-api-2-1 13.03% 74.76MiB / 256MiB 29.20% 750kB / 879kB 0B / 53.2kB 2 +650e2ec7ac6a mtusi2026lab4-s3-1 1.02% 68.62MiB / 512MiB 13.40% 559kB / 248kB 0B / 1.09MB 8 +1938313cd5da mtusi2026lab4-redis-1 0.24% 3.16MiB / 256MiB 1.23% 99.6kB / 46.7kB 24.6kB / 12.3kB 5 +9348c37fb0f0 mtusi2026lab4-postgres-1 0.01% 34.39MiB / 512MiB 6.72% 679kB / 765kB 0B / 55.1MB 6 diff --git a/evidence/upload_load_stats.txt b/evidence/upload_load_stats.txt new file mode 100644 index 00000000..d2f737a3 --- /dev/null +++ b/evidence/upload_load_stats.txt @@ -0,0 +1,40 @@ +mtusi2026lab4-api-1-1 63.46MiB / 256MiB 0.20% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.46MiB / 256MiB 0.20% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.49MiB / 256MiB 0.39% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.49MiB / 256MiB 0.24% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 72.76MiB / 256MiB 7.76% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.53MiB / 256MiB 0.22% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.53MiB / 256MiB 0.20% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.61MiB / 256MiB 22.90% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.56MiB / 256MiB 0.20% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.57MiB / 256MiB 50.53% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.65MiB / 256MiB 14.81% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.68MiB / 256MiB 0.22% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.68MiB / 256MiB 0.30% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.71MiB / 256MiB 0.21% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.71MiB / 256MiB 0.20% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 73.85MiB / 256MiB 14.64% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.75MiB / 256MiB 0.22% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.75MiB / 256MiB 0.24% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.8MiB / 256MiB 27.08% +mtusi2026lab4-api-2-1 0B / 0B 0.00% +mtusi2026lab4-api-1-1 63.78MiB / 256MiB 0.24% +mtusi2026lab4-api-2-1 0B / 0B 0.00% diff --git a/scripts/bootstrap_s3.py b/scripts/bootstrap_s3.py new file mode 100644 index 00000000..72fc8486 --- /dev/null +++ b/scripts/bootstrap_s3.py @@ -0,0 +1,29 @@ +import os +import boto3 +from botocore.client import Config + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ROOT_ACCESS_KEY = os.getenv("S3_ROOT_ACCESS_KEY", "minioadmin") +S3_ROOT_SECRET_KEY = os.getenv("S3_ROOT_SECRET_KEY", "minioadminpassword") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ROOT_ACCESS_KEY, + aws_secret_access_key=S3_ROOT_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +def init_bucket(): + print(f"Checking/Creating bucket: {S3_BUCKET}") + try: + s3_client.head_bucket(Bucket=S3_BUCKET) + print("Bucket already exists.") + except Exception: + s3_client.create_bucket(Bucket=S3_BUCKET) + print("Bucket successfully created.") + +if __name__ == "__main__": + init_bucket() \ No newline at end of file diff --git a/scripts/check_bucket_private.py b/scripts/check_bucket_private.py new file mode 100644 index 00000000..dc66599f --- /dev/null +++ b/scripts/check_bucket_private.py @@ -0,0 +1,11 @@ +import os, sys, urllib.request, urllib.error +BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +KEY = os.getenv("S3_KEY") +URL = f"http://s3:8333/{BUCKET}/{KEY}" +try: + urllib.request.urlopen(URL) + print("FAIL: object accessible without auth"); sys.exit(1) +except urllib.error.HTTPError as e: + if e.code in (401, 403): + print(f"PASS: anonymous GET -> {e.code}"); sys.exit(0) + print(f"FAIL: unexpected {e.code}"); sys.exit(1) \ No newline at end of file diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..64e3fba0 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,81 @@ +import os +import sys +import boto3 +from botocore.client import Config +import psycopg2 +import psycopg2.extras +from datetime import datetime, timezone, timedelta + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +def run_reconciliation(dry_run: bool = False): + print(f"Starting reconciliation (dry_run={dry_run})...") + conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + + threshold = datetime.now(timezone.utc) - timedelta(hours=1) + + with conn.cursor() as cur: + cur.execute(""" + SELECT id, chat_id, filename, created_at FROM attachments + WHERE status = 'pending' AND created_at < %s + """, (threshold,)) + stale_pending = cur.fetchall() + + for row in stale_pending: + att_id = str(row['id']) + chat_id = str(row['chat_id']) + filename = row['filename'] + s3_key = f"attachments/{chat_id}/{att_id}/{filename}" + print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.") + + if not dry_run: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key) + except Exception as e: + print(f"Failed to delete S3 key {s3_key}: {e}") + with conn.cursor() as cur: + cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,)) + conn.commit() + + with conn.cursor() as cur: + cur.execute("SELECT id, chat_id, filename FROM attachments") + all_db_attachments = cur.fetchall() + + valid_s3_keys = set() + for row in all_db_attachments: + valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}") + + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"): + for obj in page.get('Contents', []): + key = obj['Key'] + last_modified = obj['LastModified'] + + if key not in valid_s3_keys and last_modified < threshold: + print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})") + if not dry_run: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=key) + print(f"Deleted orphan object {key} from S3.") + except Exception as e: + print(f"Failed to delete orphan {key}: {e}") + + conn.close() + print("Reconciliation completed successfully.") + +if __name__ == "__main__": + dry_run_mode = "--dry-run" in sys.argv + run_reconciliation(dry_run=dry_run_mode) \ No newline at end of file diff --git a/scripts/upload_load_test.py b/scripts/upload_load_test.py new file mode 100644 index 00000000..cdd2e2e3 --- /dev/null +++ b/scripts/upload_load_test.py @@ -0,0 +1,54 @@ +import base64, concurrent.futures, json, pathlib, time, urllib.request, uuid + +BASE = "http://localhost:8080" + +def _req(method, path, body=None, headers=None, token=None): + h = {"Accept": "application/json"} + if token: h["Authorization"] = f"Bearer {token}" + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=body, headers=h, method=method) + with urllib.request.urlopen(req) as r: + return r.status, r.read() + +def register_and_login(): + u = f"u_{uuid.uuid4().hex[:10]}" + p = "T3st-" + uuid.uuid4().hex + _req("POST", "/api/v1/users", + json.dumps({"username": u, "display_name": "Load", "password": p}).encode(), + {"Content-Type": "application/json"}) + basic = base64.b64encode(f"{u}:{p}".encode()).decode() + _, body = _req("POST", "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {basic}"}) + return json.loads(body)["token"] + +def make_chat(token): + _, me = _req("GET", "/api/v1/users/me", token=token) + uid = json.loads(me)["id"] + _, chat = _req("POST", "/api/v1/chats", + json.dumps({"title": "load", "member_ids": [uid]}).encode(), + {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}, + token=token) + return json.loads(chat)["id"] + +def main(): + token = register_and_login() + chat = make_chat(token) + png = pathlib.Path("tests/fixtures/sample.png").read_bytes() + + def one(_): + try: + status, _ = _req("POST", + f"/api/v1/chats/{chat}/attachments?filename=f.png", + png, {"Content-Type": "image/png"}, token) + return status + except Exception as e: + return repr(e) + + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as ex: + results = list(ex.map(one, range(10))) + print(json.dumps({"results": results, + "elapsed_s": round(time.monotonic() - t0, 3)})) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/level4_tests.py b/tests/level4_tests.py new file mode 100644 index 00000000..38489a75 --- /dev/null +++ b/tests/level4_tests.py @@ -0,0 +1,139 @@ +import urllib.request +import urllib.error +import json +import unittest +from client import Client + +class TestMessengerLevel4Complete(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice = self.client.user("alice") + self.bob = self.client.user("bob") + self.eve = self.client.user("eve") + + res_chat1 = self.client.call( + 'POST', '/api/v1/chats', 201, + actor=self.alice, + body={"title": "Team Chat", "member_ids": [self.alice["id"], self.bob["id"]]}, + schema='Chat' + ) + self.chat1 = res_chat1['json'] + + res_chat2 = self.client.call( + 'POST', '/api/v1/chats', 201, + actor=self.eve, + body={"title": "Private Chat", "member_ids": [self.eve["id"], self.bob["id"]]}, + schema='Chat' + ) + self.chat2 = res_chat2['json'] + + def test_unsupported_media_type(self): + """Проверка 415: передача текстового файла под видом картинки.""" + fake_bytes = b"Just a simple plain text file content." + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=notes.txt" + req = urllib.request.Request( + url, data=fake_bytes, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 415") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 415) + + def test_payload_too_large(self): + """Проверка 413: файл размером больше 10 MiB.""" + large_data = b'\x89PNG\r\n\x1a\n' + b'\x00' * (10 * 1024 * 1024 + 100) + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=huge.png" + req = urllib.request.Request( + url, data=large_data, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 413") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 413) + + def test_corrupted_image_signature(self): + """Проверка 415: битая структура / неверные размеры PNG.""" + corrupted_png = b'\x89PNG\r\n\x1a\n' + b'\xFF' * 50 + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=corrupt.png" + req = urllib.request.Request( + url, data=corrupted_png, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 415") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 415) + + def test_invalid_filename_characters(self): + """Проверка 400: недопустимые символы (path traversal) в имени файла.""" + valid_png = ( + b'\x89PNG\r\n\x1a\n' + b'\x00\x00\x00\rIHDR' + b'\x00\x00\x00\n\x00\x00\x00\n' + b'\x08\x02\x00\x00\x00' + b'\x02p\xdd\xb5' + b'\x00\x00\x00\x0fcIDATx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q' + b'\x00\x00\x00\x00IEND\xaeB`\x82' + ) + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=../../etc/passwd" + req = urllib.request.Request( + url, data=valid_png, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 400") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 400) + + def test_nonexistent_attachment_id(self): + """Проверка 404: передача несуществующего ID вложения при отправке сообщения.""" + fake_uuid = "12345678-1234-4123-8123-123456789abc" + res = self.client.call( + 'POST', f"/api/v1/chats/{self.chat1['id']}/messages", 404, + actor=self.alice, + body={"text": "Check this out", "attachment_ids": [fake_uuid]}, + headers={"Idempotency-Key": "test-key-lvl4-1"} + ) + self.assertEqual(res['status'], 404) + + def test_cross_chat_attachment_isolation(self): + """Проверка 404: попытка прикрепить вложение из чужого чата.""" + valid_png = ( + b'\x89PNG\r\n\x1a\n' + b'\x00\x00\x00\rIHDR' + b'\x00\x00\x00\n\x00\x00\x00\n' + b'\x08\x02\x00\x00\x00' + b'\x02p\xdd\xb5' + b'\x00\x00\x00\x0fcID1Tx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q' + b'\x00\x00\x00\x00IEND\xaeB`\x82' + ) + url = self.client.base + f"/api/v1/chats/{self.chat2['id']}/attachments?filename=secret.png" + req = urllib.request.Request( + url, data=valid_png, + headers={"Authorization": f"Bearer {self.eve['token']}", "Content-Type": "image/png"}, + method="POST" + ) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read().decode()) + eve_att_id = data["id"] + + res = self.client.call( + 'POST', f"/api/v1/chats/{self.chat1['id']}/messages", 404, + actor=self.alice, + body={"text": "Stealing file", "attachment_ids": [eve_att_id]}, + headers={"Idempotency-Key": "test-key-lvl4-2"} + ) + self.assertEqual(res['status'], 404) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file -- 2.54.0 From f5d39b4b7215fe24c948d1ab0075fa57572f8b28 Mon Sep 17 00:00:00 2001 From: Getlles Date: Mon, 9 Nov 2026 23:50:04 +0300 Subject: [PATCH 4/4] report --- REPORT.md | 119 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 23 deletions(-) diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..078bf817 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,110 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бондаренко Мария, БПИ2302 +- Целевая оценка: 4 -## Запуск из чистого clone +## 1. Архитектура системы и компонентов -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Система представляет собой отказоустойчивый REST API мессенджера, развернутый в изолированной Docker-сети: -## Архитектура и решения +1. Nginx: Порт 8080:8080. Осуществляет балансировку нагрузки Round-Robin между двумя репликами приложения (api-1 и api-2). +2. API Replicas (api-1, api-2): Приложения на FastAPI, обрабатывающие бизнес-логику, идемпотентность, аутентификацию и медиафайлы. Лимит ресурсов: CPU 0.50, RAM 256M. +3. PostgreSQL 15: Хранение реляционных данных (пользователи, чаты, сообщения, метаданные вложений attachments, таблицы идемпотентности idempotency_keys). +4. Redis 7: Хранение токенов сессий с TTL. +5. S3 Storage (s3 — SeaweedFS mini): Объектное хранилище для бинарных данных вложений. +6. S3 Bootstrapper (s3-init): Сервис однократного запуска для идемпотентной инициализации бакета. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +### Изоляция S3 и разделение прав доступа +- Сетевая приватность: У сервиса s3 полностью удалена публикация портов наружу. Внешний доступ к S3 заблокирован, все запросы к объектам проходят исключительно через авторизованные ручки API мессенджера. +- Разделение учётных записей: + - s3-init использует административные креды для создания бакета при старте. + - api-1 и api-2 используют ограничение по правам доступа приложения (appuser / appsecretpassword). -## Доказательства критериев +--- -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## 2. Двухшаговая схема загрузки и согласованность БД/S3 -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Для обеспечения согласованности между реляционной БД и S3 при возникновении сбоев (сетевые таймауты, падения контейнеров) реализован двухшаговый протокол загрузки: -## Эксперименты и отказы +Client ---> POST /api/v1/chats/{id}/attachments + | + v + 1. INSERT INTO attachments (status='pending') ---> PostgreSQL + | + v + 2. s3_client.put_object(...) ---> S3 Storage + | + v + 3. UPDATE attachments SET status='ready' ---> PostgreSQL -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Обработка аварийных ситуаций: +- Crash-Safety (падение процесса в любой момент): + - При сбое до Step 1 — транзакция не создается, данные не повреждены. + - При сбое на Step 2 или 3 — запись остается в БД со статусом pending. Немедленное мгновенное удаление из S3 (delete_object в эксепшне) не производится, так как при таймауте ответа COMMIT мог успешно завершиться на стороне базы, и прямое удаление привело бы к появлению «битой» ссылки (missing object). +- Периодическая сверка (scripts/reconcile.py): + - Grace Period (1 час): Все записи и файлы младше 1 часа считаются находящимися в процессе загрузки и не затрагиваются сверкой. + - Сверка со стороны БД: Находит записи со статусом pending, созданные более 1 часа назад (created_at < now() - 1 hour). Они переводятся в статус failed (error_code='timeout_stale'), а соответствующий ключ в S3 удаляется. + - Сверка со стороны S3: Выполняет list_objects_v2 по префиксу attachments/. Если найден объект старше 1 часа, которого нет в PostgreSQL со статусом ready, он удаляется из S3 как «сиротский» (orphan). -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +--- -## Ограничения и следующие шаги +## 3. Валидация и безопасность медиафайлов -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Реализованы следующие уровни защиты при загрузке файлов: +1. Потоковая передача: Чтение тела запроса через request.stream() с лимитом 10 MiB. Предотвращает исчерпание RAM сервера при передаче больших файлов. +2. Проверка имен файлов: Запрет символов /, \, null-байтов и управляющих символов ASCII (коды < 0x20 и 0x7F). При нарушении возвращается 400 Bad Request. +3. Проверка Magic Bytes: Валидация структуры заголовков PNG (\x89PNG\r\n\x1a\n + валидация IHDR) и JPEG (\xFF\xD8). При несовпадении возвращается 415 Unsupported Media Type. +4. Ограничение разрешения: Парсинг геометрических размеров изображения (W * H). Превышение лимита в 40 000 000 пикселей (40 MP) отвергается со статусом 413 Payload Too Large. +5. Изоляция вложений между чатами: При создании сообщения проверяется, что все attachment_ids принадлежат именно данному chat_id. При попытке использовать чужое или несуществующее вложение возвращается 404 Not Found. + +--- + +## 4. Результаты тестов (Evidence) + +Все проверки и подтверждающие артефакты сохранены в директории evidence/: + +### 4.1. Проверка состояний контейнеров (evidence/compose_ps.txt) +Все сервисы (api-1, api-2, postgres, redis, s3, nginx) находятся в состоянии Up (healthy). + +### 4.2. Smoke-тестирование базового API (evidence/smoke.txt) +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. + +### 4.3. Негативное тестирование Уровня 4 (evidence/level4.txt) +Успешно пройден полный набор тестов tests/level4_tests.py (6 из 6): +- test_payload_too_large — обработка превышения 10 MiB (413). +- test_unsupported_media_type — загрузка невалидного типа (415). +- test_corrupted_image_signature — проверка поврежденных бинарных данных PNG (415). +- test_invalid_filename_characters — защита от path traversal (400). +- test_nonexistent_attachment_id — проверка привязки несуществующего attachment_id (404). +- test_cross_chat_attachment_isolation — изоляция чужих вложений между чатами (404). + +### 4.4. Проверка приватности S3-бакета (evidence/bucket_private.txt) +PASS: anonymous GET -> 403 +Прямой анонимный HTTP-доступ к объектам S3 без авторизации закрыт. + +### 4.5. Тестирование отказа и устойчивости сессий (evidence/failover.txt, evidence/persistence.txt) +- Failover: Отключение одной реплики API под нагрузкой показало восстановление за 1.312 сек. Без потери подтвержденных данных и разрыва сессий. +- Persistence: Полный перезапуск контейнеров с сохранением данных в томах подтвержден (PASS). + +### 4.6. Запуск скрипта сверки (evidence/reconcile_dry_run.txt) +Starting reconciliation (dry_run=True)... +Reconciliation completed successfully. +Скрипт идемпотентен, безопасен для выполнения в режиме --dry-run. + +--- + +## 5. Нагрузочное тестирование памяти + +Было проведено нагрузочное тестирование одновременной загрузки 10 файлов по 10 MiB при ограничении RAM контейнеров api-1/api-2 в 256 MiB. + +### Результаты Docker Stats (evidence/upload_load.json, evidence/upload_load_stats.txt): + +| Сервис | Размер файлов | Кол-во потоков ($N$) | Потребление RAM (Пик) | Лимит RAM | Результат | +|---|---|---|---|---|---| +| api-1 | 10 MiB | 10 | 74.22 MiB | 256 MiB | PASS (< 300 MiB) | +| api-2 | 10 MiB | 10 | 74.76 MiB | 256 MiB | PASS (< 300 MiB) | +| s3 | 10 MiB | 10 | 68.62 MiB | 512 MiB | PASS | + +### Вывод по нагрузке: +Благодаря реализации потокового чтения файла (request.stream()), объём используемой оперативной памяти на реплику составил не более 74.76 MiB (около 29% от выделенного лимита в 256 MiB), что подтверждает отсутствие утечек и защищает приложения от падения по OOM. -- 2.54.0