From 7d4cf242e334ffb2336c50e09cfcd41b500251c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 1 Oct 2026 21:17:29 +0300 Subject: [PATCH 01/10] chore: keep lab plans and notes local --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b5a71896..f101a4f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ # Local/private state +/docs/plans/ +/LAB_NOTES.md .env .env.* !.env.example -- 2.54.0 From bd41f2d93dc8996ee402cb16faf9143bad85274e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 12:46:12 +0300 Subject: [PATCH 02/10] feat: port replicated messenger baseline into lab4 --- Dockerfile | 12 ++ compose.yaml | 70 ++++++++ infra/haproxy.cfg | 26 +++ migrations/001_initial.sql | 36 ++++ migrations/002_sessions.sql | 7 + migrations/003_message_replays.sql | 11 ++ requirements.txt | 6 + scripts/migrate.py | 30 ++++ scripts/restart.sh | 5 + scripts/stop-one.sh | 13 ++ src/api.py | 96 ++++++++++ src/auth.py | 43 +++++ src/db.py | 38 ++++ src/main.py | 66 +++++++ src/models.py | 61 +++++++ src/server.py | 14 ++ src/store.py | 275 +++++++++++++++++++++++++++++ student-tests/test_api_limits.py | 20 +++ student-tests/test_auth.py | 174 ++++++++++++++++++ student-tests/test_database.py | 85 +++++++++ student-tests/test_idempotency.py | 53 ++++++ student-tests/test_instance.py | 54 ++++++ student-tests/test_lab3_compat.py | 27 +++ 23 files changed, 1222 insertions(+) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_sessions.sql create mode 100644 migrations/003_message_replays.sql create mode 100644 requirements.txt create mode 100644 scripts/migrate.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/api.py create mode 100644 src/auth.py create mode 100644 src/db.py create mode 100644 src/main.py create mode 100644 src/models.py create mode 100644 src/server.py create mode 100644 src/store.py create mode 100644 student-tests/test_api_limits.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_database.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_instance.py create mode 100644 student-tests/test_lab3_compat.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1ecad6dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app +COPY --chown=app:app src/ src/ +COPY --chown=app:app migrations/ migrations/ +COPY --chown=app:app scripts/migrate.py scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["python", "-m", "src.server"] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e91d213b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,70 @@ +name: mtusi2026lab4 + +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + migrate: + build: . + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + scale: 2 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url] + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + stop_grace_period: 10s + cpus: 0.5 + mem_limit: 256m + + lb: + image: haproxy:3.0.11-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + +volumes: + postgres_data: diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..40952ab9 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,26 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 2s + timeout client 15s + timeout server 15s + +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 2 + timeout retry 1s + hold valid 1s + +frontend public + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..dbfb0065 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX chats_history_idx ON chats (created_at, id); + +CREATE TABLE memberships ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_sessions.sql b/migrations/002_sessions.sql new file mode 100644 index 00000000..5d2f364a --- /dev/null +++ b/migrations/002_sessions.sql @@ -0,0 +1,7 @@ +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); diff --git a/migrations/003_message_replays.sql b/migrations/003_message_replays.sql new file mode 100644 index 00000000..e5a86270 --- /dev/null +++ b/migrations/003_message_replays.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_replays ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + request_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, request_key) +); + +CREATE INDEX message_replays_expiry_idx ON message_replays (created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..58563cf5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.118.3 +pydantic==2.12.4 +uvicorn==0.38.0 +psycopg[binary,pool]==3.2.12 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..e346d4f1 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,30 @@ +"""Apply numbered SQL migrations once, atomically and under a DB lock.""" + +from pathlib import Path +import sys + +import psycopg + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import database_url + + +def main() -> None: + migration_dir = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(2026, 2)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())" + ) + for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")): + version = int(path.name.split("_", 1)[0]) + if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone(): + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c9148ca0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api lb postgres +docker compose --env-file .env up -d --wait --scale api=2 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..99c65a25 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +for container in $(docker compose --env-file .env ps -q api); do + instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$container" >/dev/null + exit 0 + fi +done +echo "target API instance not found" >&2 +exit 1 diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..4d2965cd --- /dev/null +++ b/src/api.py @@ -0,0 +1,96 @@ +"""Messenger HTTP routes.""" + +import re +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from .auth import basic_credentials, bearer_token +from .models import CreateChat, CreateMessage, CreateUser +from .store import store + +router = APIRouter() + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + return store.ready() + + +@router.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser): + return store.register(body.username, body.display_name, body.password) + + +@router.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request, response: Response): + if await request.body(): + raise HTTPException(400) + username, password = basic_credentials(request.headers.get("authorization")) + response.headers["Cache-Control"] = "no-store" + return store.create_session(username, password) + + +def current_user(request: Request) -> dict: + return store.authenticated(bearer_token(request.headers.get("authorization"))) + + +def parse_limit(value: str) -> int: + try: + number = int(value) if value.isdecimal() else 0 + except ValueError: + raise HTTPException(400) from None + if not 1 <= number <= 100: + raise HTTPException(400) + return number + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + store.revoke_session(bearer_token(request.headers.get("authorization"))) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +def user_me(actor: dict = Depends(current_user)): + return actor + + +@router.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(current_user)): + return store.create_chat(actor["id"], body.title, body.member_ids) + + +@router.get("/api/v1/chats") +def list_chats( + limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user) +): + return store.list_chats(actor["id"], parse_limit(limit), cursor) + + +@router.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): + return store.chat(actor["id"], str(chat_id)) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): + key = request.headers.get("Idempotency-Key") + if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400) + return store.create_message(actor["id"], str(chat_id), body.text, key, body.attachment_ids) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: UUID, + limit: str = Query("50"), + cursor: str | None = Query(None, min_length=1), + actor: dict = Depends(current_user), +): + return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..5714b173 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,43 @@ +"""Parse HTTP credentials without exposing them in errors or logs.""" + +import base64 +import binascii +import hashlib +import re + +from fastapi import HTTPException + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z") + + +def unauthorized(challenge: str) -> HTTPException: + return HTTPException(401, headers={"WWW-Authenticate": challenge}) + + +def basic_credentials(header: str | None) -> tuple[str, str]: + if not header or not header.startswith("Basic "): + raise unauthorized(BASIC_CHALLENGE) + try: + raw = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeError, ValueError): + raise unauthorized(BASIC_CHALLENGE) from None + if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128: + raise unauthorized(BASIC_CHALLENGE) + return username, password + + +def bearer_token(header: str | None) -> str: + if not header or not header.startswith("Bearer "): + raise unauthorized(BEARER_CHALLENGE) + token = header[7:] + if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token): + raise unauthorized(BEARER_CHALLENGE) + return token + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..832ff074 --- /dev/null +++ b/src/db.py @@ -0,0 +1,38 @@ +"""Bounded PostgreSQL pool shared by API storage and health checks.""" + +import os +import atexit +from functools import lru_cache +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from psycopg_pool import PoolTimeout + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "postgresql:///messenger") + + +@lru_cache(maxsize=1) +def pool() -> ConnectionPool: + result = ConnectionPool( + database_url(), min_size=0, max_size=8, timeout=5, + kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row}, + open=True, + ) + atexit.register(result.close) + return result + + +def ready() -> bool: + try: + with pool().connection() as conn: + conn.execute("SELECT 1 FROM users LIMIT 0") + return True + except (psycopg.Error, PoolTimeout, OSError): + return False diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..bcac75fd --- /dev/null +++ b/src/main.py @@ -0,0 +1,66 @@ +"""FastAPI entry point and contract-wide response handling.""" + +from uuid import uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from .api import router + +app = FastAPI() +app.include_router(router) +INSTANCE_ID = uuid4().hex + +ERRORS = { + 400: ("invalid_request", "Неверный запрос"), + 401: ("unauthorized", "Требуется действительный пользователь"), + 404: ("not_found", "Объект не найден"), + 409: ("conflict", "Конфликт данных"), + 413: ("payload_too_large", "Слишком большой запрос"), + 415: ("unsupported_media_type", "Неподдерживаемый тип данных"), + 429: ("rate_limited", "Слишком много запросов"), + 503: ("unavailable", "Сервис недоступен"), +} + + +def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse: + code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка")) + request_id = request.state.request_id + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})}, + ) + + +@app.middleware("http") +async def request_contract(request: Request, call_next): + request.state.request_id = str(uuid4()) + if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions": + media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") + if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"): + return error_response(request, 415) + try: + (await request.body()).decode("utf-8") + except UnicodeDecodeError: + return error_response(request, 400) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_error(request: Request, exc: RequestValidationError): + return error_response(request, 400) + + +@app.exception_handler(HTTPException) +async def http_error(request: Request, exc: HTTPException): + return error_response(request, exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def unexpected_error(request: Request, exc: Exception): + return error_response(request, 500) diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..7ee9ec7c --- /dev/null +++ b/src/models.py @@ -0,0 +1,61 @@ +"""Request bodies from the lab 1 contract.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class StrictBody(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUser(StrictBody): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def nonblank_name(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank display name") + return value + + @field_validator("password") + @classmethod + def nonblank_password(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank password") + return value + + +class CreateChat(StrictBody): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def nonblank_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + @classmethod + def valid_members(cls, value: list[str]) -> list[str]: + members = [str(UUID(member_id)) for member_id in value] + if len(set(members)) != len(members): + raise ValueError("duplicate member") + return members + + +class CreateMessage(StrictBody): + text: str = Field(min_length=1, max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def nonblank_text(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank text") + return value diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..20fb9c9c --- /dev/null +++ b/src/server.py @@ -0,0 +1,14 @@ +"""Run the ASGI worker as the container's main process.""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + ) diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..fa918d26 --- /dev/null +++ b/src/store.py @@ -0,0 +1,275 @@ +"""PostgreSQL-backed users, chats and message history.""" + +import base64 +import binascii +import hashlib +import json +import os +import secrets +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import psycopg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException +from psycopg.errors import UniqueViolation +from psycopg_pool import PoolTimeout + +from . import db +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized + +password_hasher = PasswordHasher() + + +def timestamp(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(resource: str, item: dict) -> str: + value = [resource, item["created_at"], item["id"]] + return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=") + + +def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + value = json.loads(raw) + if not isinstance(value, list) or len(value) != 3 or value[0] != resource: + raise ValueError + date_text, item_id = value[1:] + if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str): + raise ValueError + date = datetime.fromisoformat(date_text[:-1] + "+00:00") + if date.utcoffset().total_seconds() != 0: + raise ValueError + return date, UUID(item_id) + except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError): + raise HTTPException(400) from None + + +@contextmanager +def connection(): + try: + with db.pool().connection() as conn: + yield conn + except (psycopg.Error, PoolTimeout, OSError): + raise HTTPException(503) from None + + +def user_view(row: dict) -> dict: + return { + "id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": timestamp(row["created_at"]), + } + + +def chat_view(row: dict) -> dict: + return { + "id": str(row["id"]), "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": timestamp(row["created_at"]), + } + + +def message_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), "text": row["text"], + "created_at": timestamp(row["created_at"]), + "attachment_ids": [], + } + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids""" + + +class Store: + def ready(self) -> dict: + if not db.ready(): + raise HTTPException(503) + return {"status": "ok"} + + def register(self, username: str, display_name: str, password: str) -> dict: + password_hash = password_hasher.hash(password) + with connection() as conn: + try: + row = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + (uuid4(), username, display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise HTTPException(409) from None + return user_view(row) + + def user(self, user_id: str | None) -> dict: + if not user_id: + raise HTTPException(401) + try: + user_uuid = UUID(user_id) + except (ValueError, TypeError): + raise HTTPException(401) from None + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,) + ).fetchone() + if row is None: + raise HTTPException(401) + return user_view(row) + + def create_session(self, username: str, password: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", + (username,), + ).fetchone() + try: + valid = row is not None and password_hasher.verify(row["password_hash"], password) + except (VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + expiry = conn.execute( + "INSERT INTO sessions (token_hash,user_id,expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at", + (token_hash(token), row["id"], ttl), + ).fetchone()["expires_at"] + return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)} + + def authenticated(self, token: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s " + "JOIN users u ON u.id=s.user_id " + "WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + return user_view(row) + + def revoke_session(self, token: str) -> None: + with connection() as conn: + row = conn.execute( + "DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + + def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict: + if actor_id not in member_ids: + raise HTTPException(400) + members = [UUID(member_id) for member_id in member_ids] + with connection() as conn: + count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"] + if count != len(members): + raise HTTPException(400) + row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone() + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)", + (row["id"], member_id, position), + ) + return chat_view({**row, "member_ids": members}) + + def chat(self, actor_id: str, chat_id: str) -> dict: + with connection() as conn: + row = conn.execute( + f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id " + "WHERE access.user_id=%s AND c.id=%s", + (UUID(actor_id), UUID(chat_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return chat_view(row) + + def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict: + after = decode_cursor(cursor, f"chats:{actor_id}") + query = (f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s") + params = [UUID(actor_id)] + if after: + query += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + rows = conn.execute(query, params).fetchall() + items = [chat_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} + + def create_message(self, actor_id: str, chat_id: str, text: str, key: str, attachment_ids: list[str] | None = None) -> dict: + if attachment_ids: + raise HTTPException(404) + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": []}, ensure_ascii=False, sort_keys=True).encode()).digest() + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + conn.execute( + "DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s " + "AND created_at dict: + after = decode_cursor(cursor, f"messages:{chat_id}") + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + query = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s" + params = [chat_uuid] + if after: + query += " AND (created_at,id)>(%s,%s)" + params.extend(after) + query += " ORDER BY created_at,id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + rows = conn.execute(query, params).fetchall() + items = [message_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None} + + +store = Store() diff --git a/student-tests/test_api_limits.py b/student-tests/test_api_limits.py new file mode 100644 index 00000000..4936d6f3 --- /dev/null +++ b/student-tests/test_api_limits.py @@ -0,0 +1,20 @@ +"""Invalid pagination limits must remain client errors.""" + +import unittest + +from fastapi import HTTPException + +from src.api import parse_limit + + +class LimitTests(unittest.TestCase): + def test_valid_and_extremely_long_limits(self): + self.assertEqual(parse_limit("100"), 100) + for value in ("0", "101", "-1", "x", "9" * 4301): + with self.assertRaises(HTTPException) as failure: + parse_limit(value) + self.assertEqual(failure.exception.status_code, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..e9fcdf09 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,174 @@ +"""HTTP and database checks for durable Basic/Bearer sessions.""" + +import base64 +import hashlib +import json +import os +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.request +from uuid import uuid4 + +import psycopg + + +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +class AuthTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + + @classmethod + def start_server(cls, ttl="3600"): + cls.instance_id = None + env = {**os.environ, "SESSION_TTL_SECONDS": ttl} + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API server exited during startup") + try: + cls.call("GET", "/health/ready", expected=200) + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + raise AssertionError("API server did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def tearDownClass(cls): + cls.stop_server() + + @classmethod + def call(cls, method, path, *, body=None, authorization=None, user_id=None, expected=200): + headers = {"Accept": "application/json"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode() + headers["Content-Type"] = "application/json" + if authorization is not None: + headers["Authorization"] = authorization + if user_id is not None: + headers["X-User-Id"] = user_id + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", data=data, headers=headers, method=method, + ) + try: + response = urllib.request.urlopen(request, timeout=5) + except urllib.error.HTTPError as error: + response = error + with response: + status = response.status + response_headers = response.headers + data = response.read() + assert status == expected, (method, path, expected, status) + request_id = response_headers.get("X-Request-Id") + assert request_id + instance_id = response_headers.get("X-Instance-Id") + assert instance_id + if cls.instance_id is None: + cls.instance_id = instance_id + else: + assert instance_id == cls.instance_id + payload = json.loads(data) if data else None + if status >= 400: + assert payload["error"]["request_id"] == request_id + assert "password" not in data.decode().lower() + assert "token" not in data.decode().lower() + return payload, response_headers + + def user(self, password="Пароль:очень-длинный-123"): + username = "u_" + uuid4().hex[:16] + body = {"username": username, "display_name": "Алиса", "password": password} + user, _ = self.call("POST", "/api/v1/users", body=body, expected=201) + self.assertNotIn("password", user) + return user, password + + def login(self, username, password, expected=201): + encoded = base64.b64encode(f"{username}:{password}".encode()).decode() + return self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, expected=expected) + + def test_basic_bearer_logout_and_storage(self): + alice, password = self.user() + bob, _ = self.user() + session, headers = self.login(alice["username"], password) + self.assertEqual(session["token_type"], "Bearer") + self.assertEqual(session["user"], alice) + self.assertGreaterEqual(len(base64.urlsafe_b64decode(session["token"] + "=" * (-len(session["token"]) % 4))), 32) + self.assertEqual(headers.get("Cache-Control"), "no-store") + bearer = "Bearer " + session["token"] + me, _ = self.call("GET", "/api/v1/users/me", authorization=bearer, user_id=bob["id"]) + self.assertEqual(me, alice) + with psycopg.connect(URL) as conn: + row = conn.execute("SELECT token_hash,expires_at FROM sessions WHERE user_id=%s", (alice["id"],)).fetchone() + password_hash = conn.execute("SELECT password_hash FROM users WHERE id=%s", (alice["id"],)).fetchone()[0] + self.assertEqual(row[0], hashlib.sha256(session["token"].encode()).digest()) + self.assertNotEqual(password_hash, password) + self.assertTrue(password_hash.startswith("$argon2id$")) + self.call("DELETE", "/api/v1/auth/sessions/current", authorization=bearer, expected=204) + for method, path in (("GET", "/api/v1/users/me"), ("DELETE", "/api/v1/auth/sessions/current")): + _, headers = self.call(method, path, authorization=bearer, expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"') + + def test_challenges_and_uniform_credentials(self): + user, password = self.user() + failures = [] + for username, candidate in ((user["username"], "wrong-password-123"), ("missing_user_123", password)): + payload, headers = self.login(username, candidate, expected=401) + failures.append((payload["error"]["code"], payload["error"]["message"])) + self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(failures[0], failures[1]) + encoded = base64.b64encode(f"{user['username']}:{password}".encode()).decode() + for authorization in (None, "Basic " + encoded, "Bearer garbage"): + _, headers = self.call("GET", "/api/v1/users/me", authorization=authorization, user_id=user["id"], expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"') + _, headers = self.call("POST", "/api/v1/auth/sessions", authorization="Bearer garbage", expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"') + self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, body={"unexpected": True}, expected=400) + + def test_z_short_ttl_survives_server_restart_without_extension(self): + self.stop_server() + self.start_server(ttl="2") + try: + user, password = self.user() + session, _ = self.login(user["username"], password) + bearer = "Bearer " + session["token"] + self.call("GET", "/api/v1/users/me", authorization=bearer) + with psycopg.connect(URL) as conn: + before = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0] + self.stop_server() + self.start_server(ttl="3600") + with psycopg.connect(URL) as conn: + after = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0] + self.assertEqual(before, after) + time.sleep(2.1) + _, headers = self.call("GET", "/api/v1/users/me", authorization=bearer, expected=401) + self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"') + finally: + self.stop_server() + self.start_server() + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_database.py b/student-tests/test_database.py new file mode 100644 index 00000000..132318f9 --- /dev/null +++ b/student-tests/test_database.py @@ -0,0 +1,85 @@ +"""Integration checks against a disposable PostgreSQL database.""" + +import os +import subprocess +import sys +import unittest +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import psycopg + +from src.store import Store + + +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +class DatabaseTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + cls.store = Store() + + def user(self): + name = "u_" + uuid4().hex[:16] + return self.store.register(name, "Студент", "Test-password-123") + + def test_migration_and_unique_username_race(self): + with psycopg.connect(URL) as conn: + versions = conn.execute("SELECT version FROM schema_migrations").fetchall() + self.assertEqual(versions, [(1,), (2,), (3,)]) + name = "u_" + uuid4().hex[:16] + + def register(_): + try: + return self.store.register(name, "Студент", "Test-password-123") + except Exception as exc: + return exc + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(register, range(2))) + self.assertEqual(sum(isinstance(result, dict) for result in results), 1) + self.assertEqual([result.status_code for result in results if not isinstance(result, dict)], [409]) + with psycopg.connect(URL) as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1) + password_hash = conn.execute("SELECT password_hash FROM users WHERE username=%s", (name,)).fetchone()[0] + self.assertTrue(password_hash.startswith("$argon2id$")) + self.assertNotIn("password", str(next(result for result in results if isinstance(result, dict)))) + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + with psycopg.connect(URL) as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1) + + def test_chat_rollback_and_keyset_history(self): + alice, bob, eve = self.user(), self.user(), self.user() + with psycopg.connect(URL) as conn: + before = conn.execute("SELECT count(*) FROM chats").fetchone()[0] + with self.assertRaises(Exception) as failure: + self.store.create_chat(alice["id"], "invalid", [alice["id"], str(uuid4())]) + self.assertEqual(failure.exception.status_code, 400) + with psycopg.connect(URL) as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM chats").fetchone()[0], before) + + chat = self.store.create_chat(alice["id"], "Valid", [alice["id"], bob["id"]]) + chat2 = self.store.create_chat(alice["id"], "Second", [alice["id"]]) + chat_page = self.store.list_chats(alice["id"], 1, None) + self.assertEqual(chat_page["items"], [chat]) + self.assertEqual(self.store.list_chats(alice["id"], 1, chat_page["next_cursor"]), {"items": [chat2], "next_cursor": None}) + first = self.store.create_message(alice["id"], chat["id"], "первое", uuid4().hex) + second = self.store.create_message(bob["id"], chat["id"], "второе", uuid4().hex) + page = self.store.list_messages(alice["id"], chat["id"], 1, None) + self.assertEqual(page["items"], [first]) + self.assertEqual(self.store.list_messages(bob["id"], chat["id"], 1, page["next_cursor"]), {"items": [second], "next_cursor": None}) + self.assertEqual(Store().chat(bob["id"], chat["id"]), chat) + with self.assertRaises(Exception) as wrong_cursor: + self.store.list_messages(alice["id"], chat2["id"], 1, page["next_cursor"]) + self.assertEqual(wrong_cursor.exception.status_code, 400) + with self.assertRaises(Exception) as hidden: + self.store.chat(eve["id"], chat["id"]) + self.assertEqual(hidden.exception.status_code, 404) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..3ca7a838 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,53 @@ +"""Concurrent PostgreSQL-backed message retries create one row.""" + +import os +import subprocess +import sys +import unittest +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import psycopg + +from src.store import Store + + +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +class IdempotencyTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], check=True) + cls.store = Store() + + def test_parallel_replay_conflict_and_scope(self): + alice = self.store.register("u_" + uuid4().hex[:16], "Алиса", "Test-password-123") + bob = self.store.register("u_" + uuid4().hex[:16], "Боб", "Test-password-123") + chat = self.store.create_chat(alice["id"], "Test", [alice["id"], bob["id"]]) + other_chat = self.store.create_chat(alice["id"], "Other", [alice["id"]]) + key = uuid4().hex + + def send(_): + return Store().create_message(alice["id"], chat["id"], "same body", key, [] if _ else None) + + with ThreadPoolExecutor(max_workers=2) as pool: + first, replay = list(pool.map(send, range(2))) + self.assertEqual(first, replay) + with psycopg.connect(URL) as conn: + count = conn.execute( + "SELECT count(*) FROM messages WHERE chat_id=%s AND sender_id=%s AND text=%s", + (chat["id"], alice["id"], "same body"), + ).fetchone()[0] + self.assertEqual(count, 1) + + with self.assertRaises(Exception) as conflict: + self.store.create_message(alice["id"], chat["id"], "different body", key) + self.assertEqual(conflict.exception.status_code, 409) + self.assertNotEqual(self.store.create_message(bob["id"], chat["id"], "same body", key)["id"], first["id"]) + self.assertNotEqual(self.store.create_message(alice["id"], other_chat["id"], "same body", key)["id"], first["id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py new file mode 100644 index 00000000..87e14628 --- /dev/null +++ b/student-tests/test_instance.py @@ -0,0 +1,54 @@ +"""Every API response identifies the live replica process.""" + +import json +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.request + + +class InstanceHeaderTests(unittest.TestCase): + def test_process_alias_on_success_and_error(self): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(port)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + for _ in range(100): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/live", timeout=1) as response: + live_id = response.headers.get("X-Instance-Id") + self.assertEqual(json.load(response), {"status": "ok"}) + break + except urllib.error.URLError: + if server.poll() is not None: + self.fail("API exited before readiness") + time.sleep(0.05) + else: + self.fail("API did not start") + self.assertTrue(live_id) + for path in ("/health/live", "/no-such-route"): + try: + response = urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=1) + except urllib.error.HTTPError as error: + response = error + with response: + self.assertEqual(response.headers.get("X-Instance-Id"), live_id) + self.assertTrue(response.headers.get("X-Request-Id")) + finally: + server.terminate() + try: + server.wait(timeout=5) + except subprocess.TimeoutExpired: + server.kill() + server.wait(timeout=5) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_lab3_compat.py b/student-tests/test_lab3_compat.py new file mode 100644 index 00000000..87891d4f --- /dev/null +++ b/student-tests/test_lab3_compat.py @@ -0,0 +1,27 @@ +"""Text-only messages retain lab3 behavior under the lab4 wire schema.""" + +import unittest +from datetime import datetime, timezone +from uuid import uuid4 + +from src.models import CreateMessage +from src.store import message_view + + +class Lab3CompatibilityTests(unittest.TestCase): + def test_missing_and_explicit_empty_attachments_are_equivalent(self): + missing = CreateMessage.model_validate({"text": "hello"}) + explicit = CreateMessage.model_validate({"text": "hello", "attachment_ids": []}) + self.assertEqual(missing, explicit) + self.assertEqual(missing.attachment_ids, []) + + def test_text_message_response_has_empty_attachment_ids(self): + row = { + "id": uuid4(), "chat_id": uuid4(), "sender_id": uuid4(), + "text": "hello", "created_at": datetime.now(timezone.utc), + } + self.assertEqual(message_view(row)["attachment_ids"], []) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 22f7bcddf5773ba8a5c34f2a91d46951ff698662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 13:01:50 +0300 Subject: [PATCH 03/10] feat: provision private S3 storage and attachment schema --- .gitignore | 1 + REPORT.md | 56 +++++++++++------- compose.yaml | 69 ++++++++++++++++++++++- evidence/storage-bootstrap-2026-10-02.txt | 12 ++++ infra/bootstrap-s3.sh | 16 ++++++ infra/minio-api-policy.json | 15 +++++ infra/minio-maintenance-policy.json | 15 +++++ migrations/004_attachments.sql | 32 +++++++++++ requirements.txt | 1 + scripts/setup-local-secrets.py | 42 ++++++++++++++ src/attachments.py | 25 ++++++++ student-tests/test_auth.py | 5 +- student-tests/test_database.py | 7 ++- student-tests/test_storage_bootstrap.py | 38 +++++++++++++ 14 files changed, 309 insertions(+), 25 deletions(-) create mode 100644 evidence/storage-bootstrap-2026-10-02.txt create mode 100644 infra/bootstrap-s3.sh create mode 100644 infra/minio-api-policy.json create mode 100644 infra/minio-maintenance-policy.json create mode 100644 migrations/004_attachments.sql create mode 100644 scripts/setup-local-secrets.py create mode 100644 src/attachments.py create mode 100644 student-tests/test_storage_bootstrap.py diff --git a/.gitignore b/.gitignore index f101a4f8..79b556ba 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .env .env.* !.env.example +/compose.local.yaml secrets/ certs/private/ *.key diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..332286d0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,55 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Самохвалов Андрей Ремович, БПИ-2302. +- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Пока реализованы перенос lab3 и приватное S3; остальные блоки выполняются далее. +- Ветка: `solution/bpi2302-samohvalov`; commit/PR указываются после публикации. +- Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/openapi.json`, публичные тесты, [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z), [образ](https://hub.docker.com/r/alpine/minio/tags), документация MinIO для IAM. Python/FastAPI, PostgreSQL, MinIO Python SDK; проект подготовлен с помощью ИИ Codex с локальной проверкой ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Compose и Python 3.9+. Стенд использует только синтетические данные и слушает `127.0.0.1` на `APP_PORT` (по умолчанию 8080). В репозитории нет паролей: + +```sh +cp .env.example .env +python3 scripts/setup-local-secrets.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait --scale api=2 +make check +# make test — после реализации upload в следующем блоке +``` + +`setup-local-secrets.py` создаёт отсутствующие secret-файлы с правами 0600 и не перезаписывает существующие. Миграции выполняет одноразовый `migrate` под PostgreSQL advisory lock. `s3-volume-init` задаёт владельца нового volume, затем MinIO стартует без root; одноразовый `s3-bootstrap` идемпотентно создаёт закрытый bucket и две ограниченные учётные записи. Повторный `docker compose --env-file .env run --rm s3-bootstrap` безопасен. Для остановки — `docker compose --env-file .env down` без `-v`; для намеренного удаления **только учебных данных** после проверки — `docker compose --env-file .env down -v`. + +Версии: Python 3.12.12, PostgreSQL 17.6, HAProxy 3.0.11, MinIO `RELEASE.2025-10-15T17-29-55Z` (образ закреплён digest), MinIO Client `2025.7.21-debian-12-r3` (digest). Проверенный локальный стенд — Colima/arm64 macOS, 2 CPU, 2 054 840 320 байт памяти. Локальная сеть не разрешала DNS `files.pythonhosted.org`, поэтому сборка для проверки использовала **только локальный ignored** `compose.local.yaml` с временным IP CDN; в чистом clone он не нужен при обычном доступе к PyPI. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy → две API-реплики. PostgreSQL хранит пользователей, сессии, чаты, сообщения, ключи идемпотентности и метаданные вложений. MinIO хранит только байты оригиналов в private bucket `messenger-attachments`; локальная файловая система API не хранит оригиналы. Миграция `004_attachments.sql` связывает вложение с чатом; состав сообщения хранится в `message_attachments` с FK по паре `(id, chat_id)`, чтобы исключить межчатовые ссылки на уровне БД. Статусы таблицы допускают дальнейший переход к асинхронной lab5, но lab4 будет возвращать только `ready` после обеих устойчивых записей. + +| Сервис | Образ/роль | Health и volume | Секреты и доступ | +| --- | --- | --- | --- | +| `lb` | HAProxy 3.0.11, единственный host port | проверка backend, без volume | секретов нет | +| `api ×2` | Python 3.12.12, 0,5 CPU/256 MiB на реплику | `/health/ready`, без volume | только DB URL и ограниченные S3 credentials | +| `postgres` | PostgreSQL 17.6 | `pg_isready`, `postgres_data` | только DB password | +| `migrate` | одноразовый Python | завершение 0 | только DB URL | +| `s3` | MinIO security release, 0,5 CPU/384 MiB | `/minio/health/ready`, `s3_data` | только root credentials, наружный порт и консоль не опубликованы | +| `s3-bootstrap` | одноразовый MinIO Client | завершение 0 | root и создаваемые ограниченные credentials | +| `s3-volume-init` | одноразовая настройка владельца volume | завершение 0 | секретов нет | + +`infra/minio-api-policy.json` разрешает только операции над объектами одного bucket и чтение его location; создание bucket, изменение IAM и административные операции не разрешены. Отдельная maintenance policy добавляет list для будущей сверки. Bucket не имеет анонимного чтения. Приватность bucket не заменяет проверку членства в чате на API: эта проверка добавляется в блоках upload/download. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Перенос lab3 | Compose ×2, текстовая часть `make test`, `make test-persistence`, адресный failover обеих реплик | Текстовый API совместим, прежний токен/сообщение после restart; восстановление записи после остановки каждой реплики 3,123 и 3,149 с | локальные тесты U1; полный smoke остановился на ещё не реализованном upload | +| Приватный S3 и bootstrap | `docker compose run --rm s3-bootstrap` дважды; `docker compose exec -T api python - < student-tests/test_storage_bootstrap.py` | Два повтора завершились 0; S3 PUT/GET ограниченным API пользователем совпадают по байтам, анонимный GET — 403, создание чужого bucket — AccessDenied | `evidence/storage-bootstrap-2026-10-02.txt` | +| Миграция и транзакции | `docker compose exec -T api python - < student-tests/test_database.py`; `... test_auth.py` | Миграции 1–4 применены повторяемо, 2 DB и 3 auth теста PASS | `evidence/storage-bootstrap-2026-10-02.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Запись S3/БД, компенсация после ошибки, ACL, failover файла, параллельные upload и reconciler ещё должны быть проверены. Пока `GET /health/ready` отражает обязательную PostgreSQL зависимость, а готовность MinIO проверяет Compose при запуске; после появления attachment path доступность S3 будет явно учтена в сценариях. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Полный `make test` сейчас ожидаемо получает 415 на `POST attachments`: синхронный upload, скачивание и привязка ещё не реализованы. Проверки уровня 4 и трека 5 не заявляются завершёнными до отдельных fault tests и воспроизводимых измерений. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. diff --git a/compose.yaml b/compose.yaml index e91d213b..982f3ff7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -26,6 +26,54 @@ services: condition: service_healthy restart: "no" + s3-volume-init: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: ["chown -R 100:100 /data"] + volumes: [s3_data:/data] + restart: "no" + + s3: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [minio_root_user, minio_root_password] + volumes: [s3_data:/data] + depends_on: + s3-volume-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/minio/health/ready"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 384m + + s3-bootstrap: + image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 + user: "0:0" + entrypoint: ["/bin/sh", "/bootstrap.sh"] + command: [] + volumes: + - ./infra/bootstrap-s3.sh:/bootstrap.sh:ro + - ./infra/minio-api-policy.json:/policies/minio-api-policy.json:ro + - ./infra/minio-maintenance-policy.json:/policies/minio-maintenance-policy.json:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + - s3_maintenance_key_id + - s3_maintenance_secret_key + depends_on: + s3: + condition: service_healthy + restart: "no" + api: build: . scale: 2 @@ -34,10 +82,16 @@ services: APP_PORT: 8080 DATABASE_URL_FILE: /run/secrets/database_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} - secrets: [database_url] + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, s3_access_key_id, s3_secret_access_key] depends_on: migrate: condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] interval: 2s @@ -65,6 +119,19 @@ secrets: file: ./secrets/postgres_password database_url: file: ./secrets/database_url + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_maintenance_key_id: + file: ./secrets/s3_maintenance_key_id + s3_maintenance_secret_key: + file: ./secrets/s3_maintenance_secret_key volumes: postgres_data: + s3_data: diff --git a/evidence/storage-bootstrap-2026-10-02.txt b/evidence/storage-bootstrap-2026-10-02.txt new file mode 100644 index 00000000..aeb9b8d1 --- /dev/null +++ b/evidence/storage-bootstrap-2026-10-02.txt @@ -0,0 +1,12 @@ +Lab 4 U2 local Colima/arm64; 2 CPU, 2054840320 bytes RAM; synthetic data only. +Image: alpine/minio:RELEASE.2025-10-15T17-29-55Z, digest sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4. +The official minio/minio tag could not be pulled (repository unavailable); pinned alpine/minio security-release build was used. +Build used ignored local compose.local.yaml with temporary PyPI CDN DNS override; ordinary compose.yaml remains independent of this host workaround. + +docker compose --env-file .env config --quiet: PASS. +docker compose --env-file .env up -d --wait --scale api=2: PASS after adding one-time s3-volume-init for non-root MinIO write permissions. +docker compose --env-file .env run --rm s3-bootstrap: PASS, repeated twice after initial bootstrap, no duplicate/failure. +docker compose --env-file .env exec -T api python - < student-tests/test_storage_bootstrap.py: 1 PASS; PUT/GET exact bytes, anonymous GET 403, foreign bucket create AccessDenied. +docker compose --env-file .env exec -T api python - < student-tests/test_database.py: 2 PASS; migrations 1-4, race/rollback/keyset history. +docker compose --env-file .env exec -T api python - < student-tests/test_auth.py: 3 PASS; Basic/Bearer and short TTL. +No credentials, session tokens or object keys are included in this evidence. diff --git a/infra/bootstrap-s3.sh b/infra/bootstrap-s3.sh new file mode 100644 index 00000000..becaf218 --- /dev/null +++ b/infra/bootstrap-s3.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +read_secret() { + tr -d '\n' < "/run/secrets/$1" +} + +mc alias set local http://s3:9000 "$(read_secret minio_root_user)" "$(read_secret minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin policy create local messenger-api /policies/minio-api-policy.json >/dev/null +mc admin policy create local messenger-maintenance /policies/minio-maintenance-policy.json >/dev/null +mc admin user add local "$(read_secret s3_access_key_id)" "$(read_secret s3_secret_access_key)" >/dev/null +mc admin policy attach local messenger-api --user "$(read_secret s3_access_key_id)" >/dev/null +mc admin user add local "$(read_secret s3_maintenance_key_id)" "$(read_secret s3_maintenance_secret_key)" >/dev/null +mc admin policy attach local messenger-maintenance --user "$(read_secret s3_maintenance_key_id)" >/dev/null diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 00000000..54be94d5 --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/infra/minio-maintenance-policy.json b/infra/minio-maintenance-policy.json new file mode 100644 index 00000000..6beb6bd2 --- /dev/null +++ b/infra/minio-maintenance-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..576f30b1 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,32 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/requirements.txt b/requirements.txt index 58563cf5..882107d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ uvicorn==0.38.0 psycopg[binary,pool]==3.2.12 psycopg-pool==3.3.3 argon2-cffi==25.1.0 +minio==7.2.20 diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py new file mode 100644 index 00000000..86924701 --- /dev/null +++ b/scripts/setup-local-secrets.py @@ -0,0 +1,42 @@ +"""Create local Compose secrets without printing or overwriting existing values.""" + +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] / "secrets" + + +def write(name: str, value: str) -> None: + fd = os.open(ROOT / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as out: + out.write(value) + + +def main() -> None: + ROOT.mkdir(mode=0o700, exist_ok=True) + db_names = {"postgres_password", "database_url"} + s3_names = { + "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key", + "s3_maintenance_key_id", "s3_maintenance_secret_key", + } + for names in (db_names, s3_names): + existing = {name for name in names if (ROOT / name).exists()} + if existing and existing != names: + raise SystemExit(f"Incomplete secret group: {', '.join(sorted(names - existing))}") + if not (ROOT / "postgres_password").exists(): + password = secrets.token_urlsafe(32) + write("postgres_password", password) + write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") + if not (ROOT / "minio_root_user").exists(): + write("minio_root_user", "lab4admin") + write("minio_root_password", secrets.token_urlsafe(32)) + write("s3_access_key_id", secrets.token_hex(10)) + write("s3_secret_access_key", secrets.token_urlsafe(32)) + write("s3_maintenance_key_id", secrets.token_hex(10)) + write("s3_maintenance_secret_key", secrets.token_urlsafe(32)) + + +if __name__ == "__main__": + main() diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..8f4e3c6d --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,25 @@ +"""Private S3 access shared by attachment routes and maintenance jobs.""" + +import os +from functools import lru_cache +from pathlib import Path + +from minio import Minio + + +BUCKET = "messenger-attachments" + + +def secret(name: str) -> str: + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + +@lru_cache(maxsize=1) +def storage() -> Minio: + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_ACCESS_KEY_ID"), + secret_key=secret("S3_SECRET_ACCESS_KEY"), + secure=False, + region="us-east-1", + ) diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py index e9fcdf09..ca0031a3 100644 --- a/student-tests/test_auth.py +++ b/student-tests/test_auth.py @@ -14,12 +14,13 @@ import urllib.request from uuid import uuid4 import psycopg +from src.db import database_url -URL = os.environ.get("DATABASE_URL") +URL = database_url() if os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE") else None -@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +@unittest.skipUnless(URL, "set DATABASE_URL or DATABASE_URL_FILE to a disposable PostgreSQL database") class AuthTests(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/student-tests/test_database.py b/student-tests/test_database.py index 132318f9..13067eee 100644 --- a/student-tests/test_database.py +++ b/student-tests/test_database.py @@ -9,13 +9,14 @@ from uuid import uuid4 import psycopg +from src.db import database_url from src.store import Store -URL = os.environ.get("DATABASE_URL") +URL = database_url() if os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE") else None -@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database") +@unittest.skipUnless(URL, "set DATABASE_URL or DATABASE_URL_FILE to a disposable PostgreSQL database") class DatabaseTests(unittest.TestCase): @classmethod def setUpClass(cls): @@ -30,7 +31,7 @@ class DatabaseTests(unittest.TestCase): def test_migration_and_unique_username_race(self): with psycopg.connect(URL) as conn: versions = conn.execute("SELECT version FROM schema_migrations").fetchall() - self.assertEqual(versions, [(1,), (2,), (3,)]) + self.assertEqual(versions, [(1,), (2,), (3,), (4,)]) name = "u_" + uuid4().hex[:16] def register(_): diff --git a/student-tests/test_storage_bootstrap.py b/student-tests/test_storage_bootstrap.py new file mode 100644 index 00000000..726fd51a --- /dev/null +++ b/student-tests/test_storage_bootstrap.py @@ -0,0 +1,38 @@ +"""Run inside an API container after S3 bootstrap.""" + +import io +import unittest +from uuid import uuid4 +from urllib.error import HTTPError +from urllib.request import urlopen + +from minio.error import S3Error + +from src.attachments import BUCKET, storage + + +class StorageBootstrapTests(unittest.TestCase): + def test_private_bucket_and_scoped_api_credentials(self): + client = storage() + key = "test-bootstrap/" + uuid4().hex + data = b"private attachment fixture" + client.put_object(BUCKET, key, io.BytesIO(data), len(data), content_type="application/octet-stream") + try: + response = client.get_object(BUCKET, key) + try: + self.assertEqual(response.read(), data) + finally: + response.close() + response.release_conn() + with self.assertRaises(HTTPError) as denied: + urlopen(f"http://s3:9000/{BUCKET}/{key}", timeout=3) + self.assertEqual(denied.exception.code, 403) + with self.assertRaises(S3Error) as forbidden: + client.make_bucket("forbidden-" + uuid4().hex) + self.assertEqual(forbidden.exception.code, "AccessDenied") + finally: + client.remove_object(BUCKET, key) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 412b465b855953f676f36b0476ef94ecaf122cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 13:24:51 +0300 Subject: [PATCH 04/10] Add bounded private image upload with safe S3 persistence --- REPORT.md | 16 ++- evidence/upload-memory-2026-10-02.csv | 7 ++ migrations/005_attachment_owner.sql | 2 + requirements.txt | 2 + scripts/measure-upload-memory.py | 93 +++++++++++++++++ src/api.py | 10 +- src/attachments.py | 126 ++++++++++++++++++++++++ src/main.py | 3 +- student-tests/fixtures/sample.jpg | Bin 0 -> 633 bytes student-tests/test_attachment_upload.py | 63 ++++++++++++ student-tests/test_database.py | 2 +- student-tests/test_s3_outage.py | 46 +++++++++ student-tests/test_upload_failure.py | 68 +++++++++++++ 13 files changed, 430 insertions(+), 8 deletions(-) create mode 100644 evidence/upload-memory-2026-10-02.csv create mode 100644 migrations/005_attachment_owner.sql create mode 100644 scripts/measure-upload-memory.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/test_attachment_upload.py create mode 100644 student-tests/test_s3_outage.py create mode 100644 student-tests/test_upload_failure.py diff --git a/REPORT.md b/REPORT.md index 332286d0..2b0a83ef 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Самохвалов Андрей Ремович, БПИ-2302. -- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Пока реализованы перенос lab3 и приватное S3; остальные блоки выполняются далее. +- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Реализованы перенос lab3, приватное S3 и ограниченный синхронный upload; download, связь с сообщением и reconciler выполняются далее. - Ветка: `solution/bpi2302-samohvalov`; commit/PR указываются после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/openapi.json`, публичные тесты, [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z), [образ](https://hub.docker.com/r/alpine/minio/tags), документация MinIO для IAM. Python/FastAPI, PostgreSQL, MinIO Python SDK; проект подготовлен с помощью ИИ Codex с локальной проверкой ниже. @@ -15,7 +15,7 @@ python3 scripts/setup-local-secrets.py docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build --wait --scale api=2 make check -# make test — после реализации upload в следующем блоке +# make test — после реализации download/связи с сообщением в следующем блоке ``` `setup-local-secrets.py` создаёт отсутствующие secret-файлы с правами 0600 и не перезаписывает существующие. Миграции выполняет одноразовый `migrate` под PostgreSQL advisory lock. `s3-volume-init` задаёт владельца нового volume, затем MinIO стартует без root; одноразовый `s3-bootstrap` идемпотентно создаёт закрытый bucket и две ограниченные учётные записи. Повторный `docker compose --env-file .env run --rm s3-bootstrap` безопасен. Для остановки — `docker compose --env-file .env down` без `-v`; для намеренного удаления **только учебных данных** после проверки — `docker compose --env-file .env down -v`. @@ -36,7 +36,9 @@ make check | `s3-bootstrap` | одноразовый MinIO Client | завершение 0 | root и создаваемые ограниченные credentials | | `s3-volume-init` | одноразовая настройка владельца volume | завершение 0 | секретов нет | -`infra/minio-api-policy.json` разрешает только операции над объектами одного bucket и чтение его location; создание bucket, изменение IAM и административные операции не разрешены. Отдельная maintenance policy добавляет list для будущей сверки. Bucket не имеет анонимного чтения. Приватность bucket не заменяет проверку членства в чате на API: эта проверка добавляется в блоках upload/download. +`infra/minio-api-policy.json` разрешает только операции над объектами одного bucket и чтение его location; создание bucket, изменение IAM и административные операции не разрешены. Отдельная maintenance policy добавляет list для будущей сверки. Bucket не имеет анонимного чтения. Приватность bucket не заменяет проверку членства в чате на API: upload уже проверяет членство до чтения тела; download получит такую же проверку в следующем блоке. + +Upload читает raw body в `SpooledTemporaryFile` с порогом RAM 1 MiB и останавливается после 10 MiB; проверяет формат и MIME через Pillow, до декодирования отклоняет свыше 40 млн пикселей, затем полностью проверяет PNG/JPEG. Имя файла применяется только для отображения, object key — случайный UUID в `originals/`. S3 PUT предшествует INSERT/COMMIT в PostgreSQL; `201 ready` возвращается лишь после обоих успехов. При достоверной ошибке до COMMIT объект удаляется. Если подтверждение COMMIT потеряно, удалять небезопасно: объект остаётся кандидатом для сверки после grace period. Если и компенсационное DELETE отказало, key записывается во внутренний лог; U5 добавит повторяемый reconciler. Клиенту при ошибке зависимости приходит `503`, а прозрачный retry POST не выполняется. Для S3 заданы connect/read timeouts 2/10 с без внутренних повторов; `/health/ready` проверяет БД и доступ к приватному S3 через разрешённый HEAD отсутствующего probe key. ## Доказательства критериев @@ -45,11 +47,15 @@ make check | Перенос lab3 | Compose ×2, текстовая часть `make test`, `make test-persistence`, адресный failover обеих реплик | Текстовый API совместим, прежний токен/сообщение после restart; восстановление записи после остановки каждой реплики 3,123 и 3,149 с | локальные тесты U1; полный smoke остановился на ещё не реализованном upload | | Приватный S3 и bootstrap | `docker compose run --rm s3-bootstrap` дважды; `docker compose exec -T api python - < student-tests/test_storage_bootstrap.py` | Два повтора завершились 0; S3 PUT/GET ограниченным API пользователем совпадают по байтам, анонимный GET — 403, создание чужого bucket — AccessDenied | `evidence/storage-bootstrap-2026-10-02.txt` | | Миграция и транзакции | `docker compose exec -T api python - < student-tests/test_database.py`; `... test_auth.py` | Миграции 1–4 применены повторяемо, 2 DB и 3 auth теста PASS | `evidence/storage-bootstrap-2026-10-02.txt` | +| Raw PNG/JPEG upload и ограничения | `BASE_URL=http://localhost:8084 python3 -m unittest discover -s student-tests -p 'test_attachment_upload.py' -v` | 4 набора PASS: PNG/JPEG `201 ready`, корректные hash/size/type; 10 MiB+1 и >40 млн px — 413; повреждение/MIME spoof — 415; опасное имя — 400, чужой чат — 404 | `student-tests/test_attachment_upload.py`, синтетический JPEG fixture | +| Отказ S3 | `docker compose stop s3`; `docker compose exec -T api python - < student-tests/test_s3_outage.py`; `docker compose start s3` | При реально остановленном S3 readiness=503, upload=503, число metadata строк не изменилось; S3 восстановлен и healthy | `student-tests/test_s3_outage.py` | +| Промежуток S3/БД и обрыв потока | `docker compose exec -T api python - < student-tests/test_upload_failure.py` | PUT failure не вызывает SQL; ошибка БД до INSERT вызывает компенсацию DELETE; оборванный поток не начинает PUT. Неоднозначный COMMIT оставляет объект для сверки | `student-tests/test_upload_failure.py` | +| Ограничение RAM | `BASE_URL=http://localhost:8084 python3 scripts/measure-upload-memory.py` | 10/10 одновременных upload одного PNG 9 441 653 байт, 10 разных ready ID и верный SHA-256, 6,145 с; sampled peak API-1/2 114,7/115,5 MiB при лимите 256 MiB | `evidence/upload-memory-2026-10-02.csv` | ## Эксперименты и отказы -Запись S3/БД, компенсация после ошибки, ACL, failover файла, параллельные upload и reconciler ещё должны быть проверены. Пока `GET /health/ready` отражает обязательную PostgreSQL зависимость, а готовность MinIO проверяет Compose при запуске; после появления attachment path доступность S3 будет явно учтена в сценариях. +Граница PUT/COMMIT проверена fault-инъекциями; полный reconciler и проверка восстановления при неизвестном результате COMMIT выполняются в U5/U6. Замер RAM проводился на локальном Colima/arm64, 2 CPU и 2 GiB, API ×2 с 0,5 CPU/256 MiB на экземпляр, 10 потоков с барьером старта. CSV содержит baseline и два Docker stats снимка во время 6,145 с серии: peak означает максимум **наблюдённых** снимков, а не точный аппаратный high-water mark. Все 10 ответов 201 имели уникальные ID и совпадающий SHA-256. Создание body по `os.urandom` оставляет воспроизводимыми размер/разрешение/алгоритм, но не конкретные байты и hash между прогонами. ## Ограничения и следующие шаги -Полный `make test` сейчас ожидаемо получает 415 на `POST attachments`: синхронный upload, скачивание и привязка ещё не реализованы. Проверки уровня 4 и трека 5 не заявляются завершёнными до отдельных fault tests и воспроизводимых измерений. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. +Полный `make test` пока останавливается на последующих маршрутах attachment metadata/content и привязке к сообщению; U4 реализует их отдельно. Проверки уровня 4 и трека 5 не заявляются завершёнными до повторяемой сверки. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. diff --git a/evidence/upload-memory-2026-10-02.csv b/evidence/upload-memory-2026-10-02.csv new file mode 100644 index 00000000..7fe6b47b --- /dev/null +++ b/evidence/upload-memory-2026-10-02.csv @@ -0,0 +1,7 @@ +seconds,container,used_mib,limit_mib +0,mtusi2026lab4-api-1,48.58,256.0 +0,mtusi2026lab4-api-2,51.2,256.0 +2.129,mtusi2026lab4-api-1,114.7,256.0 +2.129,mtusi2026lab4-api-2,115.5,256.0 +4.141,mtusi2026lab4-api-1,114.7,256.0 +4.141,mtusi2026lab4-api-2,115.5,256.0 diff --git a/migrations/005_attachment_owner.sql b/migrations/005_attachment_owner.sql new file mode 100644 index 00000000..6db42908 --- /dev/null +++ b/migrations/005_attachment_owner.sql @@ -0,0 +1,2 @@ +ALTER TABLE attachments ADD COLUMN owner_id uuid NOT NULL REFERENCES users(id); +ALTER TABLE attachments ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); diff --git a/requirements.txt b/requirements.txt index 882107d3..c1ae3df4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ psycopg[binary,pool]==3.2.12 psycopg-pool==3.3.3 argon2-cffi==25.1.0 minio==7.2.20 +Pillow==11.3.0 +urllib3==2.8.0 diff --git a/scripts/measure-upload-memory.py b/scripts/measure-upload-memory.py new file mode 100644 index 00000000..53a2be44 --- /dev/null +++ b/scripts/measure-upload-memory.py @@ -0,0 +1,93 @@ +"""Ten concurrent bounded PNG uploads plus Docker memory samples; BASE_URL overrides localhost:8080.""" + +import csv +import hashlib +import os +import re +import struct +import subprocess +import sys +import threading +import time +import zlib +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + +API_NAMES = ("mtusi2026lab4-api-1", "mtusi2026lab4-api-2") +EVIDENCE = ROOT / "evidence/upload-memory-2026-10-02.csv" + + +def png_chunk(name: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + name + data + struct.pack(">I", zlib.crc32(name + data)) + + +def large_png() -> bytes: + width, height = 2048, 1536 + rows = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) + return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + png_chunk(b"IDAT", zlib.compress(rows, 1)) + png_chunk(b"IEND", b"") + + +def mib(text: str) -> float: + match = re.fullmatch(r"([0-9.]+)(B|KiB|MiB|GiB)", text.strip()) + if not match: + raise ValueError("unexpected Docker memory unit") + return float(match.group(1)) * {"B": 1 / 1048576, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[match.group(2)] + + +def memory_samples(elapsed: float) -> list[tuple[float, str, float, float]]: + command = ["rtk", "proxy", "docker", "stats", "--no-stream", "--format", "{{.Name}},{{.MemUsage}}", *API_NAMES] + output = subprocess.run(command, check=True, capture_output=True, text=True, timeout=10).stdout + rows = [] + for line in output.splitlines(): + name, usage = line.split(",", 1) + used, limit = usage.split("/", 1) + rows.append((round(elapsed, 3), name, round(mib(used), 3), round(mib(limit), 3))) + if len(rows) != 2: + raise AssertionError("expected two API container memory samples") + return rows + + +def main() -> None: + client = Client(timeout=60) + actor, _, _, chat = client.bootstrap() + data = large_png() + if len(data) > 10 * 1024 * 1024: + raise AssertionError("synthetic PNG exceeds upload limit") + expected_hash = hashlib.sha256(data).hexdigest() + barrier = threading.Barrier(10) + + def send(index: int) -> dict: + barrier.wait(timeout=10) + result = Client(timeout=60).call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=load-{index}.png", 201, + actor=actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", + )["json"] + if result["sha256"] != expected_hash or result["size_bytes"] != len(data): + raise AssertionError("upload bytes/hash mismatch") + return result + + start = time.monotonic() + rows = memory_samples(0) + with ThreadPoolExecutor(max_workers=10) as workers: + futures = [workers.submit(send, index) for index in range(10)] + while not all(future.done() for future in futures): + rows.extend(memory_samples(time.monotonic() - start)) + results = [future.result() for future in futures] + rows.extend(memory_samples(time.monotonic() - start)) + if len({result["id"] for result in results}) != 10: + raise AssertionError("uploads did not create ten unique attachments") + EVIDENCE.parent.mkdir(exist_ok=True) + with EVIDENCE.open("w", newline="") as target: + writer = csv.writer(target, lineterminator="\n") + writer.writerow(("seconds", "container", "used_mib", "limit_mib")) + writer.writerows(rows) + peaks = {name: max(row[2] for row in rows if row[1] == name) for name in API_NAMES} + print(f"PASS: 10/10 uploads; PNG={len(data)} bytes; elapsed={time.monotonic()-start:.3f}s; peaks={peaks}; evidence={EVIDENCE.name}") + + +if __name__ == "__main__": + main() diff --git a/src/api.py b/src/api.py index 4d2965cd..cef93bdd 100644 --- a/src/api.py +++ b/src/api.py @@ -6,6 +6,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from .auth import basic_credentials, bearer_token +from .attachments import storage_ready, upload from .models import CreateChat, CreateMessage, CreateUser from .store import store @@ -19,7 +20,9 @@ def live(): @router.get("/health/ready") def ready(): - return store.ready() + result = store.ready() + storage_ready() + return result @router.post("/api/v1/users", status_code=201) @@ -78,6 +81,11 @@ def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): return store.chat(actor["id"], str(chat_id)) +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: UUID, request: Request, filename: str = Query(...), actor: dict = Depends(current_user)): + return await upload(request, str(chat_id), actor["id"], filename) + + @router.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): key = request.headers.get("Idempotency-Key") diff --git a/src/attachments.py b/src/attachments.py index 8f4e3c6d..de08798e 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -1,13 +1,29 @@ """Private S3 access shared by attachment routes and maintenance jobs.""" +import asyncio +import hashlib +import logging import os +import unicodedata from functools import lru_cache from pathlib import Path +from tempfile import SpooledTemporaryFile +from uuid import UUID, uuid4 +from fastapi import HTTPException, Request from minio import Minio +from minio.error import S3Error +from PIL import Image, UnidentifiedImageError +from urllib3 import PoolManager, Retry, Timeout + +from .store import connection, store, timestamp BUCKET = "messenger-attachments" +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +logger = logging.getLogger(__name__) def secret(name: str) -> str: @@ -22,4 +38,114 @@ def storage() -> Minio: secret_key=secret("S3_SECRET_ACCESS_KEY"), secure=False, region="us-east-1", + http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)), ) + + +def storage_ready() -> None: + try: + storage().stat_object(BUCKET, "health/ready-probe") + except S3Error as exc: + if exc.code != "NoSuchKey": + raise HTTPException(503) from None + except Exception: + raise HTTPException(503) from None + + +def attachment_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], + "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": [], "error_code": row["error_code"], + "created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]), + } + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename in {".", ".."}: + raise HTTPException(400) + if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename): + raise HTTPException(400) + + +def validate_image(stream, declared_type: str) -> str: + try: + with Image.open(stream) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type is None or actual_type != declared_type: + raise HTTPException(415) + if image.width * image.height > MAX_PIXELS: + raise HTTPException(413) + image.verify() + stream.seek(0) + with Image.open(stream) as image: + image.load() + return actual_type + except Image.DecompressionBombError: + raise HTTPException(413) from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise HTTPException(415) from None + + +def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict: + attachment_id = uuid4() + object_key = f"originals/{attachment_id}" + stream.seek(0) + try: + storage().put_object(BUCKET, object_key, stream, size, content_type=content_type) + except Exception: + logger.exception("S3 upload failed for attachment %s", attachment_id) + raise HTTPException(503) from None + + commit_uncertain = False + try: + with connection() as conn: + row = conn.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s) " + "RETURNING id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,error_code,created_at,updated_at", + (attachment_id, UUID(chat_id), UUID(actor_id), filename, object_key, content_type, size, sha256), + ).fetchone() + commit_uncertain = True + except Exception: + if commit_uncertain: + # The COMMIT may have succeeded despite a lost acknowledgment. Reconciliation decides later. + logger.error("Attachment commit outcome unknown; retain object %s for reconciliation", object_key) + else: + try: + storage().remove_object(BUCKET, object_key) + except Exception: + logger.exception("Orphan cleanup failed for object %s", object_key) + raise + return attachment_view(row) + + +async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict: + await asyncio.to_thread(store.chat, actor_id, chat_id) + validate_filename(filename) + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in {"image/png", "image/jpeg"}: + raise HTTPException(415) + length = request.headers.get("content-length") + if length is not None: + try: + if int(length) > MAX_BYTES: + raise HTTPException(413) + except ValueError: + raise HTTPException(400) from None + + with SpooledTemporaryFile(max_size=1024 * 1024) as stream: + digest = hashlib.sha256() + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413) + stream.write(chunk) + digest.update(chunk) + stream.seek(0) + validate_image(stream, declared_type) + return await asyncio.to_thread( + persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream + ) diff --git a/src/main.py b/src/main.py index bcac75fd..26dea4b3 100644 --- a/src/main.py +++ b/src/main.py @@ -37,7 +37,8 @@ def error_response(request: Request, status: int, extra_headers: dict | None = N @app.middleware("http") async def request_contract(request: Request, call_next): request.state.request_id = str(uuid4()) - if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions": + is_upload = request.url.path.startswith("/api/v1/chats/") and request.url.path.endswith("/attachments") + if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions" and not is_upload: media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"): return error_response(request, 415) diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c3d7571f257c5c3bffdfa8b4b70c56070bf5a2c4 GIT binary patch literal 633 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!{;oGMf+Ml`TxHO0FAZJxBvhE literal 0 HcmV?d00001 diff --git a/student-tests/test_attachment_upload.py b/student-tests/test_attachment_upload.py new file mode 100644 index 00000000..277e1fe5 --- /dev/null +++ b/student-tests/test_attachment_upload.py @@ -0,0 +1,63 @@ +"""Black-box upload and validation checks; BASE_URL defaults to localhost:8080.""" + +import hashlib +import struct +import sys +import unittest +import zlib +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + + +PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() +JPEG = (ROOT / "student-tests/fixtures/sample.jpg").read_bytes() + + +class UploadTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Client() + cls.alice, _, cls.eve, cls.chat = cls.client.bootstrap() + cls.path = f"/api/v1/chats/{cls.chat['id']}/attachments" + + def upload(self, content=PNG, media_type="image/png", filename="sample.png", expected=201, actor=None): + return self.client.call( + "POST", self.path + "?filename=" + quote(filename, safe=""), expected, + actor=actor or self.alice, raw=content, headers={"Content-Type": media_type}, + schema="Attachment" if expected == 201 else None, + ) + + def test_png_and_jpeg(self): + for content, media_type, filename in ((PNG, "image/png", "sample.png"), (JPEG, "image/jpeg", "sample.jpg")): + attachment = self.upload(content, media_type, filename)["json"] + self.assertEqual(attachment["status"], "ready") + self.assertEqual(attachment["owner_id"], self.alice["id"]) + self.assertEqual(attachment["chat_id"], self.chat["id"]) + self.assertEqual(attachment["content_type"], media_type) + self.assertEqual(attachment["size_bytes"], len(content)) + self.assertEqual(attachment["sha256"], hashlib.sha256(content).hexdigest()) + + def test_rejects_oversized_and_oversized_pixels(self): + self.upload(b"x" * (10 * 1024 * 1024 + 1), expected=413) + changed = bytearray(PNG) + changed[16:24] = struct.pack(">II", 6325, 6325) + changed[29:33] = struct.pack(">I", zlib.crc32(changed[12:29])) + self.upload(bytes(changed), expected=413) + + def test_rejects_invalid_image_and_mime_spoofing(self): + self.upload(PNG[:-10], expected=415) + self.upload(PNG, "image/jpeg", expected=415) + self.upload(b"not an image", expected=415) + + def test_rejects_unsafe_names_and_outsider(self): + for name in ("../bad.png", "folder\\bad.png", "bad\x00name.png", "bad\nname.png", ""): + self.upload(filename=name, expected=400) + self.upload(expected=404, actor=self.eve) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_database.py b/student-tests/test_database.py index 13067eee..ede3b896 100644 --- a/student-tests/test_database.py +++ b/student-tests/test_database.py @@ -31,7 +31,7 @@ class DatabaseTests(unittest.TestCase): def test_migration_and_unique_username_race(self): with psycopg.connect(URL) as conn: versions = conn.execute("SELECT version FROM schema_migrations").fetchall() - self.assertEqual(versions, [(1,), (2,), (3,), (4,)]) + self.assertEqual(versions, [(1,), (2,), (3,), (4,), (5,)]) name = "u_" + uuid4().hex[:16] def register(_): diff --git a/student-tests/test_s3_outage.py b/student-tests/test_s3_outage.py new file mode 100644 index 00000000..963548c6 --- /dev/null +++ b/student-tests/test_s3_outage.py @@ -0,0 +1,46 @@ +"""Run inside an API container while S3 is stopped; no secrets are printed.""" + +import io +import unittest +from urllib.error import HTTPError +from urllib.request import Request, urlopen +from uuid import uuid4 + +from PIL import Image + +from src.db import pool +from src.store import store + + +def status(path: str, data: bytes | None = None, headers: dict | None = None) -> int: + request = Request("http://127.0.0.1:8080" + path, data=data, headers=headers or {}, method="POST" if data is not None else "GET") + try: + with urlopen(request, timeout=15) as response: + return response.status + except HTTPError as error: + return error.code + + +class S3OutageTests(unittest.TestCase): + def test_unavailable_without_false_ready_or_metadata(self): + username = "s3_outage_" + uuid4().hex[:16] + user = store.register(username, "S3 outage", "Test-password-123") + session = store.create_session(username, "Test-password-123") + chat = store.create_chat(user["id"], "S3 outage", [user["id"]]) + with pool().connection() as conn: + before = conn.execute("SELECT count(*) AS n FROM attachments WHERE chat_id=%s", (chat["id"],)).fetchone()["n"] + data = io.BytesIO() + Image.new("RGB", (4, 3), (20, 60, 120)).save(data, format="PNG") + self.assertEqual(status("/health/ready"), 503) + self.assertEqual( + status(f"/api/v1/chats/{chat['id']}/attachments?filename=outage.png", data.getvalue(), + {"Content-Type": "image/png", "Authorization": "Bearer " + session["token"]}), + 503, + ) + with pool().connection() as conn: + after = conn.execute("SELECT count(*) AS n FROM attachments WHERE chat_id=%s", (chat["id"],)).fetchone()["n"] + self.assertEqual(after, before) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_upload_failure.py b/student-tests/test_upload_failure.py new file mode 100644 index 00000000..663d5cfb --- /dev/null +++ b/student-tests/test_upload_failure.py @@ -0,0 +1,68 @@ +"""Run inside an API image: python - < student-tests/test_upload_failure.py.""" + +import asyncio +import io +import unittest +from contextlib import contextmanager +from unittest.mock import Mock, patch +from uuid import uuid4 + +from fastapi import HTTPException + +from src import attachments + + +class UploadFailureTests(unittest.TestCase): + def test_post_put_database_failure_compensates(self): + client = Mock() + with patch.object(attachments, "storage", return_value=client), patch.object( + attachments, "connection", side_effect=HTTPException(503) + ): + with self.assertRaises(HTTPException) as failure: + attachments.persist_attachment(str(uuid4()), str(uuid4()), "test.png", "image/png", 3, "0" * 64, io.BytesIO(b"png")) + self.assertEqual(failure.exception.status_code, 503) + key = client.put_object.call_args.args[1] + client.remove_object.assert_called_once_with(attachments.BUCKET, key) + + def test_s3_failure_does_not_touch_database(self): + client = Mock() + client.put_object.side_effect = OSError("simulated outage") + with patch.object(attachments, "storage", return_value=client), patch.object(attachments, "connection") as db: + with self.assertRaises(HTTPException) as failure: + attachments.persist_attachment(str(uuid4()), str(uuid4()), "test.png", "image/png", 3, "0" * 64, io.BytesIO(b"png")) + self.assertEqual(failure.exception.status_code, 503) + db.assert_not_called() + + def test_ambiguous_commit_keeps_object_for_reconciliation(self): + @contextmanager + def ambiguous_commit(): + connection = Mock() + connection.execute.return_value.fetchone.return_value = {} + yield connection + raise HTTPException(503) + + client = Mock() + with patch.object(attachments, "storage", return_value=client), patch.object( + attachments, "connection", side_effect=ambiguous_commit + ): + with self.assertRaises(HTTPException) as failure: + attachments.persist_attachment(str(uuid4()), str(uuid4()), "test.png", "image/png", 3, "0" * 64, io.BytesIO(b"png")) + self.assertEqual(failure.exception.status_code, 503) + client.remove_object.assert_not_called() + + def test_interrupted_request_never_starts_s3_put(self): + class Interrupted: + headers = {"content-type": "image/png"} + + async def stream(self): + yield b"partial" + raise OSError("client disconnected") + + with patch.object(attachments.store, "chat"), patch.object(attachments, "storage") as client: + with self.assertRaises(OSError): + asyncio.run(attachments.upload(Interrupted(), str(uuid4()), str(uuid4()), "test.png")) + client.assert_not_called() + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 9347fe35ae9c5166d6085f41dff900965c7f88bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:00:53 +0300 Subject: [PATCH 05/10] feat: Authorize attachment reads and message links (U4) --- REPORT.md | 18 +++--- evidence/attachment-access-2026-10-02.txt | 25 ++++++++ scripts/restart.sh | 2 +- src/api.py | 13 ++++ src/attachment_read.py | 54 ++++++++++++++++ src/models.py | 21 ++++--- src/store.py | 40 +++++++++--- student-tests/test_attachment_access.py | 75 +++++++++++++++++++++++ student-tests/test_attachment_failover.py | 58 ++++++++++++++++++ 9 files changed, 282 insertions(+), 24 deletions(-) create mode 100644 evidence/attachment-access-2026-10-02.txt create mode 100644 src/attachment_read.py create mode 100644 student-tests/test_attachment_access.py create mode 100644 student-tests/test_attachment_failover.py diff --git a/REPORT.md b/REPORT.md index 2b0a83ef..97a830d6 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Самохвалов Андрей Ремович, БПИ-2302. -- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Реализованы перенос lab3, приватное S3 и ограниченный синхронный upload; download, связь с сообщением и reconciler выполняются далее. +- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Перенос lab3, приватное S3, синхронный upload, доступ к вложениям и связь с сообщениями проверены; reconciler и эксперимент трека выполняются далее. - Ветка: `solution/bpi2302-samohvalov`; commit/PR указываются после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/openapi.json`, публичные тесты, [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z), [образ](https://hub.docker.com/r/alpine/minio/tags), документация MinIO для IAM. Python/FastAPI, PostgreSQL, MinIO Python SDK; проект подготовлен с помощью ИИ Codex с локальной проверкой ниже. @@ -15,7 +15,7 @@ python3 scripts/setup-local-secrets.py docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build --wait --scale api=2 make check -# make test — после реализации download/связи с сообщением в следующем блоке +make test ``` `setup-local-secrets.py` создаёт отсутствующие secret-файлы с правами 0600 и не перезаписывает существующие. Миграции выполняет одноразовый `migrate` под PostgreSQL advisory lock. `s3-volume-init` задаёт владельца нового volume, затем MinIO стартует без root; одноразовый `s3-bootstrap` идемпотентно создаёт закрытый bucket и две ограниченные учётные записи. Повторный `docker compose --env-file .env run --rm s3-bootstrap` безопасен. Для остановки — `docker compose --env-file .env down` без `-v`; для намеренного удаления **только учебных данных** после проверки — `docker compose --env-file .env down -v`. @@ -24,7 +24,7 @@ make check ## Архитектура и решения -Клиент → HAProxy → две API-реплики. PostgreSQL хранит пользователей, сессии, чаты, сообщения, ключи идемпотентности и метаданные вложений. MinIO хранит только байты оригиналов в private bucket `messenger-attachments`; локальная файловая система API не хранит оригиналы. Миграция `004_attachments.sql` связывает вложение с чатом; состав сообщения хранится в `message_attachments` с FK по паре `(id, chat_id)`, чтобы исключить межчатовые ссылки на уровне БД. Статусы таблицы допускают дальнейший переход к асинхронной lab5, но lab4 будет возвращать только `ready` после обеих устойчивых записей. +Клиент → HAProxy → две API-реплики. PostgreSQL хранит пользователей, сессии, чаты, сообщения, ключи идемпотентности и метаданные вложений. MinIO хранит только байты оригиналов в private bucket `messenger-attachments`; локальная файловая система API не хранит оригиналы. Миграция `004_attachments.sql` связывает вложение с чатом; `005_attachment_owner.sql` добавляет владельца и время обновления по OpenAPI. Состав сообщения хранится в `message_attachments` с FK по паре `(id, chat_id)`, чтобы исключить межчатовые ссылки на уровне БД. Статусы таблицы допускают дальнейший переход к асинхронной lab5, но lab4 возвращает только `ready` после обеих устойчивых записей. | Сервис | Образ/роль | Health и volume | Секреты и доступ | | --- | --- | --- | --- | @@ -36,21 +36,25 @@ make check | `s3-bootstrap` | одноразовый MinIO Client | завершение 0 | root и создаваемые ограниченные credentials | | `s3-volume-init` | одноразовая настройка владельца volume | завершение 0 | секретов нет | -`infra/minio-api-policy.json` разрешает только операции над объектами одного bucket и чтение его location; создание bucket, изменение IAM и административные операции не разрешены. Отдельная maintenance policy добавляет list для будущей сверки. Bucket не имеет анонимного чтения. Приватность bucket не заменяет проверку членства в чате на API: upload уже проверяет членство до чтения тела; download получит такую же проверку в следующем блоке. +`infra/minio-api-policy.json` разрешает только операции над объектами одного bucket и чтение его location; создание bucket, изменение IAM и административные операции не разрешены. Отдельная maintenance policy добавляет list для будущей сверки. Bucket не имеет анонимного чтения. Приватность bucket не заменяет проверку членства в чате на API: upload проверяет его до чтения тела, metadata/content — до доступа к S3. Публичный ответ не содержит object key, endpoint и credentials. Upload читает raw body в `SpooledTemporaryFile` с порогом RAM 1 MiB и останавливается после 10 MiB; проверяет формат и MIME через Pillow, до декодирования отклоняет свыше 40 млн пикселей, затем полностью проверяет PNG/JPEG. Имя файла применяется только для отображения, object key — случайный UUID в `originals/`. S3 PUT предшествует INSERT/COMMIT в PostgreSQL; `201 ready` возвращается лишь после обоих успехов. При достоверной ошибке до COMMIT объект удаляется. Если подтверждение COMMIT потеряно, удалять небезопасно: объект остаётся кандидатом для сверки после grace period. Если и компенсационное DELETE отказало, key записывается во внутренний лог; U5 добавит повторяемый reconciler. Клиенту при ошибке зависимости приходит `503`, а прозрачный retry POST не выполняется. Для S3 заданы connect/read timeouts 2/10 с без внутренних повторов; `/health/ready` проверяет БД и доступ к приватному S3 через разрешённый HEAD отсутствующего probe key. +POST сообщения принимает до 10 уникальных ID из этого чата, включая пустой текст с вложением. Проверка вложений, запись сообщения, связей и результата идемпотентности выполняются в одной транзакции; порядок ID сохраняется. Повтор ключа с тем же содержимым возвращает прежний Message. `GET` metadata и content сначала проверяют действующую сессию и членство в чате; неизвестный и чужой ID одинаково дают 404. Исходник отдаётся потоково через API, отсутствующий вариант — 404, недоступный S3-объект — 503. + ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | | Перенос lab3 | Compose ×2, текстовая часть `make test`, `make test-persistence`, адресный failover обеих реплик | Текстовый API совместим, прежний токен/сообщение после restart; восстановление записи после остановки каждой реплики 3,123 и 3,149 с | локальные тесты U1; полный smoke остановился на ещё не реализованном upload | | Приватный S3 и bootstrap | `docker compose run --rm s3-bootstrap` дважды; `docker compose exec -T api python - < student-tests/test_storage_bootstrap.py` | Два повтора завершились 0; S3 PUT/GET ограниченным API пользователем совпадают по байтам, анонимный GET — 403, создание чужого bucket — AccessDenied | `evidence/storage-bootstrap-2026-10-02.txt` | -| Миграция и транзакции | `docker compose exec -T api python - < student-tests/test_database.py`; `... test_auth.py` | Миграции 1–4 применены повторяемо, 2 DB и 3 auth теста PASS | `evidence/storage-bootstrap-2026-10-02.txt` | +| Миграция и транзакции | `docker compose exec -T api python - < student-tests/test_database.py`; `... test_auth.py` | Миграции 1–5 применены повторяемо, 2 DB и 3 auth теста PASS | `evidence/storage-bootstrap-2026-10-02.txt` | | Raw PNG/JPEG upload и ограничения | `BASE_URL=http://localhost:8084 python3 -m unittest discover -s student-tests -p 'test_attachment_upload.py' -v` | 4 набора PASS: PNG/JPEG `201 ready`, корректные hash/size/type; 10 MiB+1 и >40 млн px — 413; повреждение/MIME spoof — 415; опасное имя — 400, чужой чат — 404 | `student-tests/test_attachment_upload.py`, синтетический JPEG fixture | | Отказ S3 | `docker compose stop s3`; `docker compose exec -T api python - < student-tests/test_s3_outage.py`; `docker compose start s3` | При реально остановленном S3 readiness=503, upload=503, число metadata строк не изменилось; S3 восстановлен и healthy | `student-tests/test_s3_outage.py` | -| Промежуток S3/БД и обрыв потока | `docker compose exec -T api python - < student-tests/test_upload_failure.py` | PUT failure не вызывает SQL; ошибка БД до INSERT вызывает компенсацию DELETE; оборванный поток не начинает PUT. Неоднозначный COMMIT оставляет объект для сверки | `student-tests/test_upload_failure.py` | +| Промежуток S3/БД и обрыв потока | `docker compose exec -T api python - < student-tests/test_upload_failure.py` | PUT failure не вызывает SQL; ошибка БД до INSERT вызывает компенсацию DELETE; оборванный поток не начинает PUT. Добавленный случай неоднозначного COMMIT требует отдельного контрольного прогона | `student-tests/test_upload_failure.py` | | Ограничение RAM | `BASE_URL=http://localhost:8084 python3 scripts/measure-upload-memory.py` | 10/10 одновременных upload одного PNG 9 441 653 байт, 10 разных ready ID и верный SHA-256, 6,145 с; sampled peak API-1/2 114,7/115,5 MiB при лимите 256 MiB | `evidence/upload-memory-2026-10-02.csv` | +| Сообщения, ACL и скачивание | `python3 -m unittest discover -s student-tests -p test_attachment_access.py -v`; `make test BASE_URL=http://127.0.0.1:8084` | 3 собственных теста OK; публичный сценарий PASS; link/replay и bytes на обеих репликах, чужой объект 404 | `evidence/attachment-access-2026-10-02.txt` | +| Сохранность S3 и отказ API | `make test-persistence ... ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ... ACTION_SCRIPT=scripts/stop-one.sh`; `python3 student-tests/test_attachment_failover.py` | Токен, Message, metadata и bytes пережили restart API/DB/S3; публичный failover восстановил запись за 2,826 и 3,020 с; адресный тест остановил каждую загрузившую реплику и скачал те же bytes через другую | `evidence/attachment-access-2026-10-02.txt` | ## Эксперименты и отказы @@ -58,4 +62,4 @@ Upload читает raw body в `SpooledTemporaryFile` с порогом RAM 1 M ## Ограничения и следующие шаги -Полный `make test` пока останавливается на последующих маршрутах attachment metadata/content и привязке к сообщению; U4 реализует их отдельно. Проверки уровня 4 и трека 5 не заявляются завершёнными до повторяемой сверки. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. +Публичный `make test` проходит; он не доказывает все отказные сценарии. Проверки уровня 4 и трека 5 не заявляются завершёнными до повторяемой сверки. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. diff --git a/evidence/attachment-access-2026-10-02.txt b/evidence/attachment-access-2026-10-02.txt new file mode 100644 index 00000000..1d4e574f --- /dev/null +++ b/evidence/attachment-access-2026-10-02.txt @@ -0,0 +1,25 @@ +Lab4 U4 local verification, 2026-10-02, Colima arm64, APP_PORT=8084. +Synthetic data only; no tokens, passwords, object keys or image bodies recorded. + +docker compose -f compose.yaml -f compose.local.yaml --env-file .env up -d --build --wait --scale api=2 +Result: postgres/s3/lb and both API replicas healthy; migrate/bootstrap one-shots exited 0. + +BASE_URL=http://127.0.0.1:8084 python3 -m unittest discover -s student-tests -p test_attachment_access.py -v +Result: 3 tests OK: message link/replay, ACL and invalid links, bytes served by both replicas. + +make test BASE_URL=http://127.0.0.1:8084 +Result: PASS lab4 public API scenario. + +make test-persistence BASE_URL=http://127.0.0.1:8084 ACTION_SCRIPT=scripts/restart.sh +Result: PASS; hook stopped API, LB, PostgreSQL and S3, then restarted without removing volumes. +Existing token, confirmed message, attachment metadata and original bytes survived. + +make test-failover BASE_URL=http://127.0.0.1:8084 ACTION_SCRIPT=scripts/stop-one.sh +Result, run 1: two instances observed; 112 confirmed writes, 1 failed attempt, +first recovery 2.826 seconds; existing token/message/original preserved. +Result, run 2 after restore: two instances observed; 113 confirmed writes, +1 failed attempt, first recovery 3.020 seconds; existing token/message/original preserved. + +BASE_URL=http://127.0.0.1:8084 python3 student-tests/test_attachment_failover.py +Result: stopped each of the two distinct replicas after it accepted an upload; +the other replica served byte-identical original; both were restored between runs. diff --git a/scripts/restart.sh b/scripts/restart.sh index c9148ca0..3e7912fb 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,5 +1,5 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -docker compose --env-file .env stop api lb postgres +docker compose --env-file .env stop api lb postgres s3 docker compose --env-file .env up -d --wait --scale api=2 diff --git a/src/api.py b/src/api.py index cef93bdd..c8776edf 100644 --- a/src/api.py +++ b/src/api.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from .auth import basic_credentials, bearer_token from .attachments import storage_ready, upload +from .attachment_read import download, get_view from .models import CreateChat, CreateMessage, CreateUser from .store import store @@ -86,6 +87,18 @@ async def upload_attachment(chat_id: UUID, request: Request, filename: str = Que return await upload(request, str(chat_id), actor["id"], filename) +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: UUID, actor: dict = Depends(current_user)): + return get_view(actor["id"], attachment_id) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def get_attachment_content( + attachment_id: UUID, variant: str = Query("original"), actor: dict = Depends(current_user) +): + return download(actor["id"], attachment_id, variant) + + @router.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): key = request.headers.get("Idempotency-Key") diff --git a/src/attachment_read.py b/src/attachment_read.py new file mode 100644 index 00000000..2171b3e8 --- /dev/null +++ b/src/attachment_read.py @@ -0,0 +1,54 @@ +"""Authorize metadata and stream originals from the private bucket.""" + +import logging +from uuid import UUID + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from .attachments import BUCKET, attachment_view, storage +from .store import connection + + +logger = logging.getLogger(__name__) + + +def metadata(actor_id: str, attachment_id: UUID) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a " + "JOIN memberships m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (attachment_id, UUID(actor_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return row + + +def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse: + row = metadata(actor_id, attachment_id) + if row["status"] != "ready": + raise HTTPException(409) + if variant != "original": + raise HTTPException(404) + try: + obj = storage().get_object(BUCKET, row["object_key"]) + except Exception as exc: + logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + + def chunks(): + try: + yield from obj.stream(64 * 1024) + finally: + obj.close() + obj.release_conn() + + return StreamingResponse( + chunks(), media_type=row["content_type"], headers={"Content-Length": str(row["size_bytes"])} + ) + + +def get_view(actor_id: str, attachment_id: UUID) -> dict: + return attachment_view(metadata(actor_id, attachment_id)) diff --git a/src/models.py b/src/models.py index 7ee9ec7c..06b16beb 100644 --- a/src/models.py +++ b/src/models.py @@ -2,7 +2,7 @@ from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class StrictBody(BaseModel): @@ -50,12 +50,19 @@ class CreateChat(StrictBody): class CreateMessage(StrictBody): - text: str = Field(min_length=1, max_length=4000) + text: str = Field(max_length=4000) attachment_ids: list[str] = Field(default_factory=list, max_length=10) - @field_validator("text") + @field_validator("attachment_ids") @classmethod - def nonblank_text(cls, value: str) -> str: - if not value.strip(): - raise ValueError("blank text") - return value + def valid_attachments(cls, value: list[str]) -> list[str]: + ids = [str(UUID(item)) for item in value] + if len(set(ids)) != len(ids): + raise ValueError("duplicate attachment") + return ids + + @model_validator(mode="after") + def text_or_attachment(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("message needs text or attachment") + return self diff --git a/src/store.py b/src/store.py index fa918d26..298109a7 100644 --- a/src/store.py +++ b/src/store.py @@ -80,13 +80,17 @@ def message_view(row: dict) -> dict: "id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": timestamp(row["created_at"]), - "attachment_ids": [], + "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], } CHAT_COLUMNS = """c.id, c.title, c.created_at, ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids""" +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id=m.id ORDER BY a.position) AS attachment_ids""" + class Store: def ready(self) -> dict: @@ -208,10 +212,13 @@ class Store: return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} def create_message(self, actor_id: str, chat_id: str, text: str, key: str, attachment_ids: list[str] | None = None) -> dict: - if attachment_ids: - raise HTTPException(404) + ids = [UUID(item) for item in attachment_ids or []] + if len(ids) > 10 or len(set(ids)) != len(ids) or (not text.strip() and not ids): + raise HTTPException(400) chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) - request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": []}, ensure_ascii=False, sort_keys=True).encode()).digest() + request_hash = hashlib.sha256(json.dumps( + {"text": text, "attachment_ids": [str(item) for item in ids]}, ensure_ascii=False, sort_keys=True, + ).encode()).digest() with connection() as conn: member = conn.execute( "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) @@ -230,7 +237,7 @@ class Store: ).fetchone() if reserved is None: replay = conn.execute( - "SELECT r.request_hash,m.id,m.chat_id,m.sender_id,m.text,m.created_at " + f"SELECT r.request_hash,{MESSAGE_COLUMNS} " "FROM message_replays r JOIN messages m ON m.id=r.message_id " "WHERE r.user_id=%s AND r.chat_id=%s AND r.request_key=%s", (actor_uuid, chat_uuid, key), @@ -240,26 +247,41 @@ class Store: if replay["request_hash"] != request_hash: raise HTTPException(409) return message_view(replay) + if ids: + attachments = conn.execute( + "SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)", + (chat_uuid, ids), + ).fetchall() + if len(attachments) != len(ids): + raise HTTPException(404) + if any(item["status"] == "failed" for item in attachments): + raise HTTPException(409) row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", (uuid4(), chat_uuid, actor_uuid, text), ).fetchone() + for position, attachment_id in enumerate(ids): + conn.execute( + "INSERT INTO message_attachments (message_id,chat_id,attachment_id,position) " + "VALUES (%s,%s,%s,%s)", + (row["id"], chat_uuid, attachment_id, position), + ) conn.execute( "UPDATE message_replays SET message_id=%s WHERE user_id=%s AND chat_id=%s AND request_key=%s", (row["id"], actor_uuid, chat_uuid, key), ) - return message_view(row) + return message_view({**row, "attachment_ids": ids}) def list_messages(self, actor_id: str, chat_id: str, limit: int, cursor: str | None) -> dict: after = decode_cursor(cursor, f"messages:{chat_id}") chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) - query = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s" + query = f"SELECT {MESSAGE_COLUMNS} FROM messages m WHERE m.chat_id=%s" params = [chat_uuid] if after: - query += " AND (created_at,id)>(%s,%s)" + query += " AND (m.created_at,m.id)>(%s,%s)" params.extend(after) - query += " ORDER BY created_at,id LIMIT %s" + query += " ORDER BY m.created_at,m.id LIMIT %s" params.append(limit + 1) with connection() as conn: member = conn.execute( diff --git a/student-tests/test_attachment_access.py b/student-tests/test_attachment_access.py new file mode 100644 index 00000000..00bca6cc --- /dev/null +++ b/student-tests/test_attachment_access.py @@ -0,0 +1,75 @@ +"""Black-box attachment ACL, message links and idempotency on a running lab4 stack.""" + +import sys +import unittest +from pathlib import Path +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client + + +class AttachmentAccessTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Client() + cls.alice, cls.bob, cls.eve, cls.chat = cls.client.bootstrap() + cls.other_chat = cls.client.call( + "POST", "/api/v1/chats", 201, actor=cls.alice, + body={"title": "Other", "member_ids": [cls.alice["id"], cls.bob["id"]]}, schema="Chat", + )["json"] + cls.attachment, cls.content = cls.client.attachment(cls.alice, cls.chat) + cls.other_attachment, _ = cls.client.attachment(cls.alice, cls.other_chat) + + def test_message_links_and_replay(self): + key = uuid4().hex + path = f"/api/v1/chats/{self.chat['id']}/messages" + body = {"text": "", "attachment_ids": [self.attachment["id"]]} + first = self.client.call("POST", path, 201, body=body, actor=self.bob, + headers={"Idempotency-Key": key}, schema="Message")["json"] + same = self.client.call("POST", path, 201, body={"attachment_ids": body["attachment_ids"], "text": ""}, + actor=self.bob, headers={"Idempotency-Key": key}, schema="Message")["json"] + self.assertEqual(first, same) + self.assertEqual(first["attachment_ids"], [self.attachment["id"]]) + items = self.client.call("GET", path, actor=self.alice, schema="MessagePage")["json"]["items"] + self.assertEqual([item["id"] for item in items].count(first["id"]), 1) + self.client.call("POST", path, 409, body={"text": "changed", "attachment_ids": body["attachment_ids"]}, + actor=self.bob, headers={"Idempotency-Key": key}) + + def test_access_and_invalid_links(self): + attachment_path = f"/api/v1/attachments/{self.attachment['id']}" + metadata = self.client.call("GET", attachment_path, actor=self.bob, schema="Attachment")["json"] + self.assertEqual(metadata["id"], self.attachment["id"]) + self.assertNotIn("object_key", metadata) + self.assertNotIn("s3", str(metadata).lower()) + self.assertEqual(self.client.call("GET", attachment_path + "/content", actor=self.bob)["bytes"], self.content) + self.client.call("GET", attachment_path + "/content?variant=thumbnail", 404, actor=self.bob) + for suffix in ("", "/content"): + self.client.call("GET", attachment_path + suffix, 401) + for suffix in ("", "/content"): + self.client.call("GET", attachment_path + suffix, 404, actor=self.eve) + path = f"/api/v1/chats/{self.chat['id']}/messages" + for attachment_id in (str(uuid4()), self.other_attachment["id"]): + self.client.call("POST", path, 404, actor=self.alice, + body={"text": "", "attachment_ids": [attachment_id]}, + headers={"Idempotency-Key": uuid4().hex}) + self.client.call("POST", path, 400, actor=self.alice, body={"text": ""}, + headers={"Idempotency-Key": uuid4().hex}) + self.client.call("POST", path, 400, actor=self.alice, + body={"text": "", "attachment_ids": [self.attachment["id"]] * 2}, + headers={"Idempotency-Key": uuid4().hex}) + + def test_other_replica_reads_same_bytes(self): + path = f"/api/v1/attachments/{self.attachment['id']}/content" + seen = set() + for _ in range(20): + result = self.client.call("GET", path, actor=self.bob) + self.assertEqual(result["bytes"], self.content) + seen.add(result["headers"]["X-Instance-Id"]) + if len(seen) >= 2: + break + self.assertGreaterEqual(len(seen), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_attachment_failover.py b/student-tests/test_attachment_failover.py new file mode 100644 index 00000000..ce87077e --- /dev/null +++ b/student-tests/test_attachment_failover.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Stop the replica that accepted an upload, then read its bytes elsewhere.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def main(): + client = Client(timeout=2) + alice, bob, _, chat = client.bootstrap() + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + observed = {client.call("GET", "/api/v1/users/me", actor=bob)["headers"]["X-Instance-Id"] for _ in range(30)} + if len(observed) != 2: + raise Failure("two API replicas were not observed before stop") + for target in sorted(observed): + for _ in range(10): + result = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", 201, + actor=alice, raw=original, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + if result["headers"]["X-Instance-Id"] == target: + break + else: + raise Failure("upload did not reach the selected replica") + attachment_id = result["json"]["id"] + env = {**os.environ, "TARGET_INSTANCE_ID": target} + try: + subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, env=env, check=True) + deadline = time.monotonic() + 10 + while True: + try: + read = client.call("GET", f"/api/v1/attachments/{attachment_id}/content", actor=bob) + if read["headers"]["X-Instance-Id"] == target: + raise Failure("stopped replica still answered") + if read["bytes"] != original: + raise Failure("attachment bytes changed after replica stop") + break + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(0.2) + finally: + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--scale", "api=2"], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + print(json.dumps({"stopped_upload_replicas": len(observed), "download_on_other_replica": True, + "bytes_equal": True}, sort_keys=True)) + + +if __name__ == "__main__": + main() -- 2.54.0 From dc4f5c6c7f057e245684c95e86f5de4730452d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:08:12 +0300 Subject: [PATCH 06/10] Add safe attachment reconciliation and scoped maintenance checks --- Dockerfile | 1 + compose.yaml | 20 +++ evidence/reconcile-u5-2026-10-02.txt | 19 +++ evidence/reconcile-u5-dry-run.json | 1 + evidence/reconcile-u5-repeat.json | 1 + evidence/reconcile-u5-restore.json | 1 + scripts/reconcile-attachments.py | 173 ++++++++++++++++++++++ student-tests/test_reconcile.py | 142 ++++++++++++++++++ student-tests/test_storage_permissions.py | 45 ++++++ 9 files changed, 403 insertions(+) create mode 100644 evidence/reconcile-u5-2026-10-02.txt create mode 100644 evidence/reconcile-u5-dry-run.json create mode 100644 evidence/reconcile-u5-repeat.json create mode 100644 evidence/reconcile-u5-restore.json create mode 100644 scripts/reconcile-attachments.py create mode 100644 student-tests/test_reconcile.py create mode 100644 student-tests/test_storage_permissions.py diff --git a/Dockerfile b/Dockerfile index 1ecad6dc..a043a5b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 100 COPY --chown=app:app src/ src/ COPY --chown=app:app migrations/ migrations/ COPY --chown=app:app scripts/migrate.py scripts/migrate.py +COPY --chown=app:app scripts/reconcile-attachments.py scripts/reconcile-attachments.py USER app EXPOSE 8080 CMD ["python", "-m", "src.server"] diff --git a/compose.yaml b/compose.yaml index 982f3ff7..0cdab49c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -102,6 +102,26 @@ services: cpus: 0.5 mem_limit: 256m + reconcile: + profiles: [maintenance] + build: . + command: ["python", "scripts/reconcile-attachments.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: s3:9000 + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_maintenance_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_maintenance_secret_key + secrets: [database_url, s3_maintenance_key_id, s3_maintenance_secret_key] + volumes: ["./tests/fixtures:/fixtures:ro"] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + restart: "no" + cpus: 0.5 + mem_limit: 256m + lb: image: haproxy:3.0.11-alpine volumes: diff --git a/evidence/reconcile-u5-2026-10-02.txt b/evidence/reconcile-u5-2026-10-02.txt new file mode 100644 index 00000000..48a45dd9 --- /dev/null +++ b/evidence/reconcile-u5-2026-10-02.txt @@ -0,0 +1,19 @@ +Lab 4 U5, local Colima/arm64, 2026-10-02, synthetic data only. +Service setup: Compose API x2 + PostgreSQL + private MinIO. Default reconciliation grace: 3600 s. +The ignored compose.local.yaml only supplied a temporary package-CDN address for local builds. + +Checks: +- `docker compose ... exec -T api python - < student-tests/test_upload_failure.py`: 4/4 PASS, including uncertain COMMIT retaining the object. +- `docker compose ... exec -T api python - < student-tests/test_storage_permissions.py`: 1/1 PASS; API identity cannot ListBucket or create a bucket, anonymous GET 403. +- `docker compose ... --profile maintenance run --rm -T reconcile python - < student-tests/test_storage_permissions.py`: 1/1 PASS; separate maintenance identity can list but cannot create a bucket, anonymous GET 403, root/API secrets absent. +- `docker compose ... --profile maintenance run --rm -T reconcile python - < student-tests/test_reconcile.py`: 2/2 PASS. Old orphan originated from S3 PUT followed by PostgreSQL FK rollback; missing originated from deleting an existing S3 object while ready metadata remained. Dry-run did not mutate. With a test-only 5-second grace, apply deleted the old orphan while preserving a fresh orphan and a message-linked attachment. A wrong fixture was rejected before any deletion; missing content returned HTTP 503. The fixture with matching size/SHA-256 restored the original key, after which API content returned the exact original bytes. Repeat restore was `already_present`. +- `docker compose ... ps s3`: healthy S3; only internal `9000-9001/tcp`, no host port published. + +CLI evidence (separate runs, production grace 3600 s): +- `reconcile-u5-dry-run.json`: 28 metadata / 28 objects; 1 fresh orphan protected, 1 missing/manual incident, 0 deleted, 5.176 ms. +- `reconcile-u5-restore.json`: verified fixture restored that synthetic missing object; 0 deleted, 9.130 ms. +- `reconcile-u5-repeat.json`: restore `already_present`, no remaining missing ID, 0 deleted, 7.657 ms. + +MinIO list timestamps include milliseconds while HEAD timestamps have second precision. Before deletion, the reconciler compares their second-level time, ETag and size, then rechecks the age and PostgreSQL reference. The first integration run exposed this representation difference; after the fix, both tests passed. + +Limits: these are single-run local timings, not a throughput estimate. The 5-second grace is used only for synthetic integration data; routine maintenance defaults to one hour. Missing bytes without a verified source remain a manual incident, with metadata retained. A test-created fresh orphan remains in the inventory until it ages beyond grace and a later apply run handles it. diff --git a/evidence/reconcile-u5-dry-run.json b/evidence/reconcile-u5-dry-run.json new file mode 100644 index 00000000..5962d08e --- /dev/null +++ b/evidence/reconcile-u5-dry-run.json @@ -0,0 +1 @@ +{"deleted": [], "duration_ms": 5.176, "fresh_orphans": ["originals/c2046814-1d5e-429b-b537-168d68d43f40"], "grace_seconds": 3600, "inventory": {"metadata": 28, "objects": 28}, "manual_incidents": ["ee0c99ff-772d-4957-91aa-8a599a67578a"], "missing_attachment_ids": ["ee0c99ff-772d-4957-91aa-8a599a67578a"], "mode": "dry_run", "old_orphans": [], "preserved_after_recheck": [], "restore": null, "unknown_keys_preserved": []} diff --git a/evidence/reconcile-u5-repeat.json b/evidence/reconcile-u5-repeat.json new file mode 100644 index 00000000..509e3733 --- /dev/null +++ b/evidence/reconcile-u5-repeat.json @@ -0,0 +1 @@ +{"deleted": [], "duration_ms": 7.657, "fresh_orphans": ["originals/c2046814-1d5e-429b-b537-168d68d43f40"], "grace_seconds": 3600, "inventory": {"metadata": 28, "objects": 29}, "manual_incidents": [], "missing_attachment_ids": [], "mode": "apply", "old_orphans": [], "preserved_after_recheck": [], "restore": "already_present", "unknown_keys_preserved": []} diff --git a/evidence/reconcile-u5-restore.json b/evidence/reconcile-u5-restore.json new file mode 100644 index 00000000..1f29879e --- /dev/null +++ b/evidence/reconcile-u5-restore.json @@ -0,0 +1 @@ +{"deleted": [], "duration_ms": 9.13, "fresh_orphans": ["originals/c2046814-1d5e-429b-b537-168d68d43f40"], "grace_seconds": 3600, "inventory": {"metadata": 28, "objects": 28}, "manual_incidents": [], "missing_attachment_ids": ["ee0c99ff-772d-4957-91aa-8a599a67578a"], "mode": "apply", "old_orphans": [], "preserved_after_recheck": [], "restore": "restored", "unknown_keys_preserved": []} diff --git a/scripts/reconcile-attachments.py b/scripts/reconcile-attachments.py new file mode 100644 index 00000000..1d42c914 --- /dev/null +++ b/scripts/reconcile-attachments.py @@ -0,0 +1,173 @@ +"""Compare PostgreSQL attachment metadata with private S3 originals. + +Dry-run is the default. Run in the Compose maintenance profile with its separate S3 identity. +""" + +import argparse +import hashlib +import json +import os +import re +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +import psycopg +from minio import Minio +from minio.error import S3Error +from psycopg.rows import dict_row +from urllib3 import PoolManager, Retry, Timeout + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import database_url # noqa: E402 + +BUCKET = "messenger-attachments" +ORIGINAL = re.compile(r"originals/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\Z") +MAX_SOURCE = 10 * 1024 * 1024 + + +def secret(name: str) -> str: + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + +def storage() -> Minio: + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_ACCESS_KEY_ID"), secret_key=secret("S3_SECRET_ACCESS_KEY"), + secure=False, region="us-east-1", + http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)), + ) + + +def age_seconds(last_modified: datetime, now: datetime) -> float: + if last_modified is None or last_modified.tzinfo is None: + return 0 # Unknown age is never safe to delete. + return max(0, (now - last_modified).total_seconds()) + + +def absent(client: Minio, key: str) -> bool: + try: + client.stat_object(BUCKET, key) + return False + except S3Error as exc: + if exc.code == "NoSuchKey": + return True + raise + + +def source_hash(path: Path) -> tuple[int, str]: + size, digest = 0, hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + size += len(chunk) + if size > MAX_SOURCE: + raise ValueError("restore source exceeds 10 MiB") + digest.update(chunk) + return size, digest.hexdigest() + + +def delete_orphan(client: Minio, conn: psycopg.Connection, item, grace: int) -> bool: + key = item.object_name + if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone(): + return False + try: + current = client.stat_object(BUCKET, key) + except S3Error as exc: + if exc.code == "NoSuchKey": + return False + raise + if ( + current.etag != item.etag or current.last_modified.replace(microsecond=0) != item.last_modified.replace(microsecond=0) + or current.size != item.size or age_seconds(current.last_modified, datetime.now(timezone.utc)) < grace + ): + return False + # Upload keys are random and S3 PUT + PostgreSQL INSERT is bounded; grace and recheck protect live uploads. + client.remove_object(BUCKET, key) + return True + + +def restore_missing(client: Minio, conn: psycopg.Connection, attachment_id: UUID, source: Path, apply: bool) -> str: + size, digest = source_hash(source) + row = conn.execute( + "SELECT id,object_key,size_bytes,sha256,content_type,status FROM attachments WHERE id=%s FOR UPDATE", + (attachment_id,), + ).fetchone() + if row is None or row["status"] != "ready": + raise ValueError("ready attachment not found") + if not ORIGINAL.fullmatch(row["object_key"]): + raise ValueError("invalid original object key") + if (size, digest) != (row["size_bytes"], row["sha256"]): + raise ValueError("restore source size/SHA-256 mismatch") + if not absent(client, row["object_key"]): + return "already_present" + if not apply: + return "would_restore" + with source.open("rb") as stream: + client.put_object(BUCKET, row["object_key"], stream, size, content_type=row["content_type"]) + if client.stat_object(BUCKET, row["object_key"]).size != size: + raise RuntimeError("restored object size mismatch") + return "restored" + + +def reconcile(client: Minio, conn: psycopg.Connection, apply: bool, grace: int, restore_id: UUID | None, source: Path | None) -> dict: + start = time.monotonic() + metadata = conn.execute( + "SELECT id,object_key,size_bytes,sha256,content_type,status FROM attachments" + ).fetchall() + objects = list(client.list_objects(BUCKET, prefix="originals/", recursive=True)) + by_key = {row["object_key"]: row for row in metadata} + by_object = {item.object_name: item for item in objects} + now = datetime.now(timezone.utc) + old = [item for item in objects if item.object_name not in by_key and ORIGINAL.fullmatch(item.object_name) + and age_seconds(item.last_modified, now) >= grace] + fresh = [item.object_name for item in objects if item.object_name not in by_key and ORIGINAL.fullmatch(item.object_name) + and age_seconds(item.last_modified, now) < grace] + unknown = [item.object_name for item in objects if item.object_name not in by_key and not ORIGINAL.fullmatch(item.object_name)] + missing = [row for row in metadata if row["status"] == "ready" and row["object_key"] not in by_object + and absent(client, row["object_key"])] + + result = { + "mode": "apply" if apply else "dry_run", "grace_seconds": grace, + "inventory": {"objects": len(objects), "metadata": len(metadata)}, + "old_orphans": [item.object_name for item in old], "fresh_orphans": fresh, + "unknown_keys_preserved": unknown, "missing_attachment_ids": [str(row["id"]) for row in missing], + "manual_incidents": [str(row["id"]) for row in missing], + "deleted": [], "preserved_after_recheck": [], "restore": None, + } + if restore_id is not None and apply: + # Reject an invalid source before any orphan deletion in this apply run. + restore_missing(client, conn, restore_id, source, False) + if apply: + for item in old: + (result["deleted"] if delete_orphan(client, conn, item, grace) else result["preserved_after_recheck"]).append(item.object_name) + if restore_id is not None: + result["restore"] = restore_missing(client, conn, restore_id, source, apply) + if result["restore"] in {"restored", "already_present"}: + result["manual_incidents"] = [item for item in result["manual_incidents"] if item != str(restore_id)] + result["duration_ms"] = round((time.monotonic() - start) * 1000, 3) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="perform safe deletions and selected verified restore") + parser.add_argument("--grace-seconds", type=int, default=3600, help="minimum age before deleting an orphan (default 3600)") + parser.add_argument("--restore-id", type=UUID, help="one missing attachment to restore") + parser.add_argument("--source", type=Path, help="read-only original fixture with matching size and SHA-256") + args = parser.parse_args() + if args.grace_seconds < 1 or (args.restore_id is None) != (args.source is None): + parser.error("grace must be positive; restore-id and source must be supplied together") + try: + with psycopg.connect(database_url(), row_factory=dict_row, connect_timeout=5, options="-c statement_timeout=30000") as conn: + result = reconcile(storage(), conn, args.apply, args.grace_seconds, args.restore_id, args.source) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + except (psycopg.Error, S3Error, OSError, ValueError, RuntimeError) as exc: + print(json.dumps({"error": type(exc).__name__, "message": "reconciliation stopped without further mutation"})) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 00000000..ad5e6b38 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,142 @@ +"""Run inside the maintenance image: python - < student-tests/test_reconcile.py.""" + +import hashlib +import importlib.util +import io +import time +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock +from urllib.error import HTTPError +from urllib.request import Request, urlopen +from uuid import UUID, uuid4 + +import psycopg +from psycopg.rows import dict_row + +from src.db import database_url +from src.store import store + +spec = importlib.util.spec_from_file_location("reconciler", "scripts/reconcile-attachments.py") +reconciler = importlib.util.module_from_spec(spec) +spec.loader.exec_module(reconciler) + + +class ReconcileTests(unittest.TestCase): + def test_delete_rechecks_database_and_object_version(self): + key = f"originals/{uuid4()}" + item = SimpleNamespace(object_name=key, etag="old", size=10, + last_modified=datetime.now(timezone.utc) - timedelta(hours=2)) + conn, client = Mock(), Mock() + conn.execute.return_value.fetchone.return_value = {"id": uuid4()} + self.assertFalse(reconciler.delete_orphan(client, conn, item, 3600)) + client.remove_object.assert_not_called() + + conn.execute.return_value.fetchone.return_value = None + client.stat_object.return_value = SimpleNamespace(etag="replaced", size=10, last_modified=item.last_modified) + self.assertFalse(reconciler.delete_orphan(client, conn, item, 3600)) + client.remove_object.assert_not_called() + + def test_orphans_missing_repair_and_live_reference(self): + client = reconciler.storage() + fixture = Path("/fixtures/sample.png") + data = fixture.read_bytes() + digest = hashlib.sha256(data).hexdigest() + username = "reconcile_" + uuid4().hex[:16] + owner = store.register(username, "Reconciler test", "Test-password-123") + session = store.create_session(username, "Test-password-123") + chat = store.create_chat(owner["id"], "Reconciler test", [owner["id"]]) + active_id, missing_id = uuid4(), uuid4() + active_key, missing_key = f"originals/{active_id}", f"originals/{missing_id}" + old_key, fresh_key = f"originals/{uuid4()}", f"originals/{uuid4()}" + + def put(key): + client.put_object(reconciler.BUCKET, key, io.BytesIO(data), len(data), content_type="image/png") + + put(active_key) + put(missing_key) + put(old_key) + with self.assertRaises(psycopg.errors.ForeignKeyViolation): + with psycopg.connect(database_url()) as failed: + failed.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", + (UUID(old_key.split("/")[1]), uuid4(), UUID(owner["id"]), "orphan.png", old_key, + "image/png", len(data), digest), + ) + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + for item_id, key in ((active_id, active_key), (missing_id, missing_key)): + conn.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", + (item_id, UUID(chat["id"]), UUID(owner["id"]), "sample.png", key, "image/png", len(data), digest), + ) + message_id = uuid4() + conn.execute( + "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", + (message_id, UUID(chat["id"]), UUID(owner["id"]), ""), + ) + conn.execute( + "INSERT INTO message_attachments (message_id,chat_id,attachment_id,position) VALUES (%s,%s,%s,0)", + (message_id, UUID(chat["id"]), active_id), + ) + + client.remove_object(reconciler.BUCKET, missing_key) + + time.sleep(6) + put(fresh_key) + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + preview = reconciler.reconcile(client, conn, False, 5, None, None) + self.assertIn(old_key, preview["old_orphans"]) + self.assertIn(fresh_key, preview["fresh_orphans"]) + self.assertIn(str(missing_id), preview["missing_attachment_ids"]) + self.assertNotIn(active_key, preview["old_orphans"]) + self.assertFalse(reconciler.absent(client, old_key)) + + wrong = Path("/tmp/reconcile-wrong-source") + wrong.write_bytes(b"not the original") + with self.assertRaisesRegex(ValueError, "size/SHA-256 mismatch"): + reconciler.reconcile(client, conn, True, 5, missing_id, wrong) + self.assertFalse(reconciler.absent(client, old_key)) + + applied = reconciler.reconcile(client, conn, True, 5, None, None) + self.assertIn(old_key, applied["deleted"]) + self.assertTrue(reconciler.absent(client, old_key)) + self.assertFalse(reconciler.absent(client, fresh_key)) + self.assertFalse(reconciler.absent(client, active_key)) + self.assertTrue(reconciler.absent(client, missing_key)) + request = Request( + f"http://api:8080/api/v1/attachments/{missing_id}/content", + headers={"Authorization": "Bearer " + session["token"]}, + ) + with self.assertRaises(HTTPError) as unavailable: + urlopen(request, timeout=5) + self.assertEqual(unavailable.exception.code, 503) + + with self.assertRaisesRegex(ValueError, "size/SHA-256 mismatch"): + reconciler.restore_missing(client, conn, missing_id, wrong, True) + self.assertTrue(reconciler.absent(client, missing_key)) + self.assertEqual(reconciler.restore_missing(client, conn, missing_id, fixture, True), "restored") + self.assertEqual(reconciler.restore_missing(client, conn, missing_id, fixture, True), "already_present") + final = reconciler.reconcile(client, conn, False, 5, None, None) + self.assertNotIn(old_key, final["old_orphans"]) + self.assertNotIn(str(missing_id), final["missing_attachment_ids"]) + self.assertIsNotNone(conn.execute( + "SELECT 1 FROM message_attachments WHERE attachment_id=%s", (active_id,) + ).fetchone()) + + response = client.get_object(reconciler.BUCKET, missing_key) + try: + self.assertEqual(hashlib.sha256(response.read()).hexdigest(), digest) + finally: + response.close() + response.release_conn() + with urlopen(request, timeout=5) as response: + self.assertEqual(response.read(), data) + print("PASS: old orphan dry-run/apply, fresh and linked objects retained, missing restored from verified fixture") + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_storage_permissions.py b/student-tests/test_storage_permissions.py new file mode 100644 index 00000000..8212e0c1 --- /dev/null +++ b/student-tests/test_storage_permissions.py @@ -0,0 +1,45 @@ +"""Run in both API and maintenance containers; tests their distinct S3 identities.""" + +import io +import os +import unittest +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import urlopen +from uuid import uuid4 + +from minio.error import S3Error + +from src.attachments import BUCKET, storage + + +class StoragePermissionsTests(unittest.TestCase): + def test_scoped_identity_and_private_bucket(self): + maintenance = os.environ["S3_ACCESS_KEY_ID_FILE"].endswith("/s3_maintenance_key_id") + self.assertFalse(Path("/run/secrets/minio_root_user").exists()) + if maintenance: + self.assertFalse(Path("/run/secrets/s3_access_key_id").exists()) + else: + self.assertFalse(Path("/run/secrets/s3_maintenance_key_id").exists()) + client = storage() + key = "permissions/" + uuid4().hex + client.put_object(BUCKET, key, io.BytesIO(b"private"), 7) + try: + if maintenance: + list(client.list_objects(BUCKET, prefix="originals/", recursive=True)) + else: + with self.assertRaises(S3Error) as denied: + list(client.list_objects(BUCKET, prefix="originals/", recursive=True)) + self.assertEqual(denied.exception.code, "AccessDenied") + with self.assertRaises(S3Error) as denied: + client.make_bucket("forbidden-" + uuid4().hex) + self.assertEqual(denied.exception.code, "AccessDenied") + with self.assertRaises(HTTPError) as denied: + urlopen(f"http://s3:9000/{BUCKET}/{key}", timeout=3) + self.assertEqual(denied.exception.code, 403) + finally: + client.remove_object(BUCKET, key) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From 55bb55d6392c5089d73df60df2fbe6ca1bee8296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:12:09 +0300 Subject: [PATCH 07/10] docs: Record level-four reconciliation evidence --- REPORT.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/REPORT.md b/REPORT.md index 97a830d6..c523f453 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Самохвалов Андрей Ремович, БПИ-2302. -- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Перенос lab3, приватное S3, синхронный upload, доступ к вложениям и связь с сообщениями проверены; reconciler и эксперимент трека выполняются далее. +- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Уровни 3 и 4 проверены; повторяемый эксперимент трека 5 выполняется далее. - Ветка: `solution/bpi2302-samohvalov`; commit/PR указываются после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/openapi.json`, публичные тесты, [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z), [образ](https://hub.docker.com/r/alpine/minio/tags), документация MinIO для IAM. Python/FastAPI, PostgreSQL, MinIO Python SDK; проект подготовлен с помощью ИИ Codex с локальной проверкой ниже. @@ -35,13 +35,16 @@ make test | `s3` | MinIO security release, 0,5 CPU/384 MiB | `/minio/health/ready`, `s3_data` | только root credentials, наружный порт и консоль не опубликованы | | `s3-bootstrap` | одноразовый MinIO Client | завершение 0 | root и создаваемые ограниченные credentials | | `s3-volume-init` | одноразовая настройка владельца volume | завершение 0 | секретов нет | +| `reconcile` | одноразовый Python, профиль `maintenance` | завершение 0, без порта и volume | DB URL и отдельные S3 maintenance credentials | `infra/minio-api-policy.json` разрешает только операции над объектами одного bucket и чтение его location; создание bucket, изменение IAM и административные операции не разрешены. Отдельная maintenance policy добавляет list для будущей сверки. Bucket не имеет анонимного чтения. Приватность bucket не заменяет проверку членства в чате на API: upload проверяет его до чтения тела, metadata/content — до доступа к S3. Публичный ответ не содержит object key, endpoint и credentials. -Upload читает raw body в `SpooledTemporaryFile` с порогом RAM 1 MiB и останавливается после 10 MiB; проверяет формат и MIME через Pillow, до декодирования отклоняет свыше 40 млн пикселей, затем полностью проверяет PNG/JPEG. Имя файла применяется только для отображения, object key — случайный UUID в `originals/`. S3 PUT предшествует INSERT/COMMIT в PostgreSQL; `201 ready` возвращается лишь после обоих успехов. При достоверной ошибке до COMMIT объект удаляется. Если подтверждение COMMIT потеряно, удалять небезопасно: объект остаётся кандидатом для сверки после grace period. Если и компенсационное DELETE отказало, key записывается во внутренний лог; U5 добавит повторяемый reconciler. Клиенту при ошибке зависимости приходит `503`, а прозрачный retry POST не выполняется. Для S3 заданы connect/read timeouts 2/10 с без внутренних повторов; `/health/ready` проверяет БД и доступ к приватному S3 через разрешённый HEAD отсутствующего probe key. +Upload читает raw body в `SpooledTemporaryFile` с порогом RAM 1 MiB и останавливается после 10 MiB; проверяет формат и MIME через Pillow, до декодирования отклоняет свыше 40 млн пикселей, затем полностью проверяет PNG/JPEG. Имя файла применяется только для отображения, object key — случайный UUID в `originals/`. S3 PUT предшествует INSERT/COMMIT в PostgreSQL; `201 ready` возвращается лишь после обоих успехов. При достоверной ошибке до COMMIT объект удаляется. Если подтверждение COMMIT потеряно, удалять небезопасно: объект остаётся кандидатом для сверки после grace period. Если и компенсационное DELETE отказало, key записывается во внутренний лог; reconciler позднее удаляет только старый объект без metadata. Клиенту при ошибке зависимости приходит `503`, а прозрачный retry POST не выполняется. Для S3 заданы connect/read timeouts 2/10 с без внутренних повторов; `/health/ready` проверяет БД и доступ к приватному S3 через разрешённый HEAD отсутствующего probe key. POST сообщения принимает до 10 уникальных ID из этого чата, включая пустой текст с вложением. Проверка вложений, запись сообщения, связей и результата идемпотентности выполняются в одной транзакции; порядок ID сохраняется. Повтор ключа с тем же содержимым возвращает прежний Message. `GET` metadata и content сначала проверяют действующую сессию и членство в чате; неизвестный и чужой ID одинаково дают 404. Исходник отдаётся потоково через API, отсутствующий вариант — 404, недоступный S3-объект — 503. +`docker compose --profile maintenance run --rm reconcile` выполняет **dry-run по умолчанию**. `--apply` удаляет только объекты `originals/` без metadata старше 3600 с: перед DELETE повторно проверяются запись БД, размер, ETag, время изменения и возраст. Неизвестные ключи не удаляются. Для metadata без S3-объекта выдаётся ручной инцидент; metadata сохраняется. Восстановление требует `--restore-id` и read-only fixture с точным размером и SHA-256. Файл проверяется до любых удалений при `--apply`; несоответствие останавливает run. Повторный apply не меняет уже восстановленный объект. У API нет `ListBucket`, у отдельной maintenance-учётной записи он есть; обе не имеют прав создавать bucket или менять IAM. + ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство | @@ -51,15 +54,19 @@ POST сообщения принимает до 10 уникальных ID из | Миграция и транзакции | `docker compose exec -T api python - < student-tests/test_database.py`; `... test_auth.py` | Миграции 1–5 применены повторяемо, 2 DB и 3 auth теста PASS | `evidence/storage-bootstrap-2026-10-02.txt` | | Raw PNG/JPEG upload и ограничения | `BASE_URL=http://localhost:8084 python3 -m unittest discover -s student-tests -p 'test_attachment_upload.py' -v` | 4 набора PASS: PNG/JPEG `201 ready`, корректные hash/size/type; 10 MiB+1 и >40 млн px — 413; повреждение/MIME spoof — 415; опасное имя — 400, чужой чат — 404 | `student-tests/test_attachment_upload.py`, синтетический JPEG fixture | | Отказ S3 | `docker compose stop s3`; `docker compose exec -T api python - < student-tests/test_s3_outage.py`; `docker compose start s3` | При реально остановленном S3 readiness=503, upload=503, число metadata строк не изменилось; S3 восстановлен и healthy | `student-tests/test_s3_outage.py` | -| Промежуток S3/БД и обрыв потока | `docker compose exec -T api python - < student-tests/test_upload_failure.py` | PUT failure не вызывает SQL; ошибка БД до INSERT вызывает компенсацию DELETE; оборванный поток не начинает PUT. Добавленный случай неоднозначного COMMIT требует отдельного контрольного прогона | `student-tests/test_upload_failure.py` | +| Промежуток S3/БД и обрыв потока | `docker compose exec -T api python - < student-tests/test_upload_failure.py` | 4/4 PASS: PUT failure не вызывает SQL; ошибка до INSERT компенсируется DELETE; оборванный поток не начинает PUT; неоднозначный COMMIT оставляет объект для сверки | `evidence/reconcile-u5-2026-10-02.txt` | | Ограничение RAM | `BASE_URL=http://localhost:8084 python3 scripts/measure-upload-memory.py` | 10/10 одновременных upload одного PNG 9 441 653 байт, 10 разных ready ID и верный SHA-256, 6,145 с; sampled peak API-1/2 114,7/115,5 MiB при лимите 256 MiB | `evidence/upload-memory-2026-10-02.csv` | | Сообщения, ACL и скачивание | `python3 -m unittest discover -s student-tests -p test_attachment_access.py -v`; `make test BASE_URL=http://127.0.0.1:8084` | 3 собственных теста OK; публичный сценарий PASS; link/replay и bytes на обеих репликах, чужой объект 404 | `evidence/attachment-access-2026-10-02.txt` | | Сохранность S3 и отказ API | `make test-persistence ... ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ... ACTION_SCRIPT=scripts/stop-one.sh`; `python3 student-tests/test_attachment_failover.py` | Токен, Message, metadata и bytes пережили restart API/DB/S3; публичный failover восстановил запись за 2,826 и 3,020 с; адресный тест остановил каждую загрузившую реплику и скачал те же bytes через другую | `evidence/attachment-access-2026-10-02.txt` | +| Сверка orphan/missing | `docker compose --profile maintenance run --rm reconcile`; `student-tests/test_reconcile.py` в maintenance-контейнере | 2/2 PASS: old orphan после PG rollback удалён по истечении grace; fresh и привязанный сохранены; missing дал 503, неверный fixture не изменил данные, верный восстановил bytes/200; повтор идемпотентен | `evidence/reconcile-u5-2026-10-02.txt`, три JSON результата | +| Права S3 | `student-tests/test_storage_permissions.py` в API и maintenance-контейнерах | 1/1 PASS для каждой роли; API не может list, обе не могут создавать bucket, анонимный GET 403, консоль не опубликована | `evidence/reconcile-u5-2026-10-02.txt` | ## Эксперименты и отказы -Граница PUT/COMMIT проверена fault-инъекциями; полный reconciler и проверка восстановления при неизвестном результате COMMIT выполняются в U5/U6. Замер RAM проводился на локальном Colima/arm64, 2 CPU и 2 GiB, API ×2 с 0,5 CPU/256 MiB на экземпляр, 10 потоков с барьером старта. CSV содержит baseline и два Docker stats снимка во время 6,145 с серии: peak означает максимум **наблюдённых** снимков, а не точный аппаратный high-water mark. Все 10 ответов 201 имели уникальные ID и совпадающий SHA-256. Создание body по `os.urandom` оставляет воспроизводимыми размер/разрешение/алгоритм, но не конкретные байты и hash между прогонами. +Граница PUT/COMMIT проверена fault-инъекциями; U5 сверяет реальные расхождения, а U6 отдельно измерит повторяемость и стоимость. Для теста удаления grace уменьшался до 5 секунд только на синтетическом наборе; обычный запуск использует 3600 секунд. Список объектов MinIO содержит миллисекунды, а HEAD округляет время до секунд: перед удалением сверяются секунды, ETag и размер, затем возраст и ссылка в БД. После канонического dry-run 28 metadata, 29 objects, 1 свежий orphan был защищён и ни один объект не удалён. + +Замер RAM проводился на локальном Colima/arm64, 2 CPU и 2 GiB, API ×2 с 0,5 CPU/256 MiB на экземпляр, 10 потоков с барьером старта. CSV содержит baseline и два Docker stats снимка во время 6,145 с серии: peak означает максимум **наблюдённых** снимков, а не точный аппаратный high-water mark. Все 10 ответов 201 имели уникальные ID и совпадающий SHA-256. Создание body по `os.urandom` оставляет воспроизводимыми размер/разрешение/алгоритм, но не конкретные байты и hash между прогонами. ## Ограничения и следующие шаги -Публичный `make test` проходит; он не доказывает все отказные сценарии. Проверки уровня 4 и трека 5 не заявляются завершёнными до повторяемой сверки. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. +Публичный `make test` проходит; он не доказывает все отказные сценарии. Уровень 4 подтверждён отдельными проверками. Трек 5 ещё не заявляется завершённым до повторяемой серии и измерения стоимости. Missing без проверенного источника остаётся ручным инцидентом, автоматической реконструкции чужих байтов нет. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. -- 2.54.0 From 9c15fc37440075949ea058f361f8dc1265d9e3e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:20:22 +0300 Subject: [PATCH 08/10] Prove attachment reconciliation under storage faults --- .../reconciliation/analysis-2026-10-02.md | 24 +++ .../reconciliation/run-20261002T111817Z.json | 1 + scripts/fault-injection.sh | 17 ++ student-tests/test_reconcile_research.py | 200 ++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 evidence/reconciliation/analysis-2026-10-02.md create mode 100644 evidence/reconciliation/run-20261002T111817Z.json create mode 100755 scripts/fault-injection.sh create mode 100644 student-tests/test_reconcile_research.py diff --git a/evidence/reconciliation/analysis-2026-10-02.md b/evidence/reconciliation/analysis-2026-10-02.md new file mode 100644 index 00000000..c48c44dc --- /dev/null +++ b/evidence/reconciliation/analysis-2026-10-02.md @@ -0,0 +1,24 @@ +# Reconciliation fault injection and inventory cost + +## Hypothesis + +The reconciler can identify an orphan left by a successful S3 PUT followed by a failed PostgreSQL write, and a missing S3 object referenced by committed metadata. Applying repairs should delete only an orphan older than the grace period, restore missing bytes only from a matching fixture, and leave fresh or linked objects untouched. A second pass should find no work for the repaired cases. + +## Reproduce + +On the local Compose stand with `.env` configured, run `sh scripts/fault-injection.sh`. The local build used `COMPOSE_FILE=compose.yaml:compose.local.yaml` because this host needed a temporary Docker DNS override; the override is ignored by Git. The script builds and runs the maintenance service and writes anonymized JSON under `evidence/reconciliation/`. It uses a synthetic user, chat, message, attachments, and object keys. Repeated runs append more synthetic inventory to persistent volumes, so compare measurements only within one run. + +The PostgreSQL fault is an invalid foreign key insert after a successful S3 PUT. The transaction rolls back, leaving an object with no metadata. The S3 fault deletes an object after its metadata commits. A 5-second grace period lets the test age the first orphan, then create a fresh one. Production maintenance defaults to one hour. A message links another attachment throughout the test. + +## Observed result + +Raw result: [run-20261002T111817Z.json](run-20261002T111817Z.json). The initial dry-run saw 283 metadata rows and 285 S3 objects, including the injected old orphan, fresh orphan, and missing object. The dry-run left them unchanged. A deliberately wrong source was rejected before deletion. Apply without a source deleted the old orphan, retained the missing metadata as a manual incident, and preserved the fresh and linked objects. A second dry-run found the old orphan gone and the missing incident present; a repeated delete made no change to the injected orphan. Apply with the correct fixture restored the exact SHA-256 bytes. Repeated restore returned `already_present`, and the final dry-run found neither injected inconsistency. + +Inventory scans used three sequential dry-run trials per size in the same maintenance process. CPU is user plus system time for that process; RSS is Linux `VmRSS`, and high-water RSS is `ru_maxrss`. Times are milliseconds, RSS is KiB. + +| Inventory | Trial wall times | Trial CPU times | Median wall | Median CPU | RSS before → after across trials | Peak RSS | +| --- | --- | --- | ---: | ---: | --- | ---: | +| 333 metadata / 334 objects | 19.390, 21.143, 23.322 | 6.447, 7.420, 8.516 | 21.143 | 7.420 | 68152 → 68512 | 126608 | +| 533 metadata / 534 objects | 16.585, 24.219, 44.148 | 9.993, 9.997, 9.500 | 24.219 | 9.993 | 68512 → 68936 | 126608 | + +The larger inventory used more CPU in this run; wall time varied widely, including one faster larger-inventory trial. These six samples on one local host do not establish a scaling law. The measurements exclude PostgreSQL and S3 server resources and include pre-existing data in persistent volumes. The process high-water RSS was reached before these scans, so it cannot isolate the scan's peak allocation; the before/after RSS readings show only retained process memory. diff --git a/evidence/reconciliation/run-20261002T111817Z.json b/evidence/reconciliation/run-20261002T111817Z.json new file mode 100644 index 00000000..31e6c2ef --- /dev/null +++ b/evidence/reconciliation/run-20261002T111817Z.json @@ -0,0 +1 @@ +{"faults": {"apply_without_source": {"fresh_preserved": true, "linked_preserved": true, "missing_incident_retained": true, "old_deleted": true}, "post_delete_dry_run": {"missing_incident_retained": true, "old_absent": true}, "post_restore_dry_run": {"missing_absent": true, "old_absent": true}, "preview": {"fresh_detected": true, "inventory": {"metadata": 283, "objects": 285}, "linked_preserved": true, "missing_detected": true, "old_detected": true}, "repeat": "already_present", "repeat_delete_no_change": true, "restore": "restored", "wrong_fixture_rejected_without_delete": true}, "limits": "Single local host, 3 sequential dry-run scans per size; CPU/RSS are maintenance-process measurements only.", "measurements": [{"cpu_system_ms": 0.031, "cpu_user_ms": 6.416, "label": "plus_50", "metadata": 333, "objects": 334, "reconciler_ms": 19.349, "rss_after_kib": 68468, "rss_before_kib": 68152, "rss_hwm_kib": 126608, "trial": 1, "wall_ms": 19.39}, {"cpu_system_ms": 0.0, "cpu_user_ms": 7.42, "label": "plus_50", "metadata": 333, "objects": 334, "reconciler_ms": 21.098, "rss_after_kib": 68500, "rss_before_kib": 68468, "rss_hwm_kib": 126608, "trial": 2, "wall_ms": 21.143}, {"cpu_system_ms": 0.02, "cpu_user_ms": 8.496, "label": "plus_50", "metadata": 333, "objects": 334, "reconciler_ms": 23.282, "rss_after_kib": 68512, "rss_before_kib": 68500, "rss_hwm_kib": 126608, "trial": 3, "wall_ms": 23.322}, {"cpu_system_ms": 0.993, "cpu_user_ms": 9.0, "label": "plus_250", "metadata": 533, "objects": 534, "reconciler_ms": 16.523, "rss_after_kib": 68908, "rss_before_kib": 68512, "rss_hwm_kib": 126608, "trial": 1, "wall_ms": 16.585}, {"cpu_system_ms": 0.007, "cpu_user_ms": 9.99, "label": "plus_250", "metadata": 533, "objects": 534, "reconciler_ms": 24.163, "rss_after_kib": 68936, "rss_before_kib": 68908, "rss_hwm_kib": 126608, "trial": 2, "wall_ms": 24.219}, {"cpu_system_ms": 0.0, "cpu_user_ms": 9.5, "label": "plus_250", "metadata": 533, "objects": 534, "reconciler_ms": 44.084, "rss_after_kib": 68936, "rss_before_kib": 68936, "rss_hwm_kib": 126608, "trial": 3, "wall_ms": 44.148}], "summary": {"plus_250": {"median_cpu_ms": 9.993, "median_wall_ms": 24.219, "metadata": 533, "objects": 534}, "plus_50": {"median_cpu_ms": 7.42, "median_wall_ms": 21.143, "metadata": 333, "objects": 334}}} diff --git a/scripts/fault-injection.sh b/scripts/fault-injection.sh new file mode 100755 index 00000000..8d449309 --- /dev/null +++ b/scripts/fault-injection.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Run on an isolated synthetic-data Compose stand. COMPOSE_FILE may include a local override. +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence/reconciliation +stamp=$(date -u +%Y%m%dT%H%M%SZ) +temporary="evidence/reconciliation/.run-$stamp.tmp" +output="evidence/reconciliation/run-$stamp.json" +trap 'rm -f "$temporary"' EXIT HUP INT TERM + +docker compose --env-file .env --profile maintenance build reconcile +docker compose --env-file .env --profile maintenance run --rm -T reconcile python - \ + < student-tests/test_reconcile_research.py > "$temporary" +python3 -m json.tool "$temporary" >/dev/null +mv "$temporary" "$output" +trap - EXIT HUP INT TERM +printf 'Reconciliation evidence: %s\n' "$output" diff --git a/student-tests/test_reconcile_research.py b/student-tests/test_reconcile_research.py new file mode 100644 index 00000000..2da32086 --- /dev/null +++ b/student-tests/test_reconcile_research.py @@ -0,0 +1,200 @@ +"""Repeatable lab 4 consistency experiment; run inside the maintenance Compose service. + +Writes one JSON result to stdout. It creates synthetic objects and metadata, but never +deletes the measured baseline or the fresh/linked objects after the experiment. +""" + +import hashlib +import importlib.util +import io +import json +import resource +import statistics +import time +from pathlib import Path +from uuid import UUID, uuid4 + +import psycopg +from PIL import Image +from psycopg.rows import dict_row + +from src.db import database_url +from src.store import store + +spec = importlib.util.spec_from_file_location("reconciler", "scripts/reconcile-attachments.py") +reconciler = importlib.util.module_from_spec(spec) +spec.loader.exec_module(reconciler) + + +def put(client, key: str, data: bytes) -> None: + client.put_object(reconciler.BUCKET, key, io.BytesIO(data), len(data), content_type="image/png") + + +def rss_kib() -> int: + for line in Path("/proc/self/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) + raise RuntimeError("VmRSS unavailable") + + +def scan(client, conn, label: str, trial: int) -> dict: + before = resource.getrusage(resource.RUSAGE_SELF) + rss_before = rss_kib() + start = time.perf_counter() + result = reconciler.reconcile(client, conn, False, 3600, None, None) + wall_ms = (time.perf_counter() - start) * 1000 + after = resource.getrusage(resource.RUSAGE_SELF) + return { + "label": label, "trial": trial, "objects": result["inventory"]["objects"], + "metadata": result["inventory"]["metadata"], + "reconciler_ms": result["duration_ms"], "wall_ms": round(wall_ms, 3), + "cpu_user_ms": round((after.ru_utime - before.ru_utime) * 1000, 3), + "cpu_system_ms": round((after.ru_stime - before.ru_stime) * 1000, 3), + "rss_before_kib": rss_before, "rss_after_kib": rss_kib(), "rss_hwm_kib": after.ru_maxrss, + } + + +def main() -> None: + client = reconciler.storage() + fixture = Path("/fixtures/sample.png") + original = fixture.read_bytes() + digest = hashlib.sha256(original).hexdigest() + username = "research_" + uuid4().hex[:16] + owner = store.register(username, "Reconciliation research", "Test-password-123") + chat = store.create_chat(owner["id"], "Reconciliation research", [owner["id"]]) + owner_id, chat_id = UUID(owner["id"]), UUID(chat["id"]) + active_id, missing_id, old_id, fresh_id = (uuid4() for _ in range(4)) + active_key, missing_key, old_key, fresh_key = (f"originals/{item}" for item in (active_id, missing_id, old_id, fresh_id)) + put(client, active_key, original) + put(client, missing_key, original) + put(client, old_key, original) + try: + with psycopg.connect(database_url()) as failed: + failed.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", + (old_id, uuid4(), owner_id, "orphan.png", old_key, "image/png", len(original), digest), + ) + except psycopg.errors.ForeignKeyViolation: + pass + else: + raise AssertionError("PostgreSQL fault injection did not roll back") + + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + for attachment_id, key in ((active_id, active_key), (missing_id, missing_key)): + conn.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", + (attachment_id, chat_id, owner_id, "sample.png", key, "image/png", len(original), digest), + ) + message_id = uuid4() + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", + (message_id, chat_id, owner_id, "")) + conn.execute("INSERT INTO message_attachments (message_id,chat_id,attachment_id,position) VALUES (%s,%s,%s,0)", + (message_id, chat_id, active_id)) + + client.remove_object(reconciler.BUCKET, missing_key) + time.sleep(6) # An explicit test-only 5-second grace; routine maintenance uses one hour. + put(client, fresh_key, original) + + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + preview = reconciler.reconcile(client, conn, False, 5, None, None) + assert old_key in preview["old_orphans"] and fresh_key in preview["fresh_orphans"] + assert str(missing_id) in preview["manual_incidents"] and active_key not in preview["old_orphans"] + assert not reconciler.absent(client, old_key) # Dry-run did not delete. + wrong = Path("/tmp/research-wrong-source") + wrong.write_bytes(b"wrong fixture") + try: + reconciler.reconcile(client, conn, True, 5, missing_id, wrong) + except ValueError: + pass + else: + raise AssertionError("wrong fixture accepted") + assert not reconciler.absent(client, old_key) + + apply_without_source = reconciler.reconcile(client, conn, True, 5, None, None) + assert old_key in apply_without_source["deleted"] + assert str(missing_id) in apply_without_source["manual_incidents"] + assert reconciler.absent(client, old_key) and reconciler.absent(client, missing_key) + assert not reconciler.absent(client, fresh_key) and not reconciler.absent(client, active_key) + assert conn.execute("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (active_id,)).fetchone() + + post_delete_preview = reconciler.reconcile(client, conn, False, 5, None, None) + assert old_key not in post_delete_preview["old_orphans"] + assert str(missing_id) in post_delete_preview["manual_incidents"] + assert fresh_key in post_delete_preview["fresh_orphans"] + repeat_delete = reconciler.reconcile(client, conn, True, 5, None, None) + assert old_key not in repeat_delete["deleted"] + + restored = reconciler.reconcile(client, conn, True, 3600, missing_id, fixture) + assert restored["restore"] == "restored" and not reconciler.absent(client, missing_key) + repeat = reconciler.reconcile(client, conn, True, 3600, missing_id, fixture) + assert repeat["restore"] == "already_present" and not repeat["deleted"] + assert str(missing_id) not in repeat["manual_incidents"] + assert not reconciler.absent(client, fresh_key) and not reconciler.absent(client, active_key) + post_restore_preview = reconciler.reconcile(client, conn, False, 5, None, None) + assert old_key not in post_restore_preview["old_orphans"] + assert str(missing_id) not in post_restore_preview["manual_incidents"] + conn.commit() + + tiny = io.BytesIO() + Image.new("RGB", (4, 4), (5, 30, 80)).save(tiny, format="PNG") + data = tiny.getvalue() + tiny_hash = hashlib.sha256(data).hexdigest() + + def add_batch(count: int) -> None: + rows = [] + for _ in range(count): + attachment_id = uuid4() + key = f"originals/{attachment_id}" + put(client, key, data) + rows.append((attachment_id, chat_id, owner_id, "inventory.png", key, "image/png", len(data), tiny_hash)) + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO attachments (id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", rows, + ) + conn.commit() + + measurements = [] + for label, additional in (("plus_50", 50), ("plus_250", 200)): + add_batch(additional) + measurements.extend(scan(client, conn, label, trial) for trial in range(1, 4)) + assert measurements[3]["objects"] >= measurements[0]["objects"] + 200 + assert measurements[3]["metadata"] >= measurements[0]["metadata"] + 200 + + response = client.get_object(reconciler.BUCKET, missing_key) + try: + assert hashlib.sha256(response.read()).hexdigest() == digest + finally: + response.close() + response.release_conn() + + summary = {label: { + "median_wall_ms": round(statistics.median(item["wall_ms"] for item in measurements if item["label"] == label), 3), + "median_cpu_ms": round(statistics.median(item["cpu_user_ms"] + item["cpu_system_ms"] for item in measurements if item["label"] == label), 3), + "objects": next(item["objects"] for item in measurements if item["label"] == label), + "metadata": next(item["metadata"] for item in measurements if item["label"] == label), + } for label in ("plus_50", "plus_250")} + print(json.dumps({ + "faults": { + "preview": {"old_detected": old_key in preview["old_orphans"], "fresh_detected": fresh_key in preview["fresh_orphans"], + "missing_detected": str(missing_id) in preview["missing_attachment_ids"], + "linked_preserved": active_key not in preview["old_orphans"], "inventory": preview["inventory"]}, + "wrong_fixture_rejected_without_delete": True, + "apply_without_source": {"old_deleted": old_key in apply_without_source["deleted"], + "missing_incident_retained": str(missing_id) in apply_without_source["manual_incidents"], + "fresh_preserved": True, "linked_preserved": True}, + "post_delete_dry_run": {"old_absent": old_key not in post_delete_preview["old_orphans"], + "missing_incident_retained": str(missing_id) in post_delete_preview["manual_incidents"]}, + "repeat_delete_no_change": old_key not in repeat_delete["deleted"], + "restore": restored["restore"], "repeat": repeat["restore"], + "post_restore_dry_run": {"old_absent": old_key not in post_restore_preview["old_orphans"], + "missing_absent": str(missing_id) not in post_restore_preview["manual_incidents"]}, + }, "measurements": measurements, "summary": summary, + "limits": "Single local host, 3 sequential dry-run scans per size; CPU/RSS are maintenance-process measurements only." + }, sort_keys=True)) + + +if __name__ == "__main__": + main() -- 2.54.0 From 255c11a8228ed07feef6474184a2d2efd5a76e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:23:33 +0300 Subject: [PATCH 09/10] Document grade-five reconciliation experiments --- REPORT.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/REPORT.md b/REPORT.md index c523f453..5b1ef80b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Самохвалов Андрей Ремович, БПИ-2302. -- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Уровни 3 и 4 проверены; повторяемый эксперимент трека 5 выполняется далее. +- Целевая оценка: 5, трек 2 «Согласованность двух хранилищ». Уровни 3 и 4 и повторяемый эксперимент трека 5 проверены. - Ветка: `solution/bpi2302-samohvalov`; commit/PR указываются после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/README.md`, `contracts/openapi.json`, публичные тесты, [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z), [образ](https://hub.docker.com/r/alpine/minio/tags), документация MinIO для IAM. Python/FastAPI, PostgreSQL, MinIO Python SDK; проект подготовлен с помощью ИИ Codex с локальной проверкой ниже. @@ -60,13 +60,16 @@ POST сообщения принимает до 10 уникальных ID из | Сохранность S3 и отказ API | `make test-persistence ... ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ... ACTION_SCRIPT=scripts/stop-one.sh`; `python3 student-tests/test_attachment_failover.py` | Токен, Message, metadata и bytes пережили restart API/DB/S3; публичный failover восстановил запись за 2,826 и 3,020 с; адресный тест остановил каждую загрузившую реплику и скачал те же bytes через другую | `evidence/attachment-access-2026-10-02.txt` | | Сверка orphan/missing | `docker compose --profile maintenance run --rm reconcile`; `student-tests/test_reconcile.py` в maintenance-контейнере | 2/2 PASS: old orphan после PG rollback удалён по истечении grace; fresh и привязанный сохранены; missing дал 503, неверный fixture не изменил данные, верный восстановил bytes/200; повтор идемпотентен | `evidence/reconcile-u5-2026-10-02.txt`, три JSON результата | | Права S3 | `student-tests/test_storage_permissions.py` в API и maintenance-контейнерах | 1/1 PASS для каждой роли; API не может list, обе не могут создавать bucket, анонимный GET 403, консоль не опубликована | `evidence/reconcile-u5-2026-10-02.txt` | +| Трек 5: два отказа и стоимость сверки | `sh scripts/fault-injection.sh`; повторные dry-run/apply; 3 замера на каждом размере inventory | Old orphan удалён, missing восстановлен только по проверенному fixture; fresh/linked сохранены; повторы без изменений. Для 333/334 и 533/534 строк/объектов медианы wall 21,143 и 24,219 мс, CPU 7,420 и 9,993 мс | `evidence/reconciliation/analysis-2026-10-02.md`, `run-20261002T111817Z.json` | ## Эксперименты и отказы -Граница PUT/COMMIT проверена fault-инъекциями; U5 сверяет реальные расхождения, а U6 отдельно измерит повторяемость и стоимость. Для теста удаления grace уменьшался до 5 секунд только на синтетическом наборе; обычный запуск использует 3600 секунд. Список объектов MinIO содержит миллисекунды, а HEAD округляет время до секунд: перед удалением сверяются секунды, ETag и размер, затем возраст и ссылка в БД. После канонического dry-run 28 metadata, 29 objects, 1 свежий orphan был защищён и ни один объект не удалён. +Граница PUT/COMMIT проверена fault-инъекциями. Повторяемый сценарий трека 5 создал orphan после успешного S3 PUT и отклонённой PostgreSQL-транзакции, а также missing после COMMIT метаданных и удаления S3-объекта. Dry-run не изменил данные. Apply удалил только старый orphan; свежий и связанный с сообщением объекты остались. Missing без fixture остался ручным инцидентом, неверный fixture был отвергнут до мутаций, проверенный по размеру и SHA-256 fixture восстановил исходные байты. Повторные dry-run/apply не сделали новых изменений. Для теста удаления grace уменьшался до 5 секунд только на синтетическом наборе; обычный запуск использует 3600 секунд. Список объектов MinIO содержит миллисекунды, а HEAD округляет время до секунд: перед удалением сверяются секунды, ETag и размер, затем возраст и ссылка в БД. После раннего канонического dry-run 28 metadata, 29 objects, 1 свежий orphan был защищён и ни один объект не удалён. + +Для двух размеров inventory сценарий выполнил по три последовательных dry-run в одном maintenance-процессе: 333 metadata/334 objects — wall 19,390/21,143/23,322 мс, CPU 6,447/7,420/8,516 мс; 533/534 — wall 16,585/24,219/44,148 мс, CPU 9,993/9,997/9,500 мс. RSS процесса между сериями изменился с 68 152 до 68 936 KiB, high-water 126 608 KiB был достигнут до этих сканов. Больший inventory потребовал больше CPU в этой серии; wall сильно колебался. Шесть замеров на одном локальном стенде не определяют закон масштабирования; ресурсы PostgreSQL и S3 в них не включены. Полные сырые результаты и способ воспроизведения приведены в `evidence/reconciliation/`. Замер RAM проводился на локальном Colima/arm64, 2 CPU и 2 GiB, API ×2 с 0,5 CPU/256 MiB на экземпляр, 10 потоков с барьером старта. CSV содержит baseline и два Docker stats снимка во время 6,145 с серии: peak означает максимум **наблюдённых** снимков, а не точный аппаратный high-water mark. Все 10 ответов 201 имели уникальные ID и совпадающий SHA-256. Создание body по `os.urandom` оставляет воспроизводимыми размер/разрешение/алгоритм, но не конкретные байты и hash между прогонами. ## Ограничения и следующие шаги -Публичный `make test` проходит; он не доказывает все отказные сценарии. Уровень 4 подтверждён отдельными проверками. Трек 5 ещё не заявляется завершённым до повторяемой серии и измерения стоимости. Missing без проверенного источника остаётся ручным инцидентом, автоматической реконструкции чужих байтов нет. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. +Публичный `make test` проходит; отказные сценарии подтверждены отдельными проверками. Missing без проверенного источника остаётся ручным инцидентом, автоматической реконструкции чужих байтов нет. Один MinIO/PostgreSQL/LB и один Docker-host остаются одиночными точками отказа. Локальный HTTP допустим только для изолированного стенда с синтетическими данными; удалённой демонстрации нужен TLS. -- 2.54.0 From f2e074d9f2c1f7d9023cdd5f8e33a78b34def881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sat, 7 Nov 2026 09:53:49 +0300 Subject: [PATCH 10/10] fix: preserve NUL characters in persisted messages Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format. (cherry picked from commit 386276532730b8758298080b49db63738ca300a4) --- migrations/009_message_text_utf8.sql | 6 + src/store.py | 4 +- student-tests/test_database.py | 2 +- student-tests/test_idempotency.py | 2 +- student-tests/test_message_unicode.py | 156 +++++++++++++++++++++++ student-tests/test_reconcile.py | 2 +- student-tests/test_reconcile_research.py | 2 +- 7 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 migrations/009_message_text_utf8.sql create mode 100644 student-tests/test_message_unicode.py diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 00000000..86196ee6 --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/src/store.py b/src/store.py index 298109a7..0ddb32d8 100644 --- a/src/store.py +++ b/src/store.py @@ -78,7 +78,7 @@ def chat_view(row: dict) -> dict: def message_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), "text": row["text"], + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), "created_at": timestamp(row["created_at"]), "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], } @@ -259,7 +259,7 @@ class Store: row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", - (uuid4(), chat_uuid, actor_uuid, text), + (uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")), ).fetchone() for position, attachment_id in enumerate(ids): conn.execute( diff --git a/student-tests/test_database.py b/student-tests/test_database.py index ede3b896..86ebc60b 100644 --- a/student-tests/test_database.py +++ b/student-tests/test_database.py @@ -31,7 +31,7 @@ class DatabaseTests(unittest.TestCase): def test_migration_and_unique_username_race(self): with psycopg.connect(URL) as conn: versions = conn.execute("SELECT version FROM schema_migrations").fetchall() - self.assertEqual(versions, [(1,), (2,), (3,), (4,), (5,)]) + self.assertEqual(versions, [(1,), (2,), (3,), (4,), (5,), (9,)]) name = "u_" + uuid4().hex[:16] def register(_): diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py index 3ca7a838..e1c5c3bf 100644 --- a/student-tests/test_idempotency.py +++ b/student-tests/test_idempotency.py @@ -38,7 +38,7 @@ class IdempotencyTests(unittest.TestCase): with psycopg.connect(URL) as conn: count = conn.execute( "SELECT count(*) FROM messages WHERE chat_id=%s AND sender_id=%s AND text=%s", - (chat["id"], alice["id"], "same body"), + (chat["id"], alice["id"], b"same body"), ).fetchone()[0] self.assertEqual(count, 1) diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 00000000..4ff79d81 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py index ad5e6b38..94758a8e 100644 --- a/student-tests/test_reconcile.py +++ b/student-tests/test_reconcile.py @@ -76,7 +76,7 @@ class ReconcileTests(unittest.TestCase): message_id = uuid4() conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", - (message_id, UUID(chat["id"]), UUID(owner["id"]), ""), + (message_id, UUID(chat["id"]), UUID(owner["id"]), b""), ) conn.execute( "INSERT INTO message_attachments (message_id,chat_id,attachment_id,position) VALUES (%s,%s,%s,0)", diff --git a/student-tests/test_reconcile_research.py b/student-tests/test_reconcile_research.py index 2da32086..4be9e1c5 100644 --- a/student-tests/test_reconcile_research.py +++ b/student-tests/test_reconcile_research.py @@ -89,7 +89,7 @@ def main() -> None: ) message_id = uuid4() conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", - (message_id, chat_id, owner_id, "")) + (message_id, chat_id, owner_id, b"")) conn.execute("INSERT INTO message_attachments (message_id,chat_id,attachment_id,position) VALUES (%s,%s,%s,0)", (message_id, chat_id, active_id)) -- 2.54.0