Files
mtusi2026lab6/src/main.py
T
2026-11-21 00:02:11 +03:00

812 lines
43 KiB
Python

"""Lab 6 REST API: lab 5 business API (FastAPI + PostgreSQL + Redis + S3 + Kafka outbox) + OpenTelemetry (traces, metrics, logs)."""
from __future__ import annotations
import asyncio
import anyio
import base64
import hashlib
import json
import logging
import os
import re
import secrets
import socket
import tempfile
import time
import unicodedata
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from urllib.parse import quote
import psycopg
from psycopg_pool import ConnectionPool, PoolTimeout
import anyio
import boto3
import redis
from botocore.config import Config as BotoConfig
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image
from redis.exceptions import RedisError
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
from starlette.requests import ClientDisconnect
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from starlette.exceptions import HTTPException as StarletteHTTPException
from opentelemetry import metrics, trace
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from . import jobs, telemetry
# Lab 6: JSON logs on stdout + OTLP (traces, metrics, logs) to the Collector. psycopg/botocore spans are automatic.
telemetry.init("messenger-api", instrument=("psycopg", "botocore"))
log = logging.getLogger("messenger")
DATABASE_URL = os.getenv("DATABASE_URL", "")
INSTANCE_ID = os.getenv("INSTANCE_ID") or socket.gethostname()
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if SESSION_TTL_SECONDS < 1:
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
# Redis uses its connection pool; session keys contain only a hash of the opaque token.
session_store = redis.Redis.from_url(REDIS_URL, decode_responses=True, socket_connect_timeout=3, socket_timeout=3, health_check_interval=30)
DB_POOL_MIN_SIZE = int(os.getenv("DB_POOL_MIN_SIZE", "1"))
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "10"))
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
if DB_POOL_MIN_SIZE < 1 or DB_POOL_MAX_SIZE < DB_POOL_MIN_SIZE:
raise RuntimeError("Invalid PostgreSQL pool size configuration")
# Every pooled connection gets bounded connect/query/idle-transaction timeouts.
db_pool = ConnectionPool(
conninfo=DATABASE_URL,
min_size=DB_POOL_MIN_SIZE,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"connect_timeout": 5,
"options": "-c statement_timeout=5000 -c idle_in_transaction_session_timeout=10000",
},
open=False,
name="messenger-postgres",
)
# ---- Lab 4: S3 / attachment limits (names follow .env.example of the lab) ----
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "")
S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "")
S3_KEY_PREFIX = "attachments/"
STAGING_PREFIX = jobs.STAGING_PREFIX # lab5: durable staging (private S3 prefix) for bytes accepted but not yet processed
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
UPLOAD_IDLE_TIMEOUT_SECONDS = float(os.getenv("UPLOAD_IDLE_TIMEOUT_SECONDS", "10"))
UPLOAD_TOTAL_TIMEOUT_SECONDS = float(os.getenv("UPLOAD_TOTAL_TIMEOUT_SECONDS", "60"))
IMAGE_DECODE_CONCURRENCY = int(os.getenv("IMAGE_DECODE_CONCURRENCY", "1"))
SPOOL_MEMORY_BYTES = 1024 * 1024 # larger bodies roll over to an anonymous temp file, never to RAM
CHUNK_BYTES = 64 * 1024
# Pillow's own bomb guard is a second line of defence behind the explicit check below.
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
# Credentials are the restricted application user created by s3_init, never the MinIO root user.
_s3_common = dict(endpoint_url=S3_ENDPOINT, region_name=S3_REGION, aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY)
s3 = boto3.client("s3", **_s3_common, config=BotoConfig(
signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=20,
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=32,
request_checksum_calculation="when_required", response_checksum_validation="when_required"))
# Readiness probe client: one attempt and tight timeouts so HAProxy's 2 s check never hangs on it.
s3_probe = boto3.client("s3", **_s3_common, config=BotoConfig(
signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=1, read_timeout=1,
retries={"max_attempts": 1}, request_checksum_calculation="when_required", response_checksum_validation="when_required"))
decode_limiter = anyio.CapacityLimiter(IMAGE_DECODE_CONCURRENCY)
IMAGE_TYPES = {"PNG": "image/png", "JPEG": "image/jpeg"}
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
PASSWORDS = PasswordHasher()
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: dict[str, str] | None = None):
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def fail(status: int, code: str, message: str, headers: dict[str, str] | None = None):
raise ApiError(status, code, message, headers)
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
class RegisterBody(StrictModel):
username: str
display_name: str
password: str
class ChatBody(StrictModel):
title: str
member_ids: list[str] = Field(min_length=1, max_length=100)
class MessageBody(StrictModel):
text: str
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
# ---- Lab 6: HTTP metrics (names/units: REPORT.md). Labels are LOW cardinality: method, route TEMPLATE, status class. ----
HTTP_DURATION = telemetry.meter().create_histogram(
"http.server.request.duration", unit="s", description="Server-side duration of business HTTP requests (health excluded)",
explicit_bucket_boundaries_advisory=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60])
KNOWN_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}
def observe_http(request: Request, status: int, started: float) -> None:
if request.url.path.startswith("/health/"):
return # error ratio denominator = business requests, health probes are excluded
route = getattr(request.scope.get("route"), "path", None) or "unmatched" # "/api/v1/chats/{chat_id}/messages", never a real UUID
HTTP_DURATION.record(time.perf_counter() - started, {
"http.request.method": request.method if request.method in KNOWN_METHODS else "OTHER",
"http.route": route, "http.response.status_class": f"{status // 100}xx"})
def _pool_gauge(field: str):
def callback(_options):
stats = db_pool.get_stats()
value = stats["pool_size"] - stats["pool_available"] if field == "used" else stats[field]
yield metrics.Observation(value, {"pool.name": "messenger-postgres"})
return callback
for _name, _field, _desc in (("messenger.db.pool.used", "used", "Connections checked out of the pool"),
("messenger.db.pool.waiting", "requests_waiting", "Requests waiting for a free pooled connection")):
telemetry.meter().create_observable_gauge(_name, callbacks=[_pool_gauge(_field)], unit="{connection}", description=_desc)
telemetry.register_process_metrics()
@app.middleware("http")
async def request_id_middleware(request: Request, call_next):
request.state.request_id = uuid.uuid4().hex
started = time.perf_counter()
# X-Request-Id <-> trace: the id is stored on the server span; logs carry both request_id and trace_id.
trace.get_current_span().set_attribute("messenger.request_id", request.state.request_id)
try:
response = await call_next(request)
except Exception as exc:
telemetry.log_event(log, logging.ERROR, "unhandled_exception", "Unhandled API exception", request_id=request.state.request_id, reason=type(exc).__name__)
response = JSONResponse(status_code=500, content={"error": {"code": "internal_error", "message": "Internal server error", "request_id": request.state.request_id}})
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
observe_http(request, response.status_code, started)
return response
# Added LAST => outermost middleware: the server span is active inside request_id_middleware. No metrics from the instrumentor:
# our own histogram above is the only HTTP metric (no duplicate instruments). Health probes produce no spans.
FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/(live|ready)", meter_provider=metrics.NoOpMeterProvider())
def error_response(status: int, code: str, message: str, request_id: str, headers: dict[str, str] | None = None):
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message, "request_id": request_id}}, headers=headers or {})
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return error_response(exc.status, exc.code, exc.message, request.state.request_id, exc.headers)
@app.exception_handler(PoolTimeout)
async def database_pool_timeout_handler(request: Request, exc: PoolTimeout):
telemetry.log_event(log, logging.WARNING, "db_pool_timeout", "Database connection pool timed out", request_id=request.state.request_id)
return error_response(503, "unavailable", "Required storage is unavailable", request.state.request_id)
@app.exception_handler(psycopg.Error)
async def database_error_handler(request: Request, exc: psycopg.Error):
telemetry.log_event(log, logging.WARNING, "db_error", "Database operation failed", request_id=request.state.request_id, reason=type(exc).__name__)
return error_response(503, "unavailable", "Required storage is unavailable", request.state.request_id)
@app.exception_handler(RedisError)
async def redis_error_handler(request: Request, exc: RedisError):
telemetry.log_event(log, logging.WARNING, "session_store_error", "Session store operation failed", request_id=request.state.request_id)
return error_response(503, "unavailable", "Required storage is unavailable", request.state.request_id)
@app.exception_handler(BotoCoreError)
async def s3_core_error_handler(request: Request, exc: BotoCoreError):
telemetry.log_event(log, logging.WARNING, "s3_error", "Object storage failure", request_id=request.state.request_id, reason=type(exc).__name__)
return error_response(503, "unavailable", "Required storage is unavailable", request.state.request_id)
@app.exception_handler(ClientError)
async def s3_client_error_handler(request: Request, exc: ClientError):
telemetry.log_event(log, logging.WARNING, "s3_error", "Object storage error", request_id=request.state.request_id, reason=str(exc.response.get("Error", {}).get("Code")))
return error_response(503, "unavailable", "Required storage is unavailable", request.state.request_id)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError):
return error_response(400, "invalid_request", "Invalid request", request.state.request_id)
@app.exception_handler(StarletteHTTPException)
async def http_error_handler(request: Request, exc: StarletteHTTPException):
status = exc.status_code
code = "not_found" if status == 404 else "invalid_request"
message = "Object not found" if status == 404 else "Invalid request"
headers = dict(exc.headers or {})
return error_response(status, code, message, request.state.request_id, headers)
def db():
"""Borrow a bounded pooled connection; context exit commits/rolls back and returns it."""
return db_pool.connection(timeout=DB_POOL_TIMEOUT_SECONDS)
@app.on_event("startup")
def startup():
# Migrations run once in the dedicated `migrate` service before API replicas.
db_pool.open(wait=False)
last_error = None
for _ in range(30):
try:
with db() as conn:
conn.execute("SELECT 1")
conn.execute("SELECT 1 FROM idempotency_records LIMIT 0")
conn.execute("SELECT 1 FROM outbox LIMIT 0")
session_store.ping()
s3.head_object(Bucket=S3_BUCKET, Key=S3_KEY_PREFIX + ".ready")
telemetry.log_event(log, logging.INFO, "api_ready", f"PostgreSQL, Redis and S3 bucket are ready; instance={INSTANCE_ID}")
return
except Exception as exc:
last_error = exc
time.sleep(1)
raise RuntimeError("PostgreSQL schema, Redis or S3 bucket unavailable during startup") from last_error
@app.on_event("shutdown")
def shutdown():
session_store.close()
db_pool.close()
def now_utc() -> datetime:
return datetime.now(timezone.utc)
def iso(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def user_dict(row: dict[str, Any]) -> dict[str, Any]:
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": iso(row["created_at"])}
def chat_dict(row: dict[str, Any], member_ids: list[str]) -> dict[str, Any]:
return {"id": str(row["id"]), "title": row["title"], "member_ids": member_ids, "created_at": iso(row["created_at"])}
def message_dict(row: dict[str, Any], attachment_ids: list[str] | None = None) -> dict[str, Any]:
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": iso(row["created_at"]), "attachment_ids": attachment_ids or []}
def validate_unicode(value: str, label: str) -> str:
"""Reject unpaired UTF-16 surrogates before sending text to PostgreSQL/KDF."""
try:
value.encode("utf-8")
except UnicodeEncodeError:
fail(400, "invalid_request", "Invalid Unicode in " + label)
return value
def parse_uuid(value: str, label: str = "UUID") -> uuid.UUID:
try:
# Keep the canonical UUID representation used by the Lab 1 API.
if not isinstance(value, str) or len(value) != 36:
raise ValueError("non-canonical UUID")
parsed = uuid.UUID(value)
if str(parsed) != value:
raise ValueError("non-canonical UUID")
return parsed
except (ValueError, TypeError, AttributeError):
fail(400, "invalid_request", "Invalid " + label)
def validate_body_type(request: Request):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
fail(415, "unsupported_media_type", "Content-Type must be application/json")
async def parse_body(request: Request, model):
validate_body_type(request)
try:
raw = await request.body()
data = json.loads(raw.decode("utf-8"))
return model.model_validate(data)
except (UnicodeDecodeError, json.JSONDecodeError, ValidationError, TypeError):
fail(400, "invalid_request", "Invalid JSON request body")
def challenge(scheme: str) -> dict[str, str]:
if scheme == "Basic":
return {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
return {"WWW-Authenticate": 'Bearer realm="messenger"'}
def current_user(request: Request) -> dict[str, Any]:
auth = request.headers.get("authorization", "")
if not auth.startswith("Bearer ") or not auth[7:].strip():
fail(401, "unauthorized", "Valid Bearer session required", challenge("Bearer"))
token_hash = hashlib.sha256(auth[7:].strip().encode("utf-8")).hexdigest()
session_json = session_store.get("session:" + token_hash)
if session_json is None:
fail(401, "unauthorized", "Valid Bearer session required", challenge("Bearer"))
try:
session = json.loads(session_json)
user_id = uuid.UUID(session["user_id"])
expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00"))
except (ValueError, KeyError, TypeError, json.JSONDecodeError):
# Corrupt session data is invalidated instead of trusted.
session_store.delete("session:" + token_hash)
fail(401, "unauthorized", "Valid Bearer session required", challenge("Bearer"))
if expires_at <= now_utc():
session_store.delete("session:" + token_hash)
fail(401, "unauthorized", "Valid Bearer session required", challenge("Bearer"))
with db() as conn:
row = conn.execute("SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_id,)).fetchone()
if row is None:
session_store.delete("session:" + token_hash)
fail(401, "unauthorized", "Valid Bearer session required", challenge("Bearer"))
return {"id": row[0], "username": row[1], "display_name": row[2], "created_at": row[3], "expires_at": expires_at, "user": {"id": str(row[0]), "username": row[1], "display_name": row[2], "created_at": iso(row[3])}, "token_hash": token_hash}
def ensure_member(actor_id: uuid.UUID, chat_id: uuid.UUID):
with db() as conn:
row = conn.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (chat_id, actor_id)).fetchone()
if row is None:
fail(404, "not_found", "Object not found")
def parse_page_args(limit: int | None, cursor: str | None, resource: str):
actual_limit = 50 if limit is None else limit
if isinstance(actual_limit, bool) or actual_limit < 1 or actual_limit > 100:
fail(400, "invalid_request", "limit must be an integer from 1 to 100")
after = None
if cursor is not None:
if not cursor:
fail(400, "invalid_request", "Invalid cursor")
try:
decoded = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("utf-8")
obj = json.loads(decoded)
if obj.get("resource") != resource:
fail(400, "invalid_request", "Cursor belongs to another resource")
after = (datetime.fromisoformat(obj["created_at"].replace("Z", "+00:00")), uuid.UUID(obj["id"]))
except ApiError:
raise
except Exception:
fail(400, "invalid_request", "Invalid cursor")
return actual_limit, after
def make_cursor(resource: str, created_at: datetime, item_id: uuid.UUID) -> str:
raw = json.dumps({"resource": resource, "created_at": iso(created_at), "id": str(item_id)}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
def json_rows(cursor, query: str, params=()):
with db() as conn:
cur = conn.execute(query, params)
names = [d.name for d in cur.description]
return [dict(zip(names, row)) for row in cur.fetchall()]
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready():
try:
with db() as conn:
conn.execute("SELECT 1")
session_store.ping()
s3_probe.head_object(Bucket=S3_BUCKET, Key=S3_KEY_PREFIX + ".ready") # S3 is critical: uploads/downloads are part of the mandatory path
return {"status": "ok"}
except (psycopg.Error, PoolTimeout, RedisError, BotoCoreError, ClientError):
fail(503, "unavailable", "Required storage is unavailable")
@app.post("/api/v1/users", status_code=201)
async def register(request: Request):
body = await parse_body(request, RegisterBody)
if not USERNAME_RE.fullmatch(body.username):
fail(400, "invalid_request", "Invalid username")
validate_unicode(body.display_name, "display_name")
validate_unicode(body.password, "password")
if not body.display_name.strip() or len(body.display_name) > 100:
fail(400, "invalid_request", "Invalid display_name")
if not 12 <= len(body.password) <= 128:
fail(400, "invalid_request", "Password must contain 12 to 128 Unicode code points")
user_id = uuid.uuid4()
password_hash = PASSWORDS.hash(body.password)
try:
with db() as conn:
row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at", (user_id, body.username, body.display_name, password_hash)).fetchone()
except psycopg.errors.UniqueViolation:
fail(409, "conflict", "Username already exists")
return user_dict(dict(zip(("id", "username", "display_name", "created_at"), row)))
@app.get("/api/v1/users/me")
def me(request: Request):
return current_user(request)["user"]
@app.post("/api/v1/auth/sessions", status_code=201)
async def create_session(request: Request):
if await request.body():
fail(400, "invalid_request", "Session creation does not accept a request body")
auth = request.headers.get("authorization", "")
if not auth[:6].lower() == "basic ":
fail(401, "unauthorized", "Valid Basic credentials required", challenge("Basic"))
try:
decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except Exception:
fail(401, "unauthorized", "Invalid credentials", challenge("Basic"))
try:
validate_unicode(password, "password")
except ApiError:
fail(401, "unauthorized", "Invalid credentials", challenge("Basic"))
if not password or len(password) < 12 or len(password) > 128:
fail(401, "unauthorized", "Invalid credentials", challenge("Basic"))
with db() as conn:
row = conn.execute("SELECT id,username,display_name,password_hash,created_at FROM users WHERE username=%s", (username,)).fetchone()
if row is None:
fail(401, "unauthorized", "Invalid credentials", challenge("Basic"))
try:
PASSWORDS.verify(row[3], password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
fail(401, "unauthorized", "Invalid credentials", challenge("Basic"))
token = secrets.token_urlsafe(32)
token_hash = hashlib.sha256(token.encode()).hexdigest()
expires_at = now_utc() + timedelta(seconds=SESSION_TTL_SECONDS)
session_json = json.dumps({"user_id": str(row[0]), "expires_at": iso(expires_at)}, separators=(",", ":"))
# Redis TTL enforces expiry; AOF + named volume make active sessions survive stack restarts.
session_store.setex("session:" + token_hash, SESSION_TTL_SECONDS, session_json)
user = user_dict({"id": row[0], "username": row[1], "display_name": row[2], "created_at": row[4]})
return {"token": token, "token_type": "Bearer", "expires_at": iso(expires_at), "user": user}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def revoke_session(request: Request):
actor = current_user(request)
session_store.delete("session:" + actor["token_hash"])
return Response(status_code=204)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
actor = current_user(request)
body = await parse_body(request, ChatBody)
validate_unicode(body.title, "title")
if not body.title.strip() or len(body.title) > 100:
fail(400, "invalid_request", "Invalid title")
ids = [parse_uuid(value, "member UUID") for value in body.member_ids]
if len(ids) != len(set(ids)) or actor["id"] not in ids:
fail(400, "invalid_request", "Members must be unique and include the caller")
chat_id = uuid.uuid4()
try:
with db() as conn:
existing = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", (ids,)).fetchall()
if len(existing) != len(ids):
fail(400, "invalid_request", "Unknown chat member")
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at", (chat_id, body.title)).fetchone()
# psycopg 3: executemany exists on cursors, not on Connection (fix of a latent lab3 bug)
with conn.cursor() as cur:
cur.executemany("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", [(chat_id, uid, position) for position, uid in enumerate(ids)])
except psycopg.errors.ForeignKeyViolation:
fail(400, "invalid_request", "Unknown chat member")
return chat_dict({"id": row[0], "title": row[1], "created_at": row[2]}, [str(x) for x in ids])
@app.get("/api/v1/chats")
def list_chats(request: Request, limit: int | None = None, cursor: str | None = None):
actor = current_user(request)
chat_list_resource = "chats:" + str(actor["id"])
actual_limit, after = parse_page_args(limit, cursor, chat_list_resource)
params: list[Any] = [actor["id"]]
where = "m.user_id=%s"
if after:
where += " AND (c.created_at,c.id)>(%s,%s)"
params.extend(after)
params.append(actual_limit + 1)
rows = json_rows(None, f"SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE {where} ORDER BY c.created_at,c.id LIMIT %s", tuple(params))
more = len(rows) > actual_limit
rows = rows[:actual_limit]
items = []
for row in rows:
members = json_rows(None, "SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position,user_id", (row["id"],))
items.append(chat_dict(row, [str(m["user_id"]) for m in members]))
next_cursor = make_cursor(chat_list_resource, rows[-1]["created_at"], rows[-1]["id"]) if more and rows else None
return {"items": items, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
actor = current_user(request)
parsed = parse_uuid(chat_id, "chat UUID")
ensure_member(actor["id"], parsed)
rows = json_rows(None, "SELECT id,title,created_at FROM chats WHERE id=%s", (parsed,))
members = json_rows(None, "SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position,user_id", (parsed,))
return chat_dict(rows[0], [str(m["user_id"]) for m in members])
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
actor = current_user(request)
parsed_chat = parse_uuid(chat_id, "chat UUID")
ensure_member(actor["id"], parsed_chat)
body = await parse_body(request, MessageBody)
validate_unicode(body.text, "text")
attachment_ids = [parse_uuid(value, "attachment UUID") for value in body.attachment_ids]
if len(attachment_ids) != len(set(attachment_ids)):
fail(400, "invalid_request", "attachment_ids must be unique")
if len(body.text) > 4000 or (not body.text.strip() and not attachment_ids):
fail(400, "invalid_request", "Message needs 1 to 4000 non-whitespace code points of text or at least one attachment")
key = request.headers.get("Idempotency-Key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
fail(400, "invalid_request", "Idempotency-Key must contain 1 to 128 ASCII letters, digits, '.', '_', ':', or '-'")
# Hash semantic JSON rather than raw bytes, so insignificant JSON formatting does not matter.
# attachment_ids is part of the hash only when non-empty: absent == [] and lab3 records stay valid.
canonical = {"text": body.text}
if attachment_ids:
canonical["attachment_ids"] = [str(x) for x in attachment_ids]
canonical_body = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).hexdigest()
scope = f"{actor['id']}|POST|/api/v1/chats/{parsed_chat}/messages|{key}"
# The advisory transaction lock serializes the same key across every API replica.
# Message and idempotency result are committed atomically in the same PostgreSQL transaction.
with db() as conn:
conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,))
existing = conn.execute(
"SELECT request_hash, response_body FROM idempotency_records "
"WHERE user_id=%s AND chat_id=%s AND idempotency_key=%s AND created_at >= now() - interval '24 hours'",
(actor["id"], parsed_chat, key),
).fetchone()
if existing is not None:
if existing[0] != request_hash:
fail(409, "conflict", "Idempotency-Key was already used with a different request body")
replay = dict(existing[1])
replay.setdefault("attachment_ids", [])
return JSONResponse(status_code=201, content=replay)
# Keys are guaranteed for 24 hours; expired records may be replaced by a new attempt.
conn.execute(
"DELETE FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND idempotency_key=%s AND created_at < now() - interval '24 hours'",
(actor["id"], parsed_chat, key),
)
if attachment_ids:
# Foreign, unknown and other-chat attachments are indistinguishable: all -> 404.
found = conn.execute("SELECT count(*) FROM attachments WHERE id = ANY(%s) AND chat_id=%s", (attachment_ids, parsed_chat)).fetchone()[0]
if found != len(attachment_ids):
fail(404, "not_found", "Object not found")
# lab5: queued/processing attachments may be attached, failed ones may not
if conn.execute("SELECT 1 FROM attachments WHERE id = ANY(%s) AND status='failed' LIMIT 1", (attachment_ids,)).fetchone():
fail(409, "conflict", "Attachment processing failed")
row = conn.execute(
"INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) "
"RETURNING id,chat_id,sender_id,text,created_at",
(uuid.uuid4(), parsed_chat, actor["id"], body.text),
).fetchone()
result = message_dict(dict(zip(("id", "chat_id", "sender_id", "text", "created_at"), row)), [str(x) for x in attachment_ids])
if attachment_ids:
with conn.cursor() as cur:
cur.executemany("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", [(row[0], aid, i) for i, aid in enumerate(attachment_ids)])
conn.execute(
"INSERT INTO idempotency_records(user_id,chat_id,idempotency_key,request_hash,response_body) "
"VALUES (%s,%s,%s,%s,%s::jsonb)",
(actor["id"], parsed_chat, key, request_hash, json.dumps(result, ensure_ascii=False)),
)
return result
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request, limit: int | None = None, cursor: str | None = None):
actor = current_user(request)
parsed_chat = parse_uuid(chat_id, "chat UUID")
ensure_member(actor["id"], parsed_chat)
actual_limit, after = parse_page_args(limit, cursor, "messages:" + str(parsed_chat))
params: list[Any] = [parsed_chat]
where = "chat_id=%s"
if after:
where += " AND (created_at,id)>(%s,%s)"
params.extend(after)
params.append(actual_limit + 1)
rows = json_rows(None, f"SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE {where} ORDER BY created_at,id LIMIT %s", tuple(params))
more = len(rows) > actual_limit
rows = rows[:actual_limit]
links: dict[Any, list[str]] = {}
if rows:
link_rows = json_rows(None, "SELECT message_id, attachment_id FROM message_attachments WHERE message_id = ANY(%s) ORDER BY message_id, position", ([r["id"] for r in rows],))
for link in link_rows:
links.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
items = [message_dict(row, links.get(row["id"])) for row in rows]
next_cursor = make_cursor("messages:" + str(parsed_chat), rows[-1]["created_at"], rows[-1]["id"]) if more and rows else None
return {"items": items, "next_cursor": next_cursor}
# ======================= Lab 4: attachments =======================
def attachment_dict(row: dict[str, Any], variants: list[dict[str, Any]] | None = None) -> dict[str, Any]:
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
"variants": variants or [], "error_code": row["error_code"] if row["status"] == "failed" else None,
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"])}
def validate_filename(values: list[str]) -> str:
"""filename is display-only: it never reaches a path or an S3 key, but dangerous values are still rejected."""
if len(values) != 1:
fail(400, "invalid_request", "Exactly one filename query parameter is required")
name = values[0]
validate_unicode(name, "filename")
if not 1 <= len(name) <= 255 or not name.strip() or name in (".", ".."):
fail(400, "invalid_request", "Invalid filename")
if "/" in name or "\\" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
fail(400, "invalid_request", "Invalid filename")
return name
def sniff_image_type(head: bytes) -> str | None:
if head.startswith(b"\x89PNG\r\n\x1a\n"):
return "image/png"
if head.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
return None
async def read_body_to_spool(request: Request, declared: str):
"""Stream the request body into a bounded spool while hashing; abort as soon as a limit is broken.
RAM per upload stays ~1 MiB: bodies above SPOOL_MEMORY_BYTES roll over to an anonymous (already unlinked) temp file.
"""
spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES)
digest, total, head = hashlib.sha256(), 0, b""
try:
iterator = request.stream().__aiter__()
with anyio.fail_after(UPLOAD_TOTAL_TIMEOUT_SECONDS):
while True:
try:
chunk = await asyncio.wait_for(iterator.__anext__(), UPLOAD_IDLE_TIMEOUT_SECONDS)
except StopAsyncIteration:
break
if not chunk:
continue
total += len(chunk)
if total > MAX_UPLOAD_BYTES:
fail(413, "payload_too_large", "Attachment exceeds the size limit")
if len(head) < 8:
head += chunk[: 8 - len(head)]
if len(head) >= 8 and sniff_image_type(head) != declared:
fail(415, "unsupported_media_type", "Content does not match the declared image type")
digest.update(chunk)
spool.write(chunk)
except ApiError:
spool.close()
raise
except (TimeoutError, asyncio.TimeoutError):
spool.close()
fail(400, "invalid_request", "Upload timed out")
except BaseException: # ClientDisconnect, cancellation: nothing was written to S3 yet
spool.close()
raise
if total == 0 or sniff_image_type(head) != declared:
spool.close()
fail(415, "unsupported_media_type", "Content is not a supported image")
return spool, total, digest.hexdigest()
def decode_image_checked(spool, declared: str) -> None:
"""Runs in a worker thread: header -> pixel limit -> full decode. Raises ApiError for every user mistake."""
try:
spool.seek(0)
with Image.open(spool) as img:
if IMAGE_TYPES.get(img.format or "") != declared:
fail(415, "unsupported_media_type", "Content does not match the declared image type")
width, height = img.size
if width < 1 or height < 1:
fail(415, "unsupported_media_type", "Corrupt image")
if width * height > MAX_IMAGE_PIXELS: # decided from the header, before any pixel buffer exists
fail(413, "payload_too_large", "Image has too many pixels")
img.load() # full decode detects truncated/corrupt data
except ApiError:
raise
except Image.DecompressionBombError:
fail(413, "payload_too_large", "Image has too many pixels")
except MemoryError:
fail(503, "unavailable", "Temporarily unable to process the image")
except Exception: # Pillow raises OSError/SyntaxError/ValueError/struct.error/... for corrupt input
fail(415, "unsupported_media_type", "Corrupt or unsupported image")
finally:
spool.seek(0)
def s3_put(key: str, spool, size: int, content_type: str) -> None:
spool.seek(0)
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=spool, ContentLength=size, ContentType=content_type)
def s3_delete_quietly(key: str) -> None:
try:
s3.delete_object(Bucket=S3_BUCKET, Key=key)
except Exception:
# The periodic sweeper (scripts/s3_sweep.py) removes it after the grace period.
log.warning("Compensation delete failed; orphan left for sweeper key=%s", key)
async def compensate(key: str) -> None:
with anyio.CancelScope(shield=True):
await run_in_threadpool(s3_delete_quietly, key)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request):
actor = current_user(request)
parsed_chat = parse_uuid(chat_id, "chat UUID")
ensure_member(actor["id"], parsed_chat)
filename = validate_filename(request.query_params.getlist("filename"))
declared = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if declared not in ("image/png", "image/jpeg"):
fail(415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg")
length = request.headers.get("content-length")
if length is not None:
if not length.isascii() or not length.isdigit():
fail(400, "invalid_request", "Invalid Content-Length")
if int(length) > MAX_UPLOAD_BYTES: # reject before reading a single body byte
fail(413, "payload_too_large", "Attachment exceeds the size limit")
spool, size, sha256 = await read_body_to_spool(request, declared)
key = STAGING_PREFIX + secrets.token_hex(16) # server-chosen random STAGING key, unrelated to filename or attachment id
try:
await anyio.to_thread.run_sync(decode_image_checked, spool, declared, limiter=decode_limiter)
await run_in_threadpool(s3_put, key, spool, size, declared)
except ApiError: # user error found by validation, before any S3 write
spool.close()
raise
except BaseException:
spool.close()
await compensate(key) # a PUT whose reply was lost may still have stored the object
raise
spool.close()
attachment_id, job_id = uuid.uuid4(), uuid.uuid4()
final_original = jobs.final_key(attachment_id, jobs.PIPELINE_VERSION, "original")
try:
# ONE transaction: metadata + job + outbox row. 202 is returned only after this COMMIT (durable handoff).
with db() as conn:
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,s3_key,staging_key,status,pipeline_version) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) "
"RETURNING id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,error_code,created_at,updated_at",
(attachment_id, parsed_chat, actor["id"], filename, declared, size, sha256, final_original, key, jobs.PIPELINE_VERSION)).fetchone()
conn.execute("INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,status,max_attempts) VALUES (%s,%s,%s,'queued',%s)",
(job_id, attachment_id, jobs.PIPELINE_VERSION, jobs.JOB_MAX_ATTEMPTS))
jobs.add_event(conn, job_id, "accepted")
jobs.enqueue(conn, job_id, attachment_id, jobs.PIPELINE_VERSION, 1)
except BaseException:
await compensate(key) # staging PUT succeeded but the COMMIT did not: remove the unaccounted object
raise
telemetry.log_event(log, logging.INFO, "attachment_accepted", "Attachment accepted, job queued", job_id=str(job_id), attachment_id=str(attachment_id))
cols = ("id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "status", "error_code", "created_at", "updated_at")
return JSONResponse(status_code=202, content=attachment_dict(dict(zip(cols, row))))
ATT_COLS = ("id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "s3_key", "status", "error_code", "created_at", "updated_at")
def load_attachment(attachment_id: str, request: Request) -> dict[str, Any]:
actor = current_user(request)
parsed = parse_uuid(attachment_id, "attachment UUID")
rows = json_rows(None, "SELECT a.id,a.chat_id,a.owner_id,a.filename,a.content_type,a.size_bytes,a.sha256,a.s3_key,a.status,a.error_code,a.created_at,a.updated_at "
"FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s", (parsed, actor["id"]))
if not rows:
fail(404, "not_found", "Object not found")
return rows[0]
def variant_rows(attachment_id, pipeline_version: int = jobs.PIPELINE_VERSION) -> list[dict[str, Any]]:
rows = json_rows(None, "SELECT name,s3_key,content_type,size_bytes,width,height,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s", (attachment_id, pipeline_version))
return sorted(rows, key=lambda r: jobs.VARIANT_ORDER.index(r["name"]))
@app.get("/api/v1/attachments/{attachment_id}")
def get_attachment(attachment_id: str, request: Request):
row = load_attachment(attachment_id, request)
variants = []
if row["status"] == "ready": # variants are published only together with ready (one COMMIT in the worker)
variants = [{k: v[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")} for v in variant_rows(row["id"])]
return attachment_dict(row, variants)
@app.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(attachment_id: str, request: Request, variant: str = "original"):
row = load_attachment(attachment_id, request)
if variant != "original" and variant not in jobs.VARIANT_ORDER:
fail(400, "invalid_request", "Unknown variant")
if row["status"] != "ready":
fail(409, "conflict", "Attachment is not ready") # queued, processing and failed: no content
key, content_type, size = row["s3_key"], row["content_type"], row["size_bytes"]
if variant != "original":
found = [v for v in variant_rows(row["id"]) if v["name"] == variant]
if not found:
fail(404, "not_found", "Object not found")
key, content_type, size = found[0]["s3_key"], found[0]["content_type"], found[0]["size_bytes"]
try:
obj = s3.get_object(Bucket=S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
log.error("INTEGRITY: metadata without object attachment=%s variant=%s", row["id"], variant)
fail(500, "internal_error", "Internal server error")
raise
body = obj["Body"]
async def stream():
try:
async for chunk in iterate_in_threadpool(body.iter_chunks(CHUNK_BYTES)):
yield chunk
finally:
body.close()
headers = {"Content-Length": str(size), "X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store",
"Content-Disposition": "inline; filename*=UTF-8''" + quote(row["filename"], safe="")}
return StreamingResponse(stream(), media_type=content_type, headers=headers)
if __name__ == "__main__":
import uvicorn
uvicorn.run("main:app", host=os.getenv("APP_HOST", "0.0.0.0"), port=int(os.getenv("APP_PORT", "8080")), timeout_graceful_shutdown=5)