Files
mtusi2026lab5/src/main.py
T

664 lines
27 KiB
Python

import base64
import binascii
import hashlib
import json
import os
import logging
import re
import time
import tempfile
import threading
import unicodedata
from io import BytesIO
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from uuid import UUID, uuid4
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
from fastapi import FastAPI, Header, Query, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from psycopg import InterfaceError, OperationalError
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from redis.exceptions import RedisError
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from src.db import pool
from src.jobs import stage_upload, queue_attachment, variants_for, fetch_variant
from src.errors import ApiError, invalid, unauthorized
from src.pagination import encode_cursor, parse_page
from src.sessions import issue_session, redis_client
from src.storage import s3, bucket, check_storage, upload_original, download_original, delete_original, StorageFailure, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, UnidentifiedImageError
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
log = logging.getLogger("messenger")
password_hasher = PasswordHasher()
INSTANCE_ID = os.getenv("INSTANCE_ID", uuid4().hex)
ready = False
IMAGE_DECODE_SLOT = threading.BoundedSemaphore(1) # prevent concurrent decoded-image RAM spikes
def parse_uuid(value):
try:
return UUID(value)
except (ValueError, TypeError, AttributeError):
invalid("Некорректный UUID")
def row_response(row):
result = {}
for key, value in row.items():
if isinstance(value, UUID):
value = str(value)
elif isinstance(value, datetime):
value = value.astimezone(timezone.utc).isoformat(
timespec="microseconds"
).replace("+00:00", "Z")
elif key in ("member_ids", "attachment_ids"):
value = [str(member) for member in value]
result[key] = value
return result
class InputModel(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
@field_validator("*", check_fields=False)
@classmethod
def reject_blank(cls, value):
if isinstance(value, str) and not value.strip():
raise ValueError("Строка не должна быть пустой")
return value
@field_validator("display_name", "title", "text", check_fields=False)
@classmethod
def reject_nul(cls, value):
if "\u0000" in value:
raise ValueError("Нулевой символ недопустим в текстовом поле")
return value
class CreateUser(InputModel):
username: str = Field(min_length=3, max_length=32)
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def validate_username(cls, value):
if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None:
raise ValueError("Некорректный username")
return value
class CreateChat(InputModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def validate_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Нужен текст или вложение")
if "\x00" in self.text:
raise ValueError("Недопустимый текст")
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Повторные attachment_ids")
for value in self.attachment_ids:
try:
UUID(value)
except (ValueError, TypeError):
raise ValueError("Некорректный attachment_id") from None
return self
def bearer_token(value):
scheme, _, token = (value or "").partition(" ")
if scheme.lower() != "bearer" or not token:
unauthorized()
return token
def current_user(value):
token = bearer_token(value)
user_id = redis_client.get("session:" + token)
if user_id is None:
unauthorized()
with pool.connection() as conn:
user = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(UUID(user_id),),
).fetchone()
if user is None:
unauthorized()
return row_response(user)
@asynccontextmanager
async def lifespan(app):
global ready
try:
await run_in_threadpool(pool.open, wait=True, timeout=10)
await run_in_threadpool(redis_client.ping)
# Container may have started before the S3 service finished bootstrap.
storage_ok = False
for _ in range(30):
try:
await run_in_threadpool(check_storage)
storage_ok = True
break
except (BotoCoreError, ClientError, StorageFailure):
import asyncio
await asyncio.sleep(2)
if not storage_ok:
raise RuntimeError("S3 not ready")
ready = True
log.info("startup ready=true")
yield
finally:
ready = False
redis_client.close()
pool.close()
log.info("shutdown complete")
app = FastAPI(title="Lab 4 Messenger", lifespan=lifespan)
def error_response(request, status, code, message):
request_id = request.state.request_id
headers = {"X-Request-Id": request_id}
if status == 401:
if request.method == "POST" and request.url.path == "/api/v1/auth/sessions":
headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"'
else:
headers["WWW-Authenticate"] = 'Bearer realm="messenger"'
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": request_id}},
headers=headers,
)
@app.exception_handler(ApiError)
async def api_error(request, exc):
return error_response(request, exc.status, exc.code, exc.message)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
@app.exception_handler((StorageFailure))
async def storage_error(request, exc):
return error_response(request, 503, "unavailable", "Хранилище недоступно")
@app.exception_handler(ClientError)
async def boto_error(request, exc):
return error_response(request, 503, "unavailable", "Хранилище недоступно")
@app.exception_handler(StarletteHTTPException)
async def http_error(request, exc):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Некорректный запрос")
@app.middleware("http")
async def process_request(request: Request, call_next):
request.state.request_id = str(uuid4())
started = time.perf_counter()
try:
is_login = request.method == "POST" and request.url.path == "/api/v1/auth/sessions"
public = request.method == "POST" and request.url.path in (
"/api/v1/users", "/api/v1/auth/sessions"
)
if request.url.path.startswith("/api/v1/") and not public:
request.state.user = await run_in_threadpool(
current_user, request.headers.get("Authorization")
)
if is_login:
if await request.body():
invalid("Для входа тело запроса должно быть пустым")
elif request.method == "POST" and request.url.path.startswith("/api/v1/"):
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
if re.fullmatch(r"/api/v1/chats/[^/]+/attachments", request.url.path):
if media_type not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Только PNG/JPEG")
elif media_type != "application/json":
raise ApiError(415, "unsupported_media_type", "Нужен application/json")
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc.status, exc.code, exc.message)
except (OperationalError, InterfaceError, RedisError, StorageFailure, BotoCoreError, ClientError):
response = error_response(request, 503, "unavailable", "Хранилище недоступно")
except Exception as exc:
log.error("request_failed type=%s request_id=%s",
type(exc).__name__, request.state.request_id)
response = error_response(request, 500, "internal_error", "Внутренняя ошибка")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
route = request.scope.get("route")
log.info("request method=%s route=%s status=%s duration_ms=%.2f request_id=%s",
request.method, getattr(route, "path", "unmatched"), response.status_code,
(time.perf_counter() - started) * 1000, request.state.request_id)
return response
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def readiness():
if not ready:
raise ApiError(503, "unavailable", "Приложение не готово")
with pool.connection() as conn:
conn.execute("SELECT 1")
redis_client.ping()
check_storage()
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
def create_user(body: CreateUser):
password_hash = password_hasher.hash(body.password)
try:
with pool.connection() as conn:
user = conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) "
"VALUES (%s, %s, %s, %s) RETURNING id, username, display_name, created_at",
(uuid4(), body.username, body.display_name, password_hash),
).fetchone()
except UniqueViolation as exc:
if exc.diag.constraint_name == "users_username_key":
raise ApiError(409, "conflict", "Username занят") from None
raise
return row_response(user)
@app.post("/api/v1/auth/sessions", status_code=201)
def login(authorization: str | None = Header(default=None)):
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic" or not encoded:
unauthorized()
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, binascii.Error):
unauthorized()
if re.fullmatch(r"[a-z0-9_]{3,32}", username) is None or not 12 <= len(password) <= 128:
unauthorized()
with pool.connection() as conn:
user = conn.execute(
"SELECT id, username, display_name, password_hash, created_at "
"FROM users WHERE username = %s", (username,),
).fetchone()
if user is None:
unauthorized()
try:
password_hasher.verify(user["password_hash"], password)
except VerifyMismatchError:
unauthorized()
token, expires_at = issue_session(user["id"])
del user["password_hash"]
return {"token": token, "token_type": "Bearer", "expires_at": expires_at,
"user": row_response(user)}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request):
token = bearer_token(request.headers.get("Authorization"))
redis_client.delete("session:" + token)
return Response(status_code=204)
@app.get("/api/v1/users/me")
def get_me(request: Request):
return request.state.user
def require_member(conn, chat_id, user_id):
found = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, UUID(user_id)),
).fetchone()
if found is None:
raise ApiError(404, "not_found", "Чат не найден")
CHAT_SELECT = """
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members
WHERE chat_id = c.id ORDER BY user_id) AS member_ids
FROM chats c
JOIN chat_members mine ON mine.chat_id = c.id
WHERE mine.user_id = %s
"""
MESSAGE_SELECT = """
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
WHERE ma.message_id=m.id ORDER BY ma.position)::uuid[] AS attachment_ids
FROM messages m
"""
HISTORY_FIRST_SQL = MESSAGE_SELECT + """
WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s
"""
HISTORY_AFTER_SQL = MESSAGE_SELECT + """
WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s)
ORDER BY m.created_at, m.id LIMIT %s
"""
def page_response(rows, size, scope):
items = [row_response(row) for row in rows[:size]]
cursor = encode_cursor(scope, items[-1]) if len(rows) > size else None
return {"items": items, "next_cursor": cursor}
@app.post("/api/v1/chats", status_code=201)
def create_chat(body: CreateChat, request: Request):
members = sorted(parse_uuid(value) for value in body.member_ids)
if len(members) != len(set(members)):
invalid("Участники повторяются")
if UUID(request.state.user["id"]) not in members:
invalid("Вызывающий должен входить в member_ids")
try:
with pool.connection() as conn:
with conn.transaction():
chat = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) "
"RETURNING id, title, created_at", (uuid4(), body.title),
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat["id"], member) for member in members],
)
except ForeignKeyViolation:
invalid("Неизвестный участник")
chat["member_ids"] = members
return row_response(chat)
@app.get("/api/v1/chats")
def list_chats(request: Request, limit: str = Query(default="50"),
cursor: str | None = Query(default=None)):
user_id = request.state.user["id"]
scope = "chats:" + user_id
size, after = parse_page(limit, cursor, scope)
query = CHAT_SELECT
params = [UUID(user_id)]
if after is not None:
query += " AND (c.created_at, c.id) > (%s, %s)"
params.extend(after)
query += " ORDER BY c.created_at, c.id LIMIT %s"
params.append(size + 1)
with pool.connection() as conn:
rows = conn.execute(query, params).fetchall()
return page_response(rows, size, scope)
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
with pool.connection() as conn:
chat = conn.execute(
CHAT_SELECT + " AND c.id = %s",
(UUID(request.state.user["id"]), parse_uuid(chat_id)),
).fetchone()
if chat is None:
raise ApiError(404, "not_found", "Чат не найден")
return row_response(chat)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(
chat_id: str,
body: CreateMessage,
request: Request,
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"),
):
if idempotency_key is None or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) is None:
invalid("Некорректный Idempotency-Key")
chat_uuid = parse_uuid(chat_id)
user_id = UUID(request.state.user["id"])
digest = hashlib.sha256(
json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()
with pool.connection() as conn:
with conn.transaction():
require_member(conn, chat_uuid, str(user_id))
if body.attachment_ids:
attachment_uuids = [UUID(v) for v in body.attachment_ids]
found = conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)",
(attachment_uuids,),
).fetchall()
if len(found) != len(attachment_uuids) or any(row["chat_id"] != chat_uuid for row in found):
raise ApiError(404, "not_found", "Вложение не найдено")
if any(row["status"] == "failed" for row in found):
raise ApiError(409, "conflict", "Вложение не готово")
conn.execute(
"DELETE FROM message_requests WHERE sender_id = %s AND chat_id = %s "
"AND request_key = %s AND created_at < now() - interval '24 hours'",
(user_id, chat_uuid, idempotency_key),
)
created = conn.execute(
"INSERT INTO message_requests (sender_id, chat_id, request_key, request_hash) "
"VALUES (%s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING request_key",
(user_id, chat_uuid, idempotency_key, digest),
).fetchone()
if created is None:
existing = conn.execute(
"SELECT request_hash, message_id FROM message_requests "
"WHERE sender_id = %s AND chat_id = %s AND request_key = %s",
(user_id, chat_uuid, idempotency_key),
).fetchone()
if existing is None:
raise ApiError(503, "unavailable", "Повторите запрос")
if existing["request_hash"] != digest:
raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения")
message = conn.execute(
MESSAGE_SELECT + " WHERE m.id = %s",
(existing["message_id"],),
).fetchone()
else:
message = conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text) "
"VALUES (%s, %s, %s, %s) "
"RETURNING id, chat_id, sender_id, text, created_at",
(uuid4(), chat_uuid, user_id, body.text),
).fetchone()
for position, attach in enumerate(body.attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message["id"], UUID(attach), position),
)
conn.execute(
"UPDATE message_requests SET message_id = %s "
"WHERE sender_id = %s AND chat_id = %s AND request_key = %s",
(message["id"], user_id, chat_uuid, idempotency_key),
)
if "attachment_ids" not in message:
message["attachment_ids"] = [UUID(v) for v in body.attachment_ids]
return row_response(message)
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request, limit: str = Query(default="50"),
cursor: str | None = Query(default=None)):
chat_uuid = parse_uuid(chat_id)
scope = "messages:" + str(chat_uuid)
size, after = parse_page(limit, cursor, scope)
with pool.connection() as conn:
require_member(conn, chat_uuid, request.state.user["id"])
if after is None:
rows = conn.execute(HISTORY_FIRST_SQL, (chat_uuid, size + 1)).fetchall()
else:
rows = conn.execute(
HISTORY_AFTER_SQL, (chat_uuid, after[0], after[1], size + 1)
).fetchall()
return page_response(rows, size, scope)
# Lab 4: binary media are kept in private S3; PostgreSQL stores only metadata.
# Raw request bodies are streamed in bounded chunks to a temporary spool file.
def validate_image(stream, advertised_type):
stream.seek(0)
try:
with IMAGE_DECODE_SLOT, Image.open(stream) as probe:
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(probe.format)
width, height = probe.size
if width * height > MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
if actual != advertised_type:
raise ApiError(415, "unsupported_media_type", "Тип изображения не совпадает с содержимым")
probe.verify()
stream.seek(0)
# Verifying full decode rejects truncated/corrupt image streams.
with IMAGE_DECODE_SLOT, Image.open(stream) as image:
image.load()
except ApiError:
raise
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None
stream.seek(0)
def attachment_response(row):
result = row_response(row)
for key in ("object_key", "staging_key", "attempts", "lease_token", "lease_until", "next_attempt_at", "pipeline_version"):
result.pop(key, None)
result["variants"] = variants_for(row["id"]) if row["status"] == "ready" else []
return result
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def create_attachment(chat_id: str, request: Request, filename: str = Query(...)):
chat_uuid = parse_uuid(chat_id)
user_id = UUID(request.state.user["id"])
if not 1 <= len(filename) <= 255 or any(c in filename for c in "/\\\x00") or any(unicodedata.category(c) == "Cc" for c in filename):
invalid("Опасное имя файла")
content_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
length_header = request.headers.get("content-length")
if length_header and length_header.isdigit() and int(length_header) > MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
# ACL before touching storage.
await run_in_threadpool(check_membership, chat_uuid, user_id)
file = tempfile.SpooledTemporaryFile(max_size=256 * 1024)
digest = hashlib.sha256()
size = 0
try:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
digest.update(chunk)
file.write(chunk)
if size == 0:
raise ApiError(415, "unsupported_media_type", "Пустой файл")
await run_in_threadpool(validate_image, file, content_type)
attachment_id = uuid4()
object_key = f"staging/{attachment_id.hex}"
file.seek(0)
# One S3 PUT; no multipart uploads left by a client disconnect.
try:
await run_in_threadpool(stage_upload, file, object_key, size, content_type)
except (StorageFailure, BotoCoreError, ClientError):
raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None
try:
result = await run_in_threadpool(queue_attachment, attachment_id, chat_uuid, user_id, filename,
content_type, size, digest.hexdigest(), object_key)
if result is None:
raise ApiError(404, "not_found", "Чат не найден")
except Exception:
# Best effort compensation; orphan reconciler handles crash between S3 PUT and DB COMMIT.
try:
await run_in_threadpool(delete_original, object_key)
except Exception:
log.warning("upload compensation failed for %s", object_key)
raise
return attachment_response(result)
finally:
file.close()
def check_membership(chat_uuid, user_id):
with pool.connection() as conn:
require_member(conn, chat_uuid, str(user_id))
def save_attachment(attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key):
with pool.connection() as conn:
with conn.transaction():
require_member(conn, chat_uuid, str(user_id))
return conn.execute(
"INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *",
(attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key)
).fetchone()
def owned_attachment(attachment_id, user_id):
with pool.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN chat_members cm ON cm.chat_id = a.chat_id "
"WHERE a.id = %s AND cm.user_id = %s",
(attachment_id, UUID(user_id)),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return row
@app.get("/api/v1/attachments/{attachment_id}")
def attachment_metadata(attachment_id: str, request: Request):
return attachment_response(owned_attachment(parse_uuid(attachment_id), request.state.user["id"]))
@app.get("/api/v1/attachments/{attachment_id}/content")
def attachment_content(attachment_id: str, request: Request, variant: str = Query(default="original")):
row = owned_attachment(parse_uuid(attachment_id), request.state.user["id"])
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение не готово")
item = None
if variant != "original":
if variant not in ("thumbnail", "crop", "optimized"):
raise ApiError(404, "not_found", "Вариант не найден")
item = fetch_variant(row["id"], variant)
if item is None:
raise ApiError(404, "not_found", "Вариант не найден")
key = row["object_key"] if item is None else item["object_key"]
mime = row["content_type"] if item is None else item["content_type"]
size = row["size_bytes"] if item is None else item["size_bytes"]
try:
s3_body = download_original(key)
except (BotoCoreError, ClientError, StorageFailure):
raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None
def chunks():
try:
for data in s3_body.iter_chunks(chunk_size=64 * 1024):
if data:
yield data
finally:
s3_body.close()
return StreamingResponse(chunks(), media_type=mime,
headers={"Content-Length": str(size), "Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff"})