forked from mtusi2026/mtusi2026lab4
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
logging.getLogger("messenger").info("environment=%s", os.environ.get("APP_ENV", "lab"))
|
||||
port = int(os.environ.get("APP_PORT", "8080"))
|
||||
if not 1 <= port <= 65535:
|
||||
raise SystemExit("APP_PORT должен быть от 1 до 65535")
|
||||
uvicorn.run(
|
||||
"src.app:create_app",
|
||||
factory=True,
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0"),
|
||||
port=port,
|
||||
workers=1,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=2,
|
||||
)
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
from contextlib import asynccontextmanager
|
||||
import logging
|
||||
import re
|
||||
from uuid import UUID, uuid4
|
||||
from urllib.parse import quote
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from .attachments import Attachments
|
||||
from .objects import Objects
|
||||
from .errors import ApiError, invalid
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .security import basic_credentials, bearer_token
|
||||
from .storage import PostgresStore
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
SESSION_PATH = "/api/v1/auth/sessions"
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
raise invalid("Неверный UUID") from None
|
||||
|
||||
|
||||
def page_args(request):
|
||||
raw = request.query_params.get("limit", "50")
|
||||
normalized = raw.lstrip("0") or "0"
|
||||
if re.fullmatch(r"[0-9]+", raw) is None or len(normalized) > 3 or not 1 <= int(normalized) <= 100:
|
||||
raise invalid("Limit должен быть от 1 до 100")
|
||||
return int(normalized), request.query_params.get("cursor")
|
||||
|
||||
|
||||
def create_app():
|
||||
store = PostgresStore()
|
||||
objects = Objects()
|
||||
attachments = Attachments(store, objects)
|
||||
instance_id = uuid4().hex
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
await run_in_threadpool(store.open)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await run_in_threadpool(store.close)
|
||||
objects.close()
|
||||
|
||||
app = FastAPI(redirect_slashes=False, docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
|
||||
app.state.store = store
|
||||
|
||||
def error_response(request, error):
|
||||
request_id = request.state.request_id
|
||||
headers = dict(error.headers)
|
||||
if error.status == 401:
|
||||
headers["WWW-Authenticate"] = (
|
||||
'Basic realm="messenger", charset="UTF-8"' if request.url.path == SESSION_PATH
|
||||
else 'Bearer realm="messenger"'
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=error.status, headers=headers,
|
||||
content={"error": {"code": error.code, "message": error.message, "request_id": request_id}},
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
path = request.url.path
|
||||
registration = path == "/api/v1/users" and request.method == "POST"
|
||||
login = path == SESSION_PATH and request.method == "POST"
|
||||
if login:
|
||||
request.state.credentials = basic_credentials(request.headers.get("Authorization"))
|
||||
async for chunk in request.stream():
|
||||
if chunk:
|
||||
raise invalid("Вход не принимает тело запроса")
|
||||
elif path.startswith("/api/v1/") and not registration:
|
||||
token = bearer_token(request.headers.get("Authorization"))
|
||||
request.state.user_id = await run_in_threadpool(store.authenticate, token)
|
||||
request.state.token = token
|
||||
upload = re.fullmatch(r"/api/v1/chats/[^/]+/attachments", path) is not None
|
||||
if request.method == "POST" and path.startswith("/api/v1/") and not login and not upload:
|
||||
media_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
||||
if media_type != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Нужен application/json")
|
||||
try:
|
||||
(await request.body()).decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
raise invalid("JSON должен быть в UTF-8") from None
|
||||
response = await call_next(request)
|
||||
except ApiError as error:
|
||||
response = error_response(request, error)
|
||||
except Exception as error:
|
||||
logger.error("request failed: %s", type(error).__name__)
|
||||
response = error_response(request, ApiError(500, "internal_error", "Внутренняя ошибка"))
|
||||
response.headers["X-Instance-Id"] = instance_id
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
logger.info("%s %s %s request_id=%s", request.method, request.url.path, response.status_code, request.state.request_id)
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error(request, error):
|
||||
return error_response(request, error)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, error):
|
||||
return error_response(request, invalid("Неверные поля запроса"))
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, error):
|
||||
code = "not_found" if error.status_code == 404 else "invalid_request"
|
||||
return error_response(request, ApiError(error.status_code, code, "Неверный запрос"))
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready():
|
||||
result = store.ready()
|
||||
objects.ready()
|
||||
return result
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def create_user(body: CreateUser):
|
||||
return store.create_user(body)
|
||||
|
||||
@app.post(SESSION_PATH, status_code=201)
|
||||
def login(request: Request):
|
||||
return store.create_session(*request.state.credentials)
|
||||
|
||||
@app.delete(SESSION_PATH + "/current", status_code=204)
|
||||
def logout(request: Request):
|
||||
store.logout(request.state.token)
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def me(request: Request):
|
||||
return store.user(request.state.user_id)
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(body: CreateChat, request: Request):
|
||||
return store.create_chat(request.state.user_id, body)
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(request: Request):
|
||||
return store.list_chats(request.state.user_id, *page_args(request))
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def chat(chat_id: str, request: Request):
|
||||
return store.chat(request.state.user_id, uuid_value(chat_id))
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def create_message(chat_id: str, body: CreateMessage, request: Request):
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key) is None:
|
||||
raise invalid("Неверный Idempotency-Key")
|
||||
return store.create_message(request.state.user_id, uuid_value(chat_id), body, key)
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def messages(chat_id: str, request: Request):
|
||||
return store.list_messages(request.state.user_id, uuid_value(chat_id), *page_args(request))
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||
async def upload(chat_id: str, filename: str, request: Request):
|
||||
return await attachments.receive(request, uuid_value(chat_id), filename)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def attachment(attachment_id: str, request: Request):
|
||||
return attachments.get(request.state.user_id, uuid_value(attachment_id))
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download(attachment_id: str, request: Request, variant: Literal["original"] = "original"):
|
||||
row = attachments.get(request.state.user_id, uuid_value(attachment_id), internal=True)
|
||||
body = objects.content(row["object_key"])
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := body.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(), media_type=row["content_type"], background=BackgroundTask(body.close),
|
||||
headers={"Content-Length": str(row["size_bytes"]), "X-Content-Type-Options": "nosniff",
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''" + quote(row["filename"], safe="")},
|
||||
)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,125 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import warnings
|
||||
from uuid import uuid4
|
||||
|
||||
import anyio
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .storage import serialize
|
||||
|
||||
|
||||
MAX_BYTES = 10485760
|
||||
MAX_PIXELS = 40000000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
FIELDS = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at"
|
||||
|
||||
|
||||
def public(row):
|
||||
return {**serialize(row), "status": "ready", "variants": [], "error_code": None}
|
||||
|
||||
|
||||
def filename_value(value):
|
||||
if not 1 <= len(value) <= 255 or not value.strip():
|
||||
raise invalid("Неверное имя файла")
|
||||
if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in value):
|
||||
raise invalid("Неверное имя файла")
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeError:
|
||||
raise invalid("Неверное имя файла") from None
|
||||
return value
|
||||
|
||||
|
||||
def validate_image(source, media_type):
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
source.seek(0)
|
||||
with Image.open(source, formats=["PNG", "JPEG"]) as image:
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format]
|
||||
if actual != media_type:
|
||||
raise ApiError(415, "unsupported_media_type", "Формат не совпадает с Content-Type")
|
||||
image.verify()
|
||||
source.seek(0)
|
||||
with Image.open(source, formats=["PNG", "JPEG"]) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, store, objects):
|
||||
self.store = store
|
||||
self.objects = objects
|
||||
self.slots = asyncio.Semaphore(2)
|
||||
|
||||
async def receive(self, request, chat_id, filename):
|
||||
user_id = request.state.user_id
|
||||
await run_in_threadpool(self.store.chat, user_id, chat_id)
|
||||
filename = filename_value(filename)
|
||||
media_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
||||
if media_type not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG")
|
||||
declared = request.headers.get("Content-Length")
|
||||
if declared is not None:
|
||||
if not declared.isascii() or not declared.isdecimal() or len(declared) > 18:
|
||||
raise invalid("Неверный Content-Length")
|
||||
if int(declared) > MAX_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
try:
|
||||
await asyncio.wait_for(self.slots.acquire(), timeout=15)
|
||||
except TimeoutError:
|
||||
raise ApiError(429, "rate_limited", "Загрузки заняты", {"Retry-After": "1"}) from None
|
||||
try:
|
||||
with tempfile.TemporaryFile(mode="w+b") as source:
|
||||
size, checksum = 0, hashlib.sha256()
|
||||
try:
|
||||
with anyio.fail_after(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
checksum.update(chunk)
|
||||
await run_in_threadpool(source.write, chunk)
|
||||
except (TimeoutError, ClientDisconnect):
|
||||
raise invalid("Загрузка прервана или слишком медленная") from None
|
||||
await run_in_threadpool(validate_image, source, media_type)
|
||||
return await run_in_threadpool(
|
||||
self.publish, user_id, chat_id, filename, media_type, size, checksum.hexdigest(), source,
|
||||
)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def publish(self, user_id, chat_id, filename, media_type, size, checksum, source):
|
||||
attachment_id = uuid4()
|
||||
key = "originals/" + attachment_id.hex
|
||||
self.objects.put(key, source, size, media_type)
|
||||
with self.store.connection() as connection:
|
||||
self.store.require_chat(connection, user_id, chat_id)
|
||||
row = connection.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING " + FIELDS,
|
||||
(attachment_id, chat_id, user_id, filename, media_type, size, checksum, key),
|
||||
).fetchone()
|
||||
return public(row)
|
||||
|
||||
def get(self, user_id, attachment_id, internal=False):
|
||||
with self.store.connection() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT " + FIELDS + (",object_key" if internal else "") + " FROM attachments a "
|
||||
"WHERE a.id=%s AND EXISTS (SELECT 1 FROM chat_members m "
|
||||
"WHERE m.chat_id=a.chat_id AND m.user_id=%s)", (attachment_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
return row if internal else public(row)
|
||||
@@ -0,0 +1,53 @@
|
||||
import logging
|
||||
import time
|
||||
from io import BytesIO
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .objects import Objects
|
||||
from .errors import ApiError
|
||||
|
||||
|
||||
def main():
|
||||
admin, app = Objects(admin=True), Objects()
|
||||
try:
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
admin.client.list_buckets()
|
||||
break
|
||||
except (ClientError, BotoCoreError):
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError("S3 startup timeout") from None
|
||||
time.sleep(1)
|
||||
try:
|
||||
admin.client.head_bucket(Bucket=admin.bucket)
|
||||
except ClientError as error:
|
||||
if error.response["ResponseMetadata"]["HTTPStatusCode"] != 404:
|
||||
raise RuntimeError("Bucket is not accessible") from None
|
||||
admin.client.create_bucket(Bucket=admin.bucket)
|
||||
acl = admin.client.get_bucket_acl(Bucket=admin.bucket)
|
||||
if any("URI" in grant["Grantee"] for grant in acl["Grants"]):
|
||||
raise RuntimeError("Bucket must be private")
|
||||
while True:
|
||||
try:
|
||||
probe = b"messenger-ready"
|
||||
app.put("system/readiness", BytesIO(probe), len(probe), "application/octet-stream")
|
||||
app.ready()
|
||||
break
|
||||
except ApiError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError("S3 object path is not ready") from None
|
||||
time.sleep(0.5)
|
||||
print("PASS: private bucket ready; application credentials work", flush=True)
|
||||
finally:
|
||||
admin.close()
|
||||
app.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
logging.error("S3 bootstrap failed: %s", type(error).__name__)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,23 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name):
|
||||
value = Path(os.environ[name]).read_text(encoding="utf-8").strip()
|
||||
if not value:
|
||||
raise ValueError(f"Empty secret file: {name}")
|
||||
return value
|
||||
|
||||
|
||||
def session_ttl():
|
||||
value = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
if not 1 <= value <= 86400:
|
||||
raise ValueError("SESSION_TTL_SECONDS must be between 1 and 86400")
|
||||
return value
|
||||
|
||||
|
||||
def idempotency_ttl():
|
||||
value = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
if value < 86400:
|
||||
raise ValueError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
|
||||
return value
|
||||
@@ -0,0 +1,15 @@
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message="Неверный запрос"):
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found():
|
||||
return ApiError(404, "not_found", "Объект не найден")
|
||||
@@ -0,0 +1,50 @@
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import secret
|
||||
|
||||
|
||||
def migrate():
|
||||
files = sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql"))
|
||||
if not files:
|
||||
raise RuntimeError("No migrations found")
|
||||
with psycopg.connect(secret("DATABASE_URL_FILE"), connect_timeout=5) as connection:
|
||||
with connection.transaction():
|
||||
connection.execute("SELECT pg_advisory_xact_lock(230402)")
|
||||
connection.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
checksum char(64) NOT NULL,
|
||||
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
""")
|
||||
for file in files:
|
||||
content = file.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
with connection.transaction():
|
||||
connection.execute("SELECT pg_advisory_xact_lock(230402)")
|
||||
old = connection.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE version = %s", (file.name,)
|
||||
).fetchone()
|
||||
if old:
|
||||
if old[0] != checksum:
|
||||
raise RuntimeError("Applied migration was changed")
|
||||
print("Already applied: " + file.name, flush=True)
|
||||
continue
|
||||
connection.execute(content)
|
||||
connection.execute(
|
||||
"INSERT INTO schema_migrations(version, checksum) VALUES (%s, %s)",
|
||||
(file.name, checksum),
|
||||
)
|
||||
print("Applied: " + file.name, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
migrate()
|
||||
except Exception as error:
|
||||
logging.error("Migration failed: %s", type(error).__name__)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,61 @@
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, StringConstraints, field_validator, model_validator
|
||||
|
||||
|
||||
def nonblank(value: str) -> str:
|
||||
if not value.strip() or "\x00" in value:
|
||||
raise ValueError("Пустая строка или NUL")
|
||||
value.encode("utf-8")
|
||||
return value
|
||||
|
||||
|
||||
Name = Annotated[str, StringConstraints(min_length=1, max_length=100), AfterValidator(nonblank)]
|
||||
Text = Annotated[str, StringConstraints(min_length=1, max_length=4000), AfterValidator(nonblank)]
|
||||
Password = Annotated[str, StringConstraints(min_length=12, max_length=128), AfterValidator(nonblank)]
|
||||
UserId = Annotated[UUID, Field(strict=False)]
|
||||
|
||||
|
||||
class RequestBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
|
||||
class CreateUser(RequestBody):
|
||||
username: str
|
||||
display_name: Name
|
||||
password: Password = Field(repr=False)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def check_username(cls, value):
|
||||
if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None:
|
||||
raise ValueError("Неверный username")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(RequestBody):
|
||||
title: Name
|
||||
member_ids: list[UserId] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def unique_members(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("Повтор участника")
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(RequestBody):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[UserId] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_message(self):
|
||||
if "\x00" in self.text or (not self.text.strip() and not self.attachment_ids):
|
||||
raise ValueError("Нужен текст или вложение")
|
||||
self.text.encode("utf-8")
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("Повтор вложения")
|
||||
return self
|
||||
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import secret
|
||||
from .errors import ApiError
|
||||
|
||||
|
||||
class Objects:
|
||||
def __init__(self, admin=False):
|
||||
prefix = "S3_ADMIN_" if admin else "S3_"
|
||||
self.bucket = os.environ.get("S3_BUCKET", "kuznetsov-lab4-attachments")
|
||||
self.client = boto3.client(
|
||||
"s3", endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"),
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=secret(prefix + "ACCESS_KEY_ID_FILE"),
|
||||
aws_secret_access_key=secret(prefix + "SECRET_ACCESS_KEY_FILE"),
|
||||
config=Config(
|
||||
signature_version="s3v4", connect_timeout=2, read_timeout=5,
|
||||
max_pool_connections=8, retries={"total_max_attempts": 1}, proxies={},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
def call(self, operation, **values):
|
||||
try:
|
||||
return getattr(self.client, operation)(Bucket=self.bucket, **values)
|
||||
except (BotoCoreError, ClientError):
|
||||
raise ApiError(503, "unavailable", "Файловое хранилище недоступно") from None
|
||||
|
||||
def ready(self):
|
||||
self.call("head_bucket")
|
||||
body = self.content("system/readiness")
|
||||
try:
|
||||
if body.read(32) != b"messenger-ready":
|
||||
raise ApiError(503, "unavailable", "Файловое хранилище не готово")
|
||||
except BotoCoreError:
|
||||
raise ApiError(503, "unavailable", "Файловое хранилище недоступно") from None
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
def put(self, key, source, size, content_type):
|
||||
source.seek(0)
|
||||
self.call("put_object", Key=key, Body=source, ContentLength=size, ContentType=content_type)
|
||||
|
||||
def content(self, key):
|
||||
return self.call("get_object", Key=key)["Body"]
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,44 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from .errors import invalid
|
||||
|
||||
|
||||
class Pagination:
|
||||
def __init__(self, secret):
|
||||
self.secret = secret
|
||||
|
||||
def encode(self, scope, item):
|
||||
data = json.dumps([1, scope, item["created_at"], item["id"]], separators=(",", ":"))
|
||||
payload = base64.urlsafe_b64encode(data.encode()).rstrip(b"=").decode()
|
||||
signature = hmac.new(self.secret, payload.encode(), hashlib.sha256).hexdigest()
|
||||
return payload + "." + signature
|
||||
|
||||
def decode(self, scope, cursor):
|
||||
try:
|
||||
if not isinstance(cursor, str) or not cursor or len(cursor) > 2048:
|
||||
raise ValueError()
|
||||
payload, signature = cursor.split(".")
|
||||
expected = hmac.new(self.secret, payload.encode("ascii"), hashlib.sha256).hexdigest()
|
||||
if not hmac.compare_digest(signature, expected):
|
||||
raise ValueError()
|
||||
data = json.loads(base64.b64decode(payload + "=" * (-len(payload) % 4), altchars=b"-_", validate=True))
|
||||
if not isinstance(data, list) or len(data) != 4 or data[:2] != [1, scope]:
|
||||
raise ValueError()
|
||||
if not isinstance(data[2], str) or not data[2].endswith("Z"):
|
||||
raise ValueError()
|
||||
created_at = datetime.fromisoformat(data[2].replace("Z", "+00:00"))
|
||||
if created_at.utcoffset() != timezone.utc.utcoffset(created_at):
|
||||
raise ValueError()
|
||||
return created_at, UUID(data[3])
|
||||
except (ValueError, TypeError, UnicodeError, KeyError, OverflowError):
|
||||
raise invalid("Неверный cursor") from None
|
||||
|
||||
def page(self, rows, scope, limit):
|
||||
items = rows[:limit]
|
||||
cursor = self.encode(scope, items[-1]) if len(rows) > limit else None
|
||||
return {"items": items, "next_cursor": cursor}
|
||||
@@ -0,0 +1,63 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from argon2.low_level import Type
|
||||
|
||||
from .errors import ApiError
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1, type=Type.ID)
|
||||
HASH_SLOTS = BoundedSemaphore(4)
|
||||
DUMMY_HASH = HASHER.hash(secrets.token_urlsafe(32))
|
||||
|
||||
|
||||
def unauthorized():
|
||||
return ApiError(401, "unauthorized", "Неверные данные авторизации")
|
||||
|
||||
|
||||
def hash_password(password):
|
||||
with HASH_SLOTS:
|
||||
return HASHER.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash, password):
|
||||
try:
|
||||
with HASH_SLOTS:
|
||||
return HASHER.verify(password_hash or DUMMY_HASH, password)
|
||||
except (VerificationError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def basic_credentials(header):
|
||||
try:
|
||||
scheme, encoded = header.split(" ", 1)
|
||||
if scheme.lower() != "basic" or len(encoded) > 2048:
|
||||
raise ValueError()
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator or re.fullmatch(r"[a-z0-9_]{3,32}", username) is None:
|
||||
raise ValueError()
|
||||
if not 12 <= len(password) <= 128 or not password.strip() or "\x00" in password:
|
||||
raise ValueError()
|
||||
return username, password
|
||||
except (ValueError, UnicodeError, AttributeError, binascii.Error):
|
||||
raise unauthorized() from None
|
||||
|
||||
|
||||
def bearer_token(header):
|
||||
try:
|
||||
scheme, token = header.split(" ", 1)
|
||||
if scheme.lower() != "bearer" or not re.fullmatch(r"[A-Za-z0-9_-]{43}", token):
|
||||
raise ValueError()
|
||||
return token
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise unauthorized() from None
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from .config import idempotency_ttl, secret, session_ttl
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .pagination import Pagination
|
||||
from .security import hash_password, token_hash, unauthorized, verify_password
|
||||
|
||||
|
||||
USER_FIELDS = "id, username, display_name, created_at"
|
||||
CHAT_FIELDS = """
|
||||
c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT cm.user_id FROM chat_members cm WHERE cm.chat_id=c.id ORDER BY cm.user_id) AS member_ids
|
||||
"""
|
||||
|
||||
|
||||
def serialize(value):
|
||||
if isinstance(value, UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
if isinstance(value, dict):
|
||||
return {key: serialize(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [serialize(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
class PostgresStore:
|
||||
def __init__(self):
|
||||
logging.getLogger("psycopg.pool").setLevel(logging.CRITICAL)
|
||||
self.pool = ConnectionPool(
|
||||
secret("DATABASE_URL_FILE"), min_size=1, max_size=10, timeout=3, max_waiting=40,
|
||||
open=False, check=ConnectionPool.check_connection,
|
||||
kwargs={"row_factory": dict_row, "connect_timeout": 3,
|
||||
"options": "-c timezone=UTC -c statement_timeout=5000 -c lock_timeout=2000"},
|
||||
)
|
||||
self.pagination = Pagination(secret("CURSOR_SECRET_FILE").encode("ascii"))
|
||||
self.ttl = session_ttl()
|
||||
self.idempotency_ttl = idempotency_ttl()
|
||||
|
||||
def open(self):
|
||||
self.pool.open()
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with self.pool.connection() as connection:
|
||||
yield connection
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests,
|
||||
psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable):
|
||||
raise ApiError(503, "unavailable", "Хранилище временно недоступно") from None
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
with self.connection() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT version FROM schema_migrations WHERE version=%s", ("003_attachments.sql",)
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise ApiError(503, "unavailable", "Схема ещё не готова")
|
||||
except psycopg.errors.UndefinedTable:
|
||||
raise ApiError(503, "unavailable", "Схема ещё не готова") from None
|
||||
return {"status": "ok"}
|
||||
|
||||
def create_user(self, body):
|
||||
password_hash = hash_password(body.password)
|
||||
try:
|
||||
with self.connection() as connection:
|
||||
row = connection.execute(
|
||||
"INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING "
|
||||
+ USER_FIELDS,
|
||||
(uuid4(), body.username, body.display_name, password_hash),
|
||||
).fetchone()
|
||||
return serialize(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, "conflict", "Username уже занят") from None
|
||||
|
||||
def create_session(self, username, password):
|
||||
with self.connection() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT " + USER_FIELDS + ", password_hash FROM users WHERE username=%s", (username,)
|
||||
).fetchone()
|
||||
if not verify_password(row["password_hash"] if row else None, password) or row is None:
|
||||
raise unauthorized()
|
||||
token = secrets.token_urlsafe(32)
|
||||
with self.connection() as connection:
|
||||
session = connection.execute("""
|
||||
INSERT INTO sessions(token_hash,user_id,created_at,expires_at)
|
||||
VALUES (%s,%s,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP + %s * INTERVAL '1 second')
|
||||
RETURNING expires_at
|
||||
""", (token_hash(token), row["id"], self.ttl)).fetchone()
|
||||
user = {key: value for key, value in row.items() if key != "password_hash"}
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": serialize(session["expires_at"]),
|
||||
"user": serialize(user)}
|
||||
|
||||
def authenticate(self, token):
|
||||
with self.connection() as connection:
|
||||
row = connection.execute("""
|
||||
SELECT user_id FROM sessions
|
||||
WHERE token_hash=%s AND expires_at > CURRENT_TIMESTAMP
|
||||
""", (token_hash(token),)).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized()
|
||||
return row["user_id"]
|
||||
|
||||
def logout(self, token):
|
||||
with self.connection() as connection:
|
||||
result = connection.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash(token),))
|
||||
if result.rowcount != 1:
|
||||
raise unauthorized()
|
||||
|
||||
def user(self, user_id):
|
||||
with self.connection() as connection:
|
||||
row = connection.execute("SELECT " + USER_FIELDS + " FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
return serialize(row)
|
||||
|
||||
def create_chat(self, user_id, body):
|
||||
members = body.member_ids
|
||||
if user_id not in members:
|
||||
raise invalid("Неверный список участников")
|
||||
with self.connection() as connection:
|
||||
found = connection.execute("SELECT id FROM users WHERE id=ANY(%s)", (members,)).fetchall()
|
||||
if len(found) != len(members):
|
||||
raise invalid("Неверный список участников")
|
||||
chat_id = uuid4()
|
||||
# чат и участники коммитятся вместе
|
||||
connection.execute("INSERT INTO chats(id,title) VALUES (%s,%s)", (chat_id, body.title))
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)",
|
||||
[(chat_id, member) for member in members],
|
||||
)
|
||||
return serialize(self.require_chat(connection, user_id, chat_id))
|
||||
|
||||
def require_chat(self, connection, user_id, chat_id):
|
||||
row = connection.execute(
|
||||
"SELECT " + CHAT_FIELDS + """ FROM chats c WHERE c.id=%s
|
||||
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)
|
||||
""", (chat_id, user_id)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
def chat(self, user_id, chat_id):
|
||||
with self.connection() as connection:
|
||||
return serialize(self.require_chat(connection, user_id, chat_id))
|
||||
|
||||
def list_chats(self, user_id, limit, cursor):
|
||||
scope = "chats:" + str(user_id)
|
||||
after = self.pagination.decode(scope, cursor) if cursor is not None else None
|
||||
condition = " AND (c.created_at,c.id) > (%s,%s)" if after else ""
|
||||
with self.connection() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT " + CHAT_FIELDS + """ FROM chats c
|
||||
WHERE EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"""
|
||||
+ condition + " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
[user_id, *(after or ()), limit + 1],
|
||||
).fetchall()
|
||||
return self.pagination.page(serialize(rows), scope, limit)
|
||||
|
||||
def create_message(self, user_id, chat_id, body, key):
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
scope = (user_id, "POST", path, key)
|
||||
canonical = json.dumps(body.model_dump(mode="json"), sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
request_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
lock_data = json.dumps([str(user_id), "POST", path, key], separators=(",", ":")).encode()
|
||||
lock_id = int.from_bytes(hashlib.sha256(lock_data).digest()[:8], "big", signed=True)
|
||||
with self.connection() as connection:
|
||||
self.require_chat(connection, user_id, chat_id)
|
||||
# один ключ блокируется сразу для обеих реплик
|
||||
connection.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
||||
saved = connection.execute("""
|
||||
SELECT request_hash,response_body,expires_at > clock_timestamp() AS active
|
||||
FROM idempotency_keys WHERE user_id=%s AND method=%s AND path=%s AND request_key=%s
|
||||
""", scope).fetchone()
|
||||
if saved and saved["active"]:
|
||||
if saved["request_hash"] != request_hash:
|
||||
raise ApiError(409, "conflict", "С этим ключом уже отправлен другой запрос")
|
||||
return saved["response_body"]
|
||||
if saved:
|
||||
connection.execute("""
|
||||
DELETE FROM idempotency_keys WHERE user_id=%s AND method=%s AND path=%s AND request_key=%s
|
||||
""", scope)
|
||||
if body.attachment_ids:
|
||||
found = connection.execute(
|
||||
"SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s)",
|
||||
(chat_id, body.attachment_ids),
|
||||
).fetchall()
|
||||
if len(found) != len(body.attachment_ids):
|
||||
raise not_found()
|
||||
row = connection.execute("""
|
||||
INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)
|
||||
RETURNING id,chat_id,sender_id,text,created_at
|
||||
""", (uuid4(), chat_id, user_id, body.text)).fetchone()
|
||||
for position, attachment_id in enumerate(body.attachment_ids):
|
||||
connection.execute(
|
||||
"INSERT INTO message_attachments(message_id,attachment_id,chat_id,position) VALUES (%s,%s,%s,%s)",
|
||||
(row["id"], attachment_id, chat_id, position),
|
||||
)
|
||||
row["attachment_ids"] = body.attachment_ids
|
||||
message = serialize(row)
|
||||
connection.execute("""
|
||||
INSERT INTO idempotency_keys
|
||||
(user_id,method,path,request_key,request_hash,message_id,response_status,response_body,expires_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,201,%s,clock_timestamp() + %s * INTERVAL '1 second')
|
||||
""", (*scope, request_hash, row["id"], Jsonb(message), self.idempotency_ttl))
|
||||
return message
|
||||
|
||||
def list_messages(self, user_id, chat_id, limit, cursor):
|
||||
scope = "messages:" + str(chat_id)
|
||||
after = self.pagination.decode(scope, cursor) if cursor is not None else None
|
||||
condition = " AND (created_at,id) > (%s,%s)" if after else ""
|
||||
with self.connection() as connection:
|
||||
self.require_chat(connection, user_id, chat_id)
|
||||
rows = connection.execute(
|
||||
"""SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,
|
||||
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
|
||||
WHERE ma.message_id=m.id ORDER BY ma.position) AS attachment_ids
|
||||
FROM messages m WHERE chat_id=%s"""
|
||||
+ condition + " ORDER BY created_at,id LIMIT %s",
|
||||
[chat_id, *(after or ()), limit + 1],
|
||||
).fetchall()
|
||||
return self.pagination.page(serialize(rows), scope, limit)
|
||||
Reference in New Issue
Block a user