Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
641 lines
25 KiB
Python
641 lines
25 KiB
Python
import asyncio
|
||
import base64
|
||
import binascii
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import resource
|
||
import signal
|
||
import tempfile
|
||
import unicodedata
|
||
import sys
|
||
import uuid
|
||
from contextlib import asynccontextmanager
|
||
from typing import List, Optional, Tuple
|
||
|
||
from fastapi import FastAPI, Request
|
||
from fastapi.responses import JSONResponse, Response
|
||
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
|
||
from starlette.responses import StreamingResponse
|
||
|
||
from app import config, passwords, reconcile, repo
|
||
from app.cursor import BadCursor, decode_cursor, encode_cursor
|
||
from app.db import Database
|
||
from app.imagecheck import inspect_image
|
||
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
|
||
from app.s3 import S3Storage
|
||
from app.sessions import SessionStore
|
||
|
||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||
log = logging.getLogger("messenger")
|
||
|
||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||
ZERO_WIDTH = str.maketrans({char: None for char in ""})
|
||
EXEMPT_PATHS = ("/health/live", "/health/ready")
|
||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||
|
||
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||
background = set()
|
||
db = Database()
|
||
sessions = SessionStore(db)
|
||
s3 = S3Storage()
|
||
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}}
|
||
|
||
|
||
def rss_mb() -> float:
|
||
try:
|
||
with open("/proc/self/statm") as handle:
|
||
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
|
||
except OSError:
|
||
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
|
||
|
||
|
||
def count_rejection(code: str) -> None:
|
||
state["rejected"][code] = state["rejected"].get(code, 0) + 1
|
||
|
||
|
||
async def sweeper() -> None:
|
||
while True:
|
||
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
|
||
try:
|
||
removed = await sessions.sweep()
|
||
if removed:
|
||
log.info("sessions_expired removed=%d", removed)
|
||
except Exception:
|
||
log.warning("session sweep skipped: storage unavailable")
|
||
try:
|
||
async with db.connection() as conn:
|
||
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
|
||
if removed:
|
||
log.info("idempotency_cleanup removed=%d", removed)
|
||
except Exception:
|
||
log.warning("idempotency cleanup skipped")
|
||
|
||
|
||
async def reconciler_loop() -> None:
|
||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||
while True:
|
||
try:
|
||
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
|
||
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
|
||
log.info("reconcile %s", result)
|
||
except Exception:
|
||
log.warning("reconcile skipped: dependency unavailable")
|
||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||
|
||
|
||
async def readiness_watch() -> None:
|
||
failures = 0
|
||
while True:
|
||
if await db.ready():
|
||
failures = 0
|
||
if not state["ready"]:
|
||
state["ready"] = True
|
||
log.info("ready again: database reachable")
|
||
else:
|
||
failures += 1
|
||
if failures >= config.READY_FAILURES and state["ready"]:
|
||
state["ready"] = False
|
||
log.warning("not ready: database unreachable, business requests get 503")
|
||
s3_ok = await run_in_threadpool(s3.probe)
|
||
if s3_ok != state["s3_ok"]:
|
||
state["s3_ok"] = s3_ok
|
||
log.info("object storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
|
||
await asyncio.sleep(config.READY_CHECK_SECONDS)
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_app: FastAPI):
|
||
previous = {}
|
||
|
||
def on_signal(signum, frame):
|
||
if not state["draining"]:
|
||
state["draining"] = True
|
||
log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name)
|
||
handler = previous.get(signum)
|
||
if callable(handler):
|
||
handler(signum, frame)
|
||
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
previous[sig] = signal.getsignal(sig)
|
||
signal.signal(sig, on_signal)
|
||
await db.open()
|
||
await sessions.open()
|
||
await run_in_threadpool(s3.open)
|
||
state["s3_ok"] = await run_in_threadpool(s3.probe)
|
||
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop())]
|
||
log.info(
|
||
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
|
||
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
|
||
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
|
||
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
|
||
)
|
||
yield
|
||
for task in tasks:
|
||
task.cancel()
|
||
await sessions.close()
|
||
await db.close()
|
||
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
|
||
|
||
|
||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||
|
||
|
||
@app.middleware("http")
|
||
async def guard(request: Request, call_next):
|
||
rid = uuid.uuid4().hex
|
||
request.state.rid = rid
|
||
exempt = request.url.path in EXEMPT_PATHS
|
||
if state["draining"] and request.url.path != "/health/live":
|
||
count_rejection("draining")
|
||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
|
||
if not exempt and not state["ready"]:
|
||
count_rejection("not_ready")
|
||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
|
||
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
|
||
count_rejection("inflight")
|
||
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
|
||
state["inflight"] += 1
|
||
try:
|
||
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
|
||
except asyncio.TimeoutError:
|
||
count_rejection("request_timeout")
|
||
log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path)
|
||
response = error_body(
|
||
rid, config.INSTANCE_ID, 503, "unavailable",
|
||
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
|
||
)
|
||
except Exception:
|
||
log.exception("unhandled error request_id=%s", rid)
|
||
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
|
||
finally:
|
||
state["inflight"] -= 1
|
||
response.headers["X-Request-Id"] = rid
|
||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||
return response
|
||
|
||
|
||
@app.exception_handler(ApiError)
|
||
async def api_error_handler(request: Request, exc: ApiError):
|
||
if exc.status in (413, 429, 503):
|
||
count_rejection(exc.code)
|
||
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
|
||
|
||
|
||
def parse_uuid(value: object) -> Optional[str]:
|
||
if not isinstance(value, str):
|
||
return None
|
||
try:
|
||
return str(uuid.UUID(value))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def non_blank(value: object, max_len: int) -> bool:
|
||
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
|
||
return False
|
||
return value.translate(ZERO_WIDTH).strip() != ""
|
||
|
||
|
||
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
|
||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"Connection": "close"})
|
||
declared = request.headers.get("content-length")
|
||
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
|
||
raise too_large
|
||
raw = bytearray()
|
||
async for chunk in request.stream():
|
||
raw += chunk
|
||
if len(raw) > config.MAX_BODY_BYTES:
|
||
raise too_large
|
||
try:
|
||
body = json.loads(bytes(raw).decode("utf-8"))
|
||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||
raise bad("Некорректный JSON") from None
|
||
if not isinstance(body, dict):
|
||
raise bad("Ожидается JSON-объект")
|
||
allowed = set(required) | set(optional)
|
||
if set(body) - allowed:
|
||
raise bad("Лишние поля в запросе")
|
||
if set(required) - set(body):
|
||
raise bad("Отсутствуют обязательные поля")
|
||
return body
|
||
|
||
|
||
def ensure_capacity() -> None:
|
||
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
|
||
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
|
||
|
||
|
||
def bearer_token(request: Request) -> str:
|
||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||
token = value.strip()
|
||
if scheme.lower() != "bearer" or not token:
|
||
raise unauthorized()
|
||
return token
|
||
|
||
|
||
async def actor(request: Request) -> dict:
|
||
user = await sessions.resolve(bearer_token(request))
|
||
if user is None:
|
||
raise unauthorized()
|
||
return user
|
||
|
||
|
||
def basic_credentials(request: Request) -> Tuple[str, str]:
|
||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||
if scheme.lower() != "basic" or not value.strip():
|
||
raise unauthorized("Basic")
|
||
try:
|
||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||
raise unauthorized("Basic") from None
|
||
username, separator, password = decoded.partition(":")
|
||
if not separator:
|
||
raise unauthorized("Basic")
|
||
return username, password
|
||
|
||
|
||
async def member_chat(user: dict, chat_id: str) -> dict:
|
||
cid = parse_uuid(chat_id)
|
||
if cid is None:
|
||
raise bad("chat_id должен быть UUID")
|
||
async with db.connection() as conn:
|
||
chat = await repo.chat_for_member(conn, cid, user["id"])
|
||
if chat is None:
|
||
raise not_found()
|
||
return chat
|
||
|
||
|
||
def page_params(request: Request) -> Tuple[int, Optional[str]]:
|
||
query = request.query_params
|
||
limit = 50
|
||
if "limit" in query:
|
||
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
|
||
raise bad("limit должен быть целым 1..100")
|
||
limit = int(query["limit"])
|
||
cursor = query.get("cursor")
|
||
if cursor is not None and cursor == "":
|
||
raise bad("Некорректный cursor")
|
||
return limit, cursor
|
||
|
||
|
||
def keyset(scope: str, cursor: Optional[str]):
|
||
if cursor is None:
|
||
return None
|
||
try:
|
||
return decode_cursor(scope, cursor)
|
||
except BadCursor:
|
||
raise bad("Некорректный cursor") from None
|
||
|
||
|
||
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
|
||
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
|
||
return JSONResponse({"items": items, "next_cursor": next_cursor})
|
||
|
||
|
||
def detect_content_type(head: bytes) -> Optional[str]:
|
||
if head[:8] == PNG_MAGIC:
|
||
return "image/png"
|
||
if head[:3] == JPEG_MAGIC:
|
||
return "image/jpeg"
|
||
return None
|
||
|
||
|
||
def validate_filename(filename: str) -> None:
|
||
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
|
||
raise bad("filename: 1..255 символов, не только пробелы")
|
||
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
|
||
raise bad("filename содержит недопустимые символы")
|
||
|
||
|
||
@app.get("/health/live")
|
||
async def live():
|
||
return {"status": "ok"}
|
||
|
||
|
||
@app.get("/health/ready")
|
||
async def ready():
|
||
if not await db.ready():
|
||
raise unavailable("База данных недоступна")
|
||
if not state["s3_ok"]:
|
||
raise unavailable("Хранилище файлов недоступно")
|
||
return {"status": "ok"}
|
||
|
||
|
||
@app.post("/api/v1/users", status_code=201)
|
||
async def create_user(request: Request):
|
||
body = await read_json(request, ("username", "display_name", "password"))
|
||
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
|
||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||
if not non_blank(body["display_name"], 100):
|
||
raise bad("display_name: 1..100 символов, не только пробелы")
|
||
password = body["password"]
|
||
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
|
||
ensure_capacity()
|
||
password_hash = await run_in_threadpool(passwords.hash_password, password)
|
||
async with db.connection() as conn:
|
||
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
|
||
if user is None:
|
||
raise ApiError(409, "conflict", "username уже занят")
|
||
log.info("user_created id=%s", user["id"])
|
||
return JSONResponse(user, status_code=201)
|
||
|
||
|
||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||
async def login(request: Request):
|
||
if await request.body():
|
||
raise bad("POST /auth/sessions не принимает тело запроса")
|
||
username, password = basic_credentials(request)
|
||
row = None
|
||
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||
async with db.connection() as conn:
|
||
row = await repo.credentials(conn, username)
|
||
stored = row["password_hash"] if row else passwords.DUMMY_HASH
|
||
matches = await run_in_threadpool(passwords.verify_password, stored, password)
|
||
if row is None or not matches:
|
||
log.info("login_failed username_known=%s", row is not None)
|
||
raise unauthorized("Basic")
|
||
user = repo.public_user(row)
|
||
token, expires_at = await sessions.create(user)
|
||
log.info("session_created user=%s", user["id"])
|
||
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
|
||
|
||
|
||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||
async def logout(request: Request):
|
||
token = bearer_token(request)
|
||
user = await sessions.resolve(token)
|
||
if user is None:
|
||
raise unauthorized()
|
||
await sessions.revoke(token)
|
||
log.info("session_revoked user=%s", user["id"])
|
||
return Response(status_code=204)
|
||
|
||
|
||
@app.get("/api/v1/users/me")
|
||
async def me(request: Request):
|
||
return await actor(request)
|
||
|
||
|
||
@app.post("/api/v1/chats", status_code=201)
|
||
async def create_chat(request: Request):
|
||
user = await actor(request)
|
||
body = await read_json(request, ("title", "member_ids"))
|
||
if not non_blank(body["title"], 100):
|
||
raise bad("title: 1..100 символов, не только пробелы")
|
||
raw_ids = body["member_ids"]
|
||
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
|
||
raise bad("member_ids: массив из 1..100 UUID")
|
||
ids: List[str] = []
|
||
for value in raw_ids:
|
||
parsed = parse_uuid(value)
|
||
if parsed is None:
|
||
raise bad("member_ids содержит некорректный UUID")
|
||
ids.append(parsed)
|
||
if len(set(ids)) != len(ids):
|
||
raise bad("member_ids содержит дубликаты")
|
||
if user["id"] not in ids:
|
||
raise bad("Создатель должен входить в member_ids")
|
||
ensure_capacity()
|
||
|
||
async def insert(conn):
|
||
return await repo.create_chat(conn, body["title"], ids)
|
||
|
||
try:
|
||
chat = await db.in_transaction(insert)
|
||
except repo.UnknownMember:
|
||
raise bad("member_ids содержит неизвестного пользователя") from None
|
||
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
|
||
return JSONResponse(chat, status_code=201)
|
||
|
||
|
||
@app.get("/api/v1/chats")
|
||
async def list_chats(request: Request):
|
||
user = await actor(request)
|
||
limit, cursor = page_params(request)
|
||
scope = "chats:" + user["id"]
|
||
after = keyset(scope, cursor)
|
||
async with db.connection() as conn:
|
||
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
|
||
return page(scope, items, has_more)
|
||
|
||
|
||
@app.get("/api/v1/chats/{chat_id}")
|
||
async def get_chat(chat_id: str, request: Request):
|
||
return await member_chat(await actor(request), chat_id)
|
||
|
||
|
||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||
async def create_message(chat_id: str, request: Request):
|
||
user = await actor(request)
|
||
cid = parse_uuid(chat_id)
|
||
if cid is None:
|
||
raise bad("chat_id должен быть UUID")
|
||
|
||
idem_key = request.headers.get("idempotency-key", "")
|
||
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
|
||
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
|
||
|
||
body = await read_json(request, ("text",), ("attachment_ids",))
|
||
|
||
text = body["text"]
|
||
attachment_ids_raw = body.get("attachment_ids", [])
|
||
|
||
if not isinstance(attachment_ids_raw, list):
|
||
raise bad("attachment_ids должен быть массивом UUID")
|
||
if len(attachment_ids_raw) > 10:
|
||
raise bad("attachment_ids: максимум 10 вложений")
|
||
|
||
attachment_ids: List[str] = []
|
||
for value in attachment_ids_raw:
|
||
parsed = parse_uuid(value)
|
||
if parsed is None:
|
||
raise bad("attachment_ids содержит некорректный UUID")
|
||
attachment_ids.append(parsed)
|
||
|
||
if len(set(attachment_ids)) != len(attachment_ids):
|
||
raise bad("attachment_ids содержит дубликаты")
|
||
|
||
has_text = isinstance(text, str) and non_blank(text, 4000)
|
||
has_attachments = len(attachment_ids) > 0
|
||
|
||
if not has_text and not has_attachments:
|
||
raise bad("text: 1..4000 символов, не только пробелы")
|
||
|
||
if isinstance(text, str) and len(text) > 4000:
|
||
raise bad("text: 1..4000 символов, не только пробелы")
|
||
|
||
if not isinstance(text, str):
|
||
raise bad("text должен быть строкой")
|
||
|
||
ensure_capacity()
|
||
|
||
normalized = {"text": text}
|
||
if attachment_ids:
|
||
normalized["attachment_ids"] = attachment_ids
|
||
body_hash = hashlib.sha256(
|
||
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
|
||
).hexdigest()
|
||
|
||
async def do(conn):
|
||
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
|
||
|
||
status, message = await db.in_transaction(do)
|
||
|
||
if status == "conflict":
|
||
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
|
||
if status == "not_member":
|
||
raise not_found()
|
||
if status == "attachment_not_found":
|
||
raise not_found()
|
||
if status == "attachment_failed":
|
||
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
|
||
return JSONResponse(message, status_code=201)
|
||
|
||
|
||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||
async def list_messages(chat_id: str, request: Request):
|
||
chat = await member_chat(await actor(request), chat_id)
|
||
limit, cursor = page_params(request)
|
||
scope = "messages:" + chat["id"]
|
||
after = keyset(scope, cursor)
|
||
async with db.connection() as conn:
|
||
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
|
||
return page(scope, items, has_more)
|
||
|
||
|
||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||
async def upload_attachment(chat_id: str, request: Request):
|
||
user = await actor(request)
|
||
chat = await member_chat(user, chat_id)
|
||
|
||
filename = request.query_params.get("filename", "")
|
||
validate_filename(filename)
|
||
|
||
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||
if declared_ct not in ALLOWED_CONTENT_TYPES:
|
||
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
|
||
|
||
declared_length = request.headers.get("content-length")
|
||
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB", {"Connection": "close"})
|
||
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
|
||
raise too_large
|
||
ensure_capacity()
|
||
|
||
sha = hashlib.sha256()
|
||
size = 0
|
||
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
|
||
head = b""
|
||
async for chunk in request.stream():
|
||
size += len(chunk)
|
||
if size > config.MAX_UPLOAD_BYTES:
|
||
raise too_large
|
||
if len(head) < 8:
|
||
head += chunk[: 8 - len(head)]
|
||
if len(head) == 8 and detect_content_type(head) is None:
|
||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||
sha.update(chunk)
|
||
spool.write(chunk)
|
||
if size == 0:
|
||
raise bad("Пустой файл")
|
||
actual_ct = detect_content_type(head)
|
||
if actual_ct is None:
|
||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||
if actual_ct != declared_ct:
|
||
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
|
||
async with upload_slots:
|
||
await run_in_threadpool(inspect_image, spool, size, actual_ct)
|
||
|
||
attachment_id = str(uuid.uuid4())
|
||
object_key = "attachments/" + attachment_id
|
||
async with db.connection() as conn:
|
||
await repo.begin_attachment(
|
||
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), object_key,
|
||
)
|
||
try:
|
||
async with upload_slots:
|
||
await run_in_threadpool(s3.put, object_key, spool, size, actual_ct)
|
||
except BaseException:
|
||
defer(compensate(attachment_id, object_key))
|
||
raise
|
||
|
||
try:
|
||
async with db.connection() as conn:
|
||
att = await repo.finish_attachment(conn, attachment_id)
|
||
except BaseException:
|
||
await compensate(attachment_id, object_key)
|
||
raise
|
||
if att is None:
|
||
raise unavailable("Загрузка отменена сверкой, повторите запрос")
|
||
log.info("attachment_uploaded id=%s chat=%s size=%d", att["id"], chat["id"], size)
|
||
return JSONResponse(att, status_code=201)
|
||
|
||
|
||
def defer(coro) -> None:
|
||
task = asyncio.ensure_future(coro)
|
||
background.add(task)
|
||
task.add_done_callback(background.discard)
|
||
|
||
|
||
async def compensate(attachment_id: str, object_key: str) -> None:
|
||
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
|
||
try:
|
||
async with db.connection() as conn:
|
||
await repo.abort_attachment(conn, attachment_id)
|
||
await run_in_threadpool(s3.delete, object_key)
|
||
except BaseException:
|
||
log.warning("compensation incomplete attachment=%s, left for reconciler", attachment_id)
|
||
|
||
|
||
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
|
||
user = await actor(request)
|
||
aid = parse_uuid(attachment_id)
|
||
if aid is None:
|
||
raise bad("attachment_id должен быть UUID")
|
||
async with db.connection() as conn:
|
||
row = await repo.get_attachment(conn, aid)
|
||
if row is None:
|
||
raise not_found()
|
||
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
|
||
if chat is None:
|
||
raise not_found()
|
||
return row
|
||
|
||
|
||
@app.get("/api/v1/attachments/{attachment_id}")
|
||
async def get_attachment(attachment_id: str, request: Request):
|
||
return repo._attachment(await accessible_attachment(request, attachment_id))
|
||
|
||
|
||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||
async def download_attachment(attachment_id: str, request: Request):
|
||
row = await accessible_attachment(request, attachment_id)
|
||
variant = request.query_params.get("variant", "original")
|
||
if variant != "original":
|
||
raise not_found()
|
||
if row["status"] != "ready":
|
||
raise ApiError(409, "conflict", "Вложение недоступно для скачивания")
|
||
opened = await run_in_threadpool(s3.open_stream, row["object_key"])
|
||
if opened is None:
|
||
log.error("object_missing attachment=%s", row["id"])
|
||
raise unavailable("Оригинал временно недоступен")
|
||
chunks, length = opened
|
||
return StreamingResponse(
|
||
iterate_in_threadpool(chunks), media_type=row["content_type"], headers={"Content-Length": str(length)},
|
||
)
|
||
|
||
|
||
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||
async def fallback(request: Request):
|
||
raise not_found()
|