forked from mtusi2026/mtusi2026lab4
882 lines
37 KiB
Python
882 lines
37 KiB
Python
"""Replicated messenger API with PostgreSQL-backed sessions and idempotency."""
|
|
|
|
import asyncio
|
|
import base64
|
|
import binascii
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import socket
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from contextlib import asynccontextmanager
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import psycopg
|
|
import redis
|
|
import urllib3
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
|
from minio import Minio
|
|
from minio.error import MinioException, S3Error
|
|
from PIL import Image, UnidentifiedImageError
|
|
from psycopg.conninfo import make_conninfo
|
|
from psycopg.rows import dict_row
|
|
from psycopg_pool import ConnectionPool, PoolTimeout
|
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
|
|
|
|
|
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger")
|
|
database_password = os.environ.get("DATABASE_PASSWORD")
|
|
if "DATABASE_PASSWORD_FILE" in os.environ:
|
|
with open(os.environ["DATABASE_PASSWORD_FILE"], encoding="utf-8") as secret_file:
|
|
database_password = secret_file.read().rstrip("\n")
|
|
if database_password is not None:
|
|
DATABASE_URL = make_conninfo(
|
|
host=os.environ.get("DATABASE_HOST", "postgres"),
|
|
dbname=os.environ.get("DATABASE_NAME", "messenger"),
|
|
user=os.environ.get("DATABASE_USER", "messenger"),
|
|
password=database_password,
|
|
)
|
|
REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
|
|
SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
|
MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384"))
|
|
POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10"))
|
|
CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1"
|
|
CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1"
|
|
CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0"))))
|
|
INSTANCE_ID = os.environ.get("INSTANCE_ID", "") or socket.gethostname()
|
|
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "minio:9000")
|
|
S3_PUBLIC_ENDPOINT = os.environ.get("S3_PUBLIC_ENDPOINT", "localhost:9000")
|
|
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "messenger-api")
|
|
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "messenger-api-secret")
|
|
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
S3_SECURE = os.environ.get("S3_SECURE", "0") == "1"
|
|
PRESIGN_TTL = min(900, max(30, int(os.environ.get("PRESIGN_TTL_SECONDS", "120"))))
|
|
MAX_ATTACHMENT = 10 * 1024 * 1024
|
|
MAX_PIXELS = 40_000_000
|
|
ready_for_traffic = False
|
|
pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"})
|
|
cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2)
|
|
s3_http = urllib3.PoolManager(
|
|
timeout=urllib3.Timeout(connect=2, read=15),
|
|
retries=False,
|
|
maxsize=max(10, POOL_SIZE * 2),
|
|
)
|
|
s3 = Minio(
|
|
S3_ENDPOINT,
|
|
access_key=S3_ACCESS_KEY,
|
|
secret_key=S3_SECRET_KEY,
|
|
secure=S3_SECURE,
|
|
http_client=s3_http,
|
|
)
|
|
s3_signer = Minio(S3_PUBLIC_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=S3_SECURE)
|
|
metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0}
|
|
metrics_lock = threading.Lock()
|
|
|
|
|
|
class ApiError(Exception):
|
|
def __init__(self, status, code, message, challenge=None):
|
|
self.status, self.code, self.message, self.challenge = status, code, message, challenge
|
|
|
|
|
|
def invalid(message="Некорректный запрос"):
|
|
raise ApiError(400, "invalid_request", message)
|
|
|
|
|
|
def unavailable():
|
|
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
|
|
|
|
|
|
def stamp(value):
|
|
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
|
|
|
|
|
def encode_cursor(scope, date, identifier):
|
|
raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode()
|
|
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
|
|
|
|
|
def decode_cursor(scope, value):
|
|
if value is None:
|
|
return None
|
|
try:
|
|
raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
|
parts = json.loads(raw)
|
|
if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope
|
|
or not isinstance(parts[1], str) or not isinstance(parts[2], str)):
|
|
invalid("Некорректный cursor")
|
|
date = datetime.fromisoformat(parts[1].replace("Z", "+00:00"))
|
|
if date.tzinfo is None:
|
|
invalid("Некорректный cursor")
|
|
return date, str(uuid.UUID(parts[2]))
|
|
except (ValueError, TypeError, binascii.Error, IndexError):
|
|
invalid("Некорректный cursor")
|
|
|
|
|
|
def identifier(value):
|
|
try:
|
|
result = str(uuid.UUID(value))
|
|
if result != value.lower() or len(value) != 36:
|
|
invalid("Некорректный UUID")
|
|
return result
|
|
except (ValueError, TypeError, AttributeError):
|
|
invalid("Некорректный UUID")
|
|
|
|
|
|
def text_field(value, maximum):
|
|
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
|
|
invalid("Некорректное текстовое поле")
|
|
return value
|
|
|
|
|
|
def exact_body(value, fields):
|
|
if not isinstance(value, dict) or set(value) != set(fields):
|
|
invalid("Некорректные поля JSON")
|
|
|
|
|
|
async def body(request, fields):
|
|
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
|
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
|
declared = request.headers.get("content-length")
|
|
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
|
|
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
|
chunks = bytearray()
|
|
async for chunk in request.stream():
|
|
chunks.extend(chunk)
|
|
if len(chunks) > MAX_BODY:
|
|
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
|
try:
|
|
value = json.loads(chunks.decode("utf-8"))
|
|
except (UnicodeError, ValueError):
|
|
invalid("Некорректный JSON")
|
|
exact_body(value, fields)
|
|
return value
|
|
|
|
|
|
def user(row):
|
|
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])}
|
|
|
|
|
|
def message(row, db=None):
|
|
attachment_ids = []
|
|
if db is not None:
|
|
linked = db.execute(
|
|
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
|
|
(row["id"],),
|
|
).fetchall()
|
|
attachment_ids = [str(item["attachment_id"]) for item in linked]
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"sender_id": str(row["sender_id"]),
|
|
"text": row["text"],
|
|
"created_at": stamp(row["created_at"]),
|
|
"attachment_ids": attachment_ids,
|
|
}
|
|
|
|
|
|
def attachment(row):
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"owner_id": str(row["owner_id"]),
|
|
"filename": row["filename"],
|
|
"content_type": row["content_type"],
|
|
"size_bytes": row["size_bytes"],
|
|
"sha256": row["sha256"],
|
|
"status": "ready",
|
|
"variants": [],
|
|
"error_code": None,
|
|
"created_at": stamp(row["created_at"]),
|
|
"updated_at": stamp(row["updated_at"]),
|
|
}
|
|
|
|
|
|
def valid_filename(value):
|
|
if (not isinstance(value, str) or not 1 <= len(value) <= 255
|
|
or any(char in value for char in ("/", "\\", "\x00"))
|
|
or any(ord(char) < 32 or ord(char) == 127 for char in value)):
|
|
invalid("Некорректный filename")
|
|
return value
|
|
|
|
|
|
def valid_content_type(value):
|
|
if value not in ("image/png", "image/jpeg"):
|
|
raise ApiError(415, "unsupported_media_type", "Допускаются PNG и JPEG")
|
|
return value
|
|
|
|
|
|
def validate_image(path, declared_type):
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
try:
|
|
with Image.open(path) as image:
|
|
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
|
|
if actual_type != declared_type:
|
|
raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому")
|
|
if image.width * image.height > MAX_PIXELS:
|
|
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
|
image.verify()
|
|
except Image.DecompressionBombError:
|
|
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None
|
|
except (UnidentifiedImageError, OSError, ValueError):
|
|
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
|
|
|
|
|
def chat(connection, row):
|
|
members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall()
|
|
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])}
|
|
|
|
|
|
def password_hash(password):
|
|
salt = os.urandom(16)
|
|
derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1)
|
|
return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex()
|
|
|
|
|
|
def password_ok(password, stored):
|
|
try:
|
|
_, n, r, p, salt, expected = stored.split(":")
|
|
actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p))
|
|
return secrets.compare_digest(actual, bytes.fromhex(expected))
|
|
except (ValueError, TypeError):
|
|
return False
|
|
|
|
|
|
def connection():
|
|
try:
|
|
return pool.connection()
|
|
except psycopg.Error:
|
|
unavailable()
|
|
|
|
|
|
def actor(request):
|
|
auth = request.headers.get("authorization", "")
|
|
if not auth.startswith("Bearer ") or not auth[7:]:
|
|
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
|
|
digest = hashlib.sha256(auth[7:].encode()).digest()
|
|
with connection() as db:
|
|
row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone()
|
|
if row is None:
|
|
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
|
|
return row["user_id"]
|
|
|
|
|
|
def member(db, uid, cid):
|
|
row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
return row
|
|
|
|
|
|
def pagination(request):
|
|
raw = request.query_params.get("limit", "50")
|
|
if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100:
|
|
invalid("Некорректный limit")
|
|
return int(raw)
|
|
|
|
|
|
def bump(key):
|
|
with metrics_lock:
|
|
metrics[key] += 1
|
|
|
|
|
|
def cache_get(key):
|
|
try:
|
|
raw = cache.get(key)
|
|
if raw is not None:
|
|
bump("history_cache_hit")
|
|
return json.loads(raw)
|
|
except (redis.RedisError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def cache_set(key, data):
|
|
try:
|
|
cache.set(key, json.dumps(data), ex=30)
|
|
except redis.RedisError:
|
|
pass
|
|
|
|
|
|
def history(db, cid, limit, cursor, version):
|
|
key = f"history:{cid}:{version}:{limit}:{cursor or '-'}"
|
|
hit = cache_get(key) if CACHE_ENABLED else None
|
|
if hit is not None:
|
|
return hit
|
|
lock_key = "lock:" + key
|
|
token = secrets.token_hex(16)
|
|
locked = False
|
|
if CACHE_ENABLED and CACHE_LOCK_ENABLED:
|
|
try:
|
|
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
|
|
if not locked:
|
|
bump("history_lock_wait")
|
|
for _ in range(20):
|
|
time.sleep(0.025)
|
|
hit = cache_get(key)
|
|
if hit is not None:
|
|
return hit
|
|
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
|
|
except redis.RedisError:
|
|
pass
|
|
if CACHE_MISS_DELAY_MS:
|
|
time.sleep(CACHE_MISS_DELAY_MS / 1000)
|
|
if cursor is None:
|
|
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall()
|
|
else:
|
|
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall()
|
|
bump("history_sql")
|
|
result = {"items": [message(row, db) for row in rows[:limit]], "next_cursor": None}
|
|
if len(rows) > limit:
|
|
last = rows[limit - 1]
|
|
result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"]))
|
|
if CACHE_ENABLED:
|
|
cache_set(key, result)
|
|
if locked:
|
|
try:
|
|
cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token)
|
|
except redis.RedisError:
|
|
pass
|
|
return result
|
|
|
|
|
|
def migrate():
|
|
from pathlib import Path
|
|
files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql"))
|
|
with pool.connection() as db:
|
|
with db.transaction():
|
|
db.execute("SELECT pg_advisory_xact_lock(2172002)")
|
|
db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)")
|
|
for file in files:
|
|
if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None:
|
|
db.execute(file.read_text())
|
|
db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,))
|
|
|
|
|
|
def cleanup_expired_uploads():
|
|
with connection() as db:
|
|
rows = db.execute(
|
|
"SELECT id,object_key FROM attachments "
|
|
"WHERE status='uploading' AND upload_expires_at < clock_timestamp()-interval '5 minutes' "
|
|
"FOR UPDATE SKIP LOCKED LIMIT 100"
|
|
).fetchall()
|
|
for row in rows:
|
|
try:
|
|
s3.remove_object(S3_BUCKET, row["object_key"])
|
|
except S3Error as error:
|
|
if error.code not in ("NoSuchKey", "NoSuchObject"):
|
|
continue
|
|
db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],))
|
|
cutoff = datetime.now(timezone.utc) - timedelta(minutes=10)
|
|
for prefix in ("originals/", "direct/"):
|
|
for item in s3.list_objects(S3_BUCKET, prefix=prefix, recursive=True):
|
|
if item.last_modified >= cutoff:
|
|
continue
|
|
known = db.execute(
|
|
"SELECT 1 FROM attachments WHERE object_key=%s", (item.object_name,)
|
|
).fetchone()
|
|
if known is None:
|
|
s3.remove_object(S3_BUCKET, item.object_name)
|
|
|
|
|
|
async def cleanup_loop():
|
|
while True:
|
|
await asyncio.sleep(60)
|
|
try:
|
|
await asyncio.to_thread(cleanup_expired_uploads)
|
|
except (psycopg.Error, MinioException, urllib3.exceptions.HTTPError, OSError):
|
|
pass
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(_app):
|
|
global ready_for_traffic
|
|
pool.open()
|
|
pool.wait(timeout=20)
|
|
if not s3.bucket_exists(S3_BUCKET):
|
|
raise RuntimeError("S3 bucket is not initialized")
|
|
ready_for_traffic = True
|
|
task = asyncio.create_task(cleanup_loop())
|
|
try:
|
|
yield
|
|
finally:
|
|
task.cancel()
|
|
ready_for_traffic = False
|
|
pool.close()
|
|
|
|
|
|
app = FastAPI(lifespan=lifespan)
|
|
|
|
|
|
@app.exception_handler(StarletteHTTPException)
|
|
async def http_error(request, error):
|
|
request_id = request.state.request_id
|
|
code = "not_found" if error.status_code == 404 else "invalid_request"
|
|
return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code)
|
|
|
|
|
|
@app.middleware("http")
|
|
async def envelope(request, call_next):
|
|
request_id = str(uuid.uuid4())
|
|
request.state.request_id = request_id
|
|
try:
|
|
response = await call_next(request)
|
|
except ApiError as exc:
|
|
response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status)
|
|
if exc.challenge:
|
|
response.headers["WWW-Authenticate"] = exc.challenge
|
|
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
|
|
psycopg.errors.LockNotAvailable, PoolTimeout, MinioException,
|
|
urllib3.exceptions.HTTPError, OSError):
|
|
response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503)
|
|
except Exception:
|
|
response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500)
|
|
response.headers["X-Request-Id"] = request_id
|
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
|
return response
|
|
|
|
|
|
@app.get("/health/live")
|
|
def live():
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.get("/health/ready")
|
|
def ready():
|
|
if not ready_for_traffic:
|
|
unavailable()
|
|
with connection() as db:
|
|
db.execute("SELECT 1")
|
|
if not s3.bucket_exists(S3_BUCKET):
|
|
unavailable()
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.post("/api/v1/users", status_code=201)
|
|
async def create_user(request: Request):
|
|
data = await body(request, ("username", "display_name", "password"))
|
|
name = data["username"]
|
|
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name):
|
|
invalid("Некорректный username")
|
|
display = text_field(data["display_name"], 100)
|
|
password = data["password"]
|
|
if not isinstance(password, str) or not password.strip() or not 12 <= len(password) <= 128:
|
|
invalid("Некорректный пароль")
|
|
try:
|
|
return await asyncio.to_thread(insert_user, name, display, password)
|
|
except psycopg.errors.UniqueViolation:
|
|
raise ApiError(409, "conflict", "Username уже занят") from None
|
|
|
|
|
|
def insert_user(name, display, password):
|
|
digest = password_hash(password)
|
|
with connection() as db:
|
|
row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone()
|
|
return user(row)
|
|
|
|
|
|
@app.post("/api/v1/auth/sessions", status_code=201)
|
|
async def login(request: Request):
|
|
async for chunk in request.stream():
|
|
if chunk:
|
|
invalid("Тело запроса не допускается")
|
|
auth = request.headers.get("authorization", "")
|
|
try:
|
|
if not auth.startswith("Basic "):
|
|
raise ValueError()
|
|
decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8")
|
|
name, password = decoded.split(":", 1)
|
|
except (ValueError, UnicodeError, binascii.Error):
|
|
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None
|
|
return await asyncio.to_thread(create_session, name, password)
|
|
|
|
|
|
def create_session(name, password):
|
|
with connection() as db:
|
|
row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone()
|
|
if row is None or not password_ok(password, row["password_hash"]):
|
|
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"')
|
|
token = secrets.token_urlsafe(32)
|
|
expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)
|
|
db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry))
|
|
return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)}
|
|
|
|
|
|
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
|
def logout(request: Request):
|
|
actor(request)
|
|
token = request.headers["authorization"][7:]
|
|
with connection() as db:
|
|
db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),))
|
|
return Response(status_code=204)
|
|
|
|
|
|
@app.get("/api/v1/users/me")
|
|
def me(request: Request):
|
|
uid = actor(request)
|
|
with connection() as db:
|
|
row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone()
|
|
return user(row)
|
|
|
|
|
|
@app.post("/api/v1/chats", status_code=201)
|
|
async def create_chat(request: Request):
|
|
uid = actor(request)
|
|
data = await body(request, ("title", "member_ids"))
|
|
title = text_field(data["title"], 100)
|
|
raw = data["member_ids"]
|
|
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
|
|
invalid("Некорректный список участников")
|
|
members = [identifier(item) for item in raw]
|
|
if len(set(members)) != len(members) or str(uid) not in members:
|
|
invalid("Некорректный список участников")
|
|
return await asyncio.to_thread(insert_chat, uid, title, members)
|
|
|
|
|
|
def insert_chat(uid, title, members):
|
|
with connection() as db:
|
|
with db.transaction():
|
|
count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"]
|
|
if count != len(members):
|
|
invalid("Неизвестный участник")
|
|
row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone()
|
|
for member_id in members:
|
|
db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id))
|
|
db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],))
|
|
return chat(db, row)
|
|
|
|
|
|
@app.get("/api/v1/chats")
|
|
def list_chats(request: Request):
|
|
uid = actor(request)
|
|
limit = pagination(request)
|
|
cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor"))
|
|
with connection() as db:
|
|
if cursor is None:
|
|
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall()
|
|
else:
|
|
valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone()
|
|
if valid is None:
|
|
invalid("Некорректный cursor")
|
|
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall()
|
|
items = [chat(db, row) for row in rows[:limit]]
|
|
next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None
|
|
return {"items": items, "next_cursor": next_cursor}
|
|
|
|
|
|
@app.get("/api/v1/chats/{chat_id}")
|
|
def get_chat(chat_id: str, request: Request):
|
|
uid = actor(request)
|
|
cid = identifier(chat_id)
|
|
with connection() as db:
|
|
return chat(db, member(db, uid, cid))
|
|
|
|
|
|
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
|
async def create_message(chat_id: str, request: Request):
|
|
uid = actor(request)
|
|
cid = identifier(chat_id)
|
|
key = request.headers.get("idempotency-key", "")
|
|
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII):
|
|
invalid("Некорректный Idempotency-Key")
|
|
data = await body(request, ("text",)) if request.headers.get("content-length") == "0" else None
|
|
if data is None:
|
|
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
|
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
|
declared = request.headers.get("content-length")
|
|
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
|
|
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
|
raw = bytearray()
|
|
async for chunk in request.stream():
|
|
raw.extend(chunk)
|
|
if len(raw) > MAX_BODY:
|
|
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
|
|
try:
|
|
data = json.loads(raw.decode("utf-8"))
|
|
except (UnicodeError, ValueError):
|
|
invalid("Некорректный JSON")
|
|
if not isinstance(data, dict) or set(data) not in ({"text"}, {"text", "attachment_ids"}):
|
|
invalid("Некорректные поля JSON")
|
|
content = data["text"]
|
|
if not isinstance(content, str) or len(content) > 4000:
|
|
invalid("Некорректный текст")
|
|
raw_ids = data.get("attachment_ids", [])
|
|
if (not isinstance(raw_ids, list) or len(raw_ids) > 10
|
|
or len(raw_ids) != len(set(raw_ids))):
|
|
invalid("Некорректные вложения")
|
|
attachment_ids = [identifier(item) for item in raw_ids]
|
|
if not content.strip() and not attachment_ids:
|
|
invalid("Пустое сообщение")
|
|
return await asyncio.to_thread(insert_message, uid, cid, content, key, attachment_ids)
|
|
|
|
|
|
def insert_message(uid, cid, content, key, attachment_ids):
|
|
with connection() as db:
|
|
with db.transaction():
|
|
member(db, uid, cid)
|
|
if attachment_ids:
|
|
rows = db.execute(
|
|
"SELECT id FROM attachments WHERE id=ANY(%s::uuid[]) AND chat_id=%s AND status='ready'",
|
|
(attachment_ids, cid),
|
|
).fetchall()
|
|
if len(rows) != len(attachment_ids):
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
# The unique constraint serializes concurrent requests on any replica.
|
|
db.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()", (uid, cid, key))
|
|
normalized = {"text": content, "attachment_ids": attachment_ids}
|
|
body_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True).encode()).digest()
|
|
claimed = db.execute("INSERT INTO message_idempotency(user_id,chat_id,key,body_hash,expires_at) VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING RETURNING key", (uid, cid, key, body_hash)).fetchone()
|
|
if claimed is None:
|
|
previous = db.execute("SELECT body_hash,message_id FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", (uid, cid, key)).fetchone()
|
|
if previous is None or previous["body_hash"] != body_hash:
|
|
raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения")
|
|
return message(db.execute("SELECT * FROM messages WHERE id=%s", (previous["message_id"],)).fetchone(), db)
|
|
row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone()
|
|
for position, attachment_id in enumerate(attachment_ids):
|
|
db.execute(
|
|
"INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)",
|
|
(row["id"], attachment_id, position),
|
|
)
|
|
db.execute("UPDATE message_idempotency SET message_id=%s WHERE user_id=%s AND chat_id=%s AND key=%s", (row["id"], uid, cid, key))
|
|
db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,))
|
|
return message(row, db)
|
|
|
|
|
|
@app.get("/api/v1/chats/{chat_id}/messages")
|
|
def list_messages(chat_id: str, request: Request):
|
|
uid = actor(request)
|
|
cid = identifier(chat_id)
|
|
limit = pagination(request)
|
|
cursor = decode_cursor(cid, request.query_params.get("cursor"))
|
|
with connection() as db:
|
|
member(db, uid, cid)
|
|
if cursor is not None:
|
|
valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone()
|
|
if valid is None:
|
|
invalid("Некорректный cursor")
|
|
version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"]
|
|
return history(db, cid, limit, cursor, version)
|
|
|
|
|
|
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
|
async def upload_attachment(chat_id: str, request: Request):
|
|
uid = actor(request)
|
|
cid = identifier(chat_id)
|
|
filename = valid_filename(request.query_params.get("filename"))
|
|
content_type = valid_content_type(
|
|
request.headers.get("content-type", "").split(";")[0].strip().lower()
|
|
)
|
|
declared = request.headers.get("content-length")
|
|
if declared is not None and declared.isdecimal() and int(declared) > MAX_ATTACHMENT:
|
|
raise ApiError(413, "payload_too_large", "Файл слишком велик")
|
|
with connection() as db:
|
|
member(db, uid, cid)
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
with tempfile.NamedTemporaryFile() as temporary:
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > MAX_ATTACHMENT:
|
|
raise ApiError(413, "payload_too_large", "Файл слишком велик")
|
|
digest.update(chunk)
|
|
temporary.write(chunk)
|
|
if size == 0:
|
|
raise ApiError(415, "unsupported_media_type", "Пустой файл")
|
|
temporary.flush()
|
|
validate_image(temporary.name, content_type)
|
|
aid = uuid.uuid4()
|
|
object_key = f"originals/{aid.hex}"
|
|
temporary.seek(0)
|
|
await asyncio.to_thread(
|
|
s3.put_object, S3_BUCKET, object_key, temporary, size, content_type
|
|
)
|
|
try:
|
|
return await asyncio.to_thread(
|
|
save_proxy_attachment,
|
|
aid,
|
|
uid,
|
|
cid,
|
|
filename,
|
|
content_type,
|
|
size,
|
|
digest.hexdigest(),
|
|
object_key,
|
|
)
|
|
except Exception:
|
|
try:
|
|
await asyncio.to_thread(s3.remove_object, S3_BUCKET, object_key)
|
|
except (MinioException, urllib3.exceptions.HTTPError, OSError):
|
|
pass
|
|
raise
|
|
|
|
|
|
def save_proxy_attachment(aid, uid, cid, filename, content_type, size, digest, object_key):
|
|
with connection() as db:
|
|
row = db.execute(
|
|
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) "
|
|
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *",
|
|
(aid, cid, uid, filename, content_type, size, digest, object_key),
|
|
).fetchone()
|
|
return attachment(row)
|
|
|
|
|
|
@app.post("/api/v1/chats/{chat_id}/attachments/presign", status_code=201)
|
|
async def presign_attachment(chat_id: str, request: Request):
|
|
uid = actor(request)
|
|
cid = identifier(chat_id)
|
|
data = await body(request, ("filename", "content_type", "size_bytes", "sha256"))
|
|
filename = valid_filename(data["filename"])
|
|
content_type = valid_content_type(data["content_type"])
|
|
size = data["size_bytes"]
|
|
digest = data["sha256"]
|
|
if not isinstance(size, int) or isinstance(size, bool) or not 1 <= size <= MAX_ATTACHMENT:
|
|
raise ApiError(413, "payload_too_large", "Некорректный размер")
|
|
if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest):
|
|
invalid("Некорректный SHA-256")
|
|
with connection() as db:
|
|
member(db, uid, cid)
|
|
aid = uuid.uuid4()
|
|
object_key = f"direct/{aid.hex}/{secrets.token_hex(16)}"
|
|
expires = datetime.now(timezone.utc) + timedelta(seconds=PRESIGN_TTL)
|
|
db.execute(
|
|
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,upload_expires_at) "
|
|
"VALUES (%s,%s,%s,%s,%s,%s,%s,'uploading',%s,%s)",
|
|
(aid, cid, uid, filename, content_type, size, digest, object_key, expires),
|
|
)
|
|
url = s3_signer.presigned_put_object(
|
|
S3_BUCKET, object_key, expires=timedelta(seconds=PRESIGN_TTL)
|
|
)
|
|
return {
|
|
"attachment_id": str(aid),
|
|
"upload_url": url,
|
|
"expires_at": stamp(expires),
|
|
"required_headers": {"Content-Type": content_type},
|
|
}
|
|
|
|
|
|
@app.post("/api/v1/attachments/{attachment_id}/complete")
|
|
async def complete_attachment(attachment_id: str, request: Request):
|
|
uid = actor(request)
|
|
aid = identifier(attachment_id)
|
|
data = await body(request, ())
|
|
if data:
|
|
invalid("Некорректные поля JSON")
|
|
return await asyncio.to_thread(complete_direct_attachment, uid, aid)
|
|
|
|
|
|
def complete_direct_attachment(uid, aid):
|
|
with connection() as db:
|
|
row = db.execute(
|
|
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
|
|
"WHERE a.id=%s AND m.user_id=%s AND a.owner_id=%s FOR UPDATE OF a",
|
|
(aid, uid, uid),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
if row["status"] == "ready":
|
|
return attachment(row)
|
|
if row["upload_expires_at"] < datetime.now(timezone.utc):
|
|
raise ApiError(409, "conflict", "Срок загрузки истёк")
|
|
try:
|
|
info = s3.stat_object(S3_BUCKET, row["object_key"])
|
|
except S3Error as error:
|
|
if error.code in ("NoSuchKey", "NoSuchObject"):
|
|
raise ApiError(409, "conflict", "Файл ещё не загружен") from None
|
|
raise
|
|
if info.size != row["size_bytes"] or info.content_type != row["content_type"]:
|
|
reject_direct_upload(db, row)
|
|
raise ApiError(409, "conflict", "Размер или Content-Type не совпадает")
|
|
response = s3.get_object(S3_BUCKET, row["object_key"])
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
try:
|
|
with tempfile.NamedTemporaryFile() as temporary:
|
|
while True:
|
|
chunk = response.read(64 * 1024)
|
|
if not chunk:
|
|
break
|
|
size += len(chunk)
|
|
if size > MAX_ATTACHMENT:
|
|
reject_direct_upload(db, row)
|
|
raise ApiError(413, "payload_too_large", "Файл слишком велик")
|
|
digest.update(chunk)
|
|
temporary.write(chunk)
|
|
temporary.flush()
|
|
validate_image(temporary.name, row["content_type"])
|
|
finally:
|
|
response.close()
|
|
response.release_conn()
|
|
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
|
reject_direct_upload(db, row)
|
|
raise ApiError(409, "conflict", "Checksum не совпадает")
|
|
ready_row = db.execute(
|
|
"UPDATE attachments SET status='ready',upload_expires_at=NULL,updated_at=clock_timestamp() "
|
|
"WHERE id=%s RETURNING *",
|
|
(aid,),
|
|
).fetchone()
|
|
return attachment(ready_row)
|
|
|
|
|
|
def reject_direct_upload(db, row):
|
|
s3.remove_object(S3_BUCKET, row["object_key"])
|
|
db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],))
|
|
|
|
|
|
def accessible_attachment(uid, aid, ready_only=True):
|
|
with connection() as db:
|
|
row = db.execute(
|
|
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
|
|
"WHERE a.id=%s AND m.user_id=%s",
|
|
(aid, uid),
|
|
).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
if ready_only and row["status"] != "ready":
|
|
raise ApiError(409, "conflict", "Вложение ещё не готово")
|
|
return row
|
|
|
|
|
|
@app.get("/api/v1/attachments/{attachment_id}")
|
|
def get_attachment(attachment_id: str, request: Request):
|
|
uid = actor(request)
|
|
return attachment(accessible_attachment(uid, identifier(attachment_id)))
|
|
|
|
|
|
@app.get("/api/v1/attachments/{attachment_id}/content")
|
|
def download_attachment(attachment_id: str, request: Request):
|
|
uid = actor(request)
|
|
if request.query_params.get("variant", "original") != "original":
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
row = accessible_attachment(uid, identifier(attachment_id))
|
|
source = s3.get_object(S3_BUCKET, row["object_key"])
|
|
|
|
def stream():
|
|
try:
|
|
while True:
|
|
chunk = source.read(64 * 1024)
|
|
if not chunk:
|
|
break
|
|
yield chunk
|
|
finally:
|
|
source.close()
|
|
source.release_conn()
|
|
|
|
return StreamingResponse(
|
|
stream(),
|
|
media_type=row["content_type"],
|
|
headers={"Content-Length": str(row["size_bytes"])},
|
|
)
|
|
|
|
|
|
@app.get("/internal/cache-metrics")
|
|
def cache_metrics(request: Request):
|
|
if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"):
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
with metrics_lock:
|
|
return dict(metrics)
|