Files
2026-11-06 15:43:25 +03:00

882 lines
37 KiB
Python

"""Replicated messenger API with PostgreSQL-backed sessions and idempotency."""
import asyncio
import base64
import binascii
import hashlib
import json
import os
import re
import secrets
import socket
import tempfile
import threading
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
import psycopg
import redis
import urllib3
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from minio import Minio
from minio.error import MinioException, S3Error
from PIL import Image, UnidentifiedImageError
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from starlette.exceptions import HTTPException as StarletteHTTPException
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger")
database_password = os.environ.get("DATABASE_PASSWORD")
if "DATABASE_PASSWORD_FILE" in os.environ:
with open(os.environ["DATABASE_PASSWORD_FILE"], encoding="utf-8") as secret_file:
database_password = secret_file.read().rstrip("\n")
if database_password is not None:
DATABASE_URL = make_conninfo(
host=os.environ.get("DATABASE_HOST", "postgres"),
dbname=os.environ.get("DATABASE_NAME", "messenger"),
user=os.environ.get("DATABASE_USER", "messenger"),
password=database_password,
)
REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384"))
POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10"))
CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1"
CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1"
CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0"))))
INSTANCE_ID = os.environ.get("INSTANCE_ID", "") or socket.gethostname()
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "minio:9000")
S3_PUBLIC_ENDPOINT = os.environ.get("S3_PUBLIC_ENDPOINT", "localhost:9000")
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "messenger-api")
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "messenger-api-secret")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_SECURE = os.environ.get("S3_SECURE", "0") == "1"
PRESIGN_TTL = min(900, max(30, int(os.environ.get("PRESIGN_TTL_SECONDS", "120"))))
MAX_ATTACHMENT = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
ready_for_traffic = False
pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"})
cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2)
s3_http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=2, read=15),
retries=False,
maxsize=max(10, POOL_SIZE * 2),
)
s3 = Minio(
S3_ENDPOINT,
access_key=S3_ACCESS_KEY,
secret_key=S3_SECRET_KEY,
secure=S3_SECURE,
http_client=s3_http,
)
s3_signer = Minio(S3_PUBLIC_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=S3_SECURE)
metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0}
metrics_lock = threading.Lock()
class ApiError(Exception):
def __init__(self, status, code, message, challenge=None):
self.status, self.code, self.message, self.challenge = status, code, message, challenge
def invalid(message="Некорректный запрос"):
raise ApiError(400, "invalid_request", message)
def unavailable():
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def encode_cursor(scope, date, identifier):
raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
def decode_cursor(scope, value):
if value is None:
return None
try:
raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
parts = json.loads(raw)
if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope
or not isinstance(parts[1], str) or not isinstance(parts[2], str)):
invalid("Некорректный cursor")
date = datetime.fromisoformat(parts[1].replace("Z", "+00:00"))
if date.tzinfo is None:
invalid("Некорректный cursor")
return date, str(uuid.UUID(parts[2]))
except (ValueError, TypeError, binascii.Error, IndexError):
invalid("Некорректный cursor")
def identifier(value):
try:
result = str(uuid.UUID(value))
if result != value.lower() or len(value) != 36:
invalid("Некорректный UUID")
return result
except (ValueError, TypeError, AttributeError):
invalid("Некорректный UUID")
def text_field(value, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
invalid("Некорректное текстовое поле")
return value
def exact_body(value, fields):
if not isinstance(value, dict) or set(value) != set(fields):
invalid("Некорректные поля JSON")
async def body(request, fields):
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
declared = request.headers.get("content-length")
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
chunks = bytearray()
async for chunk in request.stream():
chunks.extend(chunk)
if len(chunks) > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
try:
value = json.loads(chunks.decode("utf-8"))
except (UnicodeError, ValueError):
invalid("Некорректный JSON")
exact_body(value, fields)
return value
def user(row):
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])}
def message(row, db=None):
attachment_ids = []
if db is not None:
linked = db.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position",
(row["id"],),
).fetchall()
attachment_ids = [str(item["attachment_id"]) for item in linked]
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": stamp(row["created_at"]),
"attachment_ids": attachment_ids,
}
def attachment(row):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": "ready",
"variants": [],
"error_code": None,
"created_at": stamp(row["created_at"]),
"updated_at": stamp(row["updated_at"]),
}
def valid_filename(value):
if (not isinstance(value, str) or not 1 <= len(value) <= 255
or any(char in value for char in ("/", "\\", "\x00"))
or any(ord(char) < 32 or ord(char) == 127 for char in value)):
invalid("Некорректный filename")
return value
def valid_content_type(value):
if value not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Допускаются PNG и JPEG")
return value
def validate_image(path, declared_type):
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
try:
with Image.open(path) as image:
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
if actual_type != declared_type:
raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому")
if image.width * image.height > MAX_PIXELS:
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
image.verify()
except Image.DecompressionBombError:
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None
except (UnidentifiedImageError, OSError, ValueError):
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None
def chat(connection, row):
members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall()
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])}
def password_hash(password):
salt = os.urandom(16)
derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1)
return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex()
def password_ok(password, stored):
try:
_, n, r, p, salt, expected = stored.split(":")
actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p))
return secrets.compare_digest(actual, bytes.fromhex(expected))
except (ValueError, TypeError):
return False
def connection():
try:
return pool.connection()
except psycopg.Error:
unavailable()
def actor(request):
auth = request.headers.get("authorization", "")
if not auth.startswith("Bearer ") or not auth[7:]:
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
digest = hashlib.sha256(auth[7:].encode()).digest()
with connection() as db:
row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone()
if row is None:
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
return row["user_id"]
def member(db, uid, cid):
row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
return row
def pagination(request):
raw = request.query_params.get("limit", "50")
if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100:
invalid("Некорректный limit")
return int(raw)
def bump(key):
with metrics_lock:
metrics[key] += 1
def cache_get(key):
try:
raw = cache.get(key)
if raw is not None:
bump("history_cache_hit")
return json.loads(raw)
except (redis.RedisError, ValueError):
pass
return None
def cache_set(key, data):
try:
cache.set(key, json.dumps(data), ex=30)
except redis.RedisError:
pass
def history(db, cid, limit, cursor, version):
key = f"history:{cid}:{version}:{limit}:{cursor or '-'}"
hit = cache_get(key) if CACHE_ENABLED else None
if hit is not None:
return hit
lock_key = "lock:" + key
token = secrets.token_hex(16)
locked = False
if CACHE_ENABLED and CACHE_LOCK_ENABLED:
try:
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
if not locked:
bump("history_lock_wait")
for _ in range(20):
time.sleep(0.025)
hit = cache_get(key)
if hit is not None:
return hit
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
except redis.RedisError:
pass
if CACHE_MISS_DELAY_MS:
time.sleep(CACHE_MISS_DELAY_MS / 1000)
if cursor is None:
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall()
else:
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall()
bump("history_sql")
result = {"items": [message(row, db) for row in rows[:limit]], "next_cursor": None}
if len(rows) > limit:
last = rows[limit - 1]
result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"]))
if CACHE_ENABLED:
cache_set(key, result)
if locked:
try:
cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token)
except redis.RedisError:
pass
return result
def migrate():
from pathlib import Path
files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql"))
with pool.connection() as db:
with db.transaction():
db.execute("SELECT pg_advisory_xact_lock(2172002)")
db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)")
for file in files:
if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None:
db.execute(file.read_text())
db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,))
def cleanup_expired_uploads():
with connection() as db:
rows = db.execute(
"SELECT id,object_key FROM attachments "
"WHERE status='uploading' AND upload_expires_at < clock_timestamp()-interval '5 minutes' "
"FOR UPDATE SKIP LOCKED LIMIT 100"
).fetchall()
for row in rows:
try:
s3.remove_object(S3_BUCKET, row["object_key"])
except S3Error as error:
if error.code not in ("NoSuchKey", "NoSuchObject"):
continue
db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],))
cutoff = datetime.now(timezone.utc) - timedelta(minutes=10)
for prefix in ("originals/", "direct/"):
for item in s3.list_objects(S3_BUCKET, prefix=prefix, recursive=True):
if item.last_modified >= cutoff:
continue
known = db.execute(
"SELECT 1 FROM attachments WHERE object_key=%s", (item.object_name,)
).fetchone()
if known is None:
s3.remove_object(S3_BUCKET, item.object_name)
async def cleanup_loop():
while True:
await asyncio.sleep(60)
try:
await asyncio.to_thread(cleanup_expired_uploads)
except (psycopg.Error, MinioException, urllib3.exceptions.HTTPError, OSError):
pass
@asynccontextmanager
async def lifespan(_app):
global ready_for_traffic
pool.open()
pool.wait(timeout=20)
if not s3.bucket_exists(S3_BUCKET):
raise RuntimeError("S3 bucket is not initialized")
ready_for_traffic = True
task = asyncio.create_task(cleanup_loop())
try:
yield
finally:
task.cancel()
ready_for_traffic = False
pool.close()
app = FastAPI(lifespan=lifespan)
@app.exception_handler(StarletteHTTPException)
async def http_error(request, error):
request_id = request.state.request_id
code = "not_found" if error.status_code == 404 else "invalid_request"
return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code)
@app.middleware("http")
async def envelope(request, call_next):
request_id = str(uuid.uuid4())
request.state.request_id = request_id
try:
response = await call_next(request)
except ApiError as exc:
response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status)
if exc.challenge:
response.headers["WWW-Authenticate"] = exc.challenge
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable, PoolTimeout, MinioException,
urllib3.exceptions.HTTPError, OSError):
response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503)
except Exception:
response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready():
if not ready_for_traffic:
unavailable()
with connection() as db:
db.execute("SELECT 1")
if not s3.bucket_exists(S3_BUCKET):
unavailable()
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
data = await body(request, ("username", "display_name", "password"))
name = data["username"]
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name):
invalid("Некорректный username")
display = text_field(data["display_name"], 100)
password = data["password"]
if not isinstance(password, str) or not password.strip() or not 12 <= len(password) <= 128:
invalid("Некорректный пароль")
try:
return await asyncio.to_thread(insert_user, name, display, password)
except psycopg.errors.UniqueViolation:
raise ApiError(409, "conflict", "Username уже занят") from None
def insert_user(name, display, password):
digest = password_hash(password)
with connection() as db:
row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone()
return user(row)
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
async for chunk in request.stream():
if chunk:
invalid("Тело запроса не допускается")
auth = request.headers.get("authorization", "")
try:
if not auth.startswith("Basic "):
raise ValueError()
decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8")
name, password = decoded.split(":", 1)
except (ValueError, UnicodeError, binascii.Error):
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None
return await asyncio.to_thread(create_session, name, password)
def create_session(name, password):
with connection() as db:
row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone()
if row is None or not password_ok(password, row["password_hash"]):
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"')
token = secrets.token_urlsafe(32)
expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)
db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry))
return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request):
actor(request)
token = request.headers["authorization"][7:]
with connection() as db:
db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),))
return Response(status_code=204)
@app.get("/api/v1/users/me")
def me(request: Request):
uid = actor(request)
with connection() as db:
row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone()
return user(row)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
uid = actor(request)
data = await body(request, ("title", "member_ids"))
title = text_field(data["title"], 100)
raw = data["member_ids"]
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
invalid("Некорректный список участников")
members = [identifier(item) for item in raw]
if len(set(members)) != len(members) or str(uid) not in members:
invalid("Некорректный список участников")
return await asyncio.to_thread(insert_chat, uid, title, members)
def insert_chat(uid, title, members):
with connection() as db:
with db.transaction():
count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"]
if count != len(members):
invalid("Неизвестный участник")
row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone()
for member_id in members:
db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id))
db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],))
return chat(db, row)
@app.get("/api/v1/chats")
def list_chats(request: Request):
uid = actor(request)
limit = pagination(request)
cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor"))
with connection() as db:
if cursor is None:
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall()
else:
valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone()
if valid is None:
invalid("Некорректный cursor")
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall()
items = [chat(db, row) for row in rows[:limit]]
next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None
return {"items": items, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
with connection() as db:
return chat(db, member(db, uid, cid))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
key = request.headers.get("idempotency-key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII):
invalid("Некорректный Idempotency-Key")
data = await body(request, ("text",)) if request.headers.get("content-length") == "0" else None
if data is None:
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
declared = request.headers.get("content-length")
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
raw = bytearray()
async for chunk in request.stream():
raw.extend(chunk)
if len(raw) > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
try:
data = json.loads(raw.decode("utf-8"))
except (UnicodeError, ValueError):
invalid("Некорректный JSON")
if not isinstance(data, dict) or set(data) not in ({"text"}, {"text", "attachment_ids"}):
invalid("Некорректные поля JSON")
content = data["text"]
if not isinstance(content, str) or len(content) > 4000:
invalid("Некорректный текст")
raw_ids = data.get("attachment_ids", [])
if (not isinstance(raw_ids, list) or len(raw_ids) > 10
or len(raw_ids) != len(set(raw_ids))):
invalid("Некорректные вложения")
attachment_ids = [identifier(item) for item in raw_ids]
if not content.strip() and not attachment_ids:
invalid("Пустое сообщение")
return await asyncio.to_thread(insert_message, uid, cid, content, key, attachment_ids)
def insert_message(uid, cid, content, key, attachment_ids):
with connection() as db:
with db.transaction():
member(db, uid, cid)
if attachment_ids:
rows = db.execute(
"SELECT id FROM attachments WHERE id=ANY(%s::uuid[]) AND chat_id=%s AND status='ready'",
(attachment_ids, cid),
).fetchall()
if len(rows) != len(attachment_ids):
raise ApiError(404, "not_found", "Объект не найден")
# The unique constraint serializes concurrent requests on any replica.
db.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()", (uid, cid, key))
normalized = {"text": content, "attachment_ids": attachment_ids}
body_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True).encode()).digest()
claimed = db.execute("INSERT INTO message_idempotency(user_id,chat_id,key,body_hash,expires_at) VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING RETURNING key", (uid, cid, key, body_hash)).fetchone()
if claimed is None:
previous = db.execute("SELECT body_hash,message_id FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", (uid, cid, key)).fetchone()
if previous is None or previous["body_hash"] != body_hash:
raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения")
return message(db.execute("SELECT * FROM messages WHERE id=%s", (previous["message_id"],)).fetchone(), db)
row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone()
for position, attachment_id in enumerate(attachment_ids):
db.execute(
"INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)",
(row["id"], attachment_id, position),
)
db.execute("UPDATE message_idempotency SET message_id=%s WHERE user_id=%s AND chat_id=%s AND key=%s", (row["id"], uid, cid, key))
db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,))
return message(row, db)
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
limit = pagination(request)
cursor = decode_cursor(cid, request.query_params.get("cursor"))
with connection() as db:
member(db, uid, cid)
if cursor is not None:
valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone()
if valid is None:
invalid("Некорректный cursor")
version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"]
return history(db, cid, limit, cursor, version)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
async def upload_attachment(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
filename = valid_filename(request.query_params.get("filename"))
content_type = valid_content_type(
request.headers.get("content-type", "").split(";")[0].strip().lower()
)
declared = request.headers.get("content-length")
if declared is not None and declared.isdecimal() and int(declared) > MAX_ATTACHMENT:
raise ApiError(413, "payload_too_large", "Файл слишком велик")
with connection() as db:
member(db, uid, cid)
digest = hashlib.sha256()
size = 0
with tempfile.NamedTemporaryFile() as temporary:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_ATTACHMENT:
raise ApiError(413, "payload_too_large", "Файл слишком велик")
digest.update(chunk)
temporary.write(chunk)
if size == 0:
raise ApiError(415, "unsupported_media_type", "Пустой файл")
temporary.flush()
validate_image(temporary.name, content_type)
aid = uuid.uuid4()
object_key = f"originals/{aid.hex}"
temporary.seek(0)
await asyncio.to_thread(
s3.put_object, S3_BUCKET, object_key, temporary, size, content_type
)
try:
return await asyncio.to_thread(
save_proxy_attachment,
aid,
uid,
cid,
filename,
content_type,
size,
digest.hexdigest(),
object_key,
)
except Exception:
try:
await asyncio.to_thread(s3.remove_object, S3_BUCKET, object_key)
except (MinioException, urllib3.exceptions.HTTPError, OSError):
pass
raise
def save_proxy_attachment(aid, uid, cid, filename, content_type, size, digest, object_key):
with connection() as db:
row = db.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *",
(aid, cid, uid, filename, content_type, size, digest, object_key),
).fetchone()
return attachment(row)
@app.post("/api/v1/chats/{chat_id}/attachments/presign", status_code=201)
async def presign_attachment(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
data = await body(request, ("filename", "content_type", "size_bytes", "sha256"))
filename = valid_filename(data["filename"])
content_type = valid_content_type(data["content_type"])
size = data["size_bytes"]
digest = data["sha256"]
if not isinstance(size, int) or isinstance(size, bool) or not 1 <= size <= MAX_ATTACHMENT:
raise ApiError(413, "payload_too_large", "Некорректный размер")
if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest):
invalid("Некорректный SHA-256")
with connection() as db:
member(db, uid, cid)
aid = uuid.uuid4()
object_key = f"direct/{aid.hex}/{secrets.token_hex(16)}"
expires = datetime.now(timezone.utc) + timedelta(seconds=PRESIGN_TTL)
db.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key,upload_expires_at) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,'uploading',%s,%s)",
(aid, cid, uid, filename, content_type, size, digest, object_key, expires),
)
url = s3_signer.presigned_put_object(
S3_BUCKET, object_key, expires=timedelta(seconds=PRESIGN_TTL)
)
return {
"attachment_id": str(aid),
"upload_url": url,
"expires_at": stamp(expires),
"required_headers": {"Content-Type": content_type},
}
@app.post("/api/v1/attachments/{attachment_id}/complete")
async def complete_attachment(attachment_id: str, request: Request):
uid = actor(request)
aid = identifier(attachment_id)
data = await body(request, ())
if data:
invalid("Некорректные поля JSON")
return await asyncio.to_thread(complete_direct_attachment, uid, aid)
def complete_direct_attachment(uid, aid):
with connection() as db:
row = db.execute(
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s AND a.owner_id=%s FOR UPDATE OF a",
(aid, uid, uid),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
if row["status"] == "ready":
return attachment(row)
if row["upload_expires_at"] < datetime.now(timezone.utc):
raise ApiError(409, "conflict", "Срок загрузки истёк")
try:
info = s3.stat_object(S3_BUCKET, row["object_key"])
except S3Error as error:
if error.code in ("NoSuchKey", "NoSuchObject"):
raise ApiError(409, "conflict", "Файл ещё не загружен") from None
raise
if info.size != row["size_bytes"] or info.content_type != row["content_type"]:
reject_direct_upload(db, row)
raise ApiError(409, "conflict", "Размер или Content-Type не совпадает")
response = s3.get_object(S3_BUCKET, row["object_key"])
digest = hashlib.sha256()
size = 0
try:
with tempfile.NamedTemporaryFile() as temporary:
while True:
chunk = response.read(64 * 1024)
if not chunk:
break
size += len(chunk)
if size > MAX_ATTACHMENT:
reject_direct_upload(db, row)
raise ApiError(413, "payload_too_large", "Файл слишком велик")
digest.update(chunk)
temporary.write(chunk)
temporary.flush()
validate_image(temporary.name, row["content_type"])
finally:
response.close()
response.release_conn()
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
reject_direct_upload(db, row)
raise ApiError(409, "conflict", "Checksum не совпадает")
ready_row = db.execute(
"UPDATE attachments SET status='ready',upload_expires_at=NULL,updated_at=clock_timestamp() "
"WHERE id=%s RETURNING *",
(aid,),
).fetchone()
return attachment(ready_row)
def reject_direct_upload(db, row):
s3.remove_object(S3_BUCKET, row["object_key"])
db.execute("DELETE FROM attachments WHERE id=%s AND status='uploading'", (row["id"],))
def accessible_attachment(uid, aid, ready_only=True):
with connection() as db:
row = db.execute(
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(aid, uid),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
if ready_only and row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё не готово")
return row
@app.get("/api/v1/attachments/{attachment_id}")
def get_attachment(attachment_id: str, request: Request):
uid = actor(request)
return attachment(accessible_attachment(uid, identifier(attachment_id)))
@app.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(attachment_id: str, request: Request):
uid = actor(request)
if request.query_params.get("variant", "original") != "original":
raise ApiError(404, "not_found", "Объект не найден")
row = accessible_attachment(uid, identifier(attachment_id))
source = s3.get_object(S3_BUCKET, row["object_key"])
def stream():
try:
while True:
chunk = source.read(64 * 1024)
if not chunk:
break
yield chunk
finally:
source.close()
source.release_conn()
return StreamingResponse(
stream(),
media_type=row["content_type"],
headers={"Content-Length": str(row["size_bytes"])},
)
@app.get("/internal/cache-metrics")
def cache_metrics(request: Request):
if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"):
raise ApiError(404, "not_found", "Объект не найден")
with metrics_lock:
return dict(metrics)