Files
2026-10-23 19:29:40 +03:00

501 lines
21 KiB
Python

"""Replicated messenger API with PostgreSQL-backed sessions and idempotency."""
import asyncio
import base64
import binascii
import hashlib
import json
import os
import re
import secrets
import socket
import threading
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
import psycopg
import redis
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from starlette.exceptions import HTTPException as StarletteHTTPException
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger")
database_password = os.environ.get("DATABASE_PASSWORD")
if "DATABASE_PASSWORD_FILE" in os.environ:
with open(os.environ["DATABASE_PASSWORD_FILE"], encoding="utf-8") as secret_file:
database_password = secret_file.read().rstrip("\n")
if database_password is not None:
DATABASE_URL = make_conninfo(
host=os.environ.get("DATABASE_HOST", "postgres"),
dbname=os.environ.get("DATABASE_NAME", "messenger"),
user=os.environ.get("DATABASE_USER", "messenger"),
password=database_password,
)
REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384"))
POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10"))
CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1"
CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1"
CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0"))))
INSTANCE_ID = os.environ.get("INSTANCE_ID", "") or socket.gethostname()
ready_for_traffic = False
pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"})
cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2)
metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0}
metrics_lock = threading.Lock()
class ApiError(Exception):
def __init__(self, status, code, message, challenge=None):
self.status, self.code, self.message, self.challenge = status, code, message, challenge
def invalid(message="Некорректный запрос"):
raise ApiError(400, "invalid_request", message)
def unavailable():
raise ApiError(503, "unavailable", "Хранилище временно недоступно")
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def encode_cursor(scope, date, identifier):
raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
def decode_cursor(scope, value):
if value is None:
return None
try:
raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
parts = json.loads(raw)
if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope
or not isinstance(parts[1], str) or not isinstance(parts[2], str)):
invalid("Некорректный cursor")
date = datetime.fromisoformat(parts[1].replace("Z", "+00:00"))
if date.tzinfo is None:
invalid("Некорректный cursor")
return date, str(uuid.UUID(parts[2]))
except (ValueError, TypeError, binascii.Error, IndexError):
invalid("Некорректный cursor")
def identifier(value):
try:
result = str(uuid.UUID(value))
if result != value.lower() or len(value) != 36:
invalid("Некорректный UUID")
return result
except (ValueError, TypeError, AttributeError):
invalid("Некорректный UUID")
def text_field(value, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
invalid("Некорректное текстовое поле")
return value
def exact_body(value, fields):
if not isinstance(value, dict) or set(value) != set(fields):
invalid("Некорректные поля JSON")
async def body(request, fields):
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
declared = request.headers.get("content-length")
if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
chunks = bytearray()
async for chunk in request.stream():
chunks.extend(chunk)
if len(chunks) > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
try:
value = json.loads(chunks.decode("utf-8"))
except (UnicodeError, ValueError):
invalid("Некорректный JSON")
exact_body(value, fields)
return value
def user(row):
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])}
def message(row):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": stamp(row["created_at"])}
def chat(connection, row):
members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall()
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])}
def password_hash(password):
salt = os.urandom(16)
derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1)
return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex()
def password_ok(password, stored):
try:
_, n, r, p, salt, expected = stored.split(":")
actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p))
return secrets.compare_digest(actual, bytes.fromhex(expected))
except (ValueError, TypeError):
return False
def connection():
try:
return pool.connection()
except psycopg.Error:
unavailable()
def actor(request):
auth = request.headers.get("authorization", "")
if not auth.startswith("Bearer ") or not auth[7:]:
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
digest = hashlib.sha256(auth[7:].encode()).digest()
with connection() as db:
row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone()
if row is None:
raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"')
return row["user_id"]
def member(db, uid, cid):
row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
return row
def pagination(request):
raw = request.query_params.get("limit", "50")
if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100:
invalid("Некорректный limit")
return int(raw)
def bump(key):
with metrics_lock:
metrics[key] += 1
def cache_get(key):
try:
raw = cache.get(key)
if raw is not None:
bump("history_cache_hit")
return json.loads(raw)
except (redis.RedisError, ValueError):
pass
return None
def cache_set(key, data):
try:
cache.set(key, json.dumps(data), ex=30)
except redis.RedisError:
pass
def history(db, cid, limit, cursor, version):
key = f"history:{cid}:{version}:{limit}:{cursor or '-'}"
hit = cache_get(key) if CACHE_ENABLED else None
if hit is not None:
return hit
lock_key = "lock:" + key
token = secrets.token_hex(16)
locked = False
if CACHE_ENABLED and CACHE_LOCK_ENABLED:
try:
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
if not locked:
bump("history_lock_wait")
for _ in range(20):
time.sleep(0.025)
hit = cache_get(key)
if hit is not None:
return hit
locked = bool(cache.set(lock_key, token, nx=True, px=2000))
except redis.RedisError:
pass
if CACHE_MISS_DELAY_MS:
time.sleep(CACHE_MISS_DELAY_MS / 1000)
if cursor is None:
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall()
else:
rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall()
bump("history_sql")
result = {"items": [message(row) for row in rows[:limit]], "next_cursor": None}
if len(rows) > limit:
last = rows[limit - 1]
result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"]))
if CACHE_ENABLED:
cache_set(key, result)
if locked:
try:
cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token)
except redis.RedisError:
pass
return result
def migrate():
from pathlib import Path
files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql"))
with pool.connection() as db:
with db.transaction():
db.execute("SELECT pg_advisory_xact_lock(2172002)")
db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)")
for file in files:
if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None:
db.execute(file.read_text())
db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,))
@asynccontextmanager
async def lifespan(_app):
global ready_for_traffic
pool.open()
pool.wait(timeout=20)
ready_for_traffic = True
yield
ready_for_traffic = False
pool.close()
app = FastAPI(lifespan=lifespan)
@app.exception_handler(StarletteHTTPException)
async def http_error(request, error):
request_id = request.state.request_id
code = "not_found" if error.status_code == 404 else "invalid_request"
return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code)
@app.middleware("http")
async def envelope(request, call_next):
request_id = str(uuid.uuid4())
request.state.request_id = request_id
try:
response = await call_next(request)
except ApiError as exc:
response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status)
if exc.challenge:
response.headers["WWW-Authenticate"] = exc.challenge
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
psycopg.errors.LockNotAvailable, PoolTimeout):
response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503)
except Exception:
response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready():
if not ready_for_traffic:
unavailable()
with connection() as db:
db.execute("SELECT 1")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
data = await body(request, ("username", "display_name", "password"))
name = data["username"]
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name):
invalid("Некорректный username")
display = text_field(data["display_name"], 100)
password = data["password"]
if not isinstance(password, str) or not password.strip() or not 12 <= len(password) <= 128:
invalid("Некорректный пароль")
try:
return await asyncio.to_thread(insert_user, name, display, password)
except psycopg.errors.UniqueViolation:
raise ApiError(409, "conflict", "Username уже занят") from None
def insert_user(name, display, password):
digest = password_hash(password)
with connection() as db:
row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone()
return user(row)
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
async for chunk in request.stream():
if chunk:
invalid("Тело запроса не допускается")
auth = request.headers.get("authorization", "")
try:
if not auth.startswith("Basic "):
raise ValueError()
decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8")
name, password = decoded.split(":", 1)
except (ValueError, UnicodeError, binascii.Error):
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None
return await asyncio.to_thread(create_session, name, password)
def create_session(name, password):
with connection() as db:
row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone()
if row is None or not password_ok(password, row["password_hash"]):
raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"')
token = secrets.token_urlsafe(32)
expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)
db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry))
return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request):
actor(request)
token = request.headers["authorization"][7:]
with connection() as db:
db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),))
return Response(status_code=204)
@app.get("/api/v1/users/me")
def me(request: Request):
uid = actor(request)
with connection() as db:
row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone()
return user(row)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
uid = actor(request)
data = await body(request, ("title", "member_ids"))
title = text_field(data["title"], 100)
raw = data["member_ids"]
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
invalid("Некорректный список участников")
members = [identifier(item) for item in raw]
if len(set(members)) != len(members) or str(uid) not in members:
invalid("Некорректный список участников")
return await asyncio.to_thread(insert_chat, uid, title, members)
def insert_chat(uid, title, members):
with connection() as db:
with db.transaction():
count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"]
if count != len(members):
invalid("Неизвестный участник")
row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone()
for member_id in members:
db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id))
db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],))
return chat(db, row)
@app.get("/api/v1/chats")
def list_chats(request: Request):
uid = actor(request)
limit = pagination(request)
cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor"))
with connection() as db:
if cursor is None:
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall()
else:
valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone()
if valid is None:
invalid("Некорректный cursor")
rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall()
items = [chat(db, row) for row in rows[:limit]]
next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None
return {"items": items, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
with connection() as db:
return chat(db, member(db, uid, cid))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
key = request.headers.get("idempotency-key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII):
invalid("Некорректный Idempotency-Key")
data = await body(request, ("text",))
content = text_field(data["text"], 4000)
return await asyncio.to_thread(insert_message, uid, cid, content, key)
def insert_message(uid, cid, content, key):
with connection() as db:
with db.transaction():
member(db, uid, cid)
# The unique constraint serializes concurrent requests on any replica.
db.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()", (uid, cid, key))
claimed = db.execute("INSERT INTO message_idempotency(user_id,chat_id,key,body_hash,expires_at) VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING RETURNING key", (uid, cid, key, hashlib.sha256(json.dumps({"text": content}, ensure_ascii=False, sort_keys=True).encode()).digest())).fetchone()
if claimed is None:
previous = db.execute("SELECT body_hash,message_id FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", (uid, cid, key)).fetchone()
expected = hashlib.sha256(json.dumps({"text": content}, ensure_ascii=False, sort_keys=True).encode()).digest()
if previous is None or previous["body_hash"] != expected:
raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения")
return message(db.execute("SELECT * FROM messages WHERE id=%s", (previous["message_id"],)).fetchone())
row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone()
db.execute("UPDATE message_idempotency SET message_id=%s WHERE user_id=%s AND chat_id=%s AND key=%s", (row["id"], uid, cid, key))
db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,))
return message(row)
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request):
uid = actor(request)
cid = identifier(chat_id)
limit = pagination(request)
cursor = decode_cursor(cid, request.query_params.get("cursor"))
with connection() as db:
member(db, uid, cid)
if cursor is not None:
valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone()
if valid is None:
invalid("Некорректный cursor")
version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"]
return history(db, cid, limit, cursor, version)
@app.get("/internal/cache-metrics")
def cache_metrics(request: Request):
if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"):
raise ApiError(404, "not_found", "Объект не найден")
with metrics_lock:
return dict(metrics)