Перенести мессенджер в PostgreSQL и сессии в Redis.

Подтверждённые сообщения и действующие сессии переживают перезапуск, история читается по индексу, копия восстанавливается в новый том.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-10-23 12:24:59 +03:00
co-authored by Cursor
parent 65003b6119
commit c4c7a6aca2
29 changed files with 2933 additions and 35 deletions
+731
View File
@@ -0,0 +1,731 @@
import json
import os
import re
import signal
import socket
import sys
import threading
import time
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlsplit
import base64
import cursor
import db
import passwords
import sessions
from errors import DependencyDown
MAX_BODY = 65536
DEFAULT_INFLIGHT_BODIES = 32
USERNAME = re.compile(r'^[a-z0-9_]{3,32}$')
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class State:
def __init__(self, max_inflight_bodies):
self.accepting = True
slots = int(max_inflight_bodies)
if slots < 1:
slots = 1
self.body_slots = threading.BoundedSemaphore(slots)
class Limiter:
def __init__(self, max_per_second):
self.max_per_second = max_per_second
self._lock = threading.Lock()
self._recent = []
def allow(self):
if self.max_per_second <= 0:
return True
now = time.monotonic()
with self._lock:
self._recent = [stamp for stamp in self._recent if now - stamp < 1]
if len(self._recent) >= self.max_per_second:
return False
self._recent.append(now)
return True
def canonical_uuid(value):
if not isinstance(value, str):
return None
try:
return str(uuid.UUID(value))
except ValueError:
return None
def make_server(state, limiter, host, port):
class Handler(BaseHTTPRequestHandler):
def address_string(self):
return self.client_address[0]
def log_message(self, fmt, *args):
return
def do_GET(self):
self._handle()
def do_POST(self):
self._handle()
def do_PUT(self):
self._handle()
def do_PATCH(self):
self._handle()
def do_DELETE(self):
self._handle()
def _handle(self):
self.request_id = str(uuid.uuid4())
self._status = '-'
try:
self.connection.settimeout(15)
except OSError:
pass
try:
self._route()
except DependencyDown as exc:
print('dependency %s' % type(exc).__name__, flush=True)
if self._status == '-':
self._error(503, 'unavailable', 'Сервис временно недоступен')
except Exception as exc:
print('internal %s' % type(exc).__name__, flush=True)
if self._status == '-':
self._error(500, 'internal_error', 'Внутренняя ошибка')
def _route(self):
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query, keep_blank_values=True)
if path == '/health/live' and self.command == 'GET':
self._json(200, {'status': 'ok'})
return
if not state.accepting:
self._error(503, 'unavailable', 'Сервер останавливается')
return
if path == '/health/ready' and self.command == 'GET':
self._ready()
return
if path == '/api/v1/users' and self.command == 'POST':
self._register()
return
if path == '/api/v1/auth/sessions' and self.command == 'POST':
self._login()
return
if path.startswith('/api/'):
self._api(path, query)
return
if path in ('/health/live', '/health/ready', '/api/v1/users', '/api/v1/auth/sessions'):
self._error(400, 'invalid_request', 'Метод не поддерживается')
return
self._error(404, 'not_found', 'Объект не найден')
def _ready(self):
try:
db.ping()
sessions.ping()
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
self._json(200, {'status': 'ok'})
def _api(self, path, query):
if not limiter.allow():
self._discard_body()
self._error(429, 'rate_limited', 'Слишком много запросов', retry_after=1)
return
user = self._user()
if user is None:
return
if path == '/api/v1/auth/sessions/current' and self.command == 'DELETE':
self._logout()
return
if path == '/api/v1/users/me' and self.command == 'GET':
self._json(200, user)
return
if path == '/api/v1/chats' and self.command == 'POST':
self._create_chat(user)
return
if path == '/api/v1/chats' and self.command == 'GET':
self._list_chats(user, query)
return
chat_match = re.fullmatch(r'/api/v1/chats/([^/]+)', path)
if chat_match and self.command == 'GET':
self._get_chat(user, chat_match.group(1))
return
msg_match = re.fullmatch(r'/api/v1/chats/([^/]+)/messages', path)
if msg_match and self.command == 'POST':
self._create_message(user, msg_match.group(1))
return
if msg_match and self.command == 'GET':
self._list_messages(user, msg_match.group(1), query)
return
self._error(404, 'not_found', 'Объект не найден')
def _user(self):
# X-User-Id is ignored from lab2. Basic is only valid on POST /auth/sessions.
scheme, value = _authorization(self.headers.get('Authorization'))
if scheme != 'bearer' or not sessions.token_shape(value):
self._discard_body()
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BEARER_CHALLENGE)
return None
try:
user_id = sessions.read(value)
if user_id is None:
self._discard_body()
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BEARER_CHALLENGE)
return None
user = db.get_user(user_id)
except DependencyDown:
self._discard_body()
self._error(503, 'unavailable', 'Сервис временно недоступен')
return None
if user is None:
self._discard_body()
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BEARER_CHALLENGE)
return None
self._bearer = value
return user
def _logout(self):
self._discard_body()
if self._status != '-':
return
try:
gone = sessions.revoke(getattr(self, '_bearer', ''))
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if not gone:
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BEARER_CHALLENGE)
return
self._json(204, None)
def _login(self):
if not limiter.allow():
self._discard_body()
self._error(429, 'rate_limited', 'Слишком много запросов', retry_after=1)
return
raw_body, rejected = self._optional_empty_body()
if rejected:
return
if raw_body:
self._error(400, 'invalid_request', 'Тело запроса не нужно')
return
username, password = _basic_credentials(self.headers.get('Authorization'))
if username is None:
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BASIC_CHALLENGE)
return
try:
found = db.get_credentials(username)
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if found is None:
passwords.verify_password(passwords.DUMMY_HASH, password)
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BASIC_CHALLENGE)
return
user, password_hash = found
if not passwords.verify_password(password_hash, password):
self._error(401, 'unauthorized', 'Пользователь не опознан', challenge=BASIC_CHALLENGE)
return
try:
token, expires_at = sessions.create(user['id'])
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
self._json(201, {
'token': token,
'token_type': 'Bearer',
'expires_at': expires_at,
'user': user,
})
def _discard_body(self):
length, problem = _parse_content_length(self.headers.get('Content-Length'))
if problem == 'missing':
return
if problem == 'too_big':
self._reject_too_large()
return
if problem == 'bad':
self.close_connection = True
return
if length == 0:
return
data, busy = self._read_exact(length)
if busy or data is None or len(data) != length:
self.close_connection = True
def _optional_empty_body(self):
length, problem = _parse_content_length(self.headers.get('Content-Length'))
if problem == 'missing' or length == 0:
return b'', False
if problem == 'too_big':
self._reject_too_large()
self._error(413, 'payload_too_large', 'Тело запроса слишком большое')
return None, True
if problem == 'bad':
self.close_connection = True
self._error(400, 'invalid_request', 'Некорректный запрос')
return None, True
raw, busy = self._read_exact(length)
if busy:
self._error(503, 'unavailable', 'Слишком много одновременных тел запроса')
return None, True
if raw is None or len(raw) != length:
self.close_connection = True
self._error(400, 'invalid_request', 'Некорректный запрос')
return None, True
return raw.strip(), False
def _read_exact(self, length):
if length == 0:
return b'', False
if not state.body_slots.acquire(blocking=False):
self.close_connection = True
return None, True
try:
return self.rfile.read(length), False
finally:
state.body_slots.release()
def _reject_too_large(self):
self.close_connection = True
if not state.body_slots.acquire(blocking=False):
return
previous = self.connection.gettimeout()
self.connection.settimeout(0.2)
try:
remaining = MAX_BODY
while remaining > 0:
try:
chunk = self.rfile.read(min(8192, remaining))
except (TimeoutError, socket.timeout, OSError):
break
if not chunk:
break
remaining -= len(chunk)
finally:
state.body_slots.release()
try:
self.connection.settimeout(previous)
except OSError:
pass
def _register(self):
if not limiter.allow():
self._discard_body()
self._error(429, 'rate_limited', 'Слишком много запросов', retry_after=1)
return
body, problem = self._json_body({'username', 'display_name', 'password'})
if problem:
return
username = body['username']
display_name = body['display_name']
password = body['password']
if not isinstance(username, str) or USERNAME.fullmatch(username) is None:
self._error(400, 'invalid_request', 'Некорректный username')
return
if not _text_ok(display_name, 100):
self._error(400, 'invalid_request', 'Некорректный display_name')
return
if not _password_ok(password):
self._error(400, 'invalid_request', 'Некорректный пароль')
return
try:
status, user = db.create_user(username, display_name, passwords.hash_password(password))
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if status == 'conflict':
self._error(409, 'conflict', 'Такой username уже занят')
return
if status != 'ok':
self._error(400, 'invalid_request', 'Некорректные поля JSON')
return
self._json(201, user)
def _create_chat(self, user):
body, problem = self._json_body({'title', 'member_ids'})
if problem:
return
title = body['title']
member_ids = body['member_ids']
if not _text_ok(title, 100):
self._error(400, 'invalid_request', 'Некорректный title')
return
parsed_ids, ok = _member_ids(member_ids, user['id'])
if not ok:
self._error(400, 'invalid_request', 'Некорректный список участников')
return
try:
status, chat = db.create_chat(user['id'], title, parsed_ids)
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if status != 'ok':
self._error(400, 'invalid_request', 'Некорректный список участников')
return
self._json(201, chat)
def _list_chats(self, user, query):
limit, page_cursor, problem = _page_args(query, 'chats', user['id'])
if problem:
self._error(400, 'invalid_request', problem)
return
try:
items, nxt = db.list_chats(user['id'], limit, page_cursor)
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
self._json(200, {'items': items, 'next_cursor': nxt})
def _get_chat(self, user, raw_chat_id):
chat_id = canonical_uuid(raw_chat_id)
if chat_id is None:
self._error(400, 'invalid_request', 'Некорректный UUID')
return
try:
chat = db.get_chat(chat_id, user['id'])
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if chat is None:
self._error(404, 'not_found', 'Объект не найден')
return
self._json(200, chat)
def _create_message(self, user, raw_chat_id):
chat_id = canonical_uuid(raw_chat_id)
if chat_id is None:
self._error(400, 'invalid_request', 'Некорректный UUID')
return
body, problem = self._json_body({'text'})
if problem:
return
text = body['text']
if not _text_ok(text, 4000):
self._error(400, 'invalid_request', 'Некорректный текст сообщения')
return
try:
status, message = db.add_message(chat_id, user['id'], text)
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if status == 'missing':
self._error(404, 'not_found', 'Объект не найден')
return
if status == 'full':
self._error(503, 'unavailable', 'Достигнут предел числа сообщений')
return
self._json(201, message)
def _list_messages(self, user, raw_chat_id, query):
chat_id = canonical_uuid(raw_chat_id)
if chat_id is None:
self._error(400, 'invalid_request', 'Некорректный UUID')
return
limit, page_cursor, problem = _page_args(query, 'msgs', chat_id)
if problem:
self._error(400, 'invalid_request', problem)
return
try:
status, items, nxt = db.list_messages(chat_id, user['id'], limit, page_cursor)
except DependencyDown:
self._error(503, 'unavailable', 'Сервис временно недоступен')
return
if status == 'missing':
self._error(404, 'not_found', 'Объект не найден')
return
self._json(200, {'items': items, 'next_cursor': nxt})
def _json_body(self, fields):
raw, code, message = self._read_body()
if code:
self._error(code, _error_code(code), message)
return None, True
try:
body = json.loads(raw.decode('utf-8'))
except (UnicodeError, json.JSONDecodeError):
self._error(400, 'invalid_request', 'Некорректный JSON')
return None, True
if not isinstance(body, dict) or set(body) != fields:
self._error(400, 'invalid_request', 'Некорректные поля JSON')
return None, True
return body, False
def _read_body(self):
length, problem = _parse_content_length(self.headers.get('Content-Length'))
if problem == 'missing' or problem == 'bad':
self.close_connection = True
return None, 400, 'Некорректный JSON'
if problem == 'too_big':
self._reject_too_large()
return None, 413, 'Тело запроса слишком большое'
if not _json_content_type(self.headers.get('Content-Type', '')):
self._read_exact(length)
return None, 415, 'Нужен Content-Type application/json'
raw, busy = self._read_exact(length)
if busy:
return None, 503, 'Слишком много одновременных тел запроса'
if raw is None or len(raw) != length:
self.close_connection = True
return None, 400, 'Некорректный JSON'
return raw, None, None
def _error(self, status, code, message, retry_after=None, challenge=None):
self._json(status, {
'error': {
'code': code,
'message': message,
'request_id': self.request_id,
}
}, retry_after=retry_after, challenge=challenge)
def _json(self, status, payload, retry_after=None, challenge=None):
if self._status != '-':
return
if status == 204:
body = b''
else:
body = json.dumps(payload, ensure_ascii=False).encode('utf-8')
self.send_response(status)
if status != 204:
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.send_header('Content-Length', str(len(body)))
self.send_header('X-Request-Id', self.request_id)
if challenge is not None:
self.send_header('WWW-Authenticate', challenge)
if retry_after is not None:
self.send_header('Retry-After', str(retry_after))
self.end_headers()
if body:
self.wfile.write(body)
self._status = status
path = urlsplit(self.path).path
print('%s %s %s %s' % (self.request_id, self.command, path, status), flush=True)
class Server(ThreadingHTTPServer):
allow_reuse_address = True
daemon_threads = True
return Server((host, port), Handler)
def _error_code(status):
return {
400: 'invalid_request',
413: 'payload_too_large',
415: 'unsupported_media_type',
503: 'unavailable',
}.get(status, 'invalid_request')
def _json_content_type(value):
if not value:
return False
media = value.split(';', 1)[0].strip().lower()
return media == 'application/json'
def _parse_content_length(raw):
if raw is None:
return None, 'missing'
text = raw.strip()
if not re.fullmatch(r'[0-9]+', text):
return None, 'bad'
digits = text.lstrip('0') or '0'
if len(digits) > 7:
return None, 'too_big'
try:
length = int(digits)
except ValueError:
return None, 'too_big'
if length > MAX_BODY:
return length, 'too_big'
return length, None
def _text_ok(value, limit):
return (
isinstance(value, str)
and 1 <= len(value) <= limit
and value.strip() != ''
)
def _password_ok(value):
return (
isinstance(value, str)
and 12 <= len(value) <= 128
and value.strip() != ''
)
def _member_ids(value, owner_id):
if not isinstance(value, list) or not 1 <= len(value) <= 100:
return None, False
parsed = []
for item in value:
item_id = canonical_uuid(item)
if item_id is None:
return None, False
parsed.append(item_id)
if len(set(parsed)) != len(parsed):
return None, False
if owner_id not in parsed:
return None, False
return parsed, True
def _page_args(query, kind, owner_id):
if 'limit' not in query:
limit = 50
else:
raw = query['limit']
if len(raw) != 1 or re.fullmatch(r'[0-9]+', raw[0]) is None:
return None, None, 'Некорректный limit'
digits = raw[0].lstrip('0') or '0'
if len(digits) > 3:
return None, None, 'Некорректный limit'
try:
limit = int(digits)
except ValueError:
return None, None, 'Некорректный limit'
if limit < 1 or limit > 100:
return None, None, 'Некорректный limit'
if 'cursor' not in query:
return limit, None, None
raw_cursor = query['cursor']
if len(raw_cursor) != 1 or raw_cursor[0] == '':
return None, None, 'Некорректный cursor'
page_cursor = cursor.read_cursor(raw_cursor[0], kind, owner_id)
if page_cursor is None:
return None, None, 'Некорректный cursor'
return limit, page_cursor, None
def _authorization(header):
if not header:
return None, None
parts = header.split(None, 1)
if len(parts) != 2:
return None, None
return parts[0].lower(), parts[1].strip()
def _basic_credentials(header):
scheme, value = _authorization(header)
if scheme != 'basic' or not value:
return None, None
try:
raw = base64.b64decode(value, validate=True)
text = raw.decode('utf-8')
except (ValueError, UnicodeError):
return None, None
if ':' not in text:
return None, None
username, password = text.split(':', 1)
if USERNAME.fullmatch(username) is None or password == '':
return None, None
return username, password
def _positive_int(name, default):
raw = os.environ.get(name, str(default))
try:
value = int(raw)
except ValueError:
print('bad integer %s' % name, flush=True)
return None
return value
def wait_until_ready(stop):
deadline = time.monotonic() + 75
while not stop['flag']:
try:
applied = db.apply_migrations()
db.open_pool()
sessions.open_client()
sessions.ping()
db.ping()
if applied:
print('migrations applied: %s' % ','.join(applied), flush=True)
else:
print('migrations already applied', flush=True)
return True
except RuntimeError:
print('configuration error', flush=True)
return False
except Exception as exc:
print('waiting for postgres/redis: %s' % type(exc).__name__, flush=True)
db.close_pool()
sessions.close_client()
if time.monotonic() > deadline:
print('dependencies not ready', flush=True)
return False
time.sleep(1)
return False
def main():
host = os.environ.get('APP_HOST', '0.0.0.0')
port = _positive_int('APP_PORT', 8080)
message_cap = _positive_int('MAX_MESSAGES', 100000)
max_per_second = _positive_int('MAX_PER_SECOND', 10000)
max_inflight_bodies = _positive_int('MAX_INFLIGHT_BODIES', DEFAULT_INFLIGHT_BODIES)
ttl = _positive_int('SESSION_TTL_SECONDS', 3600)
if None in (port, message_cap, max_per_second, max_inflight_bodies, ttl) or ttl < 1:
sys.exit(1)
sessions.configure(ttl)
db.configure(message_cap)
stop = {'flag': False}
def stop_waiting(signum, frame):
stop['flag'] = True
print('signal %s during startup' % signum, flush=True)
signal.signal(signal.SIGTERM, stop_waiting)
signal.signal(signal.SIGINT, stop_waiting)
if not wait_until_ready(stop):
db.close_pool()
sessions.close_client()
sys.exit(0 if stop['flag'] else 1)
state = State(max_inflight_bodies)
limiter = Limiter(max_per_second)
httpd = make_server(state, limiter, host, port)
def stop_serving(signum, frame):
state.accepting = False
print('signal %s: stop accepting' % signum, flush=True)
threading.Thread(target=httpd.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, stop_serving)
signal.signal(signal.SIGINT, stop_serving)
print(
'listen http://%s:%s ttl=%s max_messages=%s pool=%s-%s'
% (host, port, ttl, message_cap, db.POOL_MIN, db.POOL_MAX),
flush=True,
)
try:
httpd.serve_forever()
finally:
httpd.server_close()
db.close_pool()
sessions.close_client()
print('process stopped', flush=True)
if __name__ == '__main__':
main()