"""Bounded threaded API; persistent correctness state is shared across replicas.""" import concurrent.futures import http.server import json import logging import os import time import signal import socket import threading import uuid from urllib.parse import parse_qs, urlsplit from objects import bounded_upload, receive, filename, unavailable from common import APIError, identifier, invalid, missing, page_args LOG = logging.getLogger('messenger') def unique_object(pairs): result = {} for key, value in pairs: if key in result: invalid() result[key] = value return result class Handler(http.server.BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' def setup(self): super().setup() self.connection.settimeout(3) def send_error(self, code, message=None, explain=None): # BaseHTTPRequestHandler otherwise emits HTML without contract headers. self.request_id = uuid.uuid4().hex self.route = '' self.close_connection = True status = 413 if code == 413 else 400 self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}}) def log_message(self, *args): pass def reply(self, status, body): if status == 200 and isinstance(body, tuple): return self.download(*body) raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') if status >= 400: self.close_connection = True self.send_response(status) self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', str(len(raw))) if self.close_connection: self.send_header('Connection', 'close') self.send_header('X-Request-Id', self.request_id) self.send_header('X-Instance-Id', self.server.instance_id) if status == 401 and self.server.store.persistent: self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') self.end_headers() if self.command != 'HEAD': self.wfile.write(raw) def download(self, row, response, chunk): stream = response['Body'] try: self.send_response(200) self.send_header('Content-Type', row['content_type']) self.send_header('Content-Length', str(row['size_bytes'])) self.send_header('X-Request-Id', self.request_id) self.send_header('X-Instance-Id', self.server.instance_id) self.send_header('X-Content-Type-Options', 'nosniff') self.send_header('Cache-Control', 'private, no-store') self.end_headers() deadline = time.monotonic() + 15 while chunk: self.wfile.write(chunk) if time.monotonic() > deadline: raise TimeoutError() chunk = stream.read(64 * 1024) finally: stream.close() def body(self): lengths = self.headers.get_all('Content-Length', []) if len(lengths) != 1 or self.headers.get('Transfer-Encoding'): invalid() try: length = int(lengths[0]) except ValueError: invalid() if length > 32768: raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт') if length <= 0: invalid() if self.headers.get_content_type() != 'application/json': raise APIError(415, 'unsupported_media_type', 'Требуется application/json') try: raw = self.rfile.read(length) if len(raw) != length: invalid() return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid()) except (ValueError, UnicodeError, RecursionError): invalid() def dispatch(self): self.request_id = uuid.uuid4().hex self.route = urlsplit(self.path).path status = 500 try: status, result = self.route_request() except APIError as exc: status = exc.status result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}} except Exception: # No bodies, credentials, tokens, SQL or exception details in logs/responses. LOG.error('request_failed request_id=%s', self.request_id) result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}} try: self.reply(status, result) except Exception: self.close_connection = True LOG.info('method=%s status=%s request_id=%s', self.command, status, self.request_id) def route_request(self): method, path, store = self.command, self.route, self.server.store if method == 'GET' and path in ('/health/live', '/health/ready'): if path.endswith('ready'): store.ready() return 200, {'status': 'ok'} if method == 'POST' and path == '/api/v1/users': return 201, store.register(self.body()) if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions': if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': invalid() return 201, store.login(self.headers) actor = store.authenticate(self.headers) if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current': store.logout(self.headers) return 204, None if method == 'GET' and path == '/api/v1/users/me': return 200, actor query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) if path == '/api/v1/chats': if method == 'POST': return 201, store.create_chat(actor, self.body()) if method == 'GET': scope = 'chats:' + actor['id'] limit, cursor = page_args(query, scope) return 200, store.list_chats(actor, limit, cursor, scope) parts = path.split('/') if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET': row = store.attachment(actor, identifier(parts[4])) if len(parts) == 5: if query: invalid() return 200, store.attachment_public(row) if parts[5] == 'content': if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()): invalid() if row['state'] != 'ready': raise APIError(409, 'conflict', 'Вложение ещё не готово') variant = query.get('variant', ['original'])[0] if variant != 'original': item = next((v for v in row['variants'] if v['name'] == variant), None) if item is None: missing() row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes']) response = store.objects.get(row['object_key']) try: chunk = response['Body'].read(64 * 1024) if response['ContentLength'] != row['size_bytes'] or not chunk: unavailable() except Exception: response['Body'].close() unavailable() return 200, (row, response, chunk) if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']: chat_id = identifier(parts[4]) if store.access_mode == 'baseline': store.chat(actor, chat_id) if len(parts) == 5 and method == 'GET': return 200, store.chat(actor, chat_id) if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST': store.chat(actor, chat_id) if set(query) != {'filename'} or len(query['filename']) != 1: invalid() name = filename(query['filename'][0]) with bounded_upload(): with receive(self) as (source, length, mime, digest): return 202, store.upload(actor, chat_id, name, source, length, mime, digest) if len(parts) == 6 and parts[5] == 'messages': if method == 'POST': keys = self.headers.get_all('Idempotency-Key', []) return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None) if method == 'GET': scope = 'messages:' + chat_id limit, cursor = page_args(query, scope) return 200, store.history(actor, chat_id, limit, cursor, scope) missing() do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch class Server(http.server.HTTPServer): request_queue_size = 64 allow_reuse_address = True def __init__(self, address, store): self.store = store self.instance_id = uuid.uuid4().hex workers = int(os.environ.get('HTTP_WORKERS', '16')) if not 1 <= workers <= 128: raise ValueError('HTTP_WORKERS must be 1..128') self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) self.slots = threading.BoundedSemaphore(workers * 2) self.connections = set() self.connections_lock = threading.Lock() super().__init__(address, Handler) def process_request(self, request, address): if not self.slots.acquire(blocking=False): rid = uuid.uuid4().hex raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode() try: request.settimeout(0.2) request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw) except OSError: pass self.shutdown_request(request) return with self.connections_lock: self.connections.add(request) self.executor.submit(self.worker, request, address) def worker(self, request, address): try: self.finish_request(request, address) except OSError: pass finally: self.shutdown_request(request) with self.connections_lock: self.connections.discard(request) self.slots.release() def close_connections(self): with self.connections_lock: connections = list(self.connections) for connection in connections: try: connection.shutdown(socket.SHUT_RDWR) except OSError: pass def server_close(self): super().server_close() # Bound draining even when a slow client keeps an incomplete request alive. deadline = threading.Timer(5, self.close_connections) deadline.start() try: self.executor.shutdown(wait=True) finally: deadline.cancel() def main(): logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s') from postgres import Store store = Store() server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store) stopping = threading.Event() def stop(signum, frame): if not stopping.is_set(): stopping.set() LOG.info('shutdown_started signal=%s', signum) threading.Thread(target=server.shutdown, daemon=True).start() signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats()))) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) LOG.info('started port=%s', server.server_address[1]) try: server.serve_forever(poll_interval=0.1) finally: server.server_close() store.close() LOG.info('shutdown_complete') if __name__ == '__main__': main()