120 lines
6.4 KiB
Python
120 lines
6.4 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import time
|
|
from uuid import uuid4
|
|
from starlette.responses import JSONResponse
|
|
from starlette.requests import ClientDisconnect
|
|
from .common import ApiError
|
|
from . import telemetry as tel
|
|
logger=logging.getLogger("messenger")
|
|
|
|
def error_response(exc, request_id):
|
|
return JSONResponse({'error': {'code': exc.code, 'message': exc.message, 'request_id': request_id}},
|
|
status_code=exc.status, headers={'X-Request-Id': request_id, **exc.headers})
|
|
|
|
|
|
class Boundaries:
|
|
"""Admission before body buffering; no unbounded application waiting queue."""
|
|
def __init__(self, app, settings):
|
|
self.app, self.cfg = app, settings
|
|
self.instance_id = uuid4().hex
|
|
self.active, self.tokens, self.updated = 0, float(settings.write_burst), time.monotonic()
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
if scope['type']!='http' or scope['path'].startswith('/health/'):
|
|
return await self._call(scope,receive,send)
|
|
status=500; started=time.monotonic()
|
|
headers={k.decode():v.decode() for k,v in scope.get('headers',[]) if k in (b'traceparent',b'tracestate')}
|
|
async def observed_send(message):
|
|
nonlocal status
|
|
if message['type']=='http.response.start': status=message['status']
|
|
await send(message)
|
|
with tel.span('HTTP '+scope['method'],kind=tel.trace.SpanKind.SERVER,carrier=headers) as current:
|
|
try:
|
|
await self._call(scope,receive,observed_send)
|
|
finally:
|
|
route=getattr(scope.get('route'),'path','unmatched')
|
|
method=scope['method'] if scope['method'] in {'GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS','TRACE','CONNECT'} else 'OTHER'
|
|
attrs={'method':method,'route':route,'status_class':str(status//100)+'xx'}
|
|
current.update_name(scope['method']+' '+route)
|
|
current.set_attributes({'http.request.method':scope['method'],'http.route':route,'http.response.status_code':status})
|
|
if status>=500: current.set_status(tel.trace.Status(tel.trace.StatusCode.ERROR))
|
|
duration=time.monotonic()-started
|
|
tel.http_count.add(1,attrs); tel.http_duration.record(duration,attrs)
|
|
tel.event('http_request',request_id=scope.get('state',{}).get('request_id'),status=status,route=route,duration_seconds=round(duration,6))
|
|
|
|
async def _call(self, scope, receive, send):
|
|
if scope['type'] != 'http':
|
|
return await self.app(scope, receive, send)
|
|
request_id, started, status = uuid4().hex, time.monotonic(), 500
|
|
scope.setdefault('state', {})['request_id'] = request_id
|
|
started_response = False
|
|
async def tracked_send(message):
|
|
nonlocal status, started_response
|
|
if message['type'] == 'http.response.start':
|
|
status, started_response = message['status'], True
|
|
message['headers'] = [(k, v) for k, v in message.get('headers', []) if k.lower() != b'x-request-id'] + [(b'x-request-id', request_id.encode()), (b'x-instance-id', self.instance_id.encode())]
|
|
await send(message)
|
|
admitted = False
|
|
try:
|
|
if self.active >= self.cfg.max_inflight:
|
|
raise ApiError(503, 'unavailable', 'Too many in-flight requests', {'Retry-After': '1'})
|
|
now = time.monotonic()
|
|
self.tokens = min(self.cfg.write_burst, self.tokens + (now - self.updated) * self.cfg.write_rps)
|
|
self.updated = now
|
|
if scope['method'] in ('POST', 'PUT', 'PATCH'):
|
|
if self.tokens < 1:
|
|
raise ApiError(429, 'rate_limited', 'Write rate exceeded', {'Retry-After': '1'})
|
|
self.tokens -= 1
|
|
self.active += 1
|
|
admitted = True
|
|
# Binary uploads are streamed by the authorized route into a bounded
|
|
# temporary file; do not copy a 10 MiB body into every API request.
|
|
if scope['method']=='POST' and scope['path'].endswith('/attachments'):
|
|
async with asyncio.timeout(self.cfg.request_timeout):
|
|
await self.app(scope,receive,tracked_send)
|
|
return
|
|
chunks, total = bytearray(), 0
|
|
async with asyncio.timeout(self.cfg.request_timeout):
|
|
while True:
|
|
message = await receive()
|
|
if message['type'] == 'http.disconnect':
|
|
return
|
|
chunk = message.get('body', b'')
|
|
total += len(chunk)
|
|
if total > self.cfg.max_body_bytes:
|
|
raise ApiError(413, 'payload_too_large', 'Request body exceeds byte limit')
|
|
chunks.extend(chunk)
|
|
if not message.get('more_body', False):
|
|
break
|
|
body = bytes(chunks)
|
|
delivered = False
|
|
async def replay():
|
|
nonlocal delivered
|
|
if delivered:
|
|
return await receive()
|
|
delivered = True
|
|
return {'type': 'http.request', 'body': body, 'more_body': False}
|
|
await self.app(scope, replay, tracked_send)
|
|
except ClientDisconnect:
|
|
status=499
|
|
except TimeoutError:
|
|
if not started_response:
|
|
await error_response(ApiError(503, 'unavailable', 'Request timed out'), request_id)(scope, receive, tracked_send)
|
|
except ApiError as exc:
|
|
if not started_response:
|
|
await error_response(exc, request_id)(scope, receive, tracked_send)
|
|
except Exception as exc:
|
|
logger.error(json.dumps({'event': 'unexpected_error', 'request_id': request_id, 'type': type(exc).__name__}))
|
|
if not started_response:
|
|
await error_response(ApiError(500, 'internal_error', 'Internal error'), request_id)(scope, receive, tracked_send)
|
|
finally:
|
|
if admitted:
|
|
self.active -= 1
|
|
# No query, body, headers, usernames, or message content in logs.
|
|
route = scope.get('route')
|
|
logger.info(json.dumps({'event': 'http', 'request_id': request_id, 'method': scope['method'],
|
|
'route': getattr(route, 'path', 'unmatched'), 'status': status,
|
|
'duration_ms': round((time.monotonic() - started) * 1000, 3)}))
|