Answer requests cancelled at shutdown with 503 in the contract format
After the drain window uvicorn waits for accepted requests for three seconds and then cancels their tasks. CancelledError is a BaseException, so neither the pipeline nor ServerErrorMiddleware saw it and uvicorn answered with its own text/plain 500 without X-Request-Id and X-Instance-Id. The pipeline now handles the cancellation next to the request timeout: it logs request.cancelled, sends 503 unavailable in the common error format when the response has not started and re-raises, so uvicorn only closes the transport after the buffered answer. The declared Content-Length check joins the guard as a rejection to keep the call within the statement limit; behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
13e856ebd4
commit
aa1485bd86
@@ -1,7 +1,7 @@
|
||||
## 1. Платформа: остановка, готовность, журнал
|
||||
|
||||
- [ ] 1.1 `RequestPipeline`: отмена задачи → `request.cancelled`, `503 unavailable` в общем формате, если ответ не начат, затем отмена пробрасывается
|
||||
- [ ] 1.2 `test_drain.py`: прерванный при остановке запрос — `503`, `application/json`, `error.code = unavailable`, `request_id` = `X-Request-Id`, непустой `X-Instance-Id`
|
||||
- [x] 1.1 `RequestPipeline`: отмена задачи → `request.cancelled`, `503 unavailable` в общем формате, если ответ не начат, затем отмена пробрасывается
|
||||
- [x] 1.2 `test_drain.py`: прерванный при остановке запрос — `503`, `application/json`, `error.code = unavailable`, `request_id` = `X-Request-Id`, непустой `X-Instance-Id`
|
||||
- [ ] 1.3 Порт `Readiness`, адаптер `DatabaseReadiness` с флагом `DRAINING` в `shared/infrastructure/readiness.py`, сборка в `PlatformProvider`, маршрут `/health/ready` через порт; тест состава модулей ядра
|
||||
- [ ] 1.4 `gunicorn_conf.py`: флаг из инфраструктуры, событие `service.draining` через `StructlogLogger`; `failover-evidence.sh` ищет его в логе
|
||||
- [ ] 1.5 `openspec/config.yaml`: карта модулей (`readiness` за портом)
|
||||
|
||||
@@ -103,12 +103,6 @@ class RequestPipeline:
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
declared = header(scope, b"content-length")
|
||||
if declared.isdigit() and int(declared) > self._max_bytes:
|
||||
await send_error(wrapped_send, 413, "payload_too_large", request_id)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
return
|
||||
|
||||
received = [0]
|
||||
|
||||
async def limited_receive() -> Message:
|
||||
@@ -121,7 +115,7 @@ class RequestPipeline:
|
||||
rejection: Rejection | None = None
|
||||
try:
|
||||
async with asyncio.timeout(self._timeout):
|
||||
rejection = await self._guard(scope)
|
||||
rejection = self._oversized(scope) or await self._guard(scope)
|
||||
if rejection is None:
|
||||
await self._app(scope, limited_receive, wrapped_send)
|
||||
except Unavailable as error:
|
||||
@@ -132,18 +126,21 @@ class RequestPipeline:
|
||||
request_id=request_id,
|
||||
)
|
||||
rejection = Rejection(503, "unavailable")
|
||||
except TimeoutError:
|
||||
except (TimeoutError, asyncio.CancelledError) as error:
|
||||
cancelled = isinstance(error, asyncio.CancelledError)
|
||||
self._logger.warning(
|
||||
"request.timeout",
|
||||
"request.cancelled" if cancelled else "request.timeout",
|
||||
method=scope["method"],
|
||||
route=self._route(scope),
|
||||
request_id=request_id,
|
||||
limit=self._timeout,
|
||||
limit=None if cancelled else self._timeout,
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 3),
|
||||
)
|
||||
if not answered[0]:
|
||||
await send_error(wrapped_send, 503, "unavailable", request_id)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
if cancelled:
|
||||
raise
|
||||
return
|
||||
except Exception:
|
||||
self._logger.error(
|
||||
@@ -160,6 +157,12 @@ class RequestPipeline:
|
||||
)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
|
||||
def _oversized(self, scope: Scope) -> Rejection | None:
|
||||
declared = header(scope, b"content-length")
|
||||
if declared.isdigit() and int(declared) > self._max_bytes:
|
||||
return Rejection(413, "payload_too_large")
|
||||
return None
|
||||
|
||||
def _route(self, scope: Scope) -> str:
|
||||
route = scope.get("route")
|
||||
return getattr(route, "path", scope["path"])
|
||||
|
||||
@@ -49,6 +49,17 @@ def status(port: int, path: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
|
||||
data = b""
|
||||
while chunk := connection.recv(4096):
|
||||
data += chunk
|
||||
connection.close()
|
||||
head, _, body = data.partition(b"\r\n\r\n")
|
||||
status, *lines = head.decode("latin-1").split("\r\n")
|
||||
headers = dict(line.lower().split(": ", 1) for line in lines)
|
||||
return status, headers, json.loads(body) if body else {}
|
||||
|
||||
|
||||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||||
body = json.dumps(new_user("drain")).encode()
|
||||
head = (
|
||||
@@ -97,17 +108,21 @@ def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests() -> Non
|
||||
pending.close()
|
||||
code = server.wait(timeout=SHUTDOWN + 2)
|
||||
exited = time.monotonic() - sent
|
||||
cancelled = stuck.recv(64).split(b"\r\n")[0].decode()
|
||||
stuck.close()
|
||||
cancelled, headers, error = answer_of(stuck)
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||||
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
|
||||
print(f" content-type {headers.get('content-type')}, тело {error}")
|
||||
assert during == (503, 200)
|
||||
assert answer == "HTTP/1.1 201 Created"
|
||||
assert cancelled == "HTTP/1.1 500 Internal Server Error"
|
||||
assert cancelled == "HTTP/1.1 503 Service Unavailable"
|
||||
assert headers["content-type"] == "application/json"
|
||||
assert headers["x-instance-id"]
|
||||
assert error["error"]["code"] == "unavailable"
|
||||
assert error["error"]["request_id"] == headers["x-request-id"]
|
||||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||||
assert exited < SHUTDOWN
|
||||
assert code == 0
|
||||
|
||||
Reference in New Issue
Block a user