Answer requests cancelled at shutdown with 503 in the contract format

After the drain window uvicorn waits for accepted requests for three
seconds and then cancels their tasks. CancelledError is a BaseException,
so neither the pipeline nor ServerErrorMiddleware saw it and uvicorn
answered with its own text/plain 500 without X-Request-Id and
X-Instance-Id. The pipeline now handles the cancellation next to the
request timeout: it logs request.cancelled, sends 503 unavailable in the
common error format when the response has not started and re-raises,
so uvicorn only closes the transport after the buffered answer.

The declared Content-Length check joins the guard as a rejection to keep
the call within the statement limit; behaviour is unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
Artyom
2026-10-30 05:39:09 +03:00
co-authored by Claude Opus 5.5
parent 13e856ebd4
commit aa1485bd86
3 changed files with 33 additions and 15 deletions
@@ -1,7 +1,7 @@
## 1. Платформа: остановка, готовность, журнал
- [ ] 1.1 `RequestPipeline`: отмена задачи → `request.cancelled`, `503 unavailable` в общем формате, если ответ не начат, затем отмена пробрасывается
- [ ] 1.2 `test_drain.py`: прерванный при остановке запрос — `503`, `application/json`, `error.code = unavailable`, `request_id` = `X-Request-Id`, непустой `X-Instance-Id`
- [x] 1.1 `RequestPipeline`: отмена задачи → `request.cancelled`, `503 unavailable` в общем формате, если ответ не начат, затем отмена пробрасывается
- [x] 1.2 `test_drain.py`: прерванный при остановке запрос — `503`, `application/json`, `error.code = unavailable`, `request_id` = `X-Request-Id`, непустой `X-Instance-Id`
- [ ] 1.3 Порт `Readiness`, адаптер `DatabaseReadiness` с флагом `DRAINING` в `shared/infrastructure/readiness.py`, сборка в `PlatformProvider`, маршрут `/health/ready` через порт; тест состава модулей ядра
- [ ] 1.4 `gunicorn_conf.py`: флаг из инфраструктуры, событие `service.draining` через `StructlogLogger`; `failover-evidence.sh` ищет его в логе
- [ ] 1.5 `openspec/config.yaml`: карта модулей (`readiness` за портом)
+13 -10
View File
@@ -103,12 +103,6 @@ class RequestPipeline:
message = {**message, "headers": headers}
await send(message)
declared = header(scope, b"content-length")
if declared.isdigit() and int(declared) > self._max_bytes:
await send_error(wrapped_send, 413, "payload_too_large", request_id)
self._log(scope, status_holder[0], request_id, started)
return
received = [0]
async def limited_receive() -> Message:
@@ -121,7 +115,7 @@ class RequestPipeline:
rejection: Rejection | None = None
try:
async with asyncio.timeout(self._timeout):
rejection = await self._guard(scope)
rejection = self._oversized(scope) or await self._guard(scope)
if rejection is None:
await self._app(scope, limited_receive, wrapped_send)
except Unavailable as error:
@@ -132,18 +126,21 @@ class RequestPipeline:
request_id=request_id,
)
rejection = Rejection(503, "unavailable")
except TimeoutError:
except (TimeoutError, asyncio.CancelledError) as error:
cancelled = isinstance(error, asyncio.CancelledError)
self._logger.warning(
"request.timeout",
"request.cancelled" if cancelled else "request.timeout",
method=scope["method"],
route=self._route(scope),
request_id=request_id,
limit=self._timeout,
limit=None if cancelled else self._timeout,
duration_ms=round((time.perf_counter() - started) * 1000, 3),
)
if not answered[0]:
await send_error(wrapped_send, 503, "unavailable", request_id)
self._log(scope, status_holder[0], request_id, started)
if cancelled:
raise
return
except Exception:
self._logger.error(
@@ -160,6 +157,12 @@ class RequestPipeline:
)
self._log(scope, status_holder[0], request_id, started)
def _oversized(self, scope: Scope) -> Rejection | None:
declared = header(scope, b"content-length")
if declared.isdigit() and int(declared) > self._max_bytes:
return Rejection(413, "payload_too_large")
return None
def _route(self, scope: Scope) -> str:
route = scope.get("route")
return getattr(route, "path", scope["path"])
+18 -3
View File
@@ -49,6 +49,17 @@ def status(port: int, path: str) -> int:
return 0
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
data = b""
while chunk := connection.recv(4096):
data += chunk
connection.close()
head, _, body = data.partition(b"\r\n\r\n")
status, *lines = head.decode("latin-1").split("\r\n")
headers = dict(line.lower().split(": ", 1) for line in lines)
return status, headers, json.loads(body) if body else {}
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
body = json.dumps(new_user("drain")).encode()
head = (
@@ -97,17 +108,21 @@ def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests() -> Non
pending.close()
code = server.wait(timeout=SHUTDOWN + 2)
exited = time.monotonic() - sent
cancelled = stuck.recv(64).split(b"\r\n")[0].decode()
stuck.close()
cancelled, headers, error = answer_of(stuck)
finally:
if server.poll() is None:
server.kill()
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
print(f" content-type {headers.get('content-type')}, тело {error}")
assert during == (503, 200)
assert answer == "HTTP/1.1 201 Created"
assert cancelled == "HTTP/1.1 500 Internal Server Error"
assert cancelled == "HTTP/1.1 503 Service Unavailable"
assert headers["content-type"] == "application/json"
assert headers["x-instance-id"]
assert error["error"]["code"] == "unavailable"
assert error["error"]["request_id"] == headers["x-request-id"]
assert DRAIN - 0.2 <= closed < SHUTDOWN
assert exited < SHUTDOWN
assert code == 0