forked from mtusi2026/mtusi2026lab2
Answer requests cancelled at shutdown with 503 in the contract format
After the drain window uvicorn waits for accepted requests for three seconds and then cancels their tasks. CancelledError is a BaseException, so neither the pipeline nor ServerErrorMiddleware saw it and uvicorn answered with its own text/plain 500 without X-Request-Id and X-Instance-Id. The pipeline now handles the cancellation next to the request timeout: it logs request.cancelled, sends 503 unavailable in the common error format when the response has not started and re-raises, so uvicorn only closes the transport after the buffered answer. The declared Content-Length check joins the guard as a rejection to keep the call within the statement limit; behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
13e856ebd4
commit
aa1485bd86
@@ -1,7 +1,7 @@
|
||||
## 1. Платформа: остановка, готовность, журнал
|
||||
|
||||
- [ ] 1.1 `RequestPipeline`: отмена задачи → `request.cancelled`, `503 unavailable` в общем формате, если ответ не начат, затем отмена пробрасывается
|
||||
- [ ] 1.2 `test_drain.py`: прерванный при остановке запрос — `503`, `application/json`, `error.code = unavailable`, `request_id` = `X-Request-Id`, непустой `X-Instance-Id`
|
||||
- [x] 1.1 `RequestPipeline`: отмена задачи → `request.cancelled`, `503 unavailable` в общем формате, если ответ не начат, затем отмена пробрасывается
|
||||
- [x] 1.2 `test_drain.py`: прерванный при остановке запрос — `503`, `application/json`, `error.code = unavailable`, `request_id` = `X-Request-Id`, непустой `X-Instance-Id`
|
||||
- [ ] 1.3 Порт `Readiness`, адаптер `DatabaseReadiness` с флагом `DRAINING` в `shared/infrastructure/readiness.py`, сборка в `PlatformProvider`, маршрут `/health/ready` через порт; тест состава модулей ядра
|
||||
- [ ] 1.4 `gunicorn_conf.py`: флаг из инфраструктуры, событие `service.draining` через `StructlogLogger`; `failover-evidence.sh` ищет его в логе
|
||||
- [ ] 1.5 `openspec/config.yaml`: карта модулей (`readiness` за портом)
|
||||
|
||||
@@ -103,12 +103,6 @@ class RequestPipeline:
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
declared = header(scope, b"content-length")
|
||||
if declared.isdigit() and int(declared) > self._max_bytes:
|
||||
await send_error(wrapped_send, 413, "payload_too_large", request_id)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
return
|
||||
|
||||
received = [0]
|
||||
|
||||
async def limited_receive() -> Message:
|
||||
@@ -121,7 +115,7 @@ class RequestPipeline:
|
||||
rejection: Rejection | None = None
|
||||
try:
|
||||
async with asyncio.timeout(self._timeout):
|
||||
rejection = await self._guard(scope)
|
||||
rejection = self._oversized(scope) or await self._guard(scope)
|
||||
if rejection is None:
|
||||
await self._app(scope, limited_receive, wrapped_send)
|
||||
except Unavailable as error:
|
||||
@@ -132,18 +126,21 @@ class RequestPipeline:
|
||||
request_id=request_id,
|
||||
)
|
||||
rejection = Rejection(503, "unavailable")
|
||||
except TimeoutError:
|
||||
except (TimeoutError, asyncio.CancelledError) as error:
|
||||
cancelled = isinstance(error, asyncio.CancelledError)
|
||||
self._logger.warning(
|
||||
"request.timeout",
|
||||
"request.cancelled" if cancelled else "request.timeout",
|
||||
method=scope["method"],
|
||||
route=self._route(scope),
|
||||
request_id=request_id,
|
||||
limit=self._timeout,
|
||||
limit=None if cancelled else self._timeout,
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 3),
|
||||
)
|
||||
if not answered[0]:
|
||||
await send_error(wrapped_send, 503, "unavailable", request_id)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
if cancelled:
|
||||
raise
|
||||
return
|
||||
except Exception:
|
||||
self._logger.error(
|
||||
@@ -160,6 +157,12 @@ class RequestPipeline:
|
||||
)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
|
||||
def _oversized(self, scope: Scope) -> Rejection | None:
|
||||
declared = header(scope, b"content-length")
|
||||
if declared.isdigit() and int(declared) > self._max_bytes:
|
||||
return Rejection(413, "payload_too_large")
|
||||
return None
|
||||
|
||||
def _route(self, scope: Scope) -> str:
|
||||
route = scope.get("route")
|
||||
return getattr(route, "path", scope["path"])
|
||||
|
||||
@@ -49,6 +49,17 @@ def status(port: int, path: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
|
||||
data = b""
|
||||
while chunk := connection.recv(4096):
|
||||
data += chunk
|
||||
connection.close()
|
||||
head, _, body = data.partition(b"\r\n\r\n")
|
||||
status, *lines = head.decode("latin-1").split("\r\n")
|
||||
headers = dict(line.lower().split(": ", 1) for line in lines)
|
||||
return status, headers, json.loads(body) if body else {}
|
||||
|
||||
|
||||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||||
body = json.dumps(new_user("drain")).encode()
|
||||
head = (
|
||||
@@ -97,17 +108,21 @@ def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests() -> Non
|
||||
pending.close()
|
||||
code = server.wait(timeout=SHUTDOWN + 2)
|
||||
exited = time.monotonic() - sent
|
||||
cancelled = stuck.recv(64).split(b"\r\n")[0].decode()
|
||||
stuck.close()
|
||||
cancelled, headers, error = answer_of(stuck)
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||||
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
|
||||
print(f" content-type {headers.get('content-type')}, тело {error}")
|
||||
assert during == (503, 200)
|
||||
assert answer == "HTTP/1.1 201 Created"
|
||||
assert cancelled == "HTTP/1.1 500 Internal Server Error"
|
||||
assert cancelled == "HTTP/1.1 503 Service Unavailable"
|
||||
assert headers["content-type"] == "application/json"
|
||||
assert headers["x-instance-id"]
|
||||
assert error["error"]["code"] == "unavailable"
|
||||
assert error["error"]["request_id"] == headers["x-request-id"]
|
||||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||||
assert exited < SHUTDOWN
|
||||
assert code == 0
|
||||
|
||||
Reference in New Issue
Block a user