Files
mtusi2026lab4/student-tests/api/test_drain.py
T
ArtyomandClaude Opus 5.5 e927afdb09 Carry over the lab 3 fixes made after 61c0c28
Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4
code, resolving each conflict by meaning and keeping everything lab 4
added:

- a message is posted with one SQL statement under the chat row lock,
  and the attachment links go into the same statement; a foreign key
  violation on the attachment link still answers 404 and leaves nothing
  behind. Text and the attachment list are checked before the write, an
  outsider gets 404 for both, as before the carry over;
- the request pipeline counts the body bytes actually received against
  the limit of the route, so a chunked body over 1 MiB gets 413 and the
  upload route keeps its MAX_UPLOAD_BYTES limit;
- a request cancelled at shutdown gets 503 unavailable in the contract
  format, the worker waits a second less than the rest of the shutdown
  budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s;
- readiness sits behind a Readiness port with the draining flag in
  shared infrastructure, and the drain window is logged through Logger;
- logout revokes through SessionRevocations and withdraws the mark when
  PostgreSQL rejects the delete; an empty cursor is 400;
- EventBus.project is gone, the sweep test subscribes and waits on a
  queue; domain tests check attachment_list instead of Chat.post;
- make up waits for every replica in the balancer, BASE_URL follows the
  lb port, the image is tagged by the compose project, retry-check and
  faults-evidence are added, detection runs three times, failover-each
  and rolling-update record the stand usage, CI checks numeric uids.
  down also takes the tools and reconcile profiles of lab 4.

Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU
pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the
database size itself instead of the track 1 sampler.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-30 06:25:23 +03:00

131 lines
4.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import os
import signal
import socket
import subprocess
import sys
import time
import httpx
from conftest import DATABASE, new_user
from messenger.shared.infrastructure.readiness import DRAINING
DRAIN = 1.0
SHUTDOWN = 3
GUNICORN = (
"-m",
"gunicorn",
"--config",
"python:messenger.gunicorn_conf",
"messenger.main:create_app()",
)
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
DRAINING.set()
try:
ready = await client.get("/health/ready")
live = await client.get("/health/live")
finally:
DRAINING.clear()
assert ready.status_code == 503
assert ready.json()["error"]["code"] == "unavailable"
assert ready.headers["X-Instance-Id"]
assert live.status_code == 200
assert (await client.get("/health/ready")).status_code == 200
def free_port() -> int:
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
return probe.getsockname()[1]
def status(port: int, path: str) -> int:
try:
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code
except httpx.TransportError:
return 0
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
data = b""
while chunk := connection.recv(4096):
data += chunk
connection.close()
head, _, body = data.partition(b"\r\n\r\n")
status, *lines = head.decode("latin-1").split("\r\n")
headers = dict(line.lower().split(": ", 1) for line in lines)
return status, headers, json.loads(body) if body else {}
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
body = json.dumps(new_user("drain")).encode()
head = (
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
)
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
connection.sendall(head.encode() + body[:8])
return connection, body[8:]
def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests() -> None:
port = free_port()
settings = {
"HOST": "127.0.0.1",
"PORT": str(port),
"WEB_CONCURRENCY": "2",
"DATABASE_URL": DATABASE["url"],
"CURSOR_SECRET": "test-secret-for-cursors",
"S3_ACCESS_KEY_ID": "test-key",
"S3_SECRET_ACCESS_KEY": "test-secret-key",
"LOG_LEVEL": "WARNING",
"DRAIN_SECONDS": str(DRAIN),
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
}
server = subprocess.Popen(
[sys.executable, *GUNICORN],
env={**os.environ, **settings},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
deadline = time.monotonic() + 30
while status(port, "/health/ready") != 200:
assert time.monotonic() < deadline, "gunicorn не стал готов"
time.sleep(0.2)
pending, rest = begin_registration(port)
stuck, _ = begin_registration(port)
sent = time.monotonic()
server.send_signal(signal.SIGTERM)
time.sleep(0.3)
during = status(port, "/health/ready"), status(port, "/health/live")
while status(port, "/health/live") == 200:
time.sleep(0.05)
closed = time.monotonic() - sent
pending.sendall(rest)
answer = pending.recv(64).split(b"\r\n")[0].decode()
pending.close()
code = server.wait(timeout=SHUTDOWN + 2)
exited = time.monotonic() - sent
cancelled, headers, error = answer_of(stuck)
finally:
if server.poll() is None:
server.kill()
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
print(f" content-type {headers.get('content-type')}, тело {error}")
assert during == (503, 200)
assert answer == "HTTP/1.1 201 Created"
assert cancelled == "HTTP/1.1 503 Service Unavailable"
assert headers["content-type"] == "application/json"
assert headers["x-instance-id"]
assert error["error"]["code"] == "unavailable"
assert error["error"]["request_id"] == headers["x-request-id"]
assert DRAIN - 0.2 <= closed < SHUTDOWN
assert exited < SHUTDOWN
assert code == 0