feat: add a driver command that stops a starting replica

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-13 16:50:18 +03:00
co-authored by Claude Opus 5.5
parent f8c81878cb
commit 42e6edf420
@@ -40,6 +40,13 @@
разморозка, одно сообщение в чате, строка журнала PostgreSQL;
триггер удаляется при любом исходе, в том числе по Ctrl+C,
SIGTERM и SIGHUP; не удалось — печатается SQL удаления
stop-at-start [--image ТЕГ] [--delays 0.2,0.3,0.4,1.5,5]
[--server-start 3] docker stop контейнеров образа (лимиты
реплики, без сети) во время старта: через паузу после
docker run и по строке «Started server process»; время
сигнала и выхода, код выхода, этап старта, кто остановил
(ранний обработчик или uvicorn); код 1, если стартующий
контейнер останавливается дольше 1 с или не с кодом 0
stop остановить свой экземпляр на порту
all start → flow → smoke → student (если есть student-tests/) → stop
@@ -1194,6 +1201,330 @@ def hang_key(port: int) -> int:
print(f"trigger {probe} removed", flush=True)
# stop-at-start: записи журнала, по которым видны этапы старта и остановки
STARTED_SERVER = "Started server process"
SERVER_RUNNING = "Uvicorn running on"
EARLY_STOP = "Stopped API before startup"
UVICORN_STOP = ("Shutting down", "Stopped API", "Finished server process")
# имена событий без переменной части («... [1]», «... on http://...»)
EVENT_PREFIXES = (STARTED_SERVER, SERVER_RUNNING, "Finished server process")
STOP_LABEL = "mtusi.stop-at-start"
# docker stop стартующего контейнера — не дольше (спек graceful-shutdown)
STOP_LIMIT_S = 1.0
# ожидание события die и строки старта сервера
ATTEMPT_TIMEOUT_S = 30
def event_name(event: str) -> str:
for prefix in EVENT_PREFIXES:
if event.startswith(prefix):
return prefix
return event
def log_time(value: str) -> float | None:
"""`timestamp` записи журнала (ISO 8601 UTC с `Z`) — секунды эпохи."""
try:
moment = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ")
except (TypeError, ValueError):
return None
return moment.replace(tzinfo=UTC).timestamp()
def docker_events(path: Path) -> list[tuple[float, str, str, str, str]]:
"""Строки потока событий: (время ВМ, действие, ID, сигнал, код)."""
rows = []
for line in path.read_text(encoding="utf-8").splitlines():
parts = line.split("|")
if len(parts) == 5 and parts[0].isdigit():
rows.append((int(parts[0]) / 1e9, *parts[1:]))
return rows
def cmd_stop_at_start(image: str, delays: list[float], server: int) -> int:
"""Сценарии спека `graceful-shutdown` «Остановка во время старта»
(решение 8 design change `fix-known-issues`).
Каждая попытка — свой контейнер образа с лимитами реплики `api`
(1 CPU / 512 МиБ без swap), без сети и без `config.toml`: приложение
стартует на дефолтах, соединения с хранилищами ленивые. `docker stop`
(срок 10 с, как `stop_grace_period`) — через паузу после `docker run`
или, у попыток `--server-start`, сразу по строке «Started server
process» (`docker logs -f`). Время — часы ВМ Docker: поток `docker
events` по метке (`start`, `kill` с сигналом 15, `die` с кодом) и
`timestamp` записей журнала. Проекты compose не трогаются, свои
контейнеры удаляются по метке при любом исходе."""
def interrupted(signum: int, _frame: object) -> None:
raise KeyboardInterrupt(signal.Signals(signum).name)
previous = {
signum: signal.signal(signum, interrupted)
for signum in (signal.SIGTERM, signal.SIGHUP)
}
tag = uuid.uuid4().hex[:8]
label = f"{STOP_LABEL}={tag}"
events_path = Path(f"/tmp/mtusi-stop-at-start-{tag}.events")
image_id = subprocess.run(
["docker", "image", "inspect", "-f", "{{.Id}}", image],
capture_output=True,
text=True,
check=False,
).stdout.strip()
if not image_id:
print(f"image {image} not found")
return 1
print(f"image {image} ({image_id[:19]}), label {label}")
with events_path.open("w") as out:
stream = subprocess.Popen(
[
"docker",
"events",
"--filter",
f"label={label}",
"--filter",
"type=container",
"--filter",
"event=start",
"--filter",
"event=kill",
"--filter",
"event=die",
"--format",
"{{.TimeNano}}|{{.Action}}|{{.Actor.ID}}|"
'{{index .Actor.Attributes "signal"}}|'
'{{index .Actor.Attributes "exitCode"}}',
],
stdout=out,
stderr=subprocess.DEVNULL,
)
try:
# CLI подключается к демону до первого контейнера
time.sleep(1)
plan = [("delay", d) for d in delays] + [("server", 0.0)] * server
attempts = []
for number, (how, delay) in enumerate(plan, start=1):
attempts.append(
stop_attempt(
number,
image,
label,
f"mtusi-stop-at-start-{tag}-{number}",
events_path,
delay=delay if how == "delay" else None,
),
)
return stop_verdict(attempts)
finally:
stream.terminate()
stream.wait()
events_path.unlink(missing_ok=True)
left = subprocess.run(
["docker", "ps", "-aq", "--filter", f"label={label}"],
capture_output=True,
text=True,
check=False,
).stdout.split()
if left:
subprocess.run(
["docker", "rm", "-f", *left],
capture_output=True,
check=False,
)
for signum, handler in previous.items():
signal.signal(signum, handler)
def stop_attempt(
number: int,
image: str,
label: str,
name: str,
events_path: Path,
*,
delay: float | None,
) -> dict[str, object]:
"""Одна попытка: контейнер, `docker stop`, события и журнал."""
run = subprocess.run(
[
"docker",
"run",
"-d",
"--network",
"none",
"--cpus",
"1",
"--memory",
"512m",
"--memory-swap",
"512m",
"--label",
label,
"--name",
name,
image,
],
capture_output=True,
text=True,
check=True,
)
container = run.stdout.strip()
if delay is not None:
time.sleep(delay)
else:
follow = subprocess.Popen(
["docker", "logs", "-f", container],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
timer = threading.Timer(ATTEMPT_TIMEOUT_S, follow.kill)
timer.start()
try:
for line in follow.stdout or []:
if STARTED_SERVER in line:
break
finally:
timer.cancel()
began = time.monotonic()
subprocess.run(["docker", "stop", container], capture_output=True)
stop_s = time.monotonic() - began
if delay is None:
follow.kill()
follow.wait()
deadline = time.monotonic() + ATTEMPT_TIMEOUT_S
mine: list[tuple[float, str, str, str, str]] = []
while time.monotonic() < deadline:
mine = [e for e in docker_events(events_path) if e[2] == container]
if any(e[1] == "die" for e in mine):
break
time.sleep(0.2)
start = next((e[0] for e in mine if e[1] == "start"), None)
kill = next(
(e[0] for e in mine if e[1] == "kill" and e[3] == "15"),
None,
)
die = next((e for e in mine if e[1] == "die"), None)
logs = subprocess.run(
["docker", "logs", container],
capture_output=True,
text=True,
check=False,
)
records = []
for line in (logs.stdout + logs.stderr).splitlines():
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(record, dict) and isinstance(record.get("event"), str):
records.append(record)
names = [event_name(r["event"]) for r in records]
started_at = next(
(
log_time(r.get("timestamp"))
for r in records
if event_name(r["event"]) == STARTED_SERVER
),
None,
)
running_at = next(
(
log_time(r.get("timestamp"))
for r in records
if event_name(r["event"]) == SERVER_RUNNING
),
None,
)
if kill is None or started_at is None or kill < started_at:
kind = "import"
elif running_at is None or kill < running_at:
kind = "server-start"
else:
kind = "running"
early = [r for r in records if r["event"] == EARLY_STOP]
if early:
by = "early"
elif all(stage in names for stage in UVICORN_STOP):
by = "uvicorn"
else:
by = "none"
# запись ранней остановки — JSON с полями записей structlog
early_ok = all(
r.get("level") == "info"
and r.get("signal") == "SIGTERM"
and log_time(r.get("timestamp")) is not None
and r.get("instance_id")
for r in early
)
code = int(die[4]) if die is not None and die[4].isdigit() else None
result = {
"number": number,
"how": f"pause {delay:g} s" if delay is not None else "server-start",
"signal_after": None
if kill is None or start is None
else kill - start,
"stop_s": stop_s,
"exit_after": None if kill is None or die is None else die[0] - kill,
"code": code,
"kind": kind,
"by": by,
"early_ok": early_ok,
"events": names,
"instance": records[0].get("instance_id") if records else None,
}
def seconds(value: object) -> str:
return "-" if value is None else f"{value:.3f} s"
print(
f"#{number} {result['how']}: signal after "
f"{seconds(result['signal_after'])}, docker stop {stop_s:.3f} s, "
f"exit after signal {seconds(result['exit_after'])}, exit code "
f"{code}, {kind}, stopped by {by}",
flush=True,
)
print(f" log: {' → '.join(names) or '(empty)'}", flush=True)
return result
def stop_verdict(attempts: list[dict[str, object]]) -> int:
"""PASS (код 0) по решению 8 design change `fix-known-issues`."""
problems = []
for a in attempts:
n = a["number"]
if a["code"] != 0:
problems.append(f"#{n}: exit code {a['code']}")
if float(a["stop_s"]) > STOP_LIMIT_S:
problems.append(f"#{n}: docker stop took {a['stop_s']:.1f} s")
if a["kind"] == "import":
# окно config.load(): сигнал до первой строки uvicorn, но уже
# под его обработчиками — штатная остановка тоже верна
if a["by"] == "early":
if STARTED_SERVER in a["events"]:
problems.append(f"#{n}: server start after early stop")
if not a["early_ok"]:
problems.append(f"#{n}: malformed early stop record")
elif a["by"] != "uvicorn":
problems.append(f"#{n}: import, nobody stopped the process")
elif a["by"] != "uvicorn":
problems.append(f"#{n}: {a['kind']}, stopped by {a['by']}")
kinds = Counter(str(a["kind"]) for a in attempts)
if not any(a["kind"] == "import" and a["by"] == "early" for a in attempts):
problems.append("no import attempt stopped early")
if not kinds["running"]:
problems.append("no running attempt")
print(
f"attempts: import {kinds['import']}, server-start "
f"{kinds['server-start']}, running {kinds['running']}",
)
if problems:
print("FAIL: " + "; ".join(problems))
return 1
print("PASS: SIGTERM during start stops the replica at once, exit 0")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
@@ -1214,6 +1545,19 @@ def main() -> int:
# контейнер LB, если --port ведёт не на LB стенда (вариант конфигурации)
idle.add_argument("--lb")
sub.add_parser("hang-key")
stop = sub.add_parser("stop-at-start")
stop.add_argument("--image", default="mtusi-messenger:lab3")
stop.add_argument(
"--delays",
default="0.2,0.3,0.4,1.5,5",
help="паузы после docker run до docker stop, с",
)
stop.add_argument(
"--server-start",
type=int,
default=3,
help="попыток со stop по строке «Started server process»",
)
req = sub.add_parser("req")
req.add_argument("method")
req.add_argument("path")
@@ -1264,6 +1608,12 @@ def main() -> int:
)
if args.cmd == "hang-key":
return cmd_hang_key(port)
if args.cmd == "stop-at-start":
return cmd_stop_at_start(
args.image,
[float(d) for d in args.delays.split(",") if d],
args.server_start,
)
handlers = {
"start": cmd_start,
"stop": cmd_stop,