feat: add a driver command that stops a starting replica
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -40,6 +40,13 @@
|
||||
разморозка, одно сообщение в чате, строка журнала PostgreSQL;
|
||||
триггер удаляется при любом исходе, в том числе по Ctrl+C,
|
||||
SIGTERM и SIGHUP; не удалось — печатается SQL удаления
|
||||
stop-at-start [--image ТЕГ] [--delays 0.2,0.3,0.4,1.5,5]
|
||||
[--server-start 3] docker stop контейнеров образа (лимиты
|
||||
реплики, без сети) во время старта: через паузу после
|
||||
docker run и по строке «Started server process»; время
|
||||
сигнала и выхода, код выхода, этап старта, кто остановил
|
||||
(ранний обработчик или uvicorn); код 1, если стартующий
|
||||
контейнер останавливается дольше 1 с или не с кодом 0
|
||||
stop остановить свой экземпляр на порту
|
||||
all start → flow → smoke → student (если есть student-tests/) → stop
|
||||
|
||||
@@ -1194,6 +1201,330 @@ def hang_key(port: int) -> int:
|
||||
print(f"trigger {probe} removed", flush=True)
|
||||
|
||||
|
||||
# stop-at-start: записи журнала, по которым видны этапы старта и остановки
|
||||
STARTED_SERVER = "Started server process"
|
||||
SERVER_RUNNING = "Uvicorn running on"
|
||||
EARLY_STOP = "Stopped API before startup"
|
||||
UVICORN_STOP = ("Shutting down", "Stopped API", "Finished server process")
|
||||
# имена событий без переменной части («... [1]», «... on http://...»)
|
||||
EVENT_PREFIXES = (STARTED_SERVER, SERVER_RUNNING, "Finished server process")
|
||||
STOP_LABEL = "mtusi.stop-at-start"
|
||||
# docker stop стартующего контейнера — не дольше (спек graceful-shutdown)
|
||||
STOP_LIMIT_S = 1.0
|
||||
# ожидание события die и строки старта сервера
|
||||
ATTEMPT_TIMEOUT_S = 30
|
||||
|
||||
|
||||
def event_name(event: str) -> str:
|
||||
for prefix in EVENT_PREFIXES:
|
||||
if event.startswith(prefix):
|
||||
return prefix
|
||||
return event
|
||||
|
||||
|
||||
def log_time(value: str) -> float | None:
|
||||
"""`timestamp` записи журнала (ISO 8601 UTC с `Z`) — секунды эпохи."""
|
||||
try:
|
||||
moment = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return moment.replace(tzinfo=UTC).timestamp()
|
||||
|
||||
|
||||
def docker_events(path: Path) -> list[tuple[float, str, str, str, str]]:
|
||||
"""Строки потока событий: (время ВМ, действие, ID, сигнал, код)."""
|
||||
rows = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
parts = line.split("|")
|
||||
if len(parts) == 5 and parts[0].isdigit():
|
||||
rows.append((int(parts[0]) / 1e9, *parts[1:]))
|
||||
return rows
|
||||
|
||||
|
||||
def cmd_stop_at_start(image: str, delays: list[float], server: int) -> int:
|
||||
"""Сценарии спека `graceful-shutdown` «Остановка во время старта»
|
||||
(решение 8 design change `fix-known-issues`).
|
||||
|
||||
Каждая попытка — свой контейнер образа с лимитами реплики `api`
|
||||
(1 CPU / 512 МиБ без swap), без сети и без `config.toml`: приложение
|
||||
стартует на дефолтах, соединения с хранилищами ленивые. `docker stop`
|
||||
(срок 10 с, как `stop_grace_period`) — через паузу после `docker run`
|
||||
или, у попыток `--server-start`, сразу по строке «Started server
|
||||
process» (`docker logs -f`). Время — часы ВМ Docker: поток `docker
|
||||
events` по метке (`start`, `kill` с сигналом 15, `die` с кодом) и
|
||||
`timestamp` записей журнала. Проекты compose не трогаются, свои
|
||||
контейнеры удаляются по метке при любом исходе."""
|
||||
|
||||
def interrupted(signum: int, _frame: object) -> None:
|
||||
raise KeyboardInterrupt(signal.Signals(signum).name)
|
||||
|
||||
previous = {
|
||||
signum: signal.signal(signum, interrupted)
|
||||
for signum in (signal.SIGTERM, signal.SIGHUP)
|
||||
}
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
label = f"{STOP_LABEL}={tag}"
|
||||
events_path = Path(f"/tmp/mtusi-stop-at-start-{tag}.events")
|
||||
image_id = subprocess.run(
|
||||
["docker", "image", "inspect", "-f", "{{.Id}}", image],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
).stdout.strip()
|
||||
if not image_id:
|
||||
print(f"image {image} not found")
|
||||
return 1
|
||||
print(f"image {image} ({image_id[:19]}), label {label}")
|
||||
with events_path.open("w") as out:
|
||||
stream = subprocess.Popen(
|
||||
[
|
||||
"docker",
|
||||
"events",
|
||||
"--filter",
|
||||
f"label={label}",
|
||||
"--filter",
|
||||
"type=container",
|
||||
"--filter",
|
||||
"event=start",
|
||||
"--filter",
|
||||
"event=kill",
|
||||
"--filter",
|
||||
"event=die",
|
||||
"--format",
|
||||
"{{.TimeNano}}|{{.Action}}|{{.Actor.ID}}|"
|
||||
'{{index .Actor.Attributes "signal"}}|'
|
||||
'{{index .Actor.Attributes "exitCode"}}',
|
||||
],
|
||||
stdout=out,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
# CLI подключается к демону до первого контейнера
|
||||
time.sleep(1)
|
||||
plan = [("delay", d) for d in delays] + [("server", 0.0)] * server
|
||||
attempts = []
|
||||
for number, (how, delay) in enumerate(plan, start=1):
|
||||
attempts.append(
|
||||
stop_attempt(
|
||||
number,
|
||||
image,
|
||||
label,
|
||||
f"mtusi-stop-at-start-{tag}-{number}",
|
||||
events_path,
|
||||
delay=delay if how == "delay" else None,
|
||||
),
|
||||
)
|
||||
return stop_verdict(attempts)
|
||||
finally:
|
||||
stream.terminate()
|
||||
stream.wait()
|
||||
events_path.unlink(missing_ok=True)
|
||||
left = subprocess.run(
|
||||
["docker", "ps", "-aq", "--filter", f"label={label}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
).stdout.split()
|
||||
if left:
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", *left],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
for signum, handler in previous.items():
|
||||
signal.signal(signum, handler)
|
||||
|
||||
|
||||
def stop_attempt(
|
||||
number: int,
|
||||
image: str,
|
||||
label: str,
|
||||
name: str,
|
||||
events_path: Path,
|
||||
*,
|
||||
delay: float | None,
|
||||
) -> dict[str, object]:
|
||||
"""Одна попытка: контейнер, `docker stop`, события и журнал."""
|
||||
run = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--network",
|
||||
"none",
|
||||
"--cpus",
|
||||
"1",
|
||||
"--memory",
|
||||
"512m",
|
||||
"--memory-swap",
|
||||
"512m",
|
||||
"--label",
|
||||
label,
|
||||
"--name",
|
||||
name,
|
||||
image,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
container = run.stdout.strip()
|
||||
if delay is not None:
|
||||
time.sleep(delay)
|
||||
else:
|
||||
follow = subprocess.Popen(
|
||||
["docker", "logs", "-f", container],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
timer = threading.Timer(ATTEMPT_TIMEOUT_S, follow.kill)
|
||||
timer.start()
|
||||
try:
|
||||
for line in follow.stdout or []:
|
||||
if STARTED_SERVER in line:
|
||||
break
|
||||
finally:
|
||||
timer.cancel()
|
||||
began = time.monotonic()
|
||||
subprocess.run(["docker", "stop", container], capture_output=True)
|
||||
stop_s = time.monotonic() - began
|
||||
if delay is None:
|
||||
follow.kill()
|
||||
follow.wait()
|
||||
deadline = time.monotonic() + ATTEMPT_TIMEOUT_S
|
||||
mine: list[tuple[float, str, str, str, str]] = []
|
||||
while time.monotonic() < deadline:
|
||||
mine = [e for e in docker_events(events_path) if e[2] == container]
|
||||
if any(e[1] == "die" for e in mine):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
start = next((e[0] for e in mine if e[1] == "start"), None)
|
||||
kill = next(
|
||||
(e[0] for e in mine if e[1] == "kill" and e[3] == "15"),
|
||||
None,
|
||||
)
|
||||
die = next((e for e in mine if e[1] == "die"), None)
|
||||
logs = subprocess.run(
|
||||
["docker", "logs", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
records = []
|
||||
for line in (logs.stdout + logs.stderr).splitlines():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(record, dict) and isinstance(record.get("event"), str):
|
||||
records.append(record)
|
||||
names = [event_name(r["event"]) for r in records]
|
||||
started_at = next(
|
||||
(
|
||||
log_time(r.get("timestamp"))
|
||||
for r in records
|
||||
if event_name(r["event"]) == STARTED_SERVER
|
||||
),
|
||||
None,
|
||||
)
|
||||
running_at = next(
|
||||
(
|
||||
log_time(r.get("timestamp"))
|
||||
for r in records
|
||||
if event_name(r["event"]) == SERVER_RUNNING
|
||||
),
|
||||
None,
|
||||
)
|
||||
if kill is None or started_at is None or kill < started_at:
|
||||
kind = "import"
|
||||
elif running_at is None or kill < running_at:
|
||||
kind = "server-start"
|
||||
else:
|
||||
kind = "running"
|
||||
early = [r for r in records if r["event"] == EARLY_STOP]
|
||||
if early:
|
||||
by = "early"
|
||||
elif all(stage in names for stage in UVICORN_STOP):
|
||||
by = "uvicorn"
|
||||
else:
|
||||
by = "none"
|
||||
# запись ранней остановки — JSON с полями записей structlog
|
||||
early_ok = all(
|
||||
r.get("level") == "info"
|
||||
and r.get("signal") == "SIGTERM"
|
||||
and log_time(r.get("timestamp")) is not None
|
||||
and r.get("instance_id")
|
||||
for r in early
|
||||
)
|
||||
code = int(die[4]) if die is not None and die[4].isdigit() else None
|
||||
result = {
|
||||
"number": number,
|
||||
"how": f"pause {delay:g} s" if delay is not None else "server-start",
|
||||
"signal_after": None
|
||||
if kill is None or start is None
|
||||
else kill - start,
|
||||
"stop_s": stop_s,
|
||||
"exit_after": None if kill is None or die is None else die[0] - kill,
|
||||
"code": code,
|
||||
"kind": kind,
|
||||
"by": by,
|
||||
"early_ok": early_ok,
|
||||
"events": names,
|
||||
"instance": records[0].get("instance_id") if records else None,
|
||||
}
|
||||
|
||||
def seconds(value: object) -> str:
|
||||
return "-" if value is None else f"{value:.3f} s"
|
||||
|
||||
print(
|
||||
f"#{number} {result['how']}: signal after "
|
||||
f"{seconds(result['signal_after'])}, docker stop {stop_s:.3f} s, "
|
||||
f"exit after signal {seconds(result['exit_after'])}, exit code "
|
||||
f"{code}, {kind}, stopped by {by}",
|
||||
flush=True,
|
||||
)
|
||||
print(f" log: {' → '.join(names) or '(empty)'}", flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def stop_verdict(attempts: list[dict[str, object]]) -> int:
|
||||
"""PASS (код 0) по решению 8 design change `fix-known-issues`."""
|
||||
problems = []
|
||||
for a in attempts:
|
||||
n = a["number"]
|
||||
if a["code"] != 0:
|
||||
problems.append(f"#{n}: exit code {a['code']}")
|
||||
if float(a["stop_s"]) > STOP_LIMIT_S:
|
||||
problems.append(f"#{n}: docker stop took {a['stop_s']:.1f} s")
|
||||
if a["kind"] == "import":
|
||||
# окно config.load(): сигнал до первой строки uvicorn, но уже
|
||||
# под его обработчиками — штатная остановка тоже верна
|
||||
if a["by"] == "early":
|
||||
if STARTED_SERVER in a["events"]:
|
||||
problems.append(f"#{n}: server start after early stop")
|
||||
if not a["early_ok"]:
|
||||
problems.append(f"#{n}: malformed early stop record")
|
||||
elif a["by"] != "uvicorn":
|
||||
problems.append(f"#{n}: import, nobody stopped the process")
|
||||
elif a["by"] != "uvicorn":
|
||||
problems.append(f"#{n}: {a['kind']}, stopped by {a['by']}")
|
||||
kinds = Counter(str(a["kind"]) for a in attempts)
|
||||
if not any(a["kind"] == "import" and a["by"] == "early" for a in attempts):
|
||||
problems.append("no import attempt stopped early")
|
||||
if not kinds["running"]:
|
||||
problems.append("no running attempt")
|
||||
print(
|
||||
f"attempts: import {kinds['import']}, server-start "
|
||||
f"{kinds['server-start']}, running {kinds['running']}",
|
||||
)
|
||||
if problems:
|
||||
print("FAIL: " + "; ".join(problems))
|
||||
return 1
|
||||
print("PASS: SIGTERM during start stops the replica at once, exit 0")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
@@ -1214,6 +1545,19 @@ def main() -> int:
|
||||
# контейнер LB, если --port ведёт не на LB стенда (вариант конфигурации)
|
||||
idle.add_argument("--lb")
|
||||
sub.add_parser("hang-key")
|
||||
stop = sub.add_parser("stop-at-start")
|
||||
stop.add_argument("--image", default="mtusi-messenger:lab3")
|
||||
stop.add_argument(
|
||||
"--delays",
|
||||
default="0.2,0.3,0.4,1.5,5",
|
||||
help="паузы после docker run до docker stop, с",
|
||||
)
|
||||
stop.add_argument(
|
||||
"--server-start",
|
||||
type=int,
|
||||
default=3,
|
||||
help="попыток со stop по строке «Started server process»",
|
||||
)
|
||||
req = sub.add_parser("req")
|
||||
req.add_argument("method")
|
||||
req.add_argument("path")
|
||||
@@ -1264,6 +1608,12 @@ def main() -> int:
|
||||
)
|
||||
if args.cmd == "hang-key":
|
||||
return cmd_hang_key(port)
|
||||
if args.cmd == "stop-at-start":
|
||||
return cmd_stop_at_start(
|
||||
args.image,
|
||||
[float(d) for d in args.delays.split(",") if d],
|
||||
args.server_start,
|
||||
)
|
||||
handlers = {
|
||||
"start": cmd_start,
|
||||
"stop": cmd_stop,
|
||||
|
||||
Reference in New Issue
Block a user