From 42e6edf420861b415b845df44e8b7296dd9352ae Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 13 Oct 2026 16:50:18 +0300 Subject: [PATCH] feat: add a driver command that stops a starting replica Co-Authored-By: Claude Opus 5.5 --- .claude/skills/run-mtusi-messenger/driver.py | 350 +++++++++++++++++++ 1 file changed, 350 insertions(+) diff --git a/.claude/skills/run-mtusi-messenger/driver.py b/.claude/skills/run-mtusi-messenger/driver.py index 58df8bbf..ef285245 100644 --- a/.claude/skills/run-mtusi-messenger/driver.py +++ b/.claude/skills/run-mtusi-messenger/driver.py @@ -40,6 +40,13 @@ разморозка, одно сообщение в чате, строка журнала PostgreSQL; триггер удаляется при любом исходе, в том числе по Ctrl+C, SIGTERM и SIGHUP; не удалось — печатается SQL удаления + stop-at-start [--image ТЕГ] [--delays 0.2,0.3,0.4,1.5,5] + [--server-start 3] docker stop контейнеров образа (лимиты + реплики, без сети) во время старта: через паузу после + docker run и по строке «Started server process»; время + сигнала и выхода, код выхода, этап старта, кто остановил + (ранний обработчик или uvicorn); код 1, если стартующий + контейнер останавливается дольше 1 с или не с кодом 0 stop остановить свой экземпляр на порту all start → flow → smoke → student (если есть student-tests/) → stop @@ -1194,6 +1201,330 @@ def hang_key(port: int) -> int: print(f"trigger {probe} removed", flush=True) +# stop-at-start: записи журнала, по которым видны этапы старта и остановки +STARTED_SERVER = "Started server process" +SERVER_RUNNING = "Uvicorn running on" +EARLY_STOP = "Stopped API before startup" +UVICORN_STOP = ("Shutting down", "Stopped API", "Finished server process") +# имена событий без переменной части («... [1]», «... on http://...») +EVENT_PREFIXES = (STARTED_SERVER, SERVER_RUNNING, "Finished server process") +STOP_LABEL = "mtusi.stop-at-start" +# docker stop стартующего контейнера — не дольше (спек graceful-shutdown) +STOP_LIMIT_S = 1.0 +# ожидание события die и строки старта сервера +ATTEMPT_TIMEOUT_S = 30 + + +def event_name(event: str) -> str: + for prefix in EVENT_PREFIXES: + if event.startswith(prefix): + return prefix + return event + + +def log_time(value: str) -> float | None: + """`timestamp` записи журнала (ISO 8601 UTC с `Z`) — секунды эпохи.""" + try: + moment = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ") + except (TypeError, ValueError): + return None + return moment.replace(tzinfo=UTC).timestamp() + + +def docker_events(path: Path) -> list[tuple[float, str, str, str, str]]: + """Строки потока событий: (время ВМ, действие, ID, сигнал, код).""" + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + parts = line.split("|") + if len(parts) == 5 and parts[0].isdigit(): + rows.append((int(parts[0]) / 1e9, *parts[1:])) + return rows + + +def cmd_stop_at_start(image: str, delays: list[float], server: int) -> int: + """Сценарии спека `graceful-shutdown` «Остановка во время старта» + (решение 8 design change `fix-known-issues`). + + Каждая попытка — свой контейнер образа с лимитами реплики `api` + (1 CPU / 512 МиБ без swap), без сети и без `config.toml`: приложение + стартует на дефолтах, соединения с хранилищами ленивые. `docker stop` + (срок 10 с, как `stop_grace_period`) — через паузу после `docker run` + или, у попыток `--server-start`, сразу по строке «Started server + process» (`docker logs -f`). Время — часы ВМ Docker: поток `docker + events` по метке (`start`, `kill` с сигналом 15, `die` с кодом) и + `timestamp` записей журнала. Проекты compose не трогаются, свои + контейнеры удаляются по метке при любом исходе.""" + + def interrupted(signum: int, _frame: object) -> None: + raise KeyboardInterrupt(signal.Signals(signum).name) + + previous = { + signum: signal.signal(signum, interrupted) + for signum in (signal.SIGTERM, signal.SIGHUP) + } + tag = uuid.uuid4().hex[:8] + label = f"{STOP_LABEL}={tag}" + events_path = Path(f"/tmp/mtusi-stop-at-start-{tag}.events") + image_id = subprocess.run( + ["docker", "image", "inspect", "-f", "{{.Id}}", image], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + if not image_id: + print(f"image {image} not found") + return 1 + print(f"image {image} ({image_id[:19]}), label {label}") + with events_path.open("w") as out: + stream = subprocess.Popen( + [ + "docker", + "events", + "--filter", + f"label={label}", + "--filter", + "type=container", + "--filter", + "event=start", + "--filter", + "event=kill", + "--filter", + "event=die", + "--format", + "{{.TimeNano}}|{{.Action}}|{{.Actor.ID}}|" + '{{index .Actor.Attributes "signal"}}|' + '{{index .Actor.Attributes "exitCode"}}', + ], + stdout=out, + stderr=subprocess.DEVNULL, + ) + try: + # CLI подключается к демону до первого контейнера + time.sleep(1) + plan = [("delay", d) for d in delays] + [("server", 0.0)] * server + attempts = [] + for number, (how, delay) in enumerate(plan, start=1): + attempts.append( + stop_attempt( + number, + image, + label, + f"mtusi-stop-at-start-{tag}-{number}", + events_path, + delay=delay if how == "delay" else None, + ), + ) + return stop_verdict(attempts) + finally: + stream.terminate() + stream.wait() + events_path.unlink(missing_ok=True) + left = subprocess.run( + ["docker", "ps", "-aq", "--filter", f"label={label}"], + capture_output=True, + text=True, + check=False, + ).stdout.split() + if left: + subprocess.run( + ["docker", "rm", "-f", *left], + capture_output=True, + check=False, + ) + for signum, handler in previous.items(): + signal.signal(signum, handler) + + +def stop_attempt( + number: int, + image: str, + label: str, + name: str, + events_path: Path, + *, + delay: float | None, +) -> dict[str, object]: + """Одна попытка: контейнер, `docker stop`, события и журнал.""" + run = subprocess.run( + [ + "docker", + "run", + "-d", + "--network", + "none", + "--cpus", + "1", + "--memory", + "512m", + "--memory-swap", + "512m", + "--label", + label, + "--name", + name, + image, + ], + capture_output=True, + text=True, + check=True, + ) + container = run.stdout.strip() + if delay is not None: + time.sleep(delay) + else: + follow = subprocess.Popen( + ["docker", "logs", "-f", container], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + timer = threading.Timer(ATTEMPT_TIMEOUT_S, follow.kill) + timer.start() + try: + for line in follow.stdout or []: + if STARTED_SERVER in line: + break + finally: + timer.cancel() + began = time.monotonic() + subprocess.run(["docker", "stop", container], capture_output=True) + stop_s = time.monotonic() - began + if delay is None: + follow.kill() + follow.wait() + deadline = time.monotonic() + ATTEMPT_TIMEOUT_S + mine: list[tuple[float, str, str, str, str]] = [] + while time.monotonic() < deadline: + mine = [e for e in docker_events(events_path) if e[2] == container] + if any(e[1] == "die" for e in mine): + break + time.sleep(0.2) + start = next((e[0] for e in mine if e[1] == "start"), None) + kill = next( + (e[0] for e in mine if e[1] == "kill" and e[3] == "15"), + None, + ) + die = next((e for e in mine if e[1] == "die"), None) + logs = subprocess.run( + ["docker", "logs", container], + capture_output=True, + text=True, + check=False, + ) + records = [] + for line in (logs.stdout + logs.stderr).splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("event"), str): + records.append(record) + names = [event_name(r["event"]) for r in records] + started_at = next( + ( + log_time(r.get("timestamp")) + for r in records + if event_name(r["event"]) == STARTED_SERVER + ), + None, + ) + running_at = next( + ( + log_time(r.get("timestamp")) + for r in records + if event_name(r["event"]) == SERVER_RUNNING + ), + None, + ) + if kill is None or started_at is None or kill < started_at: + kind = "import" + elif running_at is None or kill < running_at: + kind = "server-start" + else: + kind = "running" + early = [r for r in records if r["event"] == EARLY_STOP] + if early: + by = "early" + elif all(stage in names for stage in UVICORN_STOP): + by = "uvicorn" + else: + by = "none" + # запись ранней остановки — JSON с полями записей structlog + early_ok = all( + r.get("level") == "info" + and r.get("signal") == "SIGTERM" + and log_time(r.get("timestamp")) is not None + and r.get("instance_id") + for r in early + ) + code = int(die[4]) if die is not None and die[4].isdigit() else None + result = { + "number": number, + "how": f"pause {delay:g} s" if delay is not None else "server-start", + "signal_after": None + if kill is None or start is None + else kill - start, + "stop_s": stop_s, + "exit_after": None if kill is None or die is None else die[0] - kill, + "code": code, + "kind": kind, + "by": by, + "early_ok": early_ok, + "events": names, + "instance": records[0].get("instance_id") if records else None, + } + + def seconds(value: object) -> str: + return "-" if value is None else f"{value:.3f} s" + + print( + f"#{number} {result['how']}: signal after " + f"{seconds(result['signal_after'])}, docker stop {stop_s:.3f} s, " + f"exit after signal {seconds(result['exit_after'])}, exit code " + f"{code}, {kind}, stopped by {by}", + flush=True, + ) + print(f" log: {' → '.join(names) or '(empty)'}", flush=True) + return result + + +def stop_verdict(attempts: list[dict[str, object]]) -> int: + """PASS (код 0) по решению 8 design change `fix-known-issues`.""" + problems = [] + for a in attempts: + n = a["number"] + if a["code"] != 0: + problems.append(f"#{n}: exit code {a['code']}") + if float(a["stop_s"]) > STOP_LIMIT_S: + problems.append(f"#{n}: docker stop took {a['stop_s']:.1f} s") + if a["kind"] == "import": + # окно config.load(): сигнал до первой строки uvicorn, но уже + # под его обработчиками — штатная остановка тоже верна + if a["by"] == "early": + if STARTED_SERVER in a["events"]: + problems.append(f"#{n}: server start after early stop") + if not a["early_ok"]: + problems.append(f"#{n}: malformed early stop record") + elif a["by"] != "uvicorn": + problems.append(f"#{n}: import, nobody stopped the process") + elif a["by"] != "uvicorn": + problems.append(f"#{n}: {a['kind']}, stopped by {a['by']}") + kinds = Counter(str(a["kind"]) for a in attempts) + if not any(a["kind"] == "import" and a["by"] == "early" for a in attempts): + problems.append("no import attempt stopped early") + if not kinds["running"]: + problems.append("no running attempt") + print( + f"attempts: import {kinds['import']}, server-start " + f"{kinds['server-start']}, running {kinds['running']}", + ) + if problems: + print("FAIL: " + "; ".join(problems)) + return 1 + print("PASS: SIGTERM during start stops the replica at once, exit 0") + return 0 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--port", type=int, default=DEFAULT_PORT) @@ -1214,6 +1545,19 @@ def main() -> int: # контейнер LB, если --port ведёт не на LB стенда (вариант конфигурации) idle.add_argument("--lb") sub.add_parser("hang-key") + stop = sub.add_parser("stop-at-start") + stop.add_argument("--image", default="mtusi-messenger:lab3") + stop.add_argument( + "--delays", + default="0.2,0.3,0.4,1.5,5", + help="паузы после docker run до docker stop, с", + ) + stop.add_argument( + "--server-start", + type=int, + default=3, + help="попыток со stop по строке «Started server process»", + ) req = sub.add_parser("req") req.add_argument("method") req.add_argument("path") @@ -1264,6 +1608,12 @@ def main() -> int: ) if args.cmd == "hang-key": return cmd_hang_key(port) + if args.cmd == "stop-at-start": + return cmd_stop_at_start( + args.image, + [float(d) for d in args.delays.split(",") if d], + args.server_start, + ) handlers = { "start": cmd_start, "stop": cmd_stop,