From 8be58ce0465c5ac88e4235f70580a3ba92502e72 Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 13 Oct 2026 18:23:23 +0300 Subject: [PATCH] fix: bound docker events shutdown in failover and rollout profiles stop_stream terminates the events CLI, waits 10 s and kills it after that. The rollout profile starts k6 inside the try, so a k6 start failure no longer leaves the project events stream running. Co-Authored-By: Claude Opus 5.5 --- bench/run.py | 56 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/bench/run.py b/bench/run.py index f9ec22f1..168ea22d 100644 --- a/bench/run.py +++ b/bench/run.py @@ -787,6 +787,17 @@ def sh(*args: str, check: bool = True) -> str: return proc.stdout.strip() +def stop_stream(proc: subprocess.Popen[bytes]) -> None: + """Остановить поток `docker events`: SIGTERM, 10 с на выход, затем + SIGKILL — зависший CLI не держит уборку стенда.""" + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + def human(n: int) -> str: """10000 → 10k, 1000000 → 1M.""" if n >= 1_000_000 and n % 1_000_000 == 0: @@ -4410,7 +4421,10 @@ class Bench: duration = p.failover_before_s + p.failover_down_s + p.failover_after_s log_name = f"{phase}.log" # события цели — потоком с начала фазы, не из буфера демона после - # неё (решение 9 design change fix-known-issues) + # неё (решение 9 design change fix-known-issues). Без --since: + # подписка CLI — доли секунды, а действие — через запуск k6 и + # failover_before_s (≥ 5 с); --since потребовал бы часов хоста + # против часов ВМ, а опоздавшая подписка — громкий отказ ниже events_path = stand.work_dir / f"{phase}-events.log" stream = stand.container_events(target_id, events_path) try: @@ -4479,8 +4493,7 @@ class Bench: f"phase {phase}: replica not back: {exc}", ) from exc finally: - stream.terminate() - stream.wait() + stop_stream(stream) down = stand.down_storages() if down: raise BenchError( @@ -4719,25 +4732,27 @@ class Bench: started = time.time() events_path = stand.work_dir / "events.log" events = stand.project_events(events_path) - load = stand.k6_console( - "failover.js", - { - "RUN": run_id, - "PHASE": 1, - "DURATION": f"{duration}s", - "WRITE_RATE": p.failover_write_rate, - "READ_RATE": p.failover_read_rate, - "PRE_VUS": p.failover_pre_vus, - "MAX_VUS": p.failover_max_vus, - "SEED": p.seed + 1, - "NAME": f"{name}-k6", - }, - log_name, - ) script_out = stand.work_dir / "rollout-script.out" script_s = None script: subprocess.Popen[bytes] | None = None + load: subprocess.Popen[bytes] | None = None try: + # k6 — внутри try: при его сбое finally останавливает поток + load = stand.k6_console( + "failover.js", + { + "RUN": run_id, + "PHASE": 1, + "DURATION": f"{duration}s", + "WRITE_RATE": p.failover_write_rate, + "READ_RATE": p.failover_read_rate, + "PRE_VUS": p.failover_pre_vus, + "MAX_VUS": p.failover_max_vus, + "SEED": p.seed + 1, + "NAME": f"{name}-k6", + }, + log_name, + ) time.sleep(p.rollout_before_s) self.say("rollout: scripts/rolling-update.sh started") began = time.monotonic() @@ -4760,7 +4775,7 @@ class Bench: ) load.wait(timeout=duration + 120) finally: - if load.poll() is None: + if load is not None and load.poll() is None: load.kill() # Ctrl+C достаётся и скрипту (та же группа процессов): он # доводит текущий шаг; стенд удаляется только после него @@ -4769,8 +4784,7 @@ class Bench: script.wait(timeout=60) except subprocess.TimeoutExpired: script.kill() - events.terminate() - events.wait(timeout=10) + stop_stream(events) text = ( script_out.read_text(encoding="utf-8", errors="replace") if script_out.exists()