fix: bound docker events shutdown in failover and rollout profiles

stop_stream terminates the events CLI, waits 10 s and kills it after
that. The rollout profile starts k6 inside the try, so a k6 start
failure no longer leaves the project events stream running.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-13 18:23:23 +03:00
co-authored by Claude Opus 5.5
parent 1758790e5c
commit 8be58ce046
+35 -21
View File
@@ -787,6 +787,17 @@ def sh(*args: str, check: bool = True) -> str:
return proc.stdout.strip()
def stop_stream(proc: subprocess.Popen[bytes]) -> None:
"""Остановить поток `docker events`: SIGTERM, 10 с на выход, затем
SIGKILL — зависший CLI не держит уборку стенда."""
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
def human(n: int) -> str:
"""10000 → 10k, 1000000 → 1M."""
if n >= 1_000_000 and n % 1_000_000 == 0:
@@ -4410,7 +4421,10 @@ class Bench:
duration = p.failover_before_s + p.failover_down_s + p.failover_after_s
log_name = f"{phase}.log"
# события цели — потоком с начала фазы, не из буфера демона после
# неё (решение 9 design change fix-known-issues)
# неё (решение 9 design change fix-known-issues). Без --since:
# подписка CLI — доли секунды, а действие — через запуск k6 и
# failover_before_s (≥ 5 с); --since потребовал бы часов хоста
# против часов ВМ, а опоздавшая подписка — громкий отказ ниже
events_path = stand.work_dir / f"{phase}-events.log"
stream = stand.container_events(target_id, events_path)
try:
@@ -4479,8 +4493,7 @@ class Bench:
f"phase {phase}: replica not back: {exc}",
) from exc
finally:
stream.terminate()
stream.wait()
stop_stream(stream)
down = stand.down_storages()
if down:
raise BenchError(
@@ -4719,25 +4732,27 @@ class Bench:
started = time.time()
events_path = stand.work_dir / "events.log"
events = stand.project_events(events_path)
load = stand.k6_console(
"failover.js",
{
"RUN": run_id,
"PHASE": 1,
"DURATION": f"{duration}s",
"WRITE_RATE": p.failover_write_rate,
"READ_RATE": p.failover_read_rate,
"PRE_VUS": p.failover_pre_vus,
"MAX_VUS": p.failover_max_vus,
"SEED": p.seed + 1,
"NAME": f"{name}-k6",
},
log_name,
)
script_out = stand.work_dir / "rollout-script.out"
script_s = None
script: subprocess.Popen[bytes] | None = None
load: subprocess.Popen[bytes] | None = None
try:
# k6 — внутри try: при его сбое finally останавливает поток
load = stand.k6_console(
"failover.js",
{
"RUN": run_id,
"PHASE": 1,
"DURATION": f"{duration}s",
"WRITE_RATE": p.failover_write_rate,
"READ_RATE": p.failover_read_rate,
"PRE_VUS": p.failover_pre_vus,
"MAX_VUS": p.failover_max_vus,
"SEED": p.seed + 1,
"NAME": f"{name}-k6",
},
log_name,
)
time.sleep(p.rollout_before_s)
self.say("rollout: scripts/rolling-update.sh started")
began = time.monotonic()
@@ -4760,7 +4775,7 @@ class Bench:
)
load.wait(timeout=duration + 120)
finally:
if load.poll() is None:
if load is not None and load.poll() is None:
load.kill()
# Ctrl+C достаётся и скрипту (та же группа процессов): он
# доводит текущий шаг; стенд удаляется только после него
@@ -4769,8 +4784,7 @@ class Bench:
script.wait(timeout=60)
except subprocess.TimeoutExpired:
script.kill()
events.terminate()
events.wait(timeout=10)
stop_stream(events)
text = (
script_out.read_text(encoding="utf-8", errors="replace")
if script_out.exists()