fix: bound docker events shutdown in failover and rollout profiles
stop_stream terminates the events CLI, waits 10 s and kills it after that. The rollout profile starts k6 inside the try, so a k6 start failure no longer leaves the project events stream running. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+35
-21
@@ -787,6 +787,17 @@ def sh(*args: str, check: bool = True) -> str:
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def stop_stream(proc: subprocess.Popen[bytes]) -> None:
|
||||
"""Остановить поток `docker events`: SIGTERM, 10 с на выход, затем
|
||||
SIGKILL — зависший CLI не держит уборку стенда."""
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
|
||||
def human(n: int) -> str:
|
||||
"""10000 → 10k, 1000000 → 1M."""
|
||||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||||
@@ -4410,7 +4421,10 @@ class Bench:
|
||||
duration = p.failover_before_s + p.failover_down_s + p.failover_after_s
|
||||
log_name = f"{phase}.log"
|
||||
# события цели — потоком с начала фазы, не из буфера демона после
|
||||
# неё (решение 9 design change fix-known-issues)
|
||||
# неё (решение 9 design change fix-known-issues). Без --since:
|
||||
# подписка CLI — доли секунды, а действие — через запуск k6 и
|
||||
# failover_before_s (≥ 5 с); --since потребовал бы часов хоста
|
||||
# против часов ВМ, а опоздавшая подписка — громкий отказ ниже
|
||||
events_path = stand.work_dir / f"{phase}-events.log"
|
||||
stream = stand.container_events(target_id, events_path)
|
||||
try:
|
||||
@@ -4479,8 +4493,7 @@ class Bench:
|
||||
f"phase {phase}: replica not back: {exc}",
|
||||
) from exc
|
||||
finally:
|
||||
stream.terminate()
|
||||
stream.wait()
|
||||
stop_stream(stream)
|
||||
down = stand.down_storages()
|
||||
if down:
|
||||
raise BenchError(
|
||||
@@ -4719,25 +4732,27 @@ class Bench:
|
||||
started = time.time()
|
||||
events_path = stand.work_dir / "events.log"
|
||||
events = stand.project_events(events_path)
|
||||
load = stand.k6_console(
|
||||
"failover.js",
|
||||
{
|
||||
"RUN": run_id,
|
||||
"PHASE": 1,
|
||||
"DURATION": f"{duration}s",
|
||||
"WRITE_RATE": p.failover_write_rate,
|
||||
"READ_RATE": p.failover_read_rate,
|
||||
"PRE_VUS": p.failover_pre_vus,
|
||||
"MAX_VUS": p.failover_max_vus,
|
||||
"SEED": p.seed + 1,
|
||||
"NAME": f"{name}-k6",
|
||||
},
|
||||
log_name,
|
||||
)
|
||||
script_out = stand.work_dir / "rollout-script.out"
|
||||
script_s = None
|
||||
script: subprocess.Popen[bytes] | None = None
|
||||
load: subprocess.Popen[bytes] | None = None
|
||||
try:
|
||||
# k6 — внутри try: при его сбое finally останавливает поток
|
||||
load = stand.k6_console(
|
||||
"failover.js",
|
||||
{
|
||||
"RUN": run_id,
|
||||
"PHASE": 1,
|
||||
"DURATION": f"{duration}s",
|
||||
"WRITE_RATE": p.failover_write_rate,
|
||||
"READ_RATE": p.failover_read_rate,
|
||||
"PRE_VUS": p.failover_pre_vus,
|
||||
"MAX_VUS": p.failover_max_vus,
|
||||
"SEED": p.seed + 1,
|
||||
"NAME": f"{name}-k6",
|
||||
},
|
||||
log_name,
|
||||
)
|
||||
time.sleep(p.rollout_before_s)
|
||||
self.say("rollout: scripts/rolling-update.sh started")
|
||||
began = time.monotonic()
|
||||
@@ -4760,7 +4775,7 @@ class Bench:
|
||||
)
|
||||
load.wait(timeout=duration + 120)
|
||||
finally:
|
||||
if load.poll() is None:
|
||||
if load is not None and load.poll() is None:
|
||||
load.kill()
|
||||
# Ctrl+C достаётся и скрипту (та же группа процессов): он
|
||||
# доводит текущий шаг; стенд удаляется только после него
|
||||
@@ -4769,8 +4784,7 @@ class Bench:
|
||||
script.wait(timeout=60)
|
||||
except subprocess.TimeoutExpired:
|
||||
script.kill()
|
||||
events.terminate()
|
||||
events.wait(timeout=10)
|
||||
stop_stream(events)
|
||||
text = (
|
||||
script_out.read_text(encoding="utf-8", errors="replace")
|
||||
if script_out.exists()
|
||||
|
||||
Reference in New Issue
Block a user