diff --git a/bench/run.py b/bench/run.py index 78f62679..f9ec22f1 100644 --- a/bench/run.py +++ b/bench/run.py @@ -1387,6 +1387,34 @@ class Stand: stderr=subprocess.DEVNULL, ) + def container_events( + self, + container: str, + path: Path, + ) -> subprocess.Popen[bytes]: + """Поток событий Docker одного контейнера (часы ВМ) в файл `path`; + формат — `failover.parse_events`. + + Поток, запущенный до нагрузки фазы, а не `docker events --since` + после неё: демон хранит только последние 256 событий всех + контейнеров, а проверки здоровья и `docker exec` оркестратора (по + три события на вызов) вытесняют их (design change + `fix-known-issues`, решение 9).""" + with path.open("w") as out: + return subprocess.Popen( + [ + "docker", + "events", + "--filter", + f"container={container}", + "--format", + "{{.TimeNano}} {{.Action}} " + '{{index .Actor.Attributes "signal"}}', + ], + stdout=out, + stderr=subprocess.DEVNULL, + ) + def app_containers(self) -> list[dict[str, Any]]: """Реплики `api` (и остановленные): имя, `X-Instance-Id`, тег и digest конфигурации образа. В хранилище containerd у тега и у @@ -2378,21 +2406,6 @@ class Stand: self.env = env self.wait_ready() - def events(self, container: str, since: float, until: float) -> str: - """События Docker контейнера за окно: время — часы ВМ.""" - return sh( - "docker", - "events", - "--since", - f"{since:.3f}", - "--until", - f"{until:.3f}", - "--filter", - f"container={container}", - "--format", - '{{.TimeNano}} {{.Action}} {{index .Actor.Attributes "signal"}}', - ) - def logs(self, container: str, since: float) -> str: """Журнал контейнера с отметками Docker (часы ВМ) с момента `since`; HAProxy и PostgreSQL пишут в stdout и stderr.""" @@ -4396,76 +4409,85 @@ class Bench: ) duration = p.failover_before_s + p.failover_down_s + p.failover_after_s log_name = f"{phase}.log" - started = time.time() - load = stand.k6_console( - "failover.js", - { - "RUN": run_id, - "PHASE": number, - "DURATION": f"{duration}s", - "WRITE_RATE": p.failover_write_rate, - "READ_RATE": p.failover_read_rate, - "PRE_VUS": p.failover_pre_vus, - "MAX_VUS": p.failover_max_vus, - "SEED": p.seed + number, - "NAME": f"{phase}-k6", - }, - log_name, - ) + # события цели — потоком с начала фазы, не из буфера демона после + # неё (решение 9 design change fix-known-issues) + events_path = stand.work_dir / f"{phase}-events.log" + stream = stand.container_events(target_id, events_path) try: - time.sleep(p.failover_before_s) - acted = time.monotonic() - command = { - "stop": ("stop", "--timeout", "9"), - "kill": ("kill",), - "pause": ("pause",), - }[mode] - sh("docker", *command, target_id) - self.say(f"phase {phase}: {mode} {instance} ({slot})") - time.sleep(max(0.0, acted + p.failover_down_s - time.monotonic())) - restore = ("unpause",) if mode == "pause" else ("start",) - sh("docker", *restore, target_id) - self.say(f"phase {phase}: {restore[0]} {instance}") - load.wait(timeout=duration + 120) + started = time.time() + load = stand.k6_console( + "failover.js", + { + "RUN": run_id, + "PHASE": number, + "DURATION": f"{duration}s", + "WRITE_RATE": p.failover_write_rate, + "READ_RATE": p.failover_read_rate, + "PRE_VUS": p.failover_pre_vus, + "MAX_VUS": p.failover_max_vus, + "SEED": p.seed + number, + "NAME": f"{phase}-k6", + }, + log_name, + ) + try: + time.sleep(p.failover_before_s) + acted = time.monotonic() + command = { + "stop": ("stop", "--timeout", "9"), + "kill": ("kill",), + "pause": ("pause",), + }[mode] + sh("docker", *command, target_id) + self.say(f"phase {phase}: {mode} {instance} ({slot})") + time.sleep( + max(0.0, acted + p.failover_down_s - time.monotonic()), + ) + restore = ("unpause",) if mode == "pause" else ("start",) + sh("docker", *restore, target_id) + self.say(f"phase {phase}: {restore[0]} {instance}") + load.wait(timeout=duration + 120) + finally: + if load.poll() is None: + load.kill() + # реплика не должна остаться остановленной или замороженной + state = stand.state(target_id) + paused = sh( + "docker", + "inspect", + "-f", + "{{.State.Paused}}", + target_id, + check=False, + ) + if paused == "true": + sh("docker", "unpause", target_id, check=False) + elif not state["running"]: + sh("docker", "start", target_id, check=False) + if load.returncode != 0: + tail = (stand.work_dir / f"{log_name}.out").read_text( + encoding="utf-8", + errors="replace", + )[-3000:] + raise BenchError( + f"k6 failover.js failed ({load.returncode}):\n{tail}", + ) + try: + stand.wait_ready(timeout=30) + except BenchError as exc: + raise BenchError( + f"phase {phase}: replica not back: {exc}", + ) from exc finally: - if load.poll() is None: - load.kill() - # реплика не должна остаться остановленной или замороженной - state = stand.state(target_id) - paused = sh( - "docker", - "inspect", - "-f", - "{{.State.Paused}}", - target_id, - check=False, - ) - if paused == "true": - sh("docker", "unpause", target_id, check=False) - elif not state["running"]: - sh("docker", "start", target_id, check=False) - if load.returncode != 0: - tail = (stand.work_dir / f"{log_name}.out").read_text( - encoding="utf-8", - errors="replace", - )[-3000:] - raise BenchError( - f"k6 failover.js failed ({load.returncode}):\n{tail}", - ) - try: - stand.wait_ready(timeout=30) - except BenchError as exc: - raise BenchError( - f"phase {phase}: replica not back: {exc}", - ) from exc - ended = time.time() + stream.terminate() + stream.wait() down = stand.down_storages() if down: raise BenchError( f"{', '.join(down)} went down during phase {phase}", ) events = failover.parse_events( - stand.events(target_id, started - 5, ended + 1), + events_path.read_text(encoding="utf-8", errors="replace"), ) action = next( (e for e in events if e.action == failover.ACTIONS[mode]),