fix: stream docker events during failover phases

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-13 16:16:11 +03:00
co-authored by Claude Opus 5.5
parent 18238588b4
commit fe75fcf8a9
+99 -77
View File
@@ -1387,6 +1387,34 @@ class Stand:
stderr=subprocess.DEVNULL,
)
def container_events(
self,
container: str,
path: Path,
) -> subprocess.Popen[bytes]:
"""Поток событий Docker одного контейнера (часы ВМ) в файл `path`;
формат — `failover.parse_events`.
Поток, запущенный до нагрузки фазы, а не `docker events --since`
после неё: демон хранит только последние 256 событий всех
контейнеров, а проверки здоровья и `docker exec` оркестратора (по
три события на вызов) вытесняют их (design change
`fix-known-issues`, решение 9)."""
with path.open("w") as out:
return subprocess.Popen(
[
"docker",
"events",
"--filter",
f"container={container}",
"--format",
"{{.TimeNano}} {{.Action}} "
'{{index .Actor.Attributes "signal"}}',
],
stdout=out,
stderr=subprocess.DEVNULL,
)
def app_containers(self) -> list[dict[str, Any]]:
"""Реплики `api` (и остановленные): имя, `X-Instance-Id`, тег и
digest конфигурации образа. В хранилище containerd у тега и у
@@ -2378,21 +2406,6 @@ class Stand:
self.env = env
self.wait_ready()
def events(self, container: str, since: float, until: float) -> str:
"""События Docker контейнера за окно: время — часы ВМ."""
return sh(
"docker",
"events",
"--since",
f"{since:.3f}",
"--until",
f"{until:.3f}",
"--filter",
f"container={container}",
"--format",
'{{.TimeNano}} {{.Action}} {{index .Actor.Attributes "signal"}}',
)
def logs(self, container: str, since: float) -> str:
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
@@ -4396,76 +4409,85 @@ class Bench:
)
duration = p.failover_before_s + p.failover_down_s + p.failover_after_s
log_name = f"{phase}.log"
started = time.time()
load = stand.k6_console(
"failover.js",
{
"RUN": run_id,
"PHASE": number,
"DURATION": f"{duration}s",
"WRITE_RATE": p.failover_write_rate,
"READ_RATE": p.failover_read_rate,
"PRE_VUS": p.failover_pre_vus,
"MAX_VUS": p.failover_max_vus,
"SEED": p.seed + number,
"NAME": f"{phase}-k6",
},
log_name,
)
# события цели — потоком с начала фазы, не из буфера демона после
# неё (решение 9 design change fix-known-issues)
events_path = stand.work_dir / f"{phase}-events.log"
stream = stand.container_events(target_id, events_path)
try:
time.sleep(p.failover_before_s)
acted = time.monotonic()
command = {
"stop": ("stop", "--timeout", "9"),
"kill": ("kill",),
"pause": ("pause",),
}[mode]
sh("docker", *command, target_id)
self.say(f"phase {phase}: {mode} {instance} ({slot})")
time.sleep(max(0.0, acted + p.failover_down_s - time.monotonic()))
restore = ("unpause",) if mode == "pause" else ("start",)
sh("docker", *restore, target_id)
self.say(f"phase {phase}: {restore[0]} {instance}")
load.wait(timeout=duration + 120)
started = time.time()
load = stand.k6_console(
"failover.js",
{
"RUN": run_id,
"PHASE": number,
"DURATION": f"{duration}s",
"WRITE_RATE": p.failover_write_rate,
"READ_RATE": p.failover_read_rate,
"PRE_VUS": p.failover_pre_vus,
"MAX_VUS": p.failover_max_vus,
"SEED": p.seed + number,
"NAME": f"{phase}-k6",
},
log_name,
)
try:
time.sleep(p.failover_before_s)
acted = time.monotonic()
command = {
"stop": ("stop", "--timeout", "9"),
"kill": ("kill",),
"pause": ("pause",),
}[mode]
sh("docker", *command, target_id)
self.say(f"phase {phase}: {mode} {instance} ({slot})")
time.sleep(
max(0.0, acted + p.failover_down_s - time.monotonic()),
)
restore = ("unpause",) if mode == "pause" else ("start",)
sh("docker", *restore, target_id)
self.say(f"phase {phase}: {restore[0]} {instance}")
load.wait(timeout=duration + 120)
finally:
if load.poll() is None:
load.kill()
# реплика не должна остаться остановленной или замороженной
state = stand.state(target_id)
paused = sh(
"docker",
"inspect",
"-f",
"{{.State.Paused}}",
target_id,
check=False,
)
if paused == "true":
sh("docker", "unpause", target_id, check=False)
elif not state["running"]:
sh("docker", "start", target_id, check=False)
if load.returncode != 0:
tail = (stand.work_dir / f"{log_name}.out").read_text(
encoding="utf-8",
errors="replace",
)[-3000:]
raise BenchError(
f"k6 failover.js failed ({load.returncode}):\n{tail}",
)
try:
stand.wait_ready(timeout=30)
except BenchError as exc:
raise BenchError(
f"phase {phase}: replica not back: {exc}",
) from exc
finally:
if load.poll() is None:
load.kill()
# реплика не должна остаться остановленной или замороженной
state = stand.state(target_id)
paused = sh(
"docker",
"inspect",
"-f",
"{{.State.Paused}}",
target_id,
check=False,
)
if paused == "true":
sh("docker", "unpause", target_id, check=False)
elif not state["running"]:
sh("docker", "start", target_id, check=False)
if load.returncode != 0:
tail = (stand.work_dir / f"{log_name}.out").read_text(
encoding="utf-8",
errors="replace",
)[-3000:]
raise BenchError(
f"k6 failover.js failed ({load.returncode}):\n{tail}",
)
try:
stand.wait_ready(timeout=30)
except BenchError as exc:
raise BenchError(
f"phase {phase}: replica not back: {exc}",
) from exc
ended = time.time()
stream.terminate()
stream.wait()
down = stand.down_storages()
if down:
raise BenchError(
f"{', '.join(down)} went down during phase {phase}",
)
events = failover.parse_events(
stand.events(target_id, started - 5, ended + 1),
events_path.read_text(encoding="utf-8", errors="replace"),
)
action = next(
(e for e in events if e.action == failover.ACTIONS[mode]),