fix: stream docker events during failover phases
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+99
-77
@@ -1387,6 +1387,34 @@ class Stand:
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def container_events(
|
||||
self,
|
||||
container: str,
|
||||
path: Path,
|
||||
) -> subprocess.Popen[bytes]:
|
||||
"""Поток событий Docker одного контейнера (часы ВМ) в файл `path`;
|
||||
формат — `failover.parse_events`.
|
||||
|
||||
Поток, запущенный до нагрузки фазы, а не `docker events --since`
|
||||
после неё: демон хранит только последние 256 событий всех
|
||||
контейнеров, а проверки здоровья и `docker exec` оркестратора (по
|
||||
три события на вызов) вытесняют их (design change
|
||||
`fix-known-issues`, решение 9)."""
|
||||
with path.open("w") as out:
|
||||
return subprocess.Popen(
|
||||
[
|
||||
"docker",
|
||||
"events",
|
||||
"--filter",
|
||||
f"container={container}",
|
||||
"--format",
|
||||
"{{.TimeNano}} {{.Action}} "
|
||||
'{{index .Actor.Attributes "signal"}}',
|
||||
],
|
||||
stdout=out,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def app_containers(self) -> list[dict[str, Any]]:
|
||||
"""Реплики `api` (и остановленные): имя, `X-Instance-Id`, тег и
|
||||
digest конфигурации образа. В хранилище containerd у тега и у
|
||||
@@ -2378,21 +2406,6 @@ class Stand:
|
||||
self.env = env
|
||||
self.wait_ready()
|
||||
|
||||
def events(self, container: str, since: float, until: float) -> str:
|
||||
"""События Docker контейнера за окно: время — часы ВМ."""
|
||||
return sh(
|
||||
"docker",
|
||||
"events",
|
||||
"--since",
|
||||
f"{since:.3f}",
|
||||
"--until",
|
||||
f"{until:.3f}",
|
||||
"--filter",
|
||||
f"container={container}",
|
||||
"--format",
|
||||
'{{.TimeNano}} {{.Action}} {{index .Actor.Attributes "signal"}}',
|
||||
)
|
||||
|
||||
def logs(self, container: str, since: float) -> str:
|
||||
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
|
||||
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
|
||||
@@ -4396,76 +4409,85 @@ class Bench:
|
||||
)
|
||||
duration = p.failover_before_s + p.failover_down_s + p.failover_after_s
|
||||
log_name = f"{phase}.log"
|
||||
started = time.time()
|
||||
load = stand.k6_console(
|
||||
"failover.js",
|
||||
{
|
||||
"RUN": run_id,
|
||||
"PHASE": number,
|
||||
"DURATION": f"{duration}s",
|
||||
"WRITE_RATE": p.failover_write_rate,
|
||||
"READ_RATE": p.failover_read_rate,
|
||||
"PRE_VUS": p.failover_pre_vus,
|
||||
"MAX_VUS": p.failover_max_vus,
|
||||
"SEED": p.seed + number,
|
||||
"NAME": f"{phase}-k6",
|
||||
},
|
||||
log_name,
|
||||
)
|
||||
# события цели — потоком с начала фазы, не из буфера демона после
|
||||
# неё (решение 9 design change fix-known-issues)
|
||||
events_path = stand.work_dir / f"{phase}-events.log"
|
||||
stream = stand.container_events(target_id, events_path)
|
||||
try:
|
||||
time.sleep(p.failover_before_s)
|
||||
acted = time.monotonic()
|
||||
command = {
|
||||
"stop": ("stop", "--timeout", "9"),
|
||||
"kill": ("kill",),
|
||||
"pause": ("pause",),
|
||||
}[mode]
|
||||
sh("docker", *command, target_id)
|
||||
self.say(f"phase {phase}: {mode} {instance} ({slot})")
|
||||
time.sleep(max(0.0, acted + p.failover_down_s - time.monotonic()))
|
||||
restore = ("unpause",) if mode == "pause" else ("start",)
|
||||
sh("docker", *restore, target_id)
|
||||
self.say(f"phase {phase}: {restore[0]} {instance}")
|
||||
load.wait(timeout=duration + 120)
|
||||
started = time.time()
|
||||
load = stand.k6_console(
|
||||
"failover.js",
|
||||
{
|
||||
"RUN": run_id,
|
||||
"PHASE": number,
|
||||
"DURATION": f"{duration}s",
|
||||
"WRITE_RATE": p.failover_write_rate,
|
||||
"READ_RATE": p.failover_read_rate,
|
||||
"PRE_VUS": p.failover_pre_vus,
|
||||
"MAX_VUS": p.failover_max_vus,
|
||||
"SEED": p.seed + number,
|
||||
"NAME": f"{phase}-k6",
|
||||
},
|
||||
log_name,
|
||||
)
|
||||
try:
|
||||
time.sleep(p.failover_before_s)
|
||||
acted = time.monotonic()
|
||||
command = {
|
||||
"stop": ("stop", "--timeout", "9"),
|
||||
"kill": ("kill",),
|
||||
"pause": ("pause",),
|
||||
}[mode]
|
||||
sh("docker", *command, target_id)
|
||||
self.say(f"phase {phase}: {mode} {instance} ({slot})")
|
||||
time.sleep(
|
||||
max(0.0, acted + p.failover_down_s - time.monotonic()),
|
||||
)
|
||||
restore = ("unpause",) if mode == "pause" else ("start",)
|
||||
sh("docker", *restore, target_id)
|
||||
self.say(f"phase {phase}: {restore[0]} {instance}")
|
||||
load.wait(timeout=duration + 120)
|
||||
finally:
|
||||
if load.poll() is None:
|
||||
load.kill()
|
||||
# реплика не должна остаться остановленной или замороженной
|
||||
state = stand.state(target_id)
|
||||
paused = sh(
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.State.Paused}}",
|
||||
target_id,
|
||||
check=False,
|
||||
)
|
||||
if paused == "true":
|
||||
sh("docker", "unpause", target_id, check=False)
|
||||
elif not state["running"]:
|
||||
sh("docker", "start", target_id, check=False)
|
||||
if load.returncode != 0:
|
||||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)[-3000:]
|
||||
raise BenchError(
|
||||
f"k6 failover.js failed ({load.returncode}):\n{tail}",
|
||||
)
|
||||
try:
|
||||
stand.wait_ready(timeout=30)
|
||||
except BenchError as exc:
|
||||
raise BenchError(
|
||||
f"phase {phase}: replica not back: {exc}",
|
||||
) from exc
|
||||
finally:
|
||||
if load.poll() is None:
|
||||
load.kill()
|
||||
# реплика не должна остаться остановленной или замороженной
|
||||
state = stand.state(target_id)
|
||||
paused = sh(
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.State.Paused}}",
|
||||
target_id,
|
||||
check=False,
|
||||
)
|
||||
if paused == "true":
|
||||
sh("docker", "unpause", target_id, check=False)
|
||||
elif not state["running"]:
|
||||
sh("docker", "start", target_id, check=False)
|
||||
if load.returncode != 0:
|
||||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)[-3000:]
|
||||
raise BenchError(
|
||||
f"k6 failover.js failed ({load.returncode}):\n{tail}",
|
||||
)
|
||||
try:
|
||||
stand.wait_ready(timeout=30)
|
||||
except BenchError as exc:
|
||||
raise BenchError(
|
||||
f"phase {phase}: replica not back: {exc}",
|
||||
) from exc
|
||||
ended = time.time()
|
||||
stream.terminate()
|
||||
stream.wait()
|
||||
down = stand.down_storages()
|
||||
if down:
|
||||
raise BenchError(
|
||||
f"{', '.join(down)} went down during phase {phase}",
|
||||
)
|
||||
events = failover.parse_events(
|
||||
stand.events(target_id, started - 5, ended + 1),
|
||||
events_path.read_text(encoding="utf-8", errors="replace"),
|
||||
)
|
||||
action = next(
|
||||
(e for e in events if e.action == failover.ACTIONS[mode]),
|
||||
|
||||
Reference in New Issue
Block a user