test: cross-check failover events against the daemon buffer
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2406,6 +2406,27 @@ class Stand:
|
||||
self.env = env
|
||||
self.wait_ready()
|
||||
|
||||
def events_from_buffer(
|
||||
self,
|
||||
container: str,
|
||||
since: float,
|
||||
until: float,
|
||||
) -> str:
|
||||
"""Временная сверка: события контейнера из буфера демона за окно —
|
||||
копия прежнего `Stand.events`; время — часы ВМ."""
|
||||
return sh(
|
||||
"docker",
|
||||
"events",
|
||||
"--since",
|
||||
f"{since:.3f}",
|
||||
"--until",
|
||||
f"{until:.3f}",
|
||||
"--filter",
|
||||
f"container={container}",
|
||||
"--format",
|
||||
'{{.TimeNano}} {{.Action}} {{index .Actor.Attributes "signal"}}',
|
||||
)
|
||||
|
||||
def logs(self, container: str, since: float) -> str:
|
||||
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
|
||||
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
|
||||
@@ -4376,6 +4397,52 @@ class Bench:
|
||||
if p.sample_resources:
|
||||
self.stand.stop_sampler()
|
||||
|
||||
def cross_check_events(
|
||||
self,
|
||||
phase: str,
|
||||
mode: str,
|
||||
target_id: str,
|
||||
*,
|
||||
action: failover.DockerEvent,
|
||||
back: failover.DockerEvent,
|
||||
since: float,
|
||||
until: float,
|
||||
) -> None:
|
||||
"""Временная сверка (решение 10 design change fix-known-issues):
|
||||
моменты действия и возврата из потока против буфера демона за то
|
||||
же окно, как читал прежний `Stand.events`."""
|
||||
events = failover.parse_events(
|
||||
self.stand.events_from_buffer(target_id, since, until),
|
||||
)
|
||||
b_action = next(
|
||||
(e for e in events if e.action == failover.ACTIONS[mode]),
|
||||
None,
|
||||
)
|
||||
b_back = next(
|
||||
(
|
||||
e
|
||||
for e in events
|
||||
if e.action == failover.RESTORES[mode]
|
||||
and b_action is not None
|
||||
and e.time > b_action.time
|
||||
),
|
||||
None,
|
||||
)
|
||||
if b_action is None or b_back is None:
|
||||
self.say(f"phase {phase}: cross-check: buffer lost the events")
|
||||
return
|
||||
if (b_action.time, b_back.time) != (action.time, back.time):
|
||||
raise BenchError(
|
||||
f"phase {phase}: cross-check: stream "
|
||||
f"{action.time:.9f}/{back.time:.9f} != buffer "
|
||||
f"{b_action.time:.9f}/{b_back.time:.9f}",
|
||||
)
|
||||
self.say(
|
||||
f"phase {phase}: cross-check: stream = buffer "
|
||||
f"({action.action} {action.time:.9f}, "
|
||||
f"{back.action} {back.time:.9f})",
|
||||
)
|
||||
|
||||
def run_phase(
|
||||
self,
|
||||
number: int,
|
||||
@@ -4507,6 +4574,15 @@ class Bench:
|
||||
raise BenchError(
|
||||
f"phase {phase}: docker events of {instance} not found",
|
||||
)
|
||||
self.cross_check_events(
|
||||
phase,
|
||||
mode,
|
||||
target_id,
|
||||
action=action,
|
||||
back=back,
|
||||
since=started - 5,
|
||||
until=time.time() + 1,
|
||||
)
|
||||
lb_log = stand.logs(LB, started - 2)
|
||||
lb_events = failover.parse_lb_log(lb_log)
|
||||
(self.results.dir / "raw" / f"{phase}-lb.log").write_text(
|
||||
|
||||
Reference in New Issue
Block a user