test: cross-check failover events against the daemon buffer
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2406,6 +2406,27 @@ class Stand:
|
|||||||
self.env = env
|
self.env = env
|
||||||
self.wait_ready()
|
self.wait_ready()
|
||||||
|
|
||||||
|
def events_from_buffer(
|
||||||
|
self,
|
||||||
|
container: str,
|
||||||
|
since: float,
|
||||||
|
until: float,
|
||||||
|
) -> str:
|
||||||
|
"""Временная сверка: события контейнера из буфера демона за окно —
|
||||||
|
копия прежнего `Stand.events`; время — часы ВМ."""
|
||||||
|
return sh(
|
||||||
|
"docker",
|
||||||
|
"events",
|
||||||
|
"--since",
|
||||||
|
f"{since:.3f}",
|
||||||
|
"--until",
|
||||||
|
f"{until:.3f}",
|
||||||
|
"--filter",
|
||||||
|
f"container={container}",
|
||||||
|
"--format",
|
||||||
|
'{{.TimeNano}} {{.Action}} {{index .Actor.Attributes "signal"}}',
|
||||||
|
)
|
||||||
|
|
||||||
def logs(self, container: str, since: float) -> str:
|
def logs(self, container: str, since: float) -> str:
|
||||||
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
|
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
|
||||||
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
|
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
|
||||||
@@ -4376,6 +4397,52 @@ class Bench:
|
|||||||
if p.sample_resources:
|
if p.sample_resources:
|
||||||
self.stand.stop_sampler()
|
self.stand.stop_sampler()
|
||||||
|
|
||||||
|
def cross_check_events(
|
||||||
|
self,
|
||||||
|
phase: str,
|
||||||
|
mode: str,
|
||||||
|
target_id: str,
|
||||||
|
*,
|
||||||
|
action: failover.DockerEvent,
|
||||||
|
back: failover.DockerEvent,
|
||||||
|
since: float,
|
||||||
|
until: float,
|
||||||
|
) -> None:
|
||||||
|
"""Временная сверка (решение 10 design change fix-known-issues):
|
||||||
|
моменты действия и возврата из потока против буфера демона за то
|
||||||
|
же окно, как читал прежний `Stand.events`."""
|
||||||
|
events = failover.parse_events(
|
||||||
|
self.stand.events_from_buffer(target_id, since, until),
|
||||||
|
)
|
||||||
|
b_action = next(
|
||||||
|
(e for e in events if e.action == failover.ACTIONS[mode]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
b_back = next(
|
||||||
|
(
|
||||||
|
e
|
||||||
|
for e in events
|
||||||
|
if e.action == failover.RESTORES[mode]
|
||||||
|
and b_action is not None
|
||||||
|
and e.time > b_action.time
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if b_action is None or b_back is None:
|
||||||
|
self.say(f"phase {phase}: cross-check: buffer lost the events")
|
||||||
|
return
|
||||||
|
if (b_action.time, b_back.time) != (action.time, back.time):
|
||||||
|
raise BenchError(
|
||||||
|
f"phase {phase}: cross-check: stream "
|
||||||
|
f"{action.time:.9f}/{back.time:.9f} != buffer "
|
||||||
|
f"{b_action.time:.9f}/{b_back.time:.9f}",
|
||||||
|
)
|
||||||
|
self.say(
|
||||||
|
f"phase {phase}: cross-check: stream = buffer "
|
||||||
|
f"({action.action} {action.time:.9f}, "
|
||||||
|
f"{back.action} {back.time:.9f})",
|
||||||
|
)
|
||||||
|
|
||||||
def run_phase(
|
def run_phase(
|
||||||
self,
|
self,
|
||||||
number: int,
|
number: int,
|
||||||
@@ -4507,6 +4574,15 @@ class Bench:
|
|||||||
raise BenchError(
|
raise BenchError(
|
||||||
f"phase {phase}: docker events of {instance} not found",
|
f"phase {phase}: docker events of {instance} not found",
|
||||||
)
|
)
|
||||||
|
self.cross_check_events(
|
||||||
|
phase,
|
||||||
|
mode,
|
||||||
|
target_id,
|
||||||
|
action=action,
|
||||||
|
back=back,
|
||||||
|
since=started - 5,
|
||||||
|
until=time.time() + 1,
|
||||||
|
)
|
||||||
lb_log = stand.logs(LB, started - 2)
|
lb_log = stand.logs(LB, started - 2)
|
||||||
lb_events = failover.parse_lb_log(lb_log)
|
lb_events = failover.parse_lb_log(lb_log)
|
||||||
(self.results.dir / "raw" / f"{phase}-lb.log").write_text(
|
(self.results.dir / "raw" / f"{phase}-lb.log").write_text(
|
||||||
|
|||||||
Reference in New Issue
Block a user