diff --git a/bench/run.py b/bench/run.py index f9ec22f1..3719f9ff 100644 --- a/bench/run.py +++ b/bench/run.py @@ -2406,6 +2406,27 @@ class Stand: self.env = env self.wait_ready() + def events_from_buffer( + self, + container: str, + since: float, + until: float, + ) -> str: + """Временная сверка: события контейнера из буфера демона за окно — + копия прежнего `Stand.events`; время — часы ВМ.""" + return sh( + "docker", + "events", + "--since", + f"{since:.3f}", + "--until", + f"{until:.3f}", + "--filter", + f"container={container}", + "--format", + '{{.TimeNano}} {{.Action}} {{index .Actor.Attributes "signal"}}', + ) + def logs(self, container: str, since: float) -> str: """Журнал контейнера с отметками Docker (часы ВМ) с момента `since`; HAProxy и PostgreSQL пишут в stdout и stderr.""" @@ -4376,6 +4397,52 @@ class Bench: if p.sample_resources: self.stand.stop_sampler() + def cross_check_events( + self, + phase: str, + mode: str, + target_id: str, + *, + action: failover.DockerEvent, + back: failover.DockerEvent, + since: float, + until: float, + ) -> None: + """Временная сверка (решение 10 design change fix-known-issues): + моменты действия и возврата из потока против буфера демона за то + же окно, как читал прежний `Stand.events`.""" + events = failover.parse_events( + self.stand.events_from_buffer(target_id, since, until), + ) + b_action = next( + (e for e in events if e.action == failover.ACTIONS[mode]), + None, + ) + b_back = next( + ( + e + for e in events + if e.action == failover.RESTORES[mode] + and b_action is not None + and e.time > b_action.time + ), + None, + ) + if b_action is None or b_back is None: + self.say(f"phase {phase}: cross-check: buffer lost the events") + return + if (b_action.time, b_back.time) != (action.time, back.time): + raise BenchError( + f"phase {phase}: cross-check: stream " + f"{action.time:.9f}/{back.time:.9f} != buffer " + f"{b_action.time:.9f}/{b_back.time:.9f}", + ) + self.say( + f"phase {phase}: cross-check: stream = buffer " + f"({action.action} {action.time:.9f}, " + f"{back.action} {back.time:.9f})", + ) + def run_phase( self, number: int, @@ -4507,6 +4574,15 @@ class Bench: raise BenchError( f"phase {phase}: docker events of {instance} not found", ) + self.cross_check_events( + phase, + mode, + target_id, + action=action, + back=back, + since=started - 5, + until=time.time() + 1, + ) lb_log = stand.logs(LB, started - 2) lb_events = failover.parse_lb_log(lb_log) (self.results.dir / "raw" / f"{phase}-lb.log").write_text(