From 1d37fd0dabc607eecdea2edab4cee3b34c725cde Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:48:26 +0300 Subject: [PATCH] =?UTF-8?q?fix:=20relay/worker=20=D0=BF=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=BF=D0=BE=D0=B4=D0=BA=D0=BB=D1=8E=D1=87=D0=B0=D1=8E=D1=82?= =?UTF-8?q?=D1=81=D1=8F=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=20=D1=80=D0=B5?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D1=80=D1=82=D0=B0=20=D0=B1=D1=80=D0=BE=D0=BA?= =?UTF-8?q?=D0=B5=D1=80=D0=B0;=20heartbeat=20AMQP=2010=20=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Найдено тестом «брокер остановлен -> upload 202 -> брокер вернулся»: aio-pika не выставляет is_closed при закрытии соединения сервером, worker оставался без consumer, а healthcheck показывал healthy; relay падал на ChannelInvalidStateError. Теперь потеря соединения отслеживается через close_callbacks, heartbeat-файл пишется только при живом consumer, любая ошибка публикации оставляет строки в outbox. Внедрённые падения worker срабатывают один раз на задание. Внутренняя пометка staging не меняет публичный updated_at. --- src/messenger/relay.py | 16 +++++++++++----- src/messenger/service.py | 9 +++++++++ src/messenger/worker.py | 16 +++++++++++----- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/messenger/relay.py b/src/messenger/relay.py index 4158e2c..77a11a5 100644 --- a/src/messenger/relay.py +++ b/src/messenger/relay.py @@ -42,11 +42,17 @@ class Relay: self.connection: AbstractConnection | None = None self.channel: AbstractChannel | None = None self.exchange: AbstractExchange | None = None + self.lost: asyncio.Event | None = None + + def connected(self) -> bool: + return (self.connection is not None and self.lost is not None and not self.lost.is_set() + and self.channel is not None and not self.channel.is_closed) async def connect(self) -> None: self.connection = await aio_pika.connect( self.settings.broker_url, timeout=5, client_properties={"connection_name": "relay@" + self.settings.instance_id}) + self.lost = service.watch_connection(self.connection) # Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log). self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) # No declare: the topology belongs to the broker definitions; the relay may only write. @@ -59,7 +65,7 @@ class Relay: await self.connection.close() except Exception: # noqa: BLE001 - already broken pass - self.connection = self.channel = self.exchange = None + self.connection = self.channel = self.exchange = self.lost = None async def publish_due(self) -> int: settings = self.settings @@ -120,16 +126,16 @@ class Relay: except Unavailable: healthy = False try: - if self.exchange is None or self.connection is None or self.connection.is_closed: + if not self.connected(): await self.disconnect() await self.connect() sent = await self.publish_due() failures = 0 except Unavailable: healthy, sent = False, 0 - except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc: - # Broker down/unreachable or a message returned/nacked: nothing marked, the rows - # stay in the outbox (transaction rolled back) and are published later. + except Exception as exc: # noqa: BLE001 - every broker/channel error, never a crash + # Broker down/unreachable, channel closed, message returned/nacked: nothing marked, + # the rows stay in the outbox (transaction rolled back) and are published later. failures += 1 healthy, sent = False, 0 log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}}) diff --git a/src/messenger/service.py b/src/messenger/service.py index 8554bff..0df62f6 100644 --- a/src/messenger/service.py +++ b/src/messenger/service.py @@ -27,6 +27,15 @@ def beat() -> None: pass +def watch_connection(connection) -> asyncio.Event: + """Event set when an AMQP connection is closed by ANY side. aio-pika's `is_closed` does not turn + True when the broker drops the connection (found by the broker-restart test: the worker kept + "running" without a consumer), so the close callback is the signal to reconnect.""" + lost = asyncio.Event() + connection.close_callbacks.add(lambda *_: lost.set()) + return lost + + def stop_event() -> asyncio.Event: stop = asyncio.Event() loop = asyncio.get_running_loop() diff --git a/src/messenger/worker.py b/src/messenger/worker.py index ddd24ff..e7917c4 100644 --- a/src/messenger/worker.py +++ b/src/messenger/worker.py @@ -59,12 +59,18 @@ class Worker: self.channel: AbstractChannel | None = None self.queue: AbstractQueue | None = None self.consumer_tag: str | None = None + self.lost: asyncio.Event | None = None + + def connected(self) -> bool: + return (self.connection is not None and self.lost is not None and not self.lost.is_set() + and self.channel is not None and not self.channel.is_closed) # ---------------------------------------------------------------- broker async def connect(self) -> None: self.connection = await aio_pika.connect( self.settings.broker_url, timeout=5, client_properties={"connection_name": "worker@" + self.name}) + self.lost = service.watch_connection(self.connection) self.channel = await self.connection.channel() # Prefetch: the broker keeps at most this many unacked messages in flight to this worker. await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency) @@ -78,7 +84,7 @@ class Worker: await self.connection.close() except Exception: # noqa: BLE001 - already broken pass - self.connection = self.channel = self.queue = None + self.connection = self.channel = self.queue = self.lost = None self.consumer_tag = None async def on_message(self, message: AbstractIncomingMessage) -> None: @@ -276,15 +282,15 @@ class Worker: cleaner = asyncio.create_task(self.cleanup_loop(stop)) try: while not stop.is_set(): - if self.connection is None or self.connection.is_closed: + if not self.connected(): await self.disconnect() try: await self.connect() - except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc: + except Exception as exc: # noqa: BLE001 - broker down: retry in 2 s, never crash log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}}) await self.disconnect() - if self.connection is not None and not self.connection.is_closed: - service.beat() + if self.connected(): + service.beat() # healthy only while a consumer is really attached await service.sleep_or_stop(stop, 2.0) await self.shutdown() finally: