fix: relay/worker переподключаются после рестарта брокера; heartbeat AMQP 10 с

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Найдено тестом «брокер остановлен -> upload 202 -> брокер вернулся»: aio-pika не выставляет
is_closed при закрытии соединения сервером, worker оставался без consumer, а healthcheck
показывал healthy; relay падал на ChannelInvalidStateError. Теперь потеря соединения
отслеживается через close_callbacks, heartbeat-файл пишется только при живом consumer,
любая ошибка публикации оставляет строки в outbox. Внедрённые падения worker срабатывают
один раз на задание. Внутренняя пометка staging не меняет публичный updated_at.
This commit is contained in:
Alexey Bodanyuk
2026-10-31 16:48:26 +03:00
parent aef6eb1d4c
commit 1d37fd0dab
3 changed files with 31 additions and 10 deletions
+11 -5
View File
@@ -42,11 +42,17 @@ class Relay:
self.connection: AbstractConnection | None = None
self.channel: AbstractChannel | None = None
self.exchange: AbstractExchange | None = None
self.lost: asyncio.Event | None = None
def connected(self) -> bool:
return (self.connection is not None and self.lost is not None and not self.lost.is_set()
and self.channel is not None and not self.channel.is_closed)
async def connect(self) -> None:
self.connection = await aio_pika.connect(
self.settings.broker_url, timeout=5,
client_properties={"connection_name": "relay@" + self.settings.instance_id})
self.lost = service.watch_connection(self.connection)
# Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log).
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
# No declare: the topology belongs to the broker definitions; the relay may only write.
@@ -59,7 +65,7 @@ class Relay:
await self.connection.close()
except Exception: # noqa: BLE001 - already broken
pass
self.connection = self.channel = self.exchange = None
self.connection = self.channel = self.exchange = self.lost = None
async def publish_due(self) -> int:
settings = self.settings
@@ -120,16 +126,16 @@ class Relay:
except Unavailable:
healthy = False
try:
if self.exchange is None or self.connection is None or self.connection.is_closed:
if not self.connected():
await self.disconnect()
await self.connect()
sent = await self.publish_due()
failures = 0
except Unavailable:
healthy, sent = False, 0
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
# Broker down/unreachable or a message returned/nacked: nothing marked, the rows
# stay in the outbox (transaction rolled back) and are published later.
except Exception as exc: # noqa: BLE001 - every broker/channel error, never a crash
# Broker down/unreachable, channel closed, message returned/nacked: nothing marked,
# the rows stay in the outbox (transaction rolled back) and are published later.
failures += 1
healthy, sent = False, 0
log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}})
+9
View File
@@ -27,6 +27,15 @@ def beat() -> None:
pass
def watch_connection(connection) -> asyncio.Event:
"""Event set when an AMQP connection is closed by ANY side. aio-pika's `is_closed` does not turn
True when the broker drops the connection (found by the broker-restart test: the worker kept
"running" without a consumer), so the close callback is the signal to reconnect."""
lost = asyncio.Event()
connection.close_callbacks.add(lambda *_: lost.set())
return lost
def stop_event() -> asyncio.Event:
stop = asyncio.Event()
loop = asyncio.get_running_loop()
+11 -5
View File
@@ -59,12 +59,18 @@ class Worker:
self.channel: AbstractChannel | None = None
self.queue: AbstractQueue | None = None
self.consumer_tag: str | None = None
self.lost: asyncio.Event | None = None
def connected(self) -> bool:
return (self.connection is not None and self.lost is not None and not self.lost.is_set()
and self.channel is not None and not self.channel.is_closed)
# ---------------------------------------------------------------- broker
async def connect(self) -> None:
self.connection = await aio_pika.connect(
self.settings.broker_url, timeout=5,
client_properties={"connection_name": "worker@" + self.name})
self.lost = service.watch_connection(self.connection)
self.channel = await self.connection.channel()
# Prefetch: the broker keeps at most this many unacked messages in flight to this worker.
await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency)
@@ -78,7 +84,7 @@ class Worker:
await self.connection.close()
except Exception: # noqa: BLE001 - already broken
pass
self.connection = self.channel = self.queue = None
self.connection = self.channel = self.queue = self.lost = None
self.consumer_tag = None
async def on_message(self, message: AbstractIncomingMessage) -> None:
@@ -276,15 +282,15 @@ class Worker:
cleaner = asyncio.create_task(self.cleanup_loop(stop))
try:
while not stop.is_set():
if self.connection is None or self.connection.is_closed:
if not self.connected():
await self.disconnect()
try:
await self.connect()
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
except Exception as exc: # noqa: BLE001 - broker down: retry in 2 s, never crash
log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}})
await self.disconnect()
if self.connection is not None and not self.connection.is_closed:
service.beat()
if self.connected():
service.beat() # healthy only while a consumer is really attached
await service.sleep_or_stop(stop, 2.0)
await self.shutdown()
finally: