forked from mtusi2026/mtusi2026lab5
fix: relay/worker переподключаются после рестарта брокера; heartbeat AMQP 10 с
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Найдено тестом «брокер остановлен -> upload 202 -> брокер вернулся»: aio-pika не выставляет is_closed при закрытии соединения сервером, worker оставался без consumer, а healthcheck показывал healthy; relay падал на ChannelInvalidStateError. Теперь потеря соединения отслеживается через close_callbacks, heartbeat-файл пишется только при живом consumer, любая ошибка публикации оставляет строки в outbox. Внедрённые падения worker срабатывают один раз на задание. Внутренняя пометка staging не меняет публичный updated_at.
This commit is contained in:
+11
-5
@@ -42,11 +42,17 @@ class Relay:
|
||||
self.connection: AbstractConnection | None = None
|
||||
self.channel: AbstractChannel | None = None
|
||||
self.exchange: AbstractExchange | None = None
|
||||
self.lost: asyncio.Event | None = None
|
||||
|
||||
def connected(self) -> bool:
|
||||
return (self.connection is not None and self.lost is not None and not self.lost.is_set()
|
||||
and self.channel is not None and not self.channel.is_closed)
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.connection = await aio_pika.connect(
|
||||
self.settings.broker_url, timeout=5,
|
||||
client_properties={"connection_name": "relay@" + self.settings.instance_id})
|
||||
self.lost = service.watch_connection(self.connection)
|
||||
# Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log).
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
# No declare: the topology belongs to the broker definitions; the relay may only write.
|
||||
@@ -59,7 +65,7 @@ class Relay:
|
||||
await self.connection.close()
|
||||
except Exception: # noqa: BLE001 - already broken
|
||||
pass
|
||||
self.connection = self.channel = self.exchange = None
|
||||
self.connection = self.channel = self.exchange = self.lost = None
|
||||
|
||||
async def publish_due(self) -> int:
|
||||
settings = self.settings
|
||||
@@ -120,16 +126,16 @@ class Relay:
|
||||
except Unavailable:
|
||||
healthy = False
|
||||
try:
|
||||
if self.exchange is None or self.connection is None or self.connection.is_closed:
|
||||
if not self.connected():
|
||||
await self.disconnect()
|
||||
await self.connect()
|
||||
sent = await self.publish_due()
|
||||
failures = 0
|
||||
except Unavailable:
|
||||
healthy, sent = False, 0
|
||||
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
|
||||
# Broker down/unreachable or a message returned/nacked: nothing marked, the rows
|
||||
# stay in the outbox (transaction rolled back) and are published later.
|
||||
except Exception as exc: # noqa: BLE001 - every broker/channel error, never a crash
|
||||
# Broker down/unreachable, channel closed, message returned/nacked: nothing marked,
|
||||
# the rows stay in the outbox (transaction rolled back) and are published later.
|
||||
failures += 1
|
||||
healthy, sent = False, 0
|
||||
log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}})
|
||||
|
||||
@@ -27,6 +27,15 @@ def beat() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def watch_connection(connection) -> asyncio.Event:
|
||||
"""Event set when an AMQP connection is closed by ANY side. aio-pika's `is_closed` does not turn
|
||||
True when the broker drops the connection (found by the broker-restart test: the worker kept
|
||||
"running" without a consumer), so the close callback is the signal to reconnect."""
|
||||
lost = asyncio.Event()
|
||||
connection.close_callbacks.add(lambda *_: lost.set())
|
||||
return lost
|
||||
|
||||
|
||||
def stop_event() -> asyncio.Event:
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
+11
-5
@@ -59,12 +59,18 @@ class Worker:
|
||||
self.channel: AbstractChannel | None = None
|
||||
self.queue: AbstractQueue | None = None
|
||||
self.consumer_tag: str | None = None
|
||||
self.lost: asyncio.Event | None = None
|
||||
|
||||
def connected(self) -> bool:
|
||||
return (self.connection is not None and self.lost is not None and not self.lost.is_set()
|
||||
and self.channel is not None and not self.channel.is_closed)
|
||||
|
||||
# ---------------------------------------------------------------- broker
|
||||
async def connect(self) -> None:
|
||||
self.connection = await aio_pika.connect(
|
||||
self.settings.broker_url, timeout=5,
|
||||
client_properties={"connection_name": "worker@" + self.name})
|
||||
self.lost = service.watch_connection(self.connection)
|
||||
self.channel = await self.connection.channel()
|
||||
# Prefetch: the broker keeps at most this many unacked messages in flight to this worker.
|
||||
await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency)
|
||||
@@ -78,7 +84,7 @@ class Worker:
|
||||
await self.connection.close()
|
||||
except Exception: # noqa: BLE001 - already broken
|
||||
pass
|
||||
self.connection = self.channel = self.queue = None
|
||||
self.connection = self.channel = self.queue = self.lost = None
|
||||
self.consumer_tag = None
|
||||
|
||||
async def on_message(self, message: AbstractIncomingMessage) -> None:
|
||||
@@ -276,15 +282,15 @@ class Worker:
|
||||
cleaner = asyncio.create_task(self.cleanup_loop(stop))
|
||||
try:
|
||||
while not stop.is_set():
|
||||
if self.connection is None or self.connection.is_closed:
|
||||
if not self.connected():
|
||||
await self.disconnect()
|
||||
try:
|
||||
await self.connect()
|
||||
except (aio_pika.exceptions.AMQPError, ConnectionError, OSError, asyncio.TimeoutError) as exc:
|
||||
except Exception as exc: # noqa: BLE001 - broker down: retry in 2 s, never crash
|
||||
log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}})
|
||||
await self.disconnect()
|
||||
if self.connection is not None and not self.connection.is_closed:
|
||||
service.beat()
|
||||
if self.connected():
|
||||
service.beat() # healthy only while a consumer is really attached
|
||||
await service.sleep_or_stop(stop, 2.0)
|
||||
await self.shutdown()
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user