forked from mtusi2026/mtusi2026lab5
106 lines
3.4 KiB
Python
106 lines
3.4 KiB
Python
"""Separate worker process. It never serves the public HTTP API."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import signal
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
from app import runtime
|
|
from app.broker import QUEUE, open_consumer
|
|
from app.db import migrate
|
|
from app.outbox import start_relay
|
|
from app.pipeline import process_delivery, recover_expired_leases, sweep_staging
|
|
from app.runtime import init
|
|
from app.s3store import ObjectStore
|
|
from app.settings import load
|
|
|
|
log = logging.getLogger("messenger.worker")
|
|
|
|
|
|
def _on_term(_signum, _frame) -> None:
|
|
os._exit(0)
|
|
|
|
|
|
def _consume(index: int, broker_url: str) -> None:
|
|
worker_id = f"worker-{index}-{uuid.uuid4()}"
|
|
while True:
|
|
connection = None
|
|
try:
|
|
connection, channel = open_consumer(broker_url)
|
|
|
|
def on_message(channel, method, _properties, body, connection=connection, worker_id=worker_id):
|
|
def work() -> None:
|
|
try:
|
|
result = process_delivery(body, bool(method.redelivered), worker_id)
|
|
except Exception:
|
|
log.exception("delivery failed")
|
|
result = "nack"
|
|
if result != "ack":
|
|
time.sleep(1)
|
|
|
|
def finish() -> None:
|
|
if result == "ack":
|
|
channel.basic_ack(method.delivery_tag)
|
|
else:
|
|
channel.basic_nack(method.delivery_tag, requeue=True)
|
|
|
|
connection.add_callback_threadsafe(finish)
|
|
|
|
threading.Thread(target=work, name=f"delivery-{index}", daemon=True).start()
|
|
|
|
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
|
|
log.info("consumer %s waiting; prefetch=1", worker_id)
|
|
channel.start_consuming()
|
|
except Exception:
|
|
log.warning("consumer connection dropped", exc_info=True)
|
|
time.sleep(1)
|
|
finally:
|
|
try:
|
|
if connection is not None and connection.is_open:
|
|
connection.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _maintain() -> None:
|
|
while True:
|
|
try:
|
|
recover_expired_leases()
|
|
sweep_staging()
|
|
except Exception:
|
|
log.warning("maintenance iteration failed", exc_info=True)
|
|
time.sleep(2)
|
|
|
|
|
|
def main() -> None:
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
signal.signal(signal.SIGTERM, _on_term)
|
|
signal.signal(signal.SIGINT, _on_term)
|
|
settings = load()
|
|
migrate(settings.database_url)
|
|
store = ObjectStore(
|
|
settings.s3_endpoint,
|
|
settings.s3_region,
|
|
settings.s3_bucket,
|
|
settings.s3_access_key_id,
|
|
settings.s3_secret_access_key,
|
|
)
|
|
store.ensure_bucket()
|
|
from app.broker import Publisher
|
|
|
|
init(settings, store, Publisher(settings.broker_url))
|
|
start_relay()
|
|
threading.Thread(target=_maintain, name="maintenance", daemon=True).start()
|
|
for index in range(settings.worker_concurrency):
|
|
threading.Thread(target=_consume, args=(index, settings.broker_url), name=f"consumer-{index}", daemon=True).start()
|
|
log.info("worker started with concurrency %s", settings.worker_concurrency)
|
|
while True:
|
|
time.sleep(3600)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|