forked from mtusi2026/mtusi2026lab7
532 lines
27 KiB
Python
532 lines
27 KiB
Python
"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose).
|
||
|
||
Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1):
|
||
одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди.
|
||
|
||
Обработка одного сообщения:
|
||
1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ.
|
||
2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый
|
||
lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op.
|
||
3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает ->
|
||
аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued.
|
||
Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения.
|
||
4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД.
|
||
5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь).
|
||
6. PUT в конечный prefix: attachments/<id>/v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp.
|
||
Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи.
|
||
7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT.
|
||
8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД.
|
||
9. Удаление исходника из staging (если не вышло — доделает janitor).
|
||
Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД,
|
||
таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed.
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
import psycopg
|
||
from opentelemetry import trace
|
||
from opentelemetry.trace import NonRecordingSpan, SpanKind
|
||
from psycopg.rows import dict_row
|
||
|
||
import dbconn
|
||
import secretstore
|
||
import settings
|
||
import telemetry
|
||
|
||
telemetry.setup("messenger-worker")
|
||
|
||
import broker # noqa: E402
|
||
import faults # noqa: E402
|
||
import imaging # noqa: E402
|
||
import pipeline # noqa: E402
|
||
import storage # noqa: E402
|
||
|
||
log = logging.getLogger("messenger.worker")
|
||
|
||
# lab6: метрики фоновой обработки (contracts/observability.md). Метки: тип задания, исход, очередь.
|
||
JOB_ATTRS = {"job.type": pipeline.JOB_TYPE}
|
||
JOB_DURATION = telemetry.meter.create_histogram(
|
||
"messenger.job.duration", unit="s",
|
||
description="Attempt of an attachment job from claim to result (outcome: ready/retry/failed/stale/lease_lost)")
|
||
TIME_TO_READY = telemetry.meter.create_histogram(
|
||
"messenger.attachment.time_to_ready", unit="s",
|
||
description="From durable acceptance (job COMMIT in API) to ready COMMIT in worker")
|
||
MESSAGES = telemetry.meter.create_counter(
|
||
"messenger.messages.consumed", unit="{message}",
|
||
description="Broker deliveries by outcome: processed (claimed), noop (duplicate/not due), dead_lettered, requeued")
|
||
BUSY = {"value": 0}
|
||
BUSY_LOCK = threading.Lock()
|
||
|
||
|
||
def _observe_busy(_options):
|
||
from opentelemetry.metrics import Observation
|
||
return [Observation(BUSY["value"], {"messaging.destination.name": pipeline.QUEUE})]
|
||
|
||
|
||
def _observe_capacity(_options):
|
||
from opentelemetry.metrics import Observation
|
||
return [Observation(settings.WORKER_CONCURRENCY, {"messaging.destination.name": pipeline.QUEUE})]
|
||
|
||
|
||
telemetry.meter.create_observable_up_down_counter(
|
||
"messenger.worker.busy", [_observe_busy], unit="{consumer}", description="Consumers processing a job now")
|
||
telemetry.meter.create_observable_up_down_counter(
|
||
"messenger.worker.capacity", [_observe_capacity], unit="{consumer}",
|
||
description="Consumers of this worker process (WORKER_CONCURRENCY)")
|
||
|
||
|
||
def busy(delta: int) -> None:
|
||
with BUSY_LOCK:
|
||
BUSY["value"] += delta
|
||
|
||
STOP = threading.Event()
|
||
HERE = Path(__file__).resolve().parent
|
||
VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"}
|
||
IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large",
|
||
imaging.EXIT_MEMORY: "resource_limit"}
|
||
|
||
|
||
class JobError(Exception):
|
||
def __init__(self, code: str, permanent: bool):
|
||
super().__init__(code)
|
||
self.code = code
|
||
self.permanent = permanent
|
||
|
||
|
||
class LeaseLost(Exception):
|
||
pass
|
||
|
||
|
||
def connect_db():
|
||
# lab7: TLS verify-full, своя роль PostgreSQL, пароль из OpenBao (dbconn)
|
||
return dbconn.connect(autocommit=True, row_factory=dict_row,
|
||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||
|
||
|
||
def final_prefix(claim: pipeline.Claim) -> str:
|
||
return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/"
|
||
|
||
|
||
class LeaseKeeper(threading.Thread):
|
||
"""Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку."""
|
||
|
||
def __init__(self, claim: pipeline.Claim):
|
||
super().__init__(daemon=True)
|
||
self.claim = claim
|
||
self.done = threading.Event()
|
||
self.lost = threading.Event()
|
||
self.frozen = threading.Event() # fault worker_freeze: перестать продлевать
|
||
|
||
def run(self):
|
||
interval = max(1.0, settings.JOB_LEASE_SECONDS / 3)
|
||
conn = None
|
||
while not self.done.wait(interval):
|
||
if self.frozen.is_set():
|
||
continue
|
||
try:
|
||
if conn is None:
|
||
conn = connect_db()
|
||
if not pipeline.renew_lease(conn, self.claim):
|
||
log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id,
|
||
extra={"event": "job.lease_lost", "job.id": str(self.claim.job_id)})
|
||
self.lost.set()
|
||
return
|
||
except psycopg.Error as exc:
|
||
# Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT).
|
||
log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__)
|
||
if conn is not None:
|
||
conn.close()
|
||
conn = None
|
||
if conn is not None:
|
||
conn.close()
|
||
|
||
def check(self):
|
||
if self.lost.is_set():
|
||
raise LeaseLost()
|
||
|
||
|
||
def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict:
|
||
# Дочернему процессу — только лимиты, никаких секретов окружения.
|
||
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1",
|
||
"IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB),
|
||
"IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)}
|
||
for fault in ("hang", "invalid", "oom"):
|
||
if faults.hit("imaging_" + fault):
|
||
env["FAULT_IMAGING"] = fault
|
||
with telemetry.internal_span("imaging", {"messenger.pipeline.version": claim.pipeline_version,
|
||
"messenger.source.content_type": claim.content_type}) as span:
|
||
result = _run_imaging(env, source, out_dir, claim, keeper)
|
||
span.set_attribute("messenger.variants", len(result["variants"]))
|
||
return result
|
||
|
||
|
||
def _run_imaging(env: dict, source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict:
|
||
process = subprocess.Popen(
|
||
[sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type,
|
||
str(settings.MAX_IMAGE_PIXELS)],
|
||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir),
|
||
)
|
||
deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS
|
||
try:
|
||
while True:
|
||
try:
|
||
output, _ = process.communicate(timeout=0.5)
|
||
break
|
||
except subprocess.TimeoutExpired:
|
||
if time.monotonic() > deadline:
|
||
raise JobError("processing_timeout", permanent=False) from None
|
||
keeper.check()
|
||
finally:
|
||
if process.poll() is None:
|
||
process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку
|
||
process.wait()
|
||
code = process.returncode
|
||
if code in IMAGING_EXIT:
|
||
raise JobError(IMAGING_EXIT[code], permanent=True)
|
||
if code != 0:
|
||
# Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены.
|
||
raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False)
|
||
return json.loads(output.decode("utf-8").strip().splitlines()[-1])
|
||
|
||
|
||
def sha256_file(path: Path) -> tuple[int, str]:
|
||
digest = hashlib.sha256()
|
||
size = 0
|
||
with open(path, "rb") as file:
|
||
while chunk := file.read(1024 * 1024):
|
||
size += len(chunk)
|
||
digest.update(chunk)
|
||
return size, digest.hexdigest()
|
||
|
||
|
||
def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None:
|
||
if faults.hit("worker_s3_error"):
|
||
raise JobError("storage_unavailable", permanent=False)
|
||
with open(path, "rb") as file:
|
||
try:
|
||
storage.put(key, file, size, content_type, sha256)
|
||
except storage.StorageUnavailable:
|
||
raise JobError("storage_unavailable", permanent=False) from None
|
||
|
||
|
||
def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]:
|
||
"""Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов."""
|
||
source = workdir / "source"
|
||
try:
|
||
storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES)
|
||
except storage.ObjectMissing:
|
||
raise JobError("source_missing", permanent=True) from None
|
||
except storage.StorageUnavailable:
|
||
raise JobError("storage_unavailable", permanent=False) from None
|
||
with telemetry.internal_span("verify source sha256"):
|
||
size, digest = sha256_file(source)
|
||
if size != claim.size_bytes or digest != claim.sha256:
|
||
# Staging не совпадает с тем, что принял API: публиковать нельзя.
|
||
raise JobError("source_corrupted", permanent=True)
|
||
keeper.check()
|
||
|
||
out_dir = workdir / "out"
|
||
out_dir.mkdir()
|
||
result = run_imaging(source, out_dir, claim, keeper)
|
||
keeper.check()
|
||
|
||
prefix = final_prefix(claim)
|
||
original_key = prefix + "original"
|
||
put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes)
|
||
variants = []
|
||
for item in result["variants"]:
|
||
path = out_dir / item["file"]
|
||
v_size, v_sha = sha256_file(path)
|
||
key = prefix + VARIANT_KEYS[item["name"]]
|
||
put_file(key, path, item["content_type"], v_sha, v_size)
|
||
variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size,
|
||
"width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key})
|
||
keeper.check()
|
||
return original_key, variants
|
||
|
||
|
||
@dataclass
|
||
class Outcome:
|
||
ack: bool = True # False -> nack с requeue (не смогли даже записать claim)
|
||
reject: bool = False # True -> reject без requeue -> DLQ
|
||
cleanup_key: str | None = None
|
||
attachment_id: object = None
|
||
span_context: object = None # спан обработки: очистка staging после ack — в той же трассе
|
||
delivery: str = "processed" # метка messenger.messages.consumed: processed / noop
|
||
|
||
|
||
def carrier_from(headers, body: bytes) -> dict:
|
||
"""W3C контекст сообщения: native headers AMQP (основной carrier), иначе поле traceparent тела."""
|
||
carrier = {}
|
||
for key in ("traceparent", "tracestate"):
|
||
value = (headers or {}).get(key)
|
||
if isinstance(value, bytes):
|
||
value = value.decode("ascii", "replace")
|
||
if isinstance(value, str):
|
||
carrier[key] = value
|
||
if "traceparent" not in carrier:
|
||
try:
|
||
message = json.loads(body.decode("utf-8"))
|
||
if isinstance(message, dict) and isinstance(message.get("traceparent"), str):
|
||
carrier["traceparent"] = message["traceparent"]
|
||
except (UnicodeDecodeError, ValueError):
|
||
pass
|
||
return carrier
|
||
|
||
|
||
class Consumer(threading.Thread):
|
||
def __init__(self, number: int):
|
||
super().__init__(name=f"consumer-{number}", daemon=True)
|
||
self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}"
|
||
self.db = None
|
||
|
||
def database(self):
|
||
if self.db is None or self.db.closed:
|
||
self.db = connect_db()
|
||
return self.db
|
||
|
||
def handle(self, body: bytes, redelivered: bool, headers=None) -> Outcome:
|
||
"""Спан CONSUMER «process attachments.process.v1» — ребёнок спана publish из dispatcher."""
|
||
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
|
||
"messaging.destination.name": pipeline.QUEUE, "messaging.consumer.group.name": pipeline.QUEUE,
|
||
"messaging.rabbitmq.message.redelivered": redelivered}
|
||
with telemetry.tracer.start_as_current_span(
|
||
f"process {pipeline.QUEUE}", context=telemetry.extract(carrier_from(headers, body)),
|
||
kind=SpanKind.CONSUMER, attributes=attributes,
|
||
record_exception=False, set_status_on_exception=False) as span:
|
||
outcome = self._handle(body, redelivered, span)
|
||
outcome.span_context = span.get_span_context()
|
||
kind = "dead_lettered" if outcome.reject else "requeued" if not outcome.ack else outcome.delivery
|
||
span.set_attribute("messenger.delivery", kind)
|
||
MESSAGES.add(1, {"messaging.destination.name": pipeline.QUEUE, "outcome": kind})
|
||
return outcome
|
||
|
||
def _handle(self, body: bytes, redelivered: bool, span) -> Outcome:
|
||
try:
|
||
message = pipeline.parse_message(body)
|
||
except pipeline.InvalidMessage as exc:
|
||
log.warning("invalid message -> DLQ (%s)", exc, extra={"event": "job.invalid_message"})
|
||
telemetry.mark_error(span, "invalid_message")
|
||
return Outcome(reject=True)
|
||
span.set_attributes({"messaging.message.id": f"{message['job_id']}:{message['attempt']}",
|
||
"messenger.job.id": message["job_id"],
|
||
"messenger.attachment.id": message["attachment_id"],
|
||
"messenger.message.attempt": message["attempt"]})
|
||
try:
|
||
result = pipeline.claim(self.database(), message, self.owner, redelivered)
|
||
except psycopg.Error as exc:
|
||
log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__,
|
||
extra={"event": "job.claim_failed", "job.id": message["job_id"],
|
||
"error.type": type(exc).__name__})
|
||
telemetry.mark_error(span, type(exc).__name__)
|
||
self.db = None
|
||
STOP.wait(2)
|
||
return Outcome(ack=False)
|
||
if isinstance(result, pipeline.Reject):
|
||
log.warning("job %s: %s -> DLQ", message["job_id"], result.reason,
|
||
extra={"event": "job.rejected", "job.id": message["job_id"], "reason": result.reason})
|
||
telemetry.mark_error(span, result.reason)
|
||
return Outcome(reject=True)
|
||
if isinstance(result, pipeline.Skip):
|
||
log.info("job %s: no-op (%s), ack", message["job_id"], result.reason,
|
||
extra={"event": "job.noop", "job.id": message["job_id"], "reason": result.reason})
|
||
span.set_attribute("messenger.noop.reason", result.reason)
|
||
return Outcome(delivery="noop")
|
||
span.set_attribute("messenger.job.attempt", result.attempt)
|
||
busy(+1)
|
||
try:
|
||
return self.execute(result, span)
|
||
finally:
|
||
busy(-1)
|
||
|
||
def execute(self, claim: pipeline.Claim, span) -> Outcome:
|
||
started = time.monotonic()
|
||
fields = {"job.id": str(claim.job_id), "attachment.id": str(claim.attachment_id), "attempt": claim.attempt}
|
||
log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt,
|
||
claim.max_attempts, claim.attachment_id, extra={"event": "job.claimed", **fields})
|
||
keeper = LeaseKeeper(claim)
|
||
keeper.start()
|
||
workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR))
|
||
outcome_name = "ready"
|
||
try:
|
||
try:
|
||
original_key, variants = process_job(claim, keeper, workdir)
|
||
faults.crash("worker_crash_after_put")
|
||
if faults.hit("worker_freeze"):
|
||
keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят
|
||
time.sleep(settings.JOB_LEASE_SECONDS * 3)
|
||
with telemetry.internal_span("commit result"):
|
||
time_to_ready = pipeline.finish_ready(self.database(), claim, original_key, variants,
|
||
self.owner)
|
||
if time_to_ready is None:
|
||
outcome_name = "stale" # аренду перехватили: fencing отклонил результат
|
||
except JobError as exc:
|
||
outcome_name = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner)
|
||
telemetry.mark_error(span, exc.code)
|
||
log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code,
|
||
"permanent" if exc.permanent else "temporary", outcome_name,
|
||
extra={"event": f"job.{outcome_name}", "error.type": exc.code,
|
||
"permanent": exc.permanent, **fields})
|
||
return Outcome()
|
||
except LeaseLost:
|
||
outcome_name = "lease_lost"
|
||
telemetry.mark_error(span, "lease_lost")
|
||
log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt,
|
||
extra={"event": "job.lease_lost", **fields})
|
||
return Outcome()
|
||
except psycopg.Error:
|
||
raise
|
||
except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены
|
||
log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt,
|
||
extra={"event": "job.unexpected_error", **fields})
|
||
telemetry.mark_error(span, "processing_failed")
|
||
outcome_name = pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner)
|
||
return Outcome()
|
||
except psycopg.Error as exc:
|
||
# Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и
|
||
# dispatcher вернёт job в очередь; если прошёл — повтор увидит ready.
|
||
outcome_name = "db_error"
|
||
telemetry.mark_error(span, type(exc).__name__)
|
||
log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__,
|
||
extra={"event": "job.db_error", "error.type": type(exc).__name__, **fields})
|
||
self.db = None
|
||
return Outcome()
|
||
finally:
|
||
keeper.done.set()
|
||
shutil.rmtree(workdir, ignore_errors=True)
|
||
elapsed = time.monotonic() - started
|
||
span.set_attribute("messenger.job.outcome", outcome_name)
|
||
JOB_DURATION.record(elapsed, JOB_ATTRS | {"outcome": outcome_name})
|
||
if time_to_ready is None:
|
||
telemetry.mark_error(span, "stale_result_rejected")
|
||
log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id,
|
||
claim.attempt, extra={"event": "job.stale_result_rejected", **fields})
|
||
return Outcome()
|
||
TIME_TO_READY.record(time_to_ready, JOB_ATTRS)
|
||
faults.crash("worker_crash_after_commit")
|
||
log.info("job %s attempt %s: ready in %.0f ms (%.1f s since upload)", claim.job_id, claim.attempt,
|
||
(time.monotonic() - started) * 1000, time_to_ready,
|
||
extra={"event": "job.ready", "duration_ms": round((time.monotonic() - started) * 1000, 1),
|
||
"time_to_ready_s": round(time_to_ready, 3), **fields})
|
||
return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id)
|
||
|
||
def cleanup_staging(self, outcome: Outcome) -> None:
|
||
"""После COMMIT ready и ack: исходник в staging больше не нужен."""
|
||
parent = trace.set_span_in_context(NonRecordingSpan(outcome.span_context)) if outcome.span_context else None
|
||
with telemetry.tracer.start_as_current_span("cleanup staging", context=parent):
|
||
try:
|
||
storage.delete(outcome.cleanup_key)
|
||
self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' "
|
||
"AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key))
|
||
except (storage.StorageUnavailable, psycopg.Error) as exc:
|
||
log.info("staging cleanup postponed to janitor: %s", type(exc).__name__,
|
||
extra={"event": "staging.cleanup_postponed", "error.type": type(exc).__name__})
|
||
self.db = None
|
||
|
||
def consume_once(self) -> None:
|
||
connection = broker.connect()
|
||
try:
|
||
channel = connection.channel()
|
||
channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH)
|
||
broker.check_topology(channel, need=("queue",)) # worker видит только свою очередь
|
||
log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH,
|
||
extra={"event": "worker.consuming"})
|
||
for method, properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1):
|
||
if STOP.is_set():
|
||
break
|
||
if method is None:
|
||
continue
|
||
box = {}
|
||
headers = properties.headers if properties is not None else None
|
||
runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle(
|
||
body, method.redelivered, headers)), daemon=True)
|
||
runner.start()
|
||
while runner.is_alive():
|
||
connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа
|
||
outcome = box.get("outcome", Outcome(ack=False))
|
||
if outcome.reject:
|
||
channel.basic_reject(method.delivery_tag, requeue=False)
|
||
elif outcome.ack:
|
||
channel.basic_ack(method.delivery_tag)
|
||
else:
|
||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||
if outcome.cleanup_key:
|
||
self.cleanup_staging(outcome)
|
||
try:
|
||
channel.cancel()
|
||
except broker.BrokerError:
|
||
pass
|
||
finally:
|
||
try:
|
||
connection.close()
|
||
except broker.BrokerError:
|
||
pass
|
||
|
||
def safe_handle(self, body, redelivered, headers=None) -> Outcome:
|
||
try:
|
||
return self.handle(body, redelivered, headers)
|
||
except Exception: # noqa: BLE001
|
||
log.exception("unexpected error while handling a message; requeue",
|
||
extra={"event": "job.handler_crashed"})
|
||
self.db = None
|
||
STOP.wait(2)
|
||
return Outcome(ack=False)
|
||
|
||
def run(self):
|
||
pause = 1.0
|
||
while not STOP.is_set():
|
||
try:
|
||
self.consume_once()
|
||
pause = 1.0
|
||
except broker.BrokerError as exc:
|
||
log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name,
|
||
type(exc).__name__, pause,
|
||
extra={"event": "broker.unavailable", "error.type": type(exc).__name__})
|
||
STOP.wait(pause)
|
||
pause = min(pause * 2, 15)
|
||
|
||
|
||
def main() -> None:
|
||
if not settings.BROKER_URL:
|
||
raise SystemExit("BROKER_URL is required")
|
||
secretstore.install_metrics(telemetry.meter)
|
||
secretstore.store.start() # lab7: пароль БД и S3-ключ worker — из OpenBao (CN=worker)
|
||
Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True)
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
signal.signal(sig, lambda *_: STOP.set())
|
||
consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)]
|
||
for consumer in consumers:
|
||
consumer.start()
|
||
log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, "
|
||
"fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH,
|
||
settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB,
|
||
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
|
||
while not STOP.wait(2):
|
||
if not all(consumer.is_alive() for consumer in consumers):
|
||
log.error("a consumer thread died, exiting so that Docker restarts the worker")
|
||
os._exit(1)
|
||
try:
|
||
Path(settings.ALIVE_FILE).touch()
|
||
except OSError:
|
||
pass
|
||
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят.
|
||
for consumer in consumers:
|
||
consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10)
|
||
log.info("stopped", extra={"event": "service.stopped"})
|
||
telemetry.shutdown()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|