Перенесена реализация lab5 (API x2 за HAProxy, PostgreSQL, SeaweedFS S3, RabbitMQ, relay outbox, worker x2, janitor) без изменения бизнес-API. Наблюдаемость: - OpenTelemetry SDK в API, relay, worker, janitor: traces, metrics, logs по OTLP/HTTP; структурные JSON-логи с trace_id/span_id - OTel Collector (memory_limiter, batch, ограниченные очереди и retry), Prometheus (метрики, alert rules), Loki (логи), Tempo (трассы), Grafana с provisioning datasources и dashboard - W3C trace context API -> outbox -> RabbitMQ -> worker (parent-child, включая retry); метрики очереди, пула БД, загрузки worker, CPU/RAM - SLI/SLO, 10 alert rules с unit-тестами promtool и runbooks - инциденты: остановка worker, блокировка БД, отказ telemetry backend; по наблюдениям исправлен простой worker в цикле AMQP (разбор x2.4)
621 lines
33 KiB
Python
621 lines
33 KiB
Python
#!/usr/bin/env python3
|
||
"""Worker обработки вложений (compose-сервис `worker`, отдельный процесс от API).
|
||
|
||
WORKER_CONCURRENCY потоков-потребителей; у каждого своё соединение AMQP, prefetch=1
|
||
(не больше одного неподтверждённого сообщения на поток) и своё соединение PostgreSQL.
|
||
Итого одновременно в работе не больше WORKER_CONCURRENCY заданий на контейнер.
|
||
|
||
Обработка одного сообщения (at-least-once, ack только после устойчивого эффекта):
|
||
1. разбор по jobs.schema.json; мусор -> basic.reject(requeue=False) -> DLQ;
|
||
2. claim в транзакции (src/jobs.py): по строке jobs решаем, нужна ли работа. ready/failed,
|
||
чужая живая аренда, ранний дубль -> ack без работы. Иначе processing, attempt+1,
|
||
lease_token (fencing) и lease_until, COMMIT;
|
||
3. пока идёт работа, отдельный поток продлевает аренду (heartbeat) не дольше
|
||
JOB_TIMEOUT_SECONDS: зависший процесс аренду теряет, долгий, но живой — нет;
|
||
4. staging -> временный файл (лимит байт = size_bytes из БД, проверка SHA-256);
|
||
5. transform.py в подпроцессе (RLIMIT_AS, RLIMIT_CPU, таймаут) -> thumbnail/crop/optimized;
|
||
6. PUT оригинала и вариантов по ДЕТЕРМИНИРОВАННЫМ ключам attachments/<id>/v<ver>/<name>:
|
||
повтор перезаписывает те же объекты, второго набора не появляется;
|
||
7. complete: processing -> ready + attachment_variants одной транзакцией и только со своим
|
||
lease_token; иначе (аренду забрали) — ничего не пишем;
|
||
8. удаление staging (после COMMIT), затем ack. Крах до ack -> redelivery -> claim видит
|
||
ready -> no-op ack.
|
||
Ошибки: постоянные -> failed(error_code); временные -> queued с backoff+jitter через outbox
|
||
(relay опубликует в срок) или failed(retry_exhausted). Нет доступа к БД -> сообщение
|
||
возвращается в очередь (nack) после паузы; если так и не записалось — reaper/redrive.
|
||
|
||
Учебные отказы (WORKER_FAULT_INJECTION=1): filename вида fault-<point>.png задаёт точку:
|
||
crash-after-claim, crash-after-s3-put, crash-after-commit (os._exit один раз на job),
|
||
transient-<N> (временная ошибка в первых N попытках), transient-always (до replay),
|
||
bug (непредусмотренное исключение в 1-й попытке),
|
||
slow (пауза FAULT_SLOW_SECONDS после claim, аренда продлевается).
|
||
|
||
Lab6 (src/telemetry.py):
|
||
* каждое сообщение — CONSUMER span «attachments.process.v1 process», родитель — контекст
|
||
из AMQP headers (traceparent от relay «send»), запасной вариант — поля traceparent/tracestate
|
||
тела (jobs.schema.json). Внутри: db jobs.claim, worker.fetch_staging (s3 GetObject),
|
||
worker.transform (подпроцесс), s3 PutObject x4, db jobs.complete, s3 DeleteObject staging;
|
||
* метрики: messenger.jobs.processed{messenger.job.type, messenger.job.outcome, error.type},
|
||
messenger.job.duration{type, outcome} (s, от claim до итога), messenger.job.queue_wait (s, от
|
||
next_attempt_at до claim), messenger.attachment.time_to_ready{outcome} (s, от durable acceptance
|
||
= jobs.created_at до COMMIT ready/failed), messenger.worker.busy / messenger.worker.capacity
|
||
(мгновенно) и messenger.worker.busy_time (s, счётчик: rate/capacity = точная загрузка).
|
||
"""
|
||
import concurrent.futures
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
|
||
import psycopg
|
||
from psycopg.rows import dict_row
|
||
|
||
import common
|
||
import jobs
|
||
import objstore
|
||
import telemetry
|
||
from objstore import ObjectMissing, StorageUnavailable
|
||
from telemetry import SpanKind, Status, StatusCode
|
||
|
||
log = logging.getLogger("worker")
|
||
CONCURRENCY = common.env_int("WORKER_CONCURRENCY", 2)
|
||
PREFETCH = common.env_int("WORKER_PREFETCH", 1)
|
||
LEASE_SECONDS = common.env_int("JOB_LEASE_SECONDS", 30)
|
||
JOB_TIMEOUT = common.env_float("JOB_TIMEOUT_SECONDS", 60)
|
||
RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2)
|
||
RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60)
|
||
MAX_UPLOAD_BYTES = common.env_int("MAX_UPLOAD_BYTES", 10485760)
|
||
MAX_IMAGE_PIXELS = common.env_int("MAX_IMAGE_PIXELS", 40000000)
|
||
REQUEUE_PAUSE = common.env_float("WORKER_REQUEUE_PAUSE_SECONDS", 2)
|
||
WORK_DIR = os.environ.get("WORKER_TMP_DIR") or None
|
||
FAULT_INJECTION = os.environ.get("WORKER_FAULT_INJECTION", "0") == "1"
|
||
FAULT_SLOW_SECONDS = common.env_float("FAULT_SLOW_SECONDS", 20)
|
||
TRANSFORM = os.path.join(os.path.dirname(os.path.abspath(__file__)), "transform.py")
|
||
HEARTBEAT = "/tmp/worker.heartbeat"
|
||
EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"}
|
||
TRANSFORM_ERRORS = {3: "invalid_image", 4: "image_too_large", 5: "resource_limit"}
|
||
|
||
OBJ = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments"))
|
||
STOP = threading.Event()
|
||
|
||
# --- Lab6: метрики worker ---
|
||
_m = telemetry.meter
|
||
JOBS_PROCESSED = _m.create_counter("messenger.jobs.processed", unit="{job}",
|
||
description="Job messages handled by the worker, by outcome")
|
||
JOB_DURATION = _m.create_histogram("messenger.job.duration", unit="s",
|
||
description="Processing time of a claimed job attempt (claim -> outcome)")
|
||
QUEUE_WAIT = _m.create_histogram("messenger.job.queue_wait", unit="s",
|
||
description="Time a due job waited before a worker claimed it")
|
||
TIME_TO_READY = _m.create_histogram("messenger.attachment.time_to_ready", unit="s",
|
||
description="From durable acceptance (enqueue COMMIT) to ready/failed COMMIT")
|
||
BUSY_TIME = _m.create_counter("messenger.worker.busy_time", unit="s",
|
||
description="Cumulative consumer time spent handling messages (utilization = rate / capacity)")
|
||
_BUSY = [0]
|
||
_BUSY_LOCK = threading.Lock()
|
||
|
||
|
||
def _busy(delta):
|
||
with _BUSY_LOCK:
|
||
_BUSY[0] += delta
|
||
|
||
|
||
_m.create_observable_gauge("messenger.worker.busy", [lambda o: [telemetry.metrics.Observation(_BUSY[0])]],
|
||
unit="{consumer}", description="Consumers currently processing a job")
|
||
_m.create_observable_gauge("messenger.worker.capacity", [lambda o: [telemetry.metrics.Observation(CONCURRENCY)]],
|
||
unit="{consumer}", description="Configured consumer threads (WORKER_CONCURRENCY)")
|
||
JOB_ATTRS = {"messenger.job.type": jobs.JOB_TYPE}
|
||
|
||
|
||
class JobError(Exception):
|
||
def __init__(self, code, detail=""):
|
||
super().__init__(code)
|
||
self.code = code
|
||
self.detail = detail
|
||
|
||
|
||
class LeaseLost(Exception):
|
||
pass
|
||
|
||
|
||
class Lease:
|
||
"""Продление аренды отдельным соединением раз в LEASE_SECONDS/3, но не дольше единого
|
||
срока задания (JOB_TIMEOUT_SECONDS от claim): после него аренда больше не продлевается
|
||
и reaper вернёт задание в очередь, даже если процесс жив, но завис.
|
||
|
||
Аудит F13: срок — общий бюджет на чтение staging -> transform -> запись результатов.
|
||
check() перед каждым шагом с побочным эффектом прерывает задание, если срок вышел
|
||
(processing_timeout, обычный retry), и больше не продлевает аренду после срока."""
|
||
|
||
def __init__(self, conninfo, job, deadline):
|
||
self.conninfo = conninfo
|
||
self.job = job
|
||
self.lost = False
|
||
self.deadline = deadline
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(target=self._loop, daemon=True)
|
||
|
||
def __enter__(self):
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *exc):
|
||
self._stop.set()
|
||
self._thread.join(timeout=5)
|
||
|
||
def left(self):
|
||
return self.deadline - time.monotonic()
|
||
|
||
def check(self, conn=None):
|
||
"""Перед каждым побочным эффектом: флаг фонового heartbeat И синхронная проверка
|
||
(продление) аренды в БД. Иначе замороженный процесс после разморозки успел бы
|
||
сделать шаг до того, как фоновый поток заметит потерю аренды."""
|
||
if self.lost:
|
||
raise LeaseLost()
|
||
if self.left() <= 0:
|
||
raise JobError("processing_timeout", "job deadline %.0fs exceeded" % JOB_TIMEOUT)
|
||
if conn is not None:
|
||
with conn.transaction():
|
||
ok = jobs.heartbeat(conn, self.job["job_id"], self.job["lease_token"], LEASE_SECONDS)
|
||
if not ok:
|
||
self.lost = True
|
||
raise LeaseLost()
|
||
|
||
def _loop(self):
|
||
conn = None
|
||
while not self._stop.wait(max(1.0, LEASE_SECONDS / 3)):
|
||
if time.monotonic() > self.deadline:
|
||
log.warning("job %s exceeded hard deadline, lease is no longer renewed", self.job["job_id"],
|
||
extra={"event": "job.deadline_exceeded", "job_id": str(self.job["job_id"])})
|
||
return
|
||
try:
|
||
if conn is None or conn.closed:
|
||
conn = psycopg.connect(self.conninfo, autocommit=True)
|
||
if not jobs.heartbeat(conn, self.job["job_id"], self.job["lease_token"], LEASE_SECONDS):
|
||
self.lost = True
|
||
log.warning("lease lost for job %s", self.job["job_id"],
|
||
extra={"event": "job.lease_lost", "job_id": str(self.job["job_id"])})
|
||
return
|
||
except psycopg.Error as exc:
|
||
log.warning("lease heartbeat failed (%s)", type(exc).__name__,
|
||
extra={"event": "job.heartbeat_failed", "error.type": type(exc).__name__})
|
||
conn = None
|
||
if conn is not None:
|
||
conn.close()
|
||
|
||
|
||
def sha256_file(path):
|
||
h = hashlib.sha256()
|
||
with open(path, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
class Consumer(threading.Thread):
|
||
def __init__(self, index):
|
||
super().__init__(name="consumer-%d" % index, daemon=True)
|
||
self.conninfo = common.database_conninfo()
|
||
self.db = None
|
||
self.last_beat = time.monotonic()
|
||
self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||
|
||
# ------------------------------------------------------------------ AMQP
|
||
def run(self):
|
||
while not STOP.is_set():
|
||
conn = common.connect_broker(log, STOP)
|
||
if conn is None:
|
||
return
|
||
try:
|
||
ch = conn.channel()
|
||
ch.basic_qos(prefetch_count=PREFETCH)
|
||
log.info("%s consuming %s owner=%s", self.name, common.QUEUE, common.OWNER,
|
||
extra={"event": "worker.consuming"})
|
||
for method, props, body in ch.consume(common.QUEUE, inactivity_timeout=1):
|
||
self.last_beat = time.monotonic()
|
||
if STOP.is_set():
|
||
break
|
||
if method is None:
|
||
continue
|
||
outcome = self.run_job(conn, body, getattr(props, "headers", None) or {},
|
||
bool(getattr(method, "redelivered", False)))
|
||
if outcome == "ack":
|
||
ch.basic_ack(method.delivery_tag)
|
||
elif outcome == "reject":
|
||
ch.basic_reject(method.delivery_tag, requeue=False) # -> DLX -> DLQ
|
||
else:
|
||
self.pause(conn, REQUEUE_PAUSE)
|
||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||
ch.cancel()
|
||
conn.close()
|
||
except Exception as exc:
|
||
log.warning("%s broker loop error (%s), reconnecting", self.name, type(exc).__name__,
|
||
extra={"event": "broker.loop_error", "error.type": type(exc).__name__})
|
||
try:
|
||
conn.close()
|
||
except Exception:
|
||
pass
|
||
STOP.wait(1)
|
||
|
||
def pause(self, conn, seconds):
|
||
end = time.monotonic() + seconds
|
||
while time.monotonic() < end:
|
||
conn.process_data_events(time_limit=min(0.5, end - time.monotonic()))
|
||
self.last_beat = time.monotonic()
|
||
|
||
def run_job(self, conn, body, headers=None, redelivered=False):
|
||
"""Работа в отдельном потоке; этот поток обслуживает heartbeat AMQP-соединения."""
|
||
future = self.pool.submit(self.traced_message, body, headers or {}, redelivered)
|
||
# Lab6 (найдено по метрикам инцидента 1): process_data_events(time_limit=0.5) ждал полные
|
||
# 0.5 с и после конца задания — потребитель брал не больше 2 заданий/с (utilization 41 %).
|
||
# Пустой threadsafe-callback по завершении будит цикл AMQP сразу.
|
||
future.add_done_callback(lambda _f: conn.add_callback_threadsafe(lambda: None))
|
||
while not future.done():
|
||
conn.process_data_events(time_limit=0.5)
|
||
self.last_beat = time.monotonic()
|
||
try:
|
||
return future.result()
|
||
except Exception:
|
||
log.exception("unexpected worker error", extra={"event": "worker.unexpected_error"})
|
||
return "requeue"
|
||
|
||
# ------------------------------------------------------------------ DB
|
||
def database(self):
|
||
if self.db is None or self.db.closed:
|
||
self.db = psycopg.connect(self.conninfo, row_factory=dict_row)
|
||
return self.db
|
||
|
||
def drop_db(self):
|
||
try:
|
||
if self.db is not None:
|
||
self.db.close()
|
||
except Exception:
|
||
pass
|
||
self.db = None
|
||
|
||
# ------------------------------------------------------------------ job
|
||
@staticmethod
|
||
def _parent_context(body, headers):
|
||
carrier = {}
|
||
for key in ("traceparent", "tracestate"):
|
||
value = headers.get(key)
|
||
if isinstance(value, bytes):
|
||
value = value.decode("ascii", "replace")
|
||
if isinstance(value, str) and value:
|
||
carrier[key] = value
|
||
if "traceparent" not in carrier:
|
||
try:
|
||
raw = json.loads(body.decode("utf-8"))
|
||
for key in ("traceparent", "tracestate"):
|
||
if isinstance(raw.get(key), str):
|
||
carrier[key] = raw[key]
|
||
except Exception:
|
||
pass
|
||
return telemetry.extract(carrier)
|
||
|
||
def traced_message(self, body, headers, redelivered):
|
||
"""CONSUMER span вокруг обработки + метрики исхода. res['outcome'] заполняет обработка."""
|
||
res = {"outcome": "unknown", "error": None}
|
||
span = telemetry.tracer.start_span(
|
||
"%s process" % common.QUEUE, kind=SpanKind.CONSUMER, context=self._parent_context(body, headers),
|
||
attributes={"messaging.system": "rabbitmq", "messaging.operation.type": "process",
|
||
"messaging.operation.name": "process", "messaging.destination.name": common.QUEUE,
|
||
"messaging.rabbitmq.destination.routing_key": common.ROUTING_KEY,
|
||
"messaging.message.body.size": len(body), "messaging.message.redelivered": redelivered,
|
||
"messenger.worker.owner": common.OWNER})
|
||
_busy(1)
|
||
busy_started = time.perf_counter()
|
||
try:
|
||
with telemetry.trace.use_span(span, end_on_exit=False):
|
||
action = self.process_message(body, res, span)
|
||
return action
|
||
except BaseException as exc:
|
||
res["outcome"], res["error"] = "crashed", type(exc).__name__
|
||
span.record_exception(exc)
|
||
raise
|
||
finally:
|
||
_busy(-1)
|
||
BUSY_TIME.add(time.perf_counter() - busy_started)
|
||
span.set_attribute("messenger.job.outcome", res["outcome"])
|
||
attrs = dict(JOB_ATTRS, **{"messenger.job.outcome": res["outcome"]})
|
||
if res["error"]:
|
||
attrs["error.type"] = res["error"]
|
||
span.set_attribute("error.type", res["error"])
|
||
if res["outcome"] in ("failed", "rejected", "requeued", "crashed"):
|
||
span.set_status(Status(StatusCode.ERROR, res["error"] or res["outcome"]))
|
||
JOBS_PROCESSED.add(1, attrs)
|
||
span.end()
|
||
|
||
def process_message(self, body, res=None, span=None):
|
||
res = res if res is not None else {}
|
||
try:
|
||
msg = jobs.parse_message(body)
|
||
except jobs.InvalidMessage as exc:
|
||
log.warning("invalid message rejected to DLQ (%s)", exc,
|
||
extra={"event": "job.rejected", "reason": str(exc)})
|
||
res.update(outcome="rejected", error="invalid_message")
|
||
return "reject"
|
||
if span is not None:
|
||
span.set_attribute("messaging.message.id", "%s:%s" % (msg["job_id"], msg["attempt"]))
|
||
span.set_attribute("messenger.job.id", msg["job_id"])
|
||
span.set_attribute("messenger.attachment.id", msg["attachment_id"])
|
||
span.set_attribute("messenger.job.message_attempt", msg["attempt"])
|
||
try:
|
||
db = self.database()
|
||
with db.transaction():
|
||
action, job = jobs.claim(db, msg, common.OWNER, LEASE_SECONDS)
|
||
except psycopg.Error as exc:
|
||
log.warning("claim failed, message requeued (%s)", type(exc).__name__,
|
||
extra={"event": "job.requeued", "error.type": type(exc).__name__, "job_id": msg["job_id"]})
|
||
self.drop_db()
|
||
res.update(outcome="requeued", error="db_unavailable")
|
||
return "requeue"
|
||
if action == "reject":
|
||
log.warning("message for job %s rejected to DLQ (%s)", msg["job_id"], job,
|
||
extra={"event": "job.rejected", "job_id": msg["job_id"], "reason": str(job)})
|
||
res.update(outcome="rejected", error=str(job))
|
||
return "reject"
|
||
if action == "ack":
|
||
res.update(outcome="duplicate")
|
||
return "ack"
|
||
QUEUE_WAIT.record(float(job.get("eligible_wait_s") or 0.0), JOB_ATTRS)
|
||
if span is not None:
|
||
span.set_attribute("messenger.job.attempt", job["attempt"])
|
||
span.set_attribute("messenger.job.queue_wait_s", round(float(job.get("eligible_wait_s") or 0.0), 3))
|
||
log.info("job %s attempt %d claimed (attachment %s)", job["job_id"], job["attempt"], job["attachment_id"],
|
||
extra={"event": "job.claimed", "job_id": str(job["job_id"]),
|
||
"attachment_id": str(job["attachment_id"]), "attempt": job["attempt"]})
|
||
started = time.perf_counter()
|
||
try:
|
||
return self.execute(db, job, res)
|
||
except psycopg.Error as exc:
|
||
# Результат не записан; аренда истечёт, reaper вернёт задание. Сообщение — назад.
|
||
log.warning("database error during job %s (%s)", job["job_id"], type(exc).__name__,
|
||
extra={"event": "job.requeued", "job_id": str(job["job_id"]), "error.type": type(exc).__name__})
|
||
self.drop_db()
|
||
res.update(outcome="requeued", error="db_unavailable")
|
||
return "requeue"
|
||
finally:
|
||
JOB_DURATION.record(time.perf_counter() - started,
|
||
dict(JOB_ATTRS, **{"messenger.job.outcome": res.get("outcome", "unknown")}))
|
||
|
||
def fault_point(self, job):
|
||
if not FAULT_INJECTION:
|
||
return None
|
||
name = job["filename"] or ""
|
||
if not name.startswith("fault-"):
|
||
return None
|
||
return name[len("fault-"):].rsplit(".", 1)[0]
|
||
|
||
def crash_once(self, db, job, point, wanted):
|
||
if point != wanted:
|
||
return
|
||
with db.transaction():
|
||
seen = db.execute(
|
||
"""SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected'
|
||
AND detail->>'point' = %s""", (job["job_id"], point)).fetchone()
|
||
if seen:
|
||
return
|
||
jobs.add_event(db, job["job_id"], "fault_injected", job["attempt"], common.OWNER, {"point": point})
|
||
log.warning("FAULT INJECTION %s job=%s: process exits now", point, job["job_id"],
|
||
extra={"event": "fault.injected", "fault.point": point, "job_id": str(job["job_id"])})
|
||
telemetry.shutdown(1000) # успеть отправить spans/logs до «краха» (учебный сценарий)
|
||
os._exit(86)
|
||
|
||
def execute(self, db, job, res=None):
|
||
res = res if res is not None else {}
|
||
point = self.fault_point(job)
|
||
started = time.monotonic()
|
||
with Lease(self.conninfo, job, started + JOB_TIMEOUT) as lease:
|
||
try:
|
||
self.crash_once(db, job, point, "crash-after-claim")
|
||
if point == "slow" and job["attempt"] == 1:
|
||
log.warning("FAULT INJECTION slow job=%s sleeping %.0fs", job["job_id"], FAULT_SLOW_SECONDS)
|
||
time.sleep(FAULT_SLOW_SECONDS)
|
||
if point == "bug" and job["attempt"] == 1:
|
||
raise RuntimeError("injected programming error")
|
||
if point and point.startswith("transient-"):
|
||
n = point.split("-", 1)[1]
|
||
if (n == "always" and job["replay_count"] == 0) or (n.isdigit() and job["attempt"] <= int(n)):
|
||
raise JobError("storage_unavailable", "injected transient error")
|
||
lease.check(db)
|
||
variants = self.do_work(db, job, lease)
|
||
self.crash_once(db, job, point, "crash-after-s3-put")
|
||
lease.check(db)
|
||
except LeaseLost:
|
||
with db.transaction():
|
||
jobs.add_event(db, job["job_id"], "aborted_lease_lost", job["attempt"], common.OWNER)
|
||
res.update(outcome="lease_lost")
|
||
log.warning("job %s attempt %d: lease lost, work aborted", job["job_id"], job["attempt"],
|
||
extra={"event": "job.lease_lost", "job_id": str(job["job_id"]), "attempt": job["attempt"]})
|
||
return "ack"
|
||
except JobError as err:
|
||
with db.transaction():
|
||
result, delay = jobs.fail_or_retry(db, job, err.code, err.detail, common.OWNER,
|
||
RETRY_BASE, RETRY_MAX)
|
||
res.update(outcome=result, error=err.code)
|
||
self._record_terminal(job, result)
|
||
log.warning("job %s attempt %d: %s (%s) -> %s%s", job["job_id"], job["attempt"], err.code,
|
||
err.detail, result, " in %.1fs" % delay if delay else "",
|
||
extra={"event": "job.retry_scheduled" if result == "retry" else "job." + result,
|
||
"job_id": str(job["job_id"]), "attempt": job["attempt"],
|
||
"error_code": err.code, "retry_delay_s": round(delay or 0, 3)})
|
||
return "ack"
|
||
except psycopg.Error:
|
||
raise # нет БД: сообщение назад в очередь (process_message)
|
||
except Exception as exc:
|
||
# Непредусмотренная ошибка (баг, неожиданный вывод transform): временная ошибка
|
||
# с обычным retry/backoff, а не «висящее» processing до истечения аренды.
|
||
log.exception("job %s attempt %d: unexpected error", job["job_id"], job["attempt"],
|
||
extra={"event": "job.unexpected_error", "job_id": str(job["job_id"]),
|
||
"attempt": job["attempt"]})
|
||
with db.transaction():
|
||
result, delay = jobs.fail_or_retry(db, job, "processing_failed", type(exc).__name__,
|
||
common.OWNER, RETRY_BASE, RETRY_MAX)
|
||
res.update(outcome=result, error="processing_failed")
|
||
self._record_terminal(job, result)
|
||
return "ack"
|
||
with db.transaction():
|
||
result = jobs.complete(db, job, variants, common.OWNER)
|
||
res.update(outcome=result)
|
||
self._record_terminal(job, result)
|
||
self.crash_once(db, job, point, "crash-after-commit")
|
||
log.info("job %s attempt %d -> %s in %.0fms", job["job_id"], job["attempt"], result,
|
||
(time.monotonic() - started) * 1000,
|
||
extra={"event": "job.completed" if result == "ready" else "job." + result,
|
||
"job_id": str(job["job_id"]), "attachment_id": str(job["attachment_id"]),
|
||
"attempt": job["attempt"], "duration_ms": round((time.monotonic() - started) * 1000, 1),
|
||
"time_to_ready_s": round(job.get("time_to_ready_s") or 0.0, 3)})
|
||
if result == "ready":
|
||
self.cleanup_staging(db, job)
|
||
return "ack"
|
||
|
||
@staticmethod
|
||
def _record_terminal(job, result):
|
||
if result == "ready" and job.get("time_to_ready_s") is not None:
|
||
TIME_TO_READY.record(job["time_to_ready_s"], dict(JOB_ATTRS, outcome="ready"))
|
||
elif result == "failed" and job.get("time_to_terminal_s") is not None:
|
||
TIME_TO_READY.record(job["time_to_terminal_s"], dict(JOB_ATTRS, outcome="failed"))
|
||
|
||
def cleanup_staging(self, db, job):
|
||
"""Staging удаляется только после COMMIT ready. Не вышло — доберёт janitor."""
|
||
key = job["staging_key"]
|
||
if not key:
|
||
return
|
||
try:
|
||
OBJ.delete(key)
|
||
except StorageUnavailable:
|
||
log.warning("staging cleanup postponed for %s", job["attachment_id"],
|
||
extra={"event": "staging.cleanup_postponed", "attachment_id": str(job["attachment_id"])})
|
||
return
|
||
with db.transaction():
|
||
db.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'",
|
||
(job["attachment_id"],))
|
||
jobs.add_event(db, job["job_id"], "staging_deleted", job["attempt"], common.OWNER)
|
||
|
||
def do_work(self, db, job, lease):
|
||
tmpdir = tempfile.mkdtemp(prefix="job-", dir=WORK_DIR)
|
||
try:
|
||
src = os.path.join(tmpdir, "source")
|
||
with telemetry.child_span("worker.fetch_staging", attributes={"messenger.size_bytes": job["size_bytes"]}):
|
||
self.fetch_staging(job, src, lease)
|
||
lease.check(db)
|
||
with telemetry.child_span("worker.transform", attributes={
|
||
"messenger.content_type": job["content_type"], "messenger.pipeline_version": job["pipeline_version"],
|
||
"messenger.image.pixels": (job["width"] or 0) * (job["height"] or 0)}):
|
||
produced = self.transform(job, src, tmpdir, lease)
|
||
lease.check(db) # последняя проверка перед записью в конечный prefix
|
||
ver = job["pipeline_version"]
|
||
base = "attachments/%s/v%d/" % (job["attachment_id"], ver)
|
||
if job["object_key"] != base + "original":
|
||
raise JobError("staging_corrupted", "unexpected object key")
|
||
variants = []
|
||
for v in produced:
|
||
path = v["path"]
|
||
variants.append({"name": v["name"], "key": base + "%s.%s" % (v["name"], EXT[v["content_type"]]),
|
||
"content_type": v["content_type"], "width": v["width"], "height": v["height"],
|
||
"size_bytes": os.path.getsize(path), "sha256": sha256_file(path), "path": path})
|
||
try:
|
||
# Сначала все объекты, потом COMMIT в БД: ready никогда не ссылается на
|
||
# отсутствующий объект. Ключи детерминированы — повтор перезаписывает их же.
|
||
with open(src, "rb") as f:
|
||
OBJ.put(job["object_key"], f, job["size_bytes"], job["content_type"])
|
||
for v in variants:
|
||
with open(v["path"], "rb") as f:
|
||
OBJ.put(v["key"], f, v["size_bytes"], v["content_type"])
|
||
except StorageUnavailable:
|
||
raise JobError("storage_unavailable", "put result")
|
||
return variants
|
||
finally:
|
||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||
|
||
def fetch_staging(self, job, path, lease):
|
||
key = job["staging_key"]
|
||
if not key:
|
||
raise JobError("staging_missing", "staging already removed")
|
||
limit = min(job["size_bytes"], MAX_UPLOAD_BYTES)
|
||
try:
|
||
resp = OBJ.open(key)
|
||
except ObjectMissing:
|
||
raise JobError("staging_missing", "no staging object")
|
||
except StorageUnavailable:
|
||
raise JobError("storage_unavailable", "get staging")
|
||
h = hashlib.sha256()
|
||
size = 0
|
||
try:
|
||
with open(path, "wb") as f:
|
||
for chunk in resp.stream(objstore.STREAM_CHUNK):
|
||
if lease.left() <= 0: # медленный S3 не растягивает задание сверх срока
|
||
raise JobError("processing_timeout", "staging read exceeded job deadline")
|
||
size += len(chunk)
|
||
if size > limit:
|
||
raise JobError("staging_corrupted", "staging larger than metadata")
|
||
h.update(chunk)
|
||
f.write(chunk)
|
||
except JobError:
|
||
raise
|
||
except Exception as exc:
|
||
raise JobError("storage_unavailable", "read staging: %s" % type(exc).__name__)
|
||
finally:
|
||
resp.close()
|
||
resp.release_conn()
|
||
if size != job["size_bytes"] or h.hexdigest() != job["sha256"]:
|
||
raise JobError("staging_corrupted", "size/sha256 differ from accepted upload")
|
||
|
||
def transform(self, job, src, tmpdir, lease):
|
||
out = os.path.join(tmpdir, "out")
|
||
os.mkdir(out)
|
||
left = lease.left()
|
||
if left <= 0: # бюджет исчерпан — transform даже не запускаем (аудит F13)
|
||
raise JobError("processing_timeout", "no time left for transform")
|
||
try:
|
||
proc = subprocess.run(
|
||
[sys.executable, TRANSFORM, src, out, job["content_type"], str(MAX_IMAGE_PIXELS)],
|
||
capture_output=True, timeout=left, check=False)
|
||
except subprocess.TimeoutExpired:
|
||
raise JobError("processing_timeout", "transform exceeded %.0fs" % left)
|
||
if proc.returncode < 0:
|
||
raise JobError("resource_limit", "transform killed by signal %d" % -proc.returncode)
|
||
if proc.returncode in TRANSFORM_ERRORS:
|
||
raise JobError(TRANSFORM_ERRORS[proc.returncode], proc.stdout.decode(errors="replace")[:200])
|
||
if proc.returncode != 0:
|
||
raise JobError("processing_failed", "transform exit %d" % proc.returncode)
|
||
result = json.loads(proc.stdout)
|
||
names = sorted(v["name"] for v in result["variants"])
|
||
if names != sorted(jobs.VARIANT_NAMES):
|
||
raise JobError("processing_failed", "unexpected variants")
|
||
return result["variants"]
|
||
|
||
|
||
def main():
|
||
telemetry.setup("messenger-worker")
|
||
signal.signal(signal.SIGTERM, lambda *_: STOP.set())
|
||
signal.signal(signal.SIGINT, lambda *_: STOP.set())
|
||
log.info("worker starting owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%.0fs fault_injection=%s",
|
||
common.OWNER, CONCURRENCY, PREFETCH, LEASE_SECONDS, JOB_TIMEOUT, FAULT_INJECTION,
|
||
extra={"event": "worker.started"})
|
||
consumers = [Consumer(i) for i in range(CONCURRENCY)]
|
||
for c in consumers:
|
||
c.start()
|
||
while not STOP.is_set():
|
||
# healthcheck: файл обновляется, только пока все потребители живы и не зависли.
|
||
if all(c.is_alive() and time.monotonic() - c.last_beat < 30 for c in consumers):
|
||
common.touch(HEARTBEAT)
|
||
STOP.wait(2)
|
||
log.info("SIGTERM: finishing in-flight jobs", extra={"event": "worker.draining"})
|
||
for c in consumers:
|
||
c.join(timeout=JOB_TIMEOUT + 30)
|
||
log.info("worker stopped", extra={"event": "worker.stopped"})
|
||
telemetry.shutdown()
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|