Files
mtusi2026lab6/worker.py
T

528 lines
27 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose).
Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1):
одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди.
Обработка одного сообщения:
1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ.
2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый
lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op.
3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает ->
аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued.
Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения.
4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД.
5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь).
6. PUT в конечный prefix: attachments/<id>/v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp.
Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи.
7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT.
8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД.
9. Удаление исходника из staging (если не вышло — доделает janitor).
Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД,
таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed.
"""
import hashlib
import json
import logging
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import threading
import time
from dataclasses import dataclass
from pathlib import Path
import psycopg
from opentelemetry import trace
from opentelemetry.trace import NonRecordingSpan, SpanKind
from psycopg.rows import dict_row
import settings
import telemetry
telemetry.setup("messenger-worker")
import broker # noqa: E402
import faults # noqa: E402
import imaging # noqa: E402
import pipeline # noqa: E402
import storage # noqa: E402
log = logging.getLogger("messenger.worker")
# lab6: метрики фоновой обработки (contracts/observability.md). Метки: тип задания, исход, очередь.
JOB_ATTRS = {"job.type": pipeline.JOB_TYPE}
JOB_DURATION = telemetry.meter.create_histogram(
"messenger.job.duration", unit="s",
description="Attempt of an attachment job from claim to result (outcome: ready/retry/failed/stale/lease_lost)")
TIME_TO_READY = telemetry.meter.create_histogram(
"messenger.attachment.time_to_ready", unit="s",
description="From durable acceptance (job COMMIT in API) to ready COMMIT in worker")
MESSAGES = telemetry.meter.create_counter(
"messenger.messages.consumed", unit="{message}",
description="Broker deliveries by outcome: processed (claimed), noop (duplicate/not due), dead_lettered, requeued")
BUSY = {"value": 0}
BUSY_LOCK = threading.Lock()
def _observe_busy(_options):
from opentelemetry.metrics import Observation
return [Observation(BUSY["value"], {"messaging.destination.name": pipeline.QUEUE})]
def _observe_capacity(_options):
from opentelemetry.metrics import Observation
return [Observation(settings.WORKER_CONCURRENCY, {"messaging.destination.name": pipeline.QUEUE})]
telemetry.meter.create_observable_up_down_counter(
"messenger.worker.busy", [_observe_busy], unit="{consumer}", description="Consumers processing a job now")
telemetry.meter.create_observable_up_down_counter(
"messenger.worker.capacity", [_observe_capacity], unit="{consumer}",
description="Consumers of this worker process (WORKER_CONCURRENCY)")
def busy(delta: int) -> None:
with BUSY_LOCK:
BUSY["value"] += delta
STOP = threading.Event()
HERE = Path(__file__).resolve().parent
VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"}
IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large",
imaging.EXIT_MEMORY: "resource_limit"}
class JobError(Exception):
def __init__(self, code: str, permanent: bool):
super().__init__(code)
self.code = code
self.permanent = permanent
class LeaseLost(Exception):
pass
def connect_db():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def final_prefix(claim: pipeline.Claim) -> str:
return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/"
class LeaseKeeper(threading.Thread):
"""Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку."""
def __init__(self, claim: pipeline.Claim):
super().__init__(daemon=True)
self.claim = claim
self.done = threading.Event()
self.lost = threading.Event()
self.frozen = threading.Event() # fault worker_freeze: перестать продлевать
def run(self):
interval = max(1.0, settings.JOB_LEASE_SECONDS / 3)
conn = None
while not self.done.wait(interval):
if self.frozen.is_set():
continue
try:
if conn is None:
conn = connect_db()
if not pipeline.renew_lease(conn, self.claim):
log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id,
extra={"event": "job.lease_lost", "job.id": str(self.claim.job_id)})
self.lost.set()
return
except psycopg.Error as exc:
# Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT).
log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__)
if conn is not None:
conn.close()
conn = None
if conn is not None:
conn.close()
def check(self):
if self.lost.is_set():
raise LeaseLost()
def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict:
# Дочернему процессу — только лимиты, никаких секретов окружения.
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1",
"IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB),
"IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)}
for fault in ("hang", "invalid", "oom"):
if faults.hit("imaging_" + fault):
env["FAULT_IMAGING"] = fault
with telemetry.internal_span("imaging", {"messenger.pipeline.version": claim.pipeline_version,
"messenger.source.content_type": claim.content_type}) as span:
result = _run_imaging(env, source, out_dir, claim, keeper)
span.set_attribute("messenger.variants", len(result["variants"]))
return result
def _run_imaging(env: dict, source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict:
process = subprocess.Popen(
[sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type,
str(settings.MAX_IMAGE_PIXELS)],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir),
)
deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS
try:
while True:
try:
output, _ = process.communicate(timeout=0.5)
break
except subprocess.TimeoutExpired:
if time.monotonic() > deadline:
raise JobError("processing_timeout", permanent=False) from None
keeper.check()
finally:
if process.poll() is None:
process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку
process.wait()
code = process.returncode
if code in IMAGING_EXIT:
raise JobError(IMAGING_EXIT[code], permanent=True)
if code != 0:
# Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены.
raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False)
return json.loads(output.decode("utf-8").strip().splitlines()[-1])
def sha256_file(path: Path) -> tuple[int, str]:
digest = hashlib.sha256()
size = 0
with open(path, "rb") as file:
while chunk := file.read(1024 * 1024):
size += len(chunk)
digest.update(chunk)
return size, digest.hexdigest()
def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None:
if faults.hit("worker_s3_error"):
raise JobError("storage_unavailable", permanent=False)
with open(path, "rb") as file:
try:
storage.put(key, file, size, content_type, sha256)
except storage.StorageUnavailable:
raise JobError("storage_unavailable", permanent=False) from None
def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]:
"""Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов."""
source = workdir / "source"
try:
storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES)
except storage.ObjectMissing:
raise JobError("source_missing", permanent=True) from None
except storage.StorageUnavailable:
raise JobError("storage_unavailable", permanent=False) from None
with telemetry.internal_span("verify source sha256"):
size, digest = sha256_file(source)
if size != claim.size_bytes or digest != claim.sha256:
# Staging не совпадает с тем, что принял API: публиковать нельзя.
raise JobError("source_corrupted", permanent=True)
keeper.check()
out_dir = workdir / "out"
out_dir.mkdir()
result = run_imaging(source, out_dir, claim, keeper)
keeper.check()
prefix = final_prefix(claim)
original_key = prefix + "original"
put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes)
variants = []
for item in result["variants"]:
path = out_dir / item["file"]
v_size, v_sha = sha256_file(path)
key = prefix + VARIANT_KEYS[item["name"]]
put_file(key, path, item["content_type"], v_sha, v_size)
variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size,
"width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key})
keeper.check()
return original_key, variants
@dataclass
class Outcome:
ack: bool = True # False -> nack с requeue (не смогли даже записать claim)
reject: bool = False # True -> reject без requeue -> DLQ
cleanup_key: str | None = None
attachment_id: object = None
span_context: object = None # спан обработки: очистка staging после ack — в той же трассе
delivery: str = "processed" # метка messenger.messages.consumed: processed / noop
def carrier_from(headers, body: bytes) -> dict:
"""W3C контекст сообщения: native headers AMQP (основной carrier), иначе поле traceparent тела."""
carrier = {}
for key in ("traceparent", "tracestate"):
value = (headers or {}).get(key)
if isinstance(value, bytes):
value = value.decode("ascii", "replace")
if isinstance(value, str):
carrier[key] = value
if "traceparent" not in carrier:
try:
message = json.loads(body.decode("utf-8"))
if isinstance(message, dict) and isinstance(message.get("traceparent"), str):
carrier["traceparent"] = message["traceparent"]
except (UnicodeDecodeError, ValueError):
pass
return carrier
class Consumer(threading.Thread):
def __init__(self, number: int):
super().__init__(name=f"consumer-{number}", daemon=True)
self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}"
self.db = None
def database(self):
if self.db is None or self.db.closed:
self.db = connect_db()
return self.db
def handle(self, body: bytes, redelivered: bool, headers=None) -> Outcome:
"""Спан CONSUMER «process attachments.process.v1» — ребёнок спана publish из dispatcher."""
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
"messaging.destination.name": pipeline.QUEUE, "messaging.consumer.group.name": pipeline.QUEUE,
"messaging.rabbitmq.message.redelivered": redelivered}
with telemetry.tracer.start_as_current_span(
f"process {pipeline.QUEUE}", context=telemetry.extract(carrier_from(headers, body)),
kind=SpanKind.CONSUMER, attributes=attributes,
record_exception=False, set_status_on_exception=False) as span:
outcome = self._handle(body, redelivered, span)
outcome.span_context = span.get_span_context()
kind = "dead_lettered" if outcome.reject else "requeued" if not outcome.ack else outcome.delivery
span.set_attribute("messenger.delivery", kind)
MESSAGES.add(1, {"messaging.destination.name": pipeline.QUEUE, "outcome": kind})
return outcome
def _handle(self, body: bytes, redelivered: bool, span) -> Outcome:
try:
message = pipeline.parse_message(body)
except pipeline.InvalidMessage as exc:
log.warning("invalid message -> DLQ (%s)", exc, extra={"event": "job.invalid_message"})
telemetry.mark_error(span, "invalid_message")
return Outcome(reject=True)
span.set_attributes({"messaging.message.id": f"{message['job_id']}:{message['attempt']}",
"messenger.job.id": message["job_id"],
"messenger.attachment.id": message["attachment_id"],
"messenger.message.attempt": message["attempt"]})
try:
result = pipeline.claim(self.database(), message, self.owner, redelivered)
except psycopg.Error as exc:
log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__,
extra={"event": "job.claim_failed", "job.id": message["job_id"],
"error.type": type(exc).__name__})
telemetry.mark_error(span, type(exc).__name__)
self.db = None
STOP.wait(2)
return Outcome(ack=False)
if isinstance(result, pipeline.Reject):
log.warning("job %s: %s -> DLQ", message["job_id"], result.reason,
extra={"event": "job.rejected", "job.id": message["job_id"], "reason": result.reason})
telemetry.mark_error(span, result.reason)
return Outcome(reject=True)
if isinstance(result, pipeline.Skip):
log.info("job %s: no-op (%s), ack", message["job_id"], result.reason,
extra={"event": "job.noop", "job.id": message["job_id"], "reason": result.reason})
span.set_attribute("messenger.noop.reason", result.reason)
return Outcome(delivery="noop")
span.set_attribute("messenger.job.attempt", result.attempt)
busy(+1)
try:
return self.execute(result, span)
finally:
busy(-1)
def execute(self, claim: pipeline.Claim, span) -> Outcome:
started = time.monotonic()
fields = {"job.id": str(claim.job_id), "attachment.id": str(claim.attachment_id), "attempt": claim.attempt}
log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt,
claim.max_attempts, claim.attachment_id, extra={"event": "job.claimed", **fields})
keeper = LeaseKeeper(claim)
keeper.start()
workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR))
outcome_name = "ready"
try:
try:
original_key, variants = process_job(claim, keeper, workdir)
faults.crash("worker_crash_after_put")
if faults.hit("worker_freeze"):
keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят
time.sleep(settings.JOB_LEASE_SECONDS * 3)
with telemetry.internal_span("commit result"):
time_to_ready = pipeline.finish_ready(self.database(), claim, original_key, variants,
self.owner)
if time_to_ready is None:
outcome_name = "stale" # аренду перехватили: fencing отклонил результат
except JobError as exc:
outcome_name = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner)
telemetry.mark_error(span, exc.code)
log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code,
"permanent" if exc.permanent else "temporary", outcome_name,
extra={"event": f"job.{outcome_name}", "error.type": exc.code,
"permanent": exc.permanent, **fields})
return Outcome()
except LeaseLost:
outcome_name = "lease_lost"
telemetry.mark_error(span, "lease_lost")
log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt,
extra={"event": "job.lease_lost", **fields})
return Outcome()
except psycopg.Error:
raise
except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены
log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt,
extra={"event": "job.unexpected_error", **fields})
telemetry.mark_error(span, "processing_failed")
outcome_name = pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner)
return Outcome()
except psycopg.Error as exc:
# Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и
# dispatcher вернёт job в очередь; если прошёл — повтор увидит ready.
outcome_name = "db_error"
telemetry.mark_error(span, type(exc).__name__)
log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__,
extra={"event": "job.db_error", "error.type": type(exc).__name__, **fields})
self.db = None
return Outcome()
finally:
keeper.done.set()
shutil.rmtree(workdir, ignore_errors=True)
elapsed = time.monotonic() - started
span.set_attribute("messenger.job.outcome", outcome_name)
JOB_DURATION.record(elapsed, JOB_ATTRS | {"outcome": outcome_name})
if time_to_ready is None:
telemetry.mark_error(span, "stale_result_rejected")
log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id,
claim.attempt, extra={"event": "job.stale_result_rejected", **fields})
return Outcome()
TIME_TO_READY.record(time_to_ready, JOB_ATTRS)
faults.crash("worker_crash_after_commit")
log.info("job %s attempt %s: ready in %.0f ms (%.1f s since upload)", claim.job_id, claim.attempt,
(time.monotonic() - started) * 1000, time_to_ready,
extra={"event": "job.ready", "duration_ms": round((time.monotonic() - started) * 1000, 1),
"time_to_ready_s": round(time_to_ready, 3), **fields})
return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id)
def cleanup_staging(self, outcome: Outcome) -> None:
"""После COMMIT ready и ack: исходник в staging больше не нужен."""
parent = trace.set_span_in_context(NonRecordingSpan(outcome.span_context)) if outcome.span_context else None
with telemetry.tracer.start_as_current_span("cleanup staging", context=parent):
try:
storage.delete(outcome.cleanup_key)
self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' "
"AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key))
except (storage.StorageUnavailable, psycopg.Error) as exc:
log.info("staging cleanup postponed to janitor: %s", type(exc).__name__,
extra={"event": "staging.cleanup_postponed", "error.type": type(exc).__name__})
self.db = None
def consume_once(self) -> None:
connection = broker.connect()
try:
channel = connection.channel()
channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH)
broker.check_topology(channel)
log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH,
extra={"event": "worker.consuming"})
for method, properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1):
if STOP.is_set():
break
if method is None:
continue
box = {}
headers = properties.headers if properties is not None else None
runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle(
body, method.redelivered, headers)), daemon=True)
runner.start()
while runner.is_alive():
connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа
outcome = box.get("outcome", Outcome(ack=False))
if outcome.reject:
channel.basic_reject(method.delivery_tag, requeue=False)
elif outcome.ack:
channel.basic_ack(method.delivery_tag)
else:
channel.basic_nack(method.delivery_tag, requeue=True)
if outcome.cleanup_key:
self.cleanup_staging(outcome)
try:
channel.cancel()
except broker.BrokerError:
pass
finally:
try:
connection.close()
except broker.BrokerError:
pass
def safe_handle(self, body, redelivered, headers=None) -> Outcome:
try:
return self.handle(body, redelivered, headers)
except Exception: # noqa: BLE001
log.exception("unexpected error while handling a message; requeue",
extra={"event": "job.handler_crashed"})
self.db = None
STOP.wait(2)
return Outcome(ack=False)
def run(self):
pause = 1.0
while not STOP.is_set():
try:
self.consume_once()
pause = 1.0
except broker.BrokerError as exc:
log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name,
type(exc).__name__, pause,
extra={"event": "broker.unavailable", "error.type": type(exc).__name__})
STOP.wait(pause)
pause = min(pause * 2, 15)
def main() -> None:
if not settings.DATABASE_URL or not settings.BROKER_URL:
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True)
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)]
for consumer in consumers:
consumer.start()
log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, "
"fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH,
settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB,
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
while not STOP.wait(2):
if not all(consumer.is_alive() for consumer in consumers):
log.error("a consumer thread died, exiting so that Docker restarts the worker")
os._exit(1)
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят.
for consumer in consumers:
consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10)
log.info("stopped", extra={"event": "service.stopped"})
telemetry.shutdown()
if __name__ == "__main__":
main()