forked from mtusi2026/mtusi2026lab5
391 lines
18 KiB
Python
391 lines
18 KiB
Python
"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose).
|
||
|
||
Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1):
|
||
одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди.
|
||
|
||
Обработка одного сообщения:
|
||
1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ.
|
||
2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый
|
||
lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op.
|
||
3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает ->
|
||
аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued.
|
||
Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения.
|
||
4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД.
|
||
5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь).
|
||
6. PUT в конечный prefix: attachments/<id>/v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp.
|
||
Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи.
|
||
7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT.
|
||
8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД.
|
||
9. Удаление исходника из staging (если не вышло — доделает janitor).
|
||
Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД,
|
||
таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed.
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
import psycopg
|
||
from psycopg.rows import dict_row
|
||
|
||
import broker
|
||
import faults
|
||
import imaging
|
||
import pipeline
|
||
import settings
|
||
import storage
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||
log = logging.getLogger("messenger.worker")
|
||
|
||
STOP = threading.Event()
|
||
HERE = Path(__file__).resolve().parent
|
||
VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"}
|
||
IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large",
|
||
imaging.EXIT_MEMORY: "resource_limit"}
|
||
|
||
|
||
class JobError(Exception):
|
||
def __init__(self, code: str, permanent: bool):
|
||
super().__init__(code)
|
||
self.code = code
|
||
self.permanent = permanent
|
||
|
||
|
||
class LeaseLost(Exception):
|
||
pass
|
||
|
||
|
||
def connect_db():
|
||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||
|
||
|
||
def final_prefix(claim: pipeline.Claim) -> str:
|
||
return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/"
|
||
|
||
|
||
class LeaseKeeper(threading.Thread):
|
||
"""Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку."""
|
||
|
||
def __init__(self, claim: pipeline.Claim):
|
||
super().__init__(daemon=True)
|
||
self.claim = claim
|
||
self.done = threading.Event()
|
||
self.lost = threading.Event()
|
||
self.frozen = threading.Event() # fault worker_freeze: перестать продлевать
|
||
|
||
def run(self):
|
||
interval = max(1.0, settings.JOB_LEASE_SECONDS / 3)
|
||
conn = None
|
||
while not self.done.wait(interval):
|
||
if self.frozen.is_set():
|
||
continue
|
||
try:
|
||
if conn is None:
|
||
conn = connect_db()
|
||
if not pipeline.renew_lease(conn, self.claim):
|
||
log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id)
|
||
self.lost.set()
|
||
return
|
||
except psycopg.Error as exc:
|
||
# Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT).
|
||
log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__)
|
||
if conn is not None:
|
||
conn.close()
|
||
conn = None
|
||
if conn is not None:
|
||
conn.close()
|
||
|
||
def check(self):
|
||
if self.lost.is_set():
|
||
raise LeaseLost()
|
||
|
||
|
||
def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict:
|
||
# Дочернему процессу — только лимиты, никаких секретов окружения.
|
||
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1",
|
||
"IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB),
|
||
"IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)}
|
||
for fault in ("hang", "invalid", "oom"):
|
||
if faults.hit("imaging_" + fault):
|
||
env["FAULT_IMAGING"] = fault
|
||
process = subprocess.Popen(
|
||
[sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type,
|
||
str(settings.MAX_IMAGE_PIXELS)],
|
||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir),
|
||
)
|
||
deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS
|
||
try:
|
||
while True:
|
||
try:
|
||
output, _ = process.communicate(timeout=0.5)
|
||
break
|
||
except subprocess.TimeoutExpired:
|
||
if time.monotonic() > deadline:
|
||
raise JobError("processing_timeout", permanent=False) from None
|
||
keeper.check()
|
||
finally:
|
||
if process.poll() is None:
|
||
process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку
|
||
process.wait()
|
||
code = process.returncode
|
||
if code in IMAGING_EXIT:
|
||
raise JobError(IMAGING_EXIT[code], permanent=True)
|
||
if code != 0:
|
||
# Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены.
|
||
raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False)
|
||
return json.loads(output.decode("utf-8").strip().splitlines()[-1])
|
||
|
||
|
||
def sha256_file(path: Path) -> tuple[int, str]:
|
||
digest = hashlib.sha256()
|
||
size = 0
|
||
with open(path, "rb") as file:
|
||
while chunk := file.read(1024 * 1024):
|
||
size += len(chunk)
|
||
digest.update(chunk)
|
||
return size, digest.hexdigest()
|
||
|
||
|
||
def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None:
|
||
if faults.hit("worker_s3_error"):
|
||
raise JobError("storage_unavailable", permanent=False)
|
||
with open(path, "rb") as file:
|
||
try:
|
||
storage.put(key, file, size, content_type, sha256)
|
||
except storage.StorageUnavailable:
|
||
raise JobError("storage_unavailable", permanent=False) from None
|
||
|
||
|
||
def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]:
|
||
"""Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов."""
|
||
source = workdir / "source"
|
||
try:
|
||
storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES)
|
||
except storage.ObjectMissing:
|
||
raise JobError("source_missing", permanent=True) from None
|
||
except storage.StorageUnavailable:
|
||
raise JobError("storage_unavailable", permanent=False) from None
|
||
size, digest = sha256_file(source)
|
||
if size != claim.size_bytes or digest != claim.sha256:
|
||
# Staging не совпадает с тем, что принял API: публиковать нельзя.
|
||
raise JobError("source_corrupted", permanent=True)
|
||
keeper.check()
|
||
|
||
out_dir = workdir / "out"
|
||
out_dir.mkdir()
|
||
result = run_imaging(source, out_dir, claim, keeper)
|
||
keeper.check()
|
||
|
||
prefix = final_prefix(claim)
|
||
original_key = prefix + "original"
|
||
put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes)
|
||
variants = []
|
||
for item in result["variants"]:
|
||
path = out_dir / item["file"]
|
||
v_size, v_sha = sha256_file(path)
|
||
key = prefix + VARIANT_KEYS[item["name"]]
|
||
put_file(key, path, item["content_type"], v_sha, v_size)
|
||
variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size,
|
||
"width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key})
|
||
keeper.check()
|
||
return original_key, variants
|
||
|
||
|
||
@dataclass
|
||
class Outcome:
|
||
ack: bool = True # False -> nack с requeue (не смогли даже записать claim)
|
||
reject: bool = False # True -> reject без requeue -> DLQ
|
||
cleanup_key: str | None = None
|
||
attachment_id: object = None
|
||
|
||
|
||
class Consumer(threading.Thread):
|
||
def __init__(self, number: int):
|
||
super().__init__(name=f"consumer-{number}", daemon=True)
|
||
self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}"
|
||
self.db = None
|
||
|
||
def database(self):
|
||
if self.db is None or self.db.closed:
|
||
self.db = connect_db()
|
||
return self.db
|
||
|
||
def handle(self, body: bytes, redelivered: bool) -> Outcome:
|
||
try:
|
||
message = pipeline.parse_message(body)
|
||
except pipeline.InvalidMessage as exc:
|
||
log.warning("invalid message -> DLQ (%s)", exc)
|
||
return Outcome(reject=True)
|
||
try:
|
||
result = pipeline.claim(self.database(), message, self.owner, redelivered)
|
||
except psycopg.Error as exc:
|
||
log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__)
|
||
self.db = None
|
||
STOP.wait(2)
|
||
return Outcome(ack=False)
|
||
if isinstance(result, pipeline.Reject):
|
||
log.warning("job %s: %s -> DLQ", message["job_id"], result.reason)
|
||
return Outcome(reject=True)
|
||
if isinstance(result, pipeline.Skip):
|
||
log.info("job %s: no-op (%s), ack", message["job_id"], result.reason)
|
||
return Outcome()
|
||
return self.execute(result)
|
||
|
||
def execute(self, claim: pipeline.Claim) -> Outcome:
|
||
started = time.monotonic()
|
||
log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt,
|
||
claim.max_attempts, claim.attachment_id)
|
||
keeper = LeaseKeeper(claim)
|
||
keeper.start()
|
||
workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR))
|
||
try:
|
||
try:
|
||
original_key, variants = process_job(claim, keeper, workdir)
|
||
faults.crash("worker_crash_after_put")
|
||
if faults.hit("worker_freeze"):
|
||
keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят
|
||
time.sleep(settings.JOB_LEASE_SECONDS * 3)
|
||
committed = pipeline.finish_ready(self.database(), claim, original_key, variants, self.owner)
|
||
except JobError as exc:
|
||
outcome = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner)
|
||
log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code,
|
||
"permanent" if exc.permanent else "temporary", outcome)
|
||
return Outcome()
|
||
except LeaseLost:
|
||
log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt)
|
||
return Outcome()
|
||
except psycopg.Error:
|
||
raise
|
||
except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены
|
||
log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt)
|
||
pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner)
|
||
return Outcome()
|
||
except psycopg.Error as exc:
|
||
# Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и
|
||
# dispatcher вернёт job в очередь; если прошёл — повтор увидит ready.
|
||
log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__)
|
||
self.db = None
|
||
return Outcome()
|
||
finally:
|
||
keeper.done.set()
|
||
shutil.rmtree(workdir, ignore_errors=True)
|
||
if not committed:
|
||
log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id,
|
||
claim.attempt)
|
||
return Outcome()
|
||
faults.crash("worker_crash_after_commit")
|
||
log.info("job %s attempt %s: ready in %.0f ms", claim.job_id, claim.attempt,
|
||
(time.monotonic() - started) * 1000)
|
||
return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id)
|
||
|
||
def cleanup_staging(self, outcome: Outcome) -> None:
|
||
"""После COMMIT ready и ack: исходник в staging больше не нужен."""
|
||
try:
|
||
storage.delete(outcome.cleanup_key)
|
||
self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' "
|
||
"AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key))
|
||
except (storage.StorageUnavailable, psycopg.Error) as exc:
|
||
log.info("staging cleanup postponed to janitor: %s", type(exc).__name__)
|
||
self.db = None
|
||
|
||
def consume_once(self) -> None:
|
||
connection = broker.connect()
|
||
try:
|
||
channel = connection.channel()
|
||
channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH)
|
||
broker.check_topology(channel)
|
||
log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH)
|
||
for method, _properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1):
|
||
if STOP.is_set():
|
||
break
|
||
if method is None:
|
||
continue
|
||
box = {}
|
||
runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle(
|
||
body, method.redelivered)), daemon=True)
|
||
runner.start()
|
||
while runner.is_alive():
|
||
connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа
|
||
outcome = box.get("outcome", Outcome(ack=False))
|
||
if outcome.reject:
|
||
channel.basic_reject(method.delivery_tag, requeue=False)
|
||
elif outcome.ack:
|
||
channel.basic_ack(method.delivery_tag)
|
||
else:
|
||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||
if outcome.cleanup_key:
|
||
self.cleanup_staging(outcome)
|
||
try:
|
||
channel.cancel()
|
||
except broker.BrokerError:
|
||
pass
|
||
finally:
|
||
try:
|
||
connection.close()
|
||
except broker.BrokerError:
|
||
pass
|
||
|
||
def safe_handle(self, body, redelivered) -> Outcome:
|
||
try:
|
||
return self.handle(body, redelivered)
|
||
except Exception: # noqa: BLE001
|
||
log.exception("unexpected error while handling a message; requeue")
|
||
self.db = None
|
||
STOP.wait(2)
|
||
return Outcome(ack=False)
|
||
|
||
def run(self):
|
||
pause = 1.0
|
||
while not STOP.is_set():
|
||
try:
|
||
self.consume_once()
|
||
pause = 1.0
|
||
except broker.BrokerError as exc:
|
||
log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name,
|
||
type(exc).__name__, pause)
|
||
STOP.wait(pause)
|
||
pause = min(pause * 2, 15)
|
||
|
||
|
||
def main() -> None:
|
||
if not settings.DATABASE_URL or not settings.BROKER_URL:
|
||
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
|
||
Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True)
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
signal.signal(sig, lambda *_: STOP.set())
|
||
consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)]
|
||
for consumer in consumers:
|
||
consumer.start()
|
||
log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, "
|
||
"fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH,
|
||
settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB,
|
||
"ON" if settings.FAULT_INJECTION else "off")
|
||
while not STOP.wait(2):
|
||
if not all(consumer.is_alive() for consumer in consumers):
|
||
log.error("a consumer thread died, exiting so that Docker restarts the worker")
|
||
os._exit(1)
|
||
try:
|
||
Path(settings.ALIVE_FILE).touch()
|
||
except OSError:
|
||
pass
|
||
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят.
|
||
for consumer in consumers:
|
||
consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10)
|
||
log.info("stopped")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|