Files
mtusi2026lab6/app/worker.py
T

373 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Worker lab6: consume из RabbitMQ, обработка изображения, OTel-инструментация."""
import hashlib
import json
import logging
import signal
import sys
import threading
import time
import uuid
import pika
from opentelemetry import trace
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
from opentelemetry.trace import SpanKind
from . import broker, config, db, jobs, pipeline, repo, s3, telemetry
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
logging.getLogger("pika").setLevel(logging.WARNING)
logging.getLogger("opentelemetry").setLevel(logging.WARNING)
log = logging.getLogger("worker")
_stop = threading.Event()
# OTel-хендлы и метрики инициализируются один раз в main().
_tracer = None
_m_jobs = None
_m_duration = None
_m_time_to_ready = None
_m_busy = None
def _init_metrics():
global _tracer, _m_jobs, _m_duration, _m_time_to_ready, _m_busy
if _tracer is not None:
return
_tracer = telemetry.get_tracer("messenger-worker")
meter = telemetry.get_meter("messenger-worker")
_m_jobs = meter.create_counter(
"messenger_jobs_processed_total",
description="Jobs processed, by outcome")
_m_duration = meter.create_histogram(
"messenger_job_processing_duration_seconds",
description="Job processing duration",
unit="s")
_m_time_to_ready = meter.create_histogram(
"messenger_time_to_ready_seconds",
description="Upload accepted -> status ready",
unit="s")
_m_busy = meter.create_up_down_counter(
"messenger_worker_busy",
description="Worker slots in use")
def _on_sigterm(signum, frame):
log.info("SIGTERM received, stopping after current message")
_stop.set()
def _backoff(attempt: int) -> float:
"""Экспоненциальный backoff с jitter: 2^attempt секунд ± 50%."""
base = min(60.0, 2.0 ** min(attempt, 6))
jitter = 0.5 + 0.5 * (uuid.uuid4().int & 0xFFFF) / 0xFFFF
return base * jitter
def _is_permanent(exc: Exception) -> bool:
if isinstance(exc, ValueError):
return True
return False
def _load_attachment(conn, attachment_id: uuid.UUID):
return conn.execute(
"""SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, staging_key, status
FROM attachments WHERE id = %s""", (attachment_id,)).fetchone()
def _already_ready(conn, attachment_id: uuid.UUID) -> bool:
"""Идемпотентность: если ready и есть все три варианта — no-op."""
row = conn.execute(
"""SELECT a.status,
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id) AS n
FROM attachments a WHERE a.id = %s""", (attachment_id,)).fetchone()
if row is None:
return False
return row["status"] == "ready" and row["n"] >= 3
def _process_one(conn, job_row) -> None:
"""Один job: скачать staging, обработать, залить финал+варианты, транзакционно записать."""
attachment_id = job_row["attachment_id"]
job_id = job_row["job_id"]
if _already_ready(conn, attachment_id):
jobs.mark_ready(conn, job_id)
log.info("job %s: already ready, ack", job_id)
return
att = _load_attachment(conn, attachment_id)
if att is None:
jobs.mark_failed(conn, job_id, "attachment_missing")
return
if not att["staging_key"]:
jobs.mark_failed(conn, job_id, "staging_missing")
return
# 1. Прочитать staging из S3. BotocoreInstrumentor создаст client span.
body = s3.get_object(att["staging_key"])
try:
data = body.read()
finally:
body.close()
if hashlib.sha256(data).hexdigest() != att["sha256"]:
raise RuntimeError("staging checksum mismatch")
# 2. Обработать (thumbnail, crop, optimized). Не инструментируется отдельно —
# CPU-bound, метрика duration уже покрывает всё.
variants = pipeline.process(data)
# 3. Финализировать: original + варианты в S3.
final_key = "originals/%s/%s" % (att["chat_id"], uuid.uuid4().hex)
s3.put_object(final_key, data, att["content_type"], content_length=len(data))
for v in variants:
v_key = "variants/%s/%s" % (attachment_id, v["name"])
s3.put_object(v_key, v["bytes"], v["content_type"], content_length=len(v["bytes"]))
v["object_key"] = v_key
# 4. Зафиксировать в БД атомарно.
with db.connection() as txn:
check = txn.execute(
"""SELECT status FROM attachments WHERE id = %s FOR UPDATE""",
(attachment_id,)).fetchone()
if check["status"] == "ready":
jobs.mark_ready(txn, job_id)
return
for v in variants:
txn.execute(
"""INSERT INTO attachment_variants
(attachment_id, name, object_key, content_type, size_bytes, width, height, sha256)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, name) DO UPDATE
SET object_key = EXCLUDED.object_key,
content_type = EXCLUDED.content_type,
size_bytes = EXCLUDED.size_bytes,
width = EXCLUDED.width,
height = EXCLUDED.height,
sha256 = EXCLUDED.sha256""",
(attachment_id, v["name"], v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"]))
txn.execute(
"""UPDATE attachments
SET status = 'ready', object_key = %s, error_code = NULL,
staging_key = NULL, updated_at = clock_timestamp()
WHERE id = %s""",
(final_key, attachment_id))
jobs.mark_ready(txn, job_id)
# 5. После COMMIT — удалить staging (best-effort).
try:
s3.delete_object(att["staging_key"])
except Exception:
log.warning("staging delete failed for %s", att["staging_key"])
def _record_time_to_ready(attachment_id: uuid.UUID) -> None:
"""histogram: from attachments.created_at (upload accepted) to now (ready)."""
try:
with db.connection() as conn:
row = conn.execute(
"""SELECT EXTRACT(EPOCH FROM now() - created_at)::float AS delta
FROM attachments WHERE id = %s""",
(attachment_id,)).fetchone()
if row and row["delta"] is not None:
_m_time_to_ready.record(float(row["delta"]),
{"pipeline.version": "1"})
except Exception:
pass
def _handle_message(ch, method, properties, body):
"""Consumer: span + W3C trace context + ack только после COMMIT."""
_init_metrics()
# W3C trace context из headers. Pika хранит значения как bytes — декодируем.
raw_headers = (properties.headers or {}) if properties else {}
headers = {}
for k, v in raw_headers.items():
if isinstance(v, bytes):
headers[k] = v.decode("utf-8", errors="replace")
else:
headers[k] = v
ctx = telemetry.extract_context(headers)
try:
payload = json.loads(body)
job_id = uuid.UUID(payload["job_id"])
except Exception:
log.error("malformed job payload, drop")
ch.basic_nack(method.delivery_tag, requeue=False)
return
with _tracer.start_as_current_span(
"attachment.process",
context=ctx,
kind=SpanKind.CONSUMER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": broker.QUEUE,
"messaging.message.id": str(job_id),
"messaging.operation": "process",
"job.id": str(job_id),
"attachment.id": payload.get("attachment_id", ""),
"service.instance.id": config.INSTANCE_ID,
}) as span:
t0 = time.monotonic()
_m_busy.add(1, {"worker": config.INSTANCE_ID})
try:
with db.connection() as conn:
job_row = jobs.claim_job(conn, job_id)
if job_row is None:
span.set_attribute("job.skip", "not_claimable")
ch.basic_ack(method.delivery_tag)
return
if job_row["attempt"] > job_row["max_attempts"]:
with db.connection() as conn:
jobs.mark_failed(conn, job_id, "retry_exhausted")
_m_jobs.add(1, {"outcome": "failed", "job.type": "attachment.process.v1"})
span.set_attribute("job.outcome", "failed")
span.set_attribute("job.error_code", "retry_exhausted")
ch.basic_ack(method.delivery_tag)
return
with db.connection() as conn:
_process_one(conn, job_row)
duration = time.monotonic() - t0
_m_duration.record(duration,
{"outcome": "ready", "job.type": "attachment.process.v1"})
_m_jobs.add(1, {"outcome": "ready", "job.type": "attachment.process.v1"})
_record_time_to_ready(job_row["attachment_id"])
span.set_attribute("job.outcome", "ready")
span.set_attribute("job.attempt", job_row["attempt"])
span.set_attribute("job.duration_seconds", round(duration, 3))
ch.basic_ack(method.delivery_tag)
log.info("job %s: done (attempt %d, %.2fs)", job_id, job_row["attempt"], duration)
except Exception as exc:
span.set_attribute("job.outcome", "error")
span.record_exception(exc)
log.warning("job %s: error %s", job_id, type(exc).__name__)
permanent = _is_permanent(exc)
attempt = 1
max_attempts = config.JOB_MAX_ATTEMPTS
try:
with db.connection() as conn:
row = conn.execute(
"SELECT attempt, max_attempts FROM jobs WHERE job_id = %s",
(job_id,)).fetchone()
if row:
attempt = row["attempt"]
max_attempts = row["max_attempts"]
if permanent or attempt >= max_attempts:
jobs.mark_failed(conn, job_id,
"invalid_image" if permanent else "retry_exhausted")
_m_jobs.add(1, {"outcome": "failed", "job.type": "attachment.process.v1"})
span.set_attribute("job.error_code",
"invalid_image" if permanent else "retry_exhausted")
else:
jobs.mark_retry(conn, job_id, "processing_failed", _backoff(attempt))
_m_jobs.add(1, {"outcome": "retry", "job.type": "attachment.process.v1"})
span.set_attribute("job.error_code", "processing_failed")
span.set_attribute("job.retry_after", round(_backoff(attempt), 1))
finally:
ch.basic_ack(method.delivery_tag)
finally:
_m_busy.add(-1, {"worker": config.INSTANCE_ID})
def _reaper_loop():
"""Возвращает в queued задачи с истёкшим lease."""
while not _stop.wait(30):
try:
with db.connection() as conn:
rows = jobs.find_expired_leases(conn)
for row in rows:
outcome = jobs.requeue_expired(conn, row)
log.info("lease reaper: job %s -> %s", row["job_id"], outcome)
except Exception as exc:
log.warning("reaper error: %s", type(exc).__name__)
def _outbox_loop():
"""Публикует outbox — на случай, если API упал между COMMIT и publish."""
while not _stop.wait(3):
try:
with db.connection() as conn:
jobs.publish_pending_outbox(conn, broker.publish)
except Exception as exc:
log.warning("outbox error: %s", type(exc).__name__)
def main():
signal.signal(signal.SIGTERM, _on_sigterm)
signal.signal(signal.SIGINT, _on_sigterm)
# lab6: OTel. service.name=messenger-worker (отличается от API).
telemetry.init("messenger-worker")
PsycopgInstrumentor().instrument()
BotocoreInstrumentor().instrument()
_init_metrics()
# Ждём broker до 120 секунд — на случай рестарта RabbitMQ.
ready = False
for attempt in range(60):
if _stop.is_set():
return
try:
broker.declare_only()
ready = True
break
except Exception as exc:
log.warning("broker not ready (attempt %d): %s",
attempt + 1, type(exc).__name__)
if _stop.wait(2):
return
if not ready:
log.error("broker unavailable after 120s, exiting")
sys.exit(1)
log.info("worker started, queue=%s concurrency=%d",
broker.QUEUE, config.WORKER_CONCURRENCY)
threading.Thread(target=_reaper_loop, daemon=True).start()
threading.Thread(target=_outbox_loop, daemon=True).start()
# Consume с ручным ack и ограниченным prefetch.
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
ch = conn.channel()
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
while not _stop.is_set():
try:
conn.process_data_events(time_limit=1)
except pika.exceptions.AMQPConnectionError:
log.warning("broker connection lost, reconnecting in 2s")
time.sleep(2)
try:
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
ch = conn.channel()
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message,
auto_ack=False)
except Exception as exc:
log.warning("reconnect failed: %s", type(exc).__name__)
try:
ch.stop_consuming()
conn.close()
except Exception:
pass
db.close()
log.info("worker stopped")
if __name__ == "__main__":
main()