forked from mtusi2026/mtusi2026lab6
373 lines
15 KiB
Python
373 lines
15 KiB
Python
"""Worker lab6: consume из RabbitMQ, обработка изображения, OTel-инструментация."""
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import signal
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
|
||
import pika
|
||
from opentelemetry import trace
|
||
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
||
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
|
||
from opentelemetry.trace import SpanKind
|
||
|
||
from . import broker, config, db, jobs, pipeline, repo, s3, telemetry
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||
logging.getLogger("pika").setLevel(logging.WARNING)
|
||
logging.getLogger("opentelemetry").setLevel(logging.WARNING)
|
||
log = logging.getLogger("worker")
|
||
|
||
_stop = threading.Event()
|
||
|
||
# OTel-хендлы и метрики инициализируются один раз в main().
|
||
_tracer = None
|
||
_m_jobs = None
|
||
_m_duration = None
|
||
_m_time_to_ready = None
|
||
_m_busy = None
|
||
|
||
|
||
def _init_metrics():
|
||
global _tracer, _m_jobs, _m_duration, _m_time_to_ready, _m_busy
|
||
if _tracer is not None:
|
||
return
|
||
_tracer = telemetry.get_tracer("messenger-worker")
|
||
meter = telemetry.get_meter("messenger-worker")
|
||
_m_jobs = meter.create_counter(
|
||
"messenger_jobs_processed_total",
|
||
description="Jobs processed, by outcome")
|
||
_m_duration = meter.create_histogram(
|
||
"messenger_job_processing_duration_seconds",
|
||
description="Job processing duration",
|
||
unit="s")
|
||
_m_time_to_ready = meter.create_histogram(
|
||
"messenger_time_to_ready_seconds",
|
||
description="Upload accepted -> status ready",
|
||
unit="s")
|
||
_m_busy = meter.create_up_down_counter(
|
||
"messenger_worker_busy",
|
||
description="Worker slots in use")
|
||
|
||
|
||
def _on_sigterm(signum, frame):
|
||
log.info("SIGTERM received, stopping after current message")
|
||
_stop.set()
|
||
|
||
|
||
def _backoff(attempt: int) -> float:
|
||
"""Экспоненциальный backoff с jitter: 2^attempt секунд ± 50%."""
|
||
base = min(60.0, 2.0 ** min(attempt, 6))
|
||
jitter = 0.5 + 0.5 * (uuid.uuid4().int & 0xFFFF) / 0xFFFF
|
||
return base * jitter
|
||
|
||
|
||
def _is_permanent(exc: Exception) -> bool:
|
||
if isinstance(exc, ValueError):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _load_attachment(conn, attachment_id: uuid.UUID):
|
||
return conn.execute(
|
||
"""SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||
object_key, staging_key, status
|
||
FROM attachments WHERE id = %s""", (attachment_id,)).fetchone()
|
||
|
||
|
||
def _already_ready(conn, attachment_id: uuid.UUID) -> bool:
|
||
"""Идемпотентность: если ready и есть все три варианта — no-op."""
|
||
row = conn.execute(
|
||
"""SELECT a.status,
|
||
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id) AS n
|
||
FROM attachments a WHERE a.id = %s""", (attachment_id,)).fetchone()
|
||
if row is None:
|
||
return False
|
||
return row["status"] == "ready" and row["n"] >= 3
|
||
|
||
|
||
def _process_one(conn, job_row) -> None:
|
||
"""Один job: скачать staging, обработать, залить финал+варианты, транзакционно записать."""
|
||
attachment_id = job_row["attachment_id"]
|
||
job_id = job_row["job_id"]
|
||
|
||
if _already_ready(conn, attachment_id):
|
||
jobs.mark_ready(conn, job_id)
|
||
log.info("job %s: already ready, ack", job_id)
|
||
return
|
||
|
||
att = _load_attachment(conn, attachment_id)
|
||
if att is None:
|
||
jobs.mark_failed(conn, job_id, "attachment_missing")
|
||
return
|
||
if not att["staging_key"]:
|
||
jobs.mark_failed(conn, job_id, "staging_missing")
|
||
return
|
||
|
||
# 1. Прочитать staging из S3. BotocoreInstrumentor создаст client span.
|
||
body = s3.get_object(att["staging_key"])
|
||
try:
|
||
data = body.read()
|
||
finally:
|
||
body.close()
|
||
|
||
if hashlib.sha256(data).hexdigest() != att["sha256"]:
|
||
raise RuntimeError("staging checksum mismatch")
|
||
|
||
# 2. Обработать (thumbnail, crop, optimized). Не инструментируется отдельно —
|
||
# CPU-bound, метрика duration уже покрывает всё.
|
||
variants = pipeline.process(data)
|
||
|
||
# 3. Финализировать: original + варианты в S3.
|
||
final_key = "originals/%s/%s" % (att["chat_id"], uuid.uuid4().hex)
|
||
s3.put_object(final_key, data, att["content_type"], content_length=len(data))
|
||
for v in variants:
|
||
v_key = "variants/%s/%s" % (attachment_id, v["name"])
|
||
s3.put_object(v_key, v["bytes"], v["content_type"], content_length=len(v["bytes"]))
|
||
v["object_key"] = v_key
|
||
|
||
# 4. Зафиксировать в БД атомарно.
|
||
with db.connection() as txn:
|
||
check = txn.execute(
|
||
"""SELECT status FROM attachments WHERE id = %s FOR UPDATE""",
|
||
(attachment_id,)).fetchone()
|
||
if check["status"] == "ready":
|
||
jobs.mark_ready(txn, job_id)
|
||
return
|
||
for v in variants:
|
||
txn.execute(
|
||
"""INSERT INTO attachment_variants
|
||
(attachment_id, name, object_key, content_type, size_bytes, width, height, sha256)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (attachment_id, name) DO UPDATE
|
||
SET object_key = EXCLUDED.object_key,
|
||
content_type = EXCLUDED.content_type,
|
||
size_bytes = EXCLUDED.size_bytes,
|
||
width = EXCLUDED.width,
|
||
height = EXCLUDED.height,
|
||
sha256 = EXCLUDED.sha256""",
|
||
(attachment_id, v["name"], v["object_key"], v["content_type"],
|
||
v["size_bytes"], v["width"], v["height"], v["sha256"]))
|
||
txn.execute(
|
||
"""UPDATE attachments
|
||
SET status = 'ready', object_key = %s, error_code = NULL,
|
||
staging_key = NULL, updated_at = clock_timestamp()
|
||
WHERE id = %s""",
|
||
(final_key, attachment_id))
|
||
jobs.mark_ready(txn, job_id)
|
||
|
||
# 5. После COMMIT — удалить staging (best-effort).
|
||
try:
|
||
s3.delete_object(att["staging_key"])
|
||
except Exception:
|
||
log.warning("staging delete failed for %s", att["staging_key"])
|
||
|
||
|
||
def _record_time_to_ready(attachment_id: uuid.UUID) -> None:
|
||
"""histogram: from attachments.created_at (upload accepted) to now (ready)."""
|
||
try:
|
||
with db.connection() as conn:
|
||
row = conn.execute(
|
||
"""SELECT EXTRACT(EPOCH FROM now() - created_at)::float AS delta
|
||
FROM attachments WHERE id = %s""",
|
||
(attachment_id,)).fetchone()
|
||
if row and row["delta"] is not None:
|
||
_m_time_to_ready.record(float(row["delta"]),
|
||
{"pipeline.version": "1"})
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _handle_message(ch, method, properties, body):
|
||
"""Consumer: span + W3C trace context + ack только после COMMIT."""
|
||
_init_metrics()
|
||
|
||
# W3C trace context из headers. Pika хранит значения как bytes — декодируем.
|
||
raw_headers = (properties.headers or {}) if properties else {}
|
||
headers = {}
|
||
for k, v in raw_headers.items():
|
||
if isinstance(v, bytes):
|
||
headers[k] = v.decode("utf-8", errors="replace")
|
||
else:
|
||
headers[k] = v
|
||
ctx = telemetry.extract_context(headers)
|
||
|
||
try:
|
||
payload = json.loads(body)
|
||
job_id = uuid.UUID(payload["job_id"])
|
||
except Exception:
|
||
log.error("malformed job payload, drop")
|
||
ch.basic_nack(method.delivery_tag, requeue=False)
|
||
return
|
||
|
||
with _tracer.start_as_current_span(
|
||
"attachment.process",
|
||
context=ctx,
|
||
kind=SpanKind.CONSUMER,
|
||
attributes={
|
||
"messaging.system": "rabbitmq",
|
||
"messaging.destination.name": broker.QUEUE,
|
||
"messaging.message.id": str(job_id),
|
||
"messaging.operation": "process",
|
||
"job.id": str(job_id),
|
||
"attachment.id": payload.get("attachment_id", ""),
|
||
"service.instance.id": config.INSTANCE_ID,
|
||
}) as span:
|
||
t0 = time.monotonic()
|
||
_m_busy.add(1, {"worker": config.INSTANCE_ID})
|
||
try:
|
||
with db.connection() as conn:
|
||
job_row = jobs.claim_job(conn, job_id)
|
||
if job_row is None:
|
||
span.set_attribute("job.skip", "not_claimable")
|
||
ch.basic_ack(method.delivery_tag)
|
||
return
|
||
|
||
if job_row["attempt"] > job_row["max_attempts"]:
|
||
with db.connection() as conn:
|
||
jobs.mark_failed(conn, job_id, "retry_exhausted")
|
||
_m_jobs.add(1, {"outcome": "failed", "job.type": "attachment.process.v1"})
|
||
span.set_attribute("job.outcome", "failed")
|
||
span.set_attribute("job.error_code", "retry_exhausted")
|
||
ch.basic_ack(method.delivery_tag)
|
||
return
|
||
|
||
with db.connection() as conn:
|
||
_process_one(conn, job_row)
|
||
|
||
duration = time.monotonic() - t0
|
||
_m_duration.record(duration,
|
||
{"outcome": "ready", "job.type": "attachment.process.v1"})
|
||
_m_jobs.add(1, {"outcome": "ready", "job.type": "attachment.process.v1"})
|
||
_record_time_to_ready(job_row["attachment_id"])
|
||
|
||
span.set_attribute("job.outcome", "ready")
|
||
span.set_attribute("job.attempt", job_row["attempt"])
|
||
span.set_attribute("job.duration_seconds", round(duration, 3))
|
||
ch.basic_ack(method.delivery_tag)
|
||
log.info("job %s: done (attempt %d, %.2fs)", job_id, job_row["attempt"], duration)
|
||
|
||
except Exception as exc:
|
||
span.set_attribute("job.outcome", "error")
|
||
span.record_exception(exc)
|
||
log.warning("job %s: error %s", job_id, type(exc).__name__)
|
||
permanent = _is_permanent(exc)
|
||
attempt = 1
|
||
max_attempts = config.JOB_MAX_ATTEMPTS
|
||
try:
|
||
with db.connection() as conn:
|
||
row = conn.execute(
|
||
"SELECT attempt, max_attempts FROM jobs WHERE job_id = %s",
|
||
(job_id,)).fetchone()
|
||
if row:
|
||
attempt = row["attempt"]
|
||
max_attempts = row["max_attempts"]
|
||
if permanent or attempt >= max_attempts:
|
||
jobs.mark_failed(conn, job_id,
|
||
"invalid_image" if permanent else "retry_exhausted")
|
||
_m_jobs.add(1, {"outcome": "failed", "job.type": "attachment.process.v1"})
|
||
span.set_attribute("job.error_code",
|
||
"invalid_image" if permanent else "retry_exhausted")
|
||
else:
|
||
jobs.mark_retry(conn, job_id, "processing_failed", _backoff(attempt))
|
||
_m_jobs.add(1, {"outcome": "retry", "job.type": "attachment.process.v1"})
|
||
span.set_attribute("job.error_code", "processing_failed")
|
||
span.set_attribute("job.retry_after", round(_backoff(attempt), 1))
|
||
finally:
|
||
ch.basic_ack(method.delivery_tag)
|
||
finally:
|
||
_m_busy.add(-1, {"worker": config.INSTANCE_ID})
|
||
|
||
|
||
def _reaper_loop():
|
||
"""Возвращает в queued задачи с истёкшим lease."""
|
||
while not _stop.wait(30):
|
||
try:
|
||
with db.connection() as conn:
|
||
rows = jobs.find_expired_leases(conn)
|
||
for row in rows:
|
||
outcome = jobs.requeue_expired(conn, row)
|
||
log.info("lease reaper: job %s -> %s", row["job_id"], outcome)
|
||
except Exception as exc:
|
||
log.warning("reaper error: %s", type(exc).__name__)
|
||
|
||
|
||
def _outbox_loop():
|
||
"""Публикует outbox — на случай, если API упал между COMMIT и publish."""
|
||
while not _stop.wait(3):
|
||
try:
|
||
with db.connection() as conn:
|
||
jobs.publish_pending_outbox(conn, broker.publish)
|
||
except Exception as exc:
|
||
log.warning("outbox error: %s", type(exc).__name__)
|
||
|
||
|
||
def main():
|
||
signal.signal(signal.SIGTERM, _on_sigterm)
|
||
signal.signal(signal.SIGINT, _on_sigterm)
|
||
|
||
# lab6: OTel. service.name=messenger-worker (отличается от API).
|
||
telemetry.init("messenger-worker")
|
||
PsycopgInstrumentor().instrument()
|
||
BotocoreInstrumentor().instrument()
|
||
_init_metrics()
|
||
|
||
# Ждём broker до 120 секунд — на случай рестарта RabbitMQ.
|
||
ready = False
|
||
for attempt in range(60):
|
||
if _stop.is_set():
|
||
return
|
||
try:
|
||
broker.declare_only()
|
||
ready = True
|
||
break
|
||
except Exception as exc:
|
||
log.warning("broker not ready (attempt %d): %s",
|
||
attempt + 1, type(exc).__name__)
|
||
if _stop.wait(2):
|
||
return
|
||
if not ready:
|
||
log.error("broker unavailable after 120s, exiting")
|
||
sys.exit(1)
|
||
|
||
log.info("worker started, queue=%s concurrency=%d",
|
||
broker.QUEUE, config.WORKER_CONCURRENCY)
|
||
|
||
threading.Thread(target=_reaper_loop, daemon=True).start()
|
||
threading.Thread(target=_outbox_loop, daemon=True).start()
|
||
|
||
# Consume с ручным ack и ограниченным prefetch.
|
||
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
|
||
ch = conn.channel()
|
||
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
|
||
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
|
||
|
||
while not _stop.is_set():
|
||
try:
|
||
conn.process_data_events(time_limit=1)
|
||
except pika.exceptions.AMQPConnectionError:
|
||
log.warning("broker connection lost, reconnecting in 2s")
|
||
time.sleep(2)
|
||
try:
|
||
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
|
||
ch = conn.channel()
|
||
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
|
||
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message,
|
||
auto_ack=False)
|
||
except Exception as exc:
|
||
log.warning("reconnect failed: %s", type(exc).__name__)
|
||
|
||
try:
|
||
ch.stop_consuming()
|
||
conn.close()
|
||
except Exception:
|
||
pass
|
||
db.close()
|
||
log.info("worker stopped")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |