Files
mtusi2026lab5/app/worker.py
T

266 lines
10 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Worker: consume из RabbitMQ, обработка изображения, идемпотентная фиксация."""
import hashlib
import json
import logging
import signal
import sys
import threading
import time
import uuid
import pika
from . import broker, config, db, jobs, pipeline, repo, s3
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
logging.getLogger("pika").setLevel(logging.WARNING)
log = logging.getLogger("worker")
_stop = threading.Event()
def _on_sigterm(signum, frame):
log.info("SIGTERM received, stopping after current message")
_stop.set()
def _backoff(attempt: int) -> float:
"""Экспоненциальный backoff с jitter: 2^attempt секунд ± 50%."""
base = min(60.0, 2.0 ** min(attempt, 6))
jitter = 0.5 + 0.5 * (uuid.uuid4().int & 0xFFFF) / 0xFFFF
return base * jitter
def _is_permanent(exc: Exception) -> bool:
"""Постоянные ошибки не ретраим: пользовательский мусор в файле."""
if isinstance(exc, ValueError):
return True
return False
def _load_attachment(conn, attachment_id: uuid.UUID):
return conn.execute(
"""SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, staging_key, status
FROM attachments WHERE id = %s""", (attachment_id,)).fetchone()
def _already_ready(conn, attachment_id: uuid.UUID) -> bool:
"""Идемпотентность: если ready и есть все три варианта — no-op."""
row = conn.execute(
"""SELECT a.status,
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id) AS n
FROM attachments a WHERE a.id = %s""", (attachment_id,)).fetchone()
if row is None:
return False
return row["status"] == "ready" and row["n"] >= 3
def _process_one(conn, job_row) -> None:
"""Один job: скачать staging, обработать, залить финал+варианты, транзакционно записать."""
attachment_id = job_row["attachment_id"]
job_id = job_row["job_id"]
if _already_ready(conn, attachment_id):
jobs.mark_ready(conn, job_id)
log.info("job %s: already ready, ack", job_id)
return
att = _load_attachment(conn, attachment_id)
if att is None:
jobs.mark_failed(conn, job_id, "attachment_missing")
return
if not att["staging_key"]:
jobs.mark_failed(conn, job_id, "staging_missing")
return
# 1. Прочитать staging из S3.
body = s3.get_object(att["staging_key"])
try:
data = body.read()
finally:
body.close()
if hashlib.sha256(data).hexdigest() != att["sha256"]:
raise RuntimeError("staging checksum mismatch")
# 2. Обработать (thumbnail, crop, optimized).
variants = pipeline.process(data)
# 3. Финализировать: original в originals/... + варианты в variants/<id>/...
final_key = "originals/%s/%s" % (att["chat_id"], uuid.uuid4().hex)
s3.put_object(final_key, data, att["content_type"], content_length=len(data))
for v in variants:
v_key = "variants/%s/%s" % (attachment_id, v["name"])
s3.put_object(v_key, v["bytes"], v["content_type"], content_length=len(v["bytes"]))
v["object_key"] = v_key
# 4. Зафиксировать в БД атомарно; конкурирующие попытки не перезаписывают ready.
with db.connection() as txn:
check = txn.execute(
"""SELECT status FROM attachments WHERE id = %s FOR UPDATE""",
(attachment_id,)).fetchone()
if check["status"] == "ready":
jobs.mark_ready(txn, job_id)
return
for v in variants:
txn.execute(
"""INSERT INTO attachment_variants
(attachment_id, name, object_key, content_type, size_bytes, width, height, sha256)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, name) DO UPDATE
SET object_key = EXCLUDED.object_key,
content_type = EXCLUDED.content_type,
size_bytes = EXCLUDED.size_bytes,
width = EXCLUDED.width,
height = EXCLUDED.height,
sha256 = EXCLUDED.sha256""",
(attachment_id, v["name"], v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"]))
txn.execute(
"""UPDATE attachments
SET status = 'ready', object_key = %s, error_code = NULL,
staging_key = NULL, updated_at = clock_timestamp()
WHERE id = %s""",
(final_key, attachment_id))
jobs.mark_ready(txn, job_id)
# 5. После COMMIT — удалить staging (best-effort, ошибки логируем).
try:
s3.delete_object(att["staging_key"])
except Exception:
log.warning("staging delete failed for %s", att["staging_key"])
def _handle_message(ch, method, properties, body):
"""Consumer callback: ack только после устойчивого эффекта."""
try:
payload = json.loads(body)
job_id = uuid.UUID(payload["job_id"])
except Exception:
log.error("malformed job payload, drop")
ch.basic_nack(method.delivery_tag, requeue=False)
return
try:
with db.connection() as conn:
job_row = jobs.claim_job(conn, job_id)
if job_row is None:
log.info("job %s: not claimable, ack (idempotent)", job_id)
ch.basic_ack(method.delivery_tag)
return
if job_row["attempt"] > job_row["max_attempts"]:
with db.connection() as conn:
jobs.mark_failed(conn, job_id, "retry_exhausted")
ch.basic_ack(method.delivery_tag)
return
with db.connection() as conn:
_process_one(conn, job_row)
ch.basic_ack(method.delivery_tag)
log.info("job %s: done (attempt %d)", job_id, job_row["attempt"])
except Exception as exc:
log.warning("job %s: error %s", job_id, type(exc).__name__)
permanent = _is_permanent(exc)
attempt = 1
max_attempts = config.JOB_MAX_ATTEMPTS
try:
with db.connection() as conn:
row = conn.execute(
"SELECT attempt, max_attempts FROM jobs WHERE job_id = %s",
(job_id,)).fetchone()
if row:
attempt = row["attempt"]
max_attempts = row["max_attempts"]
if permanent or attempt >= max_attempts:
jobs.mark_failed(conn, job_id, "invalid_image" if permanent else "retry_exhausted")
else:
jobs.mark_retry(conn, job_id, "processing_failed", _backoff(attempt))
finally:
ch.basic_ack(method.delivery_tag)
def _reaper_loop():
"""Периодически возвращает в queued задачи с истёкшим lease (зависший worker)."""
while not _stop.wait(30):
try:
with db.connection() as conn:
rows = jobs.find_expired_leases(conn)
for row in rows:
outcome = jobs.requeue_expired(conn, row)
log.info("lease reaper: job %s → %s", row["job_id"], outcome)
except Exception as exc:
log.warning("reaper error: %s", type(exc).__name__)
def _outbox_loop():
"""Публикует outbox. В API тоже есть такой поток — здесь на случай, если API упал."""
while not _stop.wait(3):
try:
with db.connection() as conn:
jobs.publish_pending_outbox(conn, broker.publish)
except Exception as exc:
log.warning("outbox error: %s", type(exc).__name__)
def main():
signal.signal(signal.SIGTERM, _on_sigterm)
signal.signal(signal.SIGINT, _on_sigterm)
# Ждём broker до 120 секунд — на случай рестарта RabbitMQ.
# Это нужно, потому что `docker compose restart` не соблюдает depends_on.
ready = False
for attempt in range(60):
if _stop.is_set():
return
try:
broker.declare_only()
ready = True
break
except Exception as exc:
log.warning("broker not ready (attempt %d): %s",
attempt + 1, type(exc).__name__)
if _stop.wait(2):
return
if not ready:
log.error("broker unavailable after 120s, exiting")
sys.exit(1)
log.info("worker started, queue=%s concurrency=%d", broker.QUEUE, config.WORKER_CONCURRENCY)
threading.Thread(target=_reaper_loop, daemon=True).start()
threading.Thread(target=_outbox_loop, daemon=True).start()
# Consume с ручным ack и ограниченным prefetch.
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
ch = conn.channel()
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
while not _stop.is_set():
try:
conn.process_data_events(time_limit=1)
except pika.exceptions.AMQPConnectionError:
log.warning("broker connection lost, reconnecting in 2s")
time.sleep(2)
try:
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
ch = conn.channel()
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
except Exception as exc:
log.warning("reconnect failed: %s", type(exc).__name__)
try:
ch.stop_consuming()
conn.close()
except Exception:
pass
db.close()
log.info("worker stopped")
if __name__ == "__main__":
main()