forked from mtusi2026/mtusi2026lab5
266 lines
10 KiB
Python
266 lines
10 KiB
Python
"""Worker: consume из RabbitMQ, обработка изображения, идемпотентная фиксация."""
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import signal
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
|
||
import pika
|
||
|
||
from . import broker, config, db, jobs, pipeline, repo, s3
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||
logging.getLogger("pika").setLevel(logging.WARNING)
|
||
log = logging.getLogger("worker")
|
||
|
||
_stop = threading.Event()
|
||
|
||
|
||
def _on_sigterm(signum, frame):
|
||
log.info("SIGTERM received, stopping after current message")
|
||
_stop.set()
|
||
|
||
|
||
def _backoff(attempt: int) -> float:
|
||
"""Экспоненциальный backoff с jitter: 2^attempt секунд ± 50%."""
|
||
base = min(60.0, 2.0 ** min(attempt, 6))
|
||
jitter = 0.5 + 0.5 * (uuid.uuid4().int & 0xFFFF) / 0xFFFF
|
||
return base * jitter
|
||
|
||
|
||
def _is_permanent(exc: Exception) -> bool:
|
||
"""Постоянные ошибки не ретраим: пользовательский мусор в файле."""
|
||
if isinstance(exc, ValueError):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _load_attachment(conn, attachment_id: uuid.UUID):
|
||
return conn.execute(
|
||
"""SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||
object_key, staging_key, status
|
||
FROM attachments WHERE id = %s""", (attachment_id,)).fetchone()
|
||
|
||
|
||
def _already_ready(conn, attachment_id: uuid.UUID) -> bool:
|
||
"""Идемпотентность: если ready и есть все три варианта — no-op."""
|
||
row = conn.execute(
|
||
"""SELECT a.status,
|
||
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id) AS n
|
||
FROM attachments a WHERE a.id = %s""", (attachment_id,)).fetchone()
|
||
if row is None:
|
||
return False
|
||
return row["status"] == "ready" and row["n"] >= 3
|
||
|
||
|
||
def _process_one(conn, job_row) -> None:
|
||
"""Один job: скачать staging, обработать, залить финал+варианты, транзакционно записать."""
|
||
attachment_id = job_row["attachment_id"]
|
||
job_id = job_row["job_id"]
|
||
|
||
if _already_ready(conn, attachment_id):
|
||
jobs.mark_ready(conn, job_id)
|
||
log.info("job %s: already ready, ack", job_id)
|
||
return
|
||
|
||
att = _load_attachment(conn, attachment_id)
|
||
if att is None:
|
||
jobs.mark_failed(conn, job_id, "attachment_missing")
|
||
return
|
||
if not att["staging_key"]:
|
||
jobs.mark_failed(conn, job_id, "staging_missing")
|
||
return
|
||
|
||
# 1. Прочитать staging из S3.
|
||
body = s3.get_object(att["staging_key"])
|
||
try:
|
||
data = body.read()
|
||
finally:
|
||
body.close()
|
||
|
||
if hashlib.sha256(data).hexdigest() != att["sha256"]:
|
||
raise RuntimeError("staging checksum mismatch")
|
||
|
||
# 2. Обработать (thumbnail, crop, optimized).
|
||
variants = pipeline.process(data)
|
||
|
||
# 3. Финализировать: original в originals/... + варианты в variants/<id>/...
|
||
final_key = "originals/%s/%s" % (att["chat_id"], uuid.uuid4().hex)
|
||
s3.put_object(final_key, data, att["content_type"], content_length=len(data))
|
||
for v in variants:
|
||
v_key = "variants/%s/%s" % (attachment_id, v["name"])
|
||
s3.put_object(v_key, v["bytes"], v["content_type"], content_length=len(v["bytes"]))
|
||
v["object_key"] = v_key
|
||
|
||
# 4. Зафиксировать в БД атомарно; конкурирующие попытки не перезаписывают ready.
|
||
with db.connection() as txn:
|
||
check = txn.execute(
|
||
"""SELECT status FROM attachments WHERE id = %s FOR UPDATE""",
|
||
(attachment_id,)).fetchone()
|
||
if check["status"] == "ready":
|
||
jobs.mark_ready(txn, job_id)
|
||
return
|
||
for v in variants:
|
||
txn.execute(
|
||
"""INSERT INTO attachment_variants
|
||
(attachment_id, name, object_key, content_type, size_bytes, width, height, sha256)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (attachment_id, name) DO UPDATE
|
||
SET object_key = EXCLUDED.object_key,
|
||
content_type = EXCLUDED.content_type,
|
||
size_bytes = EXCLUDED.size_bytes,
|
||
width = EXCLUDED.width,
|
||
height = EXCLUDED.height,
|
||
sha256 = EXCLUDED.sha256""",
|
||
(attachment_id, v["name"], v["object_key"], v["content_type"],
|
||
v["size_bytes"], v["width"], v["height"], v["sha256"]))
|
||
txn.execute(
|
||
"""UPDATE attachments
|
||
SET status = 'ready', object_key = %s, error_code = NULL,
|
||
staging_key = NULL, updated_at = clock_timestamp()
|
||
WHERE id = %s""",
|
||
(final_key, attachment_id))
|
||
jobs.mark_ready(txn, job_id)
|
||
|
||
# 5. После COMMIT — удалить staging (best-effort, ошибки логируем).
|
||
try:
|
||
s3.delete_object(att["staging_key"])
|
||
except Exception:
|
||
log.warning("staging delete failed for %s", att["staging_key"])
|
||
|
||
|
||
def _handle_message(ch, method, properties, body):
|
||
"""Consumer callback: ack только после устойчивого эффекта."""
|
||
try:
|
||
payload = json.loads(body)
|
||
job_id = uuid.UUID(payload["job_id"])
|
||
except Exception:
|
||
log.error("malformed job payload, drop")
|
||
ch.basic_nack(method.delivery_tag, requeue=False)
|
||
return
|
||
|
||
try:
|
||
with db.connection() as conn:
|
||
job_row = jobs.claim_job(conn, job_id)
|
||
if job_row is None:
|
||
log.info("job %s: not claimable, ack (idempotent)", job_id)
|
||
ch.basic_ack(method.delivery_tag)
|
||
return
|
||
|
||
if job_row["attempt"] > job_row["max_attempts"]:
|
||
with db.connection() as conn:
|
||
jobs.mark_failed(conn, job_id, "retry_exhausted")
|
||
ch.basic_ack(method.delivery_tag)
|
||
return
|
||
|
||
with db.connection() as conn:
|
||
_process_one(conn, job_row)
|
||
ch.basic_ack(method.delivery_tag)
|
||
log.info("job %s: done (attempt %d)", job_id, job_row["attempt"])
|
||
|
||
except Exception as exc:
|
||
log.warning("job %s: error %s", job_id, type(exc).__name__)
|
||
permanent = _is_permanent(exc)
|
||
attempt = 1
|
||
max_attempts = config.JOB_MAX_ATTEMPTS
|
||
try:
|
||
with db.connection() as conn:
|
||
row = conn.execute(
|
||
"SELECT attempt, max_attempts FROM jobs WHERE job_id = %s",
|
||
(job_id,)).fetchone()
|
||
if row:
|
||
attempt = row["attempt"]
|
||
max_attempts = row["max_attempts"]
|
||
if permanent or attempt >= max_attempts:
|
||
jobs.mark_failed(conn, job_id, "invalid_image" if permanent else "retry_exhausted")
|
||
else:
|
||
jobs.mark_retry(conn, job_id, "processing_failed", _backoff(attempt))
|
||
finally:
|
||
ch.basic_ack(method.delivery_tag)
|
||
|
||
|
||
def _reaper_loop():
|
||
"""Периодически возвращает в queued задачи с истёкшим lease (зависший worker)."""
|
||
while not _stop.wait(30):
|
||
try:
|
||
with db.connection() as conn:
|
||
rows = jobs.find_expired_leases(conn)
|
||
for row in rows:
|
||
outcome = jobs.requeue_expired(conn, row)
|
||
log.info("lease reaper: job %s → %s", row["job_id"], outcome)
|
||
except Exception as exc:
|
||
log.warning("reaper error: %s", type(exc).__name__)
|
||
|
||
|
||
def _outbox_loop():
|
||
"""Публикует outbox. В API тоже есть такой поток — здесь на случай, если API упал."""
|
||
while not _stop.wait(3):
|
||
try:
|
||
with db.connection() as conn:
|
||
jobs.publish_pending_outbox(conn, broker.publish)
|
||
except Exception as exc:
|
||
log.warning("outbox error: %s", type(exc).__name__)
|
||
|
||
|
||
def main():
|
||
signal.signal(signal.SIGTERM, _on_sigterm)
|
||
signal.signal(signal.SIGINT, _on_sigterm)
|
||
|
||
# Ждём broker до 120 секунд — на случай рестарта RabbitMQ.
|
||
# Это нужно, потому что `docker compose restart` не соблюдает depends_on.
|
||
ready = False
|
||
for attempt in range(60):
|
||
if _stop.is_set():
|
||
return
|
||
try:
|
||
broker.declare_only()
|
||
ready = True
|
||
break
|
||
except Exception as exc:
|
||
log.warning("broker not ready (attempt %d): %s",
|
||
attempt + 1, type(exc).__name__)
|
||
if _stop.wait(2):
|
||
return
|
||
if not ready:
|
||
log.error("broker unavailable after 120s, exiting")
|
||
sys.exit(1)
|
||
|
||
log.info("worker started, queue=%s concurrency=%d", broker.QUEUE, config.WORKER_CONCURRENCY)
|
||
|
||
threading.Thread(target=_reaper_loop, daemon=True).start()
|
||
threading.Thread(target=_outbox_loop, daemon=True).start()
|
||
|
||
# Consume с ручным ack и ограниченным prefetch.
|
||
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
|
||
ch = conn.channel()
|
||
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
|
||
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
|
||
|
||
while not _stop.is_set():
|
||
try:
|
||
conn.process_data_events(time_limit=1)
|
||
except pika.exceptions.AMQPConnectionError:
|
||
log.warning("broker connection lost, reconnecting in 2s")
|
||
time.sleep(2)
|
||
try:
|
||
conn = pika.BlockingConnection(pika.URLParameters(config.BROKER_URL))
|
||
ch = conn.channel()
|
||
ch.basic_qos(prefetch_count=config.WORKER_CONCURRENCY)
|
||
ch.basic_consume(queue=broker.QUEUE, on_message_callback=_handle_message, auto_ack=False)
|
||
except Exception as exc:
|
||
log.warning("reconnect failed: %s", type(exc).__name__)
|
||
|
||
try:
|
||
ch.stop_consuming()
|
||
conn.close()
|
||
except Exception:
|
||
pass
|
||
db.close()
|
||
log.info("worker stopped")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |