forked from mtusi2026/mtusi2026lab6
346 lines
19 KiB
Python
346 lines
19 KiB
Python
"""Worker вложений: python -m app.worker
|
||
|
||
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
|
||
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
|
||
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
|
||
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
|
||
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
|
||
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
|
||
"""
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import random
|
||
import signal
|
||
import tempfile
|
||
import time
|
||
import uuid
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
import aio_pika
|
||
|
||
from . import config, pipeline, telemetry
|
||
from .broker import QUEUE, Broker
|
||
from .db import Database
|
||
from .outbox import Relay
|
||
from .s3 import S3
|
||
from .web import log
|
||
|
||
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
|
||
|
||
|
||
def backoff_seconds(attempt):
|
||
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
|
||
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
|
||
return random.uniform(ceiling * 0.5, ceiling)
|
||
|
||
|
||
class Worker:
|
||
def __init__(self):
|
||
self.db, self.s3, self.broker = Database(), S3(), Broker()
|
||
self.relay = Relay(self.db, self.broker)
|
||
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
|
||
self.busy = 0
|
||
self.stopping = False
|
||
self.tasks = []
|
||
telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке")
|
||
telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker")
|
||
|
||
# --- claim / finalize ---------------------------------------------------------
|
||
async def claim(self, job):
|
||
return await self.db.fetchrow(
|
||
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
|
||
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
|
||
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
|
||
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
|
||
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
|
||
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
|
||
|
||
async def fail(self, att_id, job, code, event="failed"):
|
||
await self.db.execute(
|
||
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
|
||
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
|
||
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
|
||
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
|
||
|
||
async def reschedule(self, att_id, job, reason):
|
||
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
|
||
nxt = job["attempt"] + 1
|
||
if nxt > config.MAX_ATTEMPTS:
|
||
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
|
||
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
|
||
return
|
||
delay = backoff_seconds(job["attempt"])
|
||
await self.db.execute(
|
||
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
|
||
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
|
||
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
|
||
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) "
|
||
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
|
||
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate"))
|
||
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
|
||
|
||
async def publish_result(self, row, job, variants):
|
||
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
|
||
async with self.db.conn() as c:
|
||
async with c.transaction():
|
||
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
|
||
"FOR UPDATE", row["id"], WORKER_ID)
|
||
if owner is None:
|
||
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
|
||
for v in variants:
|
||
await c.execute(
|
||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
|
||
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
|
||
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
|
||
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
|
||
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
|
||
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
|
||
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
|
||
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
|
||
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
|
||
return True
|
||
|
||
# --- обработка ----------------------------------------------------------------
|
||
async def process(self, job):
|
||
att_id = uuid.UUID(job["attachment_id"])
|
||
row = await self.claim(job)
|
||
if row is None:
|
||
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
|
||
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
|
||
telemetry.record_job("attachment.process.v1", "noop", 0.0)
|
||
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
|
||
started = time.perf_counter()
|
||
outcome = "ready"
|
||
try:
|
||
await self.fault("after_claim")
|
||
stream, _ = await self.s3.open_stream(row["staging_key"])
|
||
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
|
||
try:
|
||
try:
|
||
while True:
|
||
chunk = await stream.read(256 * 1024)
|
||
if not chunk:
|
||
break
|
||
spool.write(chunk)
|
||
finally:
|
||
stream.close()
|
||
loop = asyncio.get_running_loop()
|
||
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
|
||
finally:
|
||
spool.close()
|
||
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
|
||
await self.copy_original(row)
|
||
for v in variants:
|
||
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
|
||
await self.fault("after_s3_before_commit")
|
||
if not await self.publish_result(row, job, variants):
|
||
log("warning", "lease_lost", attachment_id=job["attachment_id"])
|
||
return
|
||
await self.fault("after_commit_before_ack")
|
||
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
|
||
duration_ms=round((time.perf_counter() - started) * 1000, 1))
|
||
try:
|
||
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
|
||
except Exception as exc:
|
||
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
|
||
except pipeline.PermanentError as exc:
|
||
outcome = "failed"
|
||
await self.fail(att_id, job, exc.code)
|
||
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
|
||
except MemoryError:
|
||
outcome = "failed"
|
||
await self.fail(att_id, job, "processing_failed")
|
||
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except InjectedCrash:
|
||
raise
|
||
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
|
||
outcome = "retried"
|
||
await self.reschedule(att_id, job, type(exc).__name__)
|
||
finally:
|
||
ttr = None
|
||
if outcome == "ready":
|
||
ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 "
|
||
"AND status = 'ready'", att_id)
|
||
telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None)
|
||
|
||
async def copy_original(self, row):
|
||
stream, size = await self.s3.open_stream(row["staging_key"])
|
||
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
|
||
try:
|
||
while True:
|
||
chunk = await stream.read(256 * 1024)
|
||
if not chunk:
|
||
break
|
||
spool.write(chunk)
|
||
stream.close()
|
||
spool.seek(0)
|
||
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
|
||
finally:
|
||
spool.close()
|
||
|
||
async def fault(self, point):
|
||
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
|
||
if "crash_once:" + point in config.FAULT_INJECTION:
|
||
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
|
||
if not os.path.exists(marker):
|
||
open(marker, "w").close()
|
||
log("error", "fault_crash_once", point=point)
|
||
os._exit(137)
|
||
if "crash:" + point in config.FAULT_INJECTION:
|
||
log("error", "fault_crash", point=point)
|
||
os._exit(137)
|
||
if "fail:" + point in config.FAULT_INJECTION:
|
||
raise OSError("injected failure at " + point)
|
||
|
||
async def on_message(self, message):
|
||
try:
|
||
job = json.loads(message.body)
|
||
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
|
||
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
|
||
raise ValueError("bad job")
|
||
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
|
||
except Exception:
|
||
log("error", "job_unreadable")
|
||
await message.reject(requeue=False) # -> DLQ
|
||
return
|
||
self.busy += 1
|
||
carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job}
|
||
try:
|
||
with telemetry.span("attachment.process process", "consumer",
|
||
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
|
||
"messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier):
|
||
await self.process(job)
|
||
await message.ack()
|
||
except InjectedCrash:
|
||
raise
|
||
except Exception as exc:
|
||
log("error", "job_unhandled", error_type=type(exc).__name__)
|
||
await message.nack(requeue=True)
|
||
finally:
|
||
self.busy -= 1
|
||
|
||
# --- sweeper -------------------------------------------------------------------
|
||
async def sweep_once(self):
|
||
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
|
||
rows = await self.db.fetch(
|
||
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
|
||
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
|
||
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
|
||
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
|
||
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
|
||
"SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, "
|
||
"(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), "
|
||
"(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), "
|
||
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
|
||
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
|
||
for r in rows:
|
||
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
|
||
if rows:
|
||
self.relay.kick()
|
||
|
||
async def sweeper(self):
|
||
while not self.stopping:
|
||
try:
|
||
await self.sweep_once()
|
||
except Exception as exc:
|
||
log("warning", "sweeper_failed", error_type=type(exc).__name__)
|
||
await asyncio.sleep(config.SWEEP_INTERVAL)
|
||
|
||
async def staging_janitor(self):
|
||
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
|
||
while not self.stopping:
|
||
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
|
||
try:
|
||
rows = await self.db.fetch(
|
||
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
|
||
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
|
||
float(config.STAGING_FAILED_RETENTION_SECONDS))
|
||
for r in rows:
|
||
await self.s3.delete(r["staging_key"])
|
||
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
|
||
except Exception as exc:
|
||
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
|
||
|
||
# --- жизненный цикл ------------------------------------------------------------
|
||
async def connect_broker(self):
|
||
while not self.stopping:
|
||
try:
|
||
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
|
||
return
|
||
except Exception as exc:
|
||
log("warning", "broker_unavailable", error_type=type(exc).__name__)
|
||
await asyncio.sleep(2)
|
||
|
||
async def consume(self):
|
||
while not self.stopping:
|
||
await self.connect_broker()
|
||
try:
|
||
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
|
||
tag = await queue.consume(self.on_message)
|
||
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
|
||
while not self.stopping and not self.broker.channel.is_closed:
|
||
await asyncio.sleep(1)
|
||
if self.stopping:
|
||
await queue.cancel(tag)
|
||
return
|
||
except Exception as exc:
|
||
log("warning", "consume_failed", error_type=type(exc).__name__)
|
||
await asyncio.sleep(2)
|
||
|
||
async def run(self):
|
||
telemetry.setup(config.SERVICE_NAME)
|
||
telemetry.process_metrics()
|
||
await self.db.start()
|
||
await self.s3.start()
|
||
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
|
||
max_attempts=config.MAX_ATTEMPTS)
|
||
loop = asyncio.get_running_loop()
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
try:
|
||
loop.add_signal_handler(sig, self.request_stop)
|
||
except NotImplementedError:
|
||
signal.signal(sig, lambda *_: self.request_stop())
|
||
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
|
||
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
|
||
beat = 0.0
|
||
while not self.stopping:
|
||
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
|
||
beat = time.monotonic()
|
||
try:
|
||
with open("/tmp/worker.alive", "w") as fh:
|
||
fh.write(str(time.time()))
|
||
except OSError:
|
||
pass
|
||
await asyncio.sleep(0.5)
|
||
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
|
||
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
|
||
while self.busy and time.monotonic() < deadline:
|
||
await asyncio.sleep(0.2)
|
||
self.relay.stop()
|
||
for t in self.tasks:
|
||
t.cancel()
|
||
await self.broker.stop()
|
||
await self.s3.stop()
|
||
await self.db.stop()
|
||
log("info", "worker_stopped", unfinished=self.busy)
|
||
telemetry.shutdown()
|
||
|
||
def request_stop(self):
|
||
log("info", "worker_stopping")
|
||
self.stopping = True
|
||
|
||
|
||
class InjectedCrash(Exception):
|
||
pass
|
||
|
||
|
||
def main():
|
||
asyncio.run(Worker().run())
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|