Files
mtusi2026lab6/src/app/worker.py
T

346 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Worker вложений: python -m app.worker
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
"""
import asyncio
import json
import os
import random
import signal
import tempfile
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import aio_pika
from . import config, pipeline, telemetry
from .broker import QUEUE, Broker
from .db import Database
from .outbox import Relay
from .s3 import S3
from .web import log
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
def backoff_seconds(attempt):
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
return random.uniform(ceiling * 0.5, ceiling)
class Worker:
def __init__(self):
self.db, self.s3, self.broker = Database(), S3(), Broker()
self.relay = Relay(self.db, self.broker)
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
self.busy = 0
self.stopping = False
self.tasks = []
telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке")
telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker")
# --- claim / finalize ---------------------------------------------------------
async def claim(self, job):
return await self.db.fetchrow(
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
async def fail(self, att_id, job, code, event="failed"):
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
async def reschedule(self, att_id, job, reason):
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
nxt = job["attempt"] + 1
if nxt > config.MAX_ATTEMPTS:
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
return
delay = backoff_seconds(job["attempt"])
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate"))
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
async def publish_result(self, row, job, variants):
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
async with self.db.conn() as c:
async with c.transaction():
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
"FOR UPDATE", row["id"], WORKER_ID)
if owner is None:
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
for v in variants:
await c.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
return True
# --- обработка ----------------------------------------------------------------
async def process(self, job):
att_id = uuid.UUID(job["attachment_id"])
row = await self.claim(job)
if row is None:
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
telemetry.record_job("attachment.process.v1", "noop", 0.0)
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
started = time.perf_counter()
outcome = "ready"
try:
await self.fault("after_claim")
stream, _ = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
finally:
stream.close()
loop = asyncio.get_running_loop()
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
finally:
spool.close()
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
await self.copy_original(row)
for v in variants:
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
await self.fault("after_s3_before_commit")
if not await self.publish_result(row, job, variants):
log("warning", "lease_lost", attachment_id=job["attachment_id"])
return
await self.fault("after_commit_before_ack")
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
duration_ms=round((time.perf_counter() - started) * 1000, 1))
try:
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
except Exception as exc:
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
except pipeline.PermanentError as exc:
outcome = "failed"
await self.fail(att_id, job, exc.code)
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
except MemoryError:
outcome = "failed"
await self.fail(att_id, job, "processing_failed")
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
except asyncio.CancelledError:
raise
except InjectedCrash:
raise
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
outcome = "retried"
await self.reschedule(att_id, job, type(exc).__name__)
finally:
ttr = None
if outcome == "ready":
ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 "
"AND status = 'ready'", att_id)
telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None)
async def copy_original(self, row):
stream, size = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
stream.close()
spool.seek(0)
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
finally:
spool.close()
async def fault(self, point):
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
if "crash_once:" + point in config.FAULT_INJECTION:
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
if not os.path.exists(marker):
open(marker, "w").close()
log("error", "fault_crash_once", point=point)
os._exit(137)
if "crash:" + point in config.FAULT_INJECTION:
log("error", "fault_crash", point=point)
os._exit(137)
if "fail:" + point in config.FAULT_INJECTION:
raise OSError("injected failure at " + point)
async def on_message(self, message):
try:
job = json.loads(message.body)
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
raise ValueError("bad job")
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
except Exception:
log("error", "job_unreadable")
await message.reject(requeue=False) # -> DLQ
return
self.busy += 1
carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job}
try:
with telemetry.span("attachment.process process", "consumer",
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
"messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier):
await self.process(job)
await message.ack()
except InjectedCrash:
raise
except Exception as exc:
log("error", "job_unhandled", error_type=type(exc).__name__)
await message.nack(requeue=True)
finally:
self.busy -= 1
# --- sweeper -------------------------------------------------------------------
async def sweep_once(self):
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
rows = await self.db.fetch(
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, "
"(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), "
"(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), "
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
for r in rows:
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
if rows:
self.relay.kick()
async def sweeper(self):
while not self.stopping:
try:
await self.sweep_once()
except Exception as exc:
log("warning", "sweeper_failed", error_type=type(exc).__name__)
await asyncio.sleep(config.SWEEP_INTERVAL)
async def staging_janitor(self):
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
while not self.stopping:
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
try:
rows = await self.db.fetch(
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
float(config.STAGING_FAILED_RETENTION_SECONDS))
for r in rows:
await self.s3.delete(r["staging_key"])
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
except Exception as exc:
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
# --- жизненный цикл ------------------------------------------------------------
async def connect_broker(self):
while not self.stopping:
try:
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
return
except Exception as exc:
log("warning", "broker_unavailable", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def consume(self):
while not self.stopping:
await self.connect_broker()
try:
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
tag = await queue.consume(self.on_message)
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
while not self.stopping and not self.broker.channel.is_closed:
await asyncio.sleep(1)
if self.stopping:
await queue.cancel(tag)
return
except Exception as exc:
log("warning", "consume_failed", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def run(self):
telemetry.setup(config.SERVICE_NAME)
telemetry.process_metrics()
await self.db.start()
await self.s3.start()
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
max_attempts=config.MAX_ATTEMPTS)
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, self.request_stop)
except NotImplementedError:
signal.signal(sig, lambda *_: self.request_stop())
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
beat = 0.0
while not self.stopping:
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
beat = time.monotonic()
try:
with open("/tmp/worker.alive", "w") as fh:
fh.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(0.5)
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
while self.busy and time.monotonic() < deadline:
await asyncio.sleep(0.2)
self.relay.stop()
for t in self.tasks:
t.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "worker_stopped", unfinished=self.busy)
telemetry.shutdown()
def request_stop(self):
log("info", "worker_stopping")
self.stopping = True
class InjectedCrash(Exception):
pass
def main():
asyncio.run(Worker().run())
if __name__ == "__main__":
main()