RabbitMQ (quorum queue, publisher confirms, manual ack, DLX) + отдельный worker: staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued; relay outbox, lease с heartbeat и fencing (worker_id, attempt), retry с backoff+jitter, failed-реестр, безопасный replay с audit; thumbnail/crop/WebP в изолированном процессе с лимитами. Тесты окон отказа (fault injection), трек 1 «Очередь под перегрузкой» с backpressure. Перенесено исправление замечания lab3 PR #99 (U+0000 в тексте). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
308 lines
16 KiB
Python
308 lines
16 KiB
Python
"""Worker обработки вложений: python -m app.worker (отдельный сервис, не API).
|
||
|
||
Одна доставка сообщения:
|
||
1. проверить сообщение по contracts/jobs.schema.json (иначе reject без requeue -> DLQ);
|
||
2. claim в PostgreSQL: queued (или processing с истёкшей арендой) -> processing, attempt+1, lease;
|
||
done/failed/чужая живая аренда/ещё не время retry -> no-op + ack;
|
||
3. heartbeat продлевает lease, пока идёт работа (отличает долгую обработку от зависшей/мёртвой);
|
||
4. оригинал из staging, сверка size/sha256 с доверенными метаданными из БД;
|
||
5. варианты в дочернем процессе (лимиты памяти/времени), запись в S3 по ДЕТЕРМИНИРОВАННЫМ ключам
|
||
variants/<id>/v<pipeline>/<name> + server-side copy staging -> originals/<id>;
|
||
6. COMMIT результата с fencing по (worker_id, attempt): variants (ON CONFLICT DO NOTHING) + ready;
|
||
7. удалить staging (после безопасной фиксации), затем ack.
|
||
Сбой на любом шаге до ack -> redelivery/reaper -> повтор шагов 4–6 перезаписывает те же ключи тем же
|
||
содержимым и не создаёт второй набор вариантов (ключ эффекта (attachment_id, pipeline_version, variant)).
|
||
"""
|
||
import asyncio
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import secrets
|
||
import signal
|
||
import socket
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import jsonschema
|
||
|
||
from . import broker, config, faults, imaging, jobs
|
||
from .db import Database
|
||
from .errors import ApiError
|
||
from .storage import ObjectMissing, Storage
|
||
|
||
log = logging.getLogger('messenger.worker')
|
||
SCHEMA = json.loads((Path(__file__).resolve().parent.parent / 'contracts/jobs.schema.json').read_text(encoding='utf-8'))
|
||
VALIDATOR = jsonschema.Draft202012Validator(SCHEMA, format_checker=jsonschema.FormatChecker())
|
||
ALIVE_FILE = Path('/tmp/worker-alive')
|
||
|
||
|
||
class LostLease(Exception):
|
||
"""Аренду перехватили (истекла): результат фиксирует новый владелец, этот — только ack."""
|
||
|
||
|
||
class Worker:
|
||
def __init__(self):
|
||
self.settings = config.load()
|
||
self.db = Database(self.settings)
|
||
self.storage = Storage(self.settings)
|
||
self.worker_id = f'{socket.gethostname()}-{secrets.token_hex(3)}'
|
||
self.slots = asyncio.Semaphore(self.settings.worker_concurrency)
|
||
self.inflight: set[asyncio.Task] = set()
|
||
self.stopping = asyncio.Event()
|
||
self.connection = None
|
||
|
||
# ---------- приём сообщения ----------
|
||
|
||
async def on_message(self, message) -> None:
|
||
task = asyncio.current_task()
|
||
self.inflight.add(task)
|
||
try:
|
||
try:
|
||
payload = json.loads(message.body)
|
||
VALIDATOR.validate(payload)
|
||
except (ValueError, jsonschema.ValidationError):
|
||
log.error('invalid job message %s -> DLQ', message.message_id)
|
||
await message.reject(requeue=False)
|
||
return
|
||
async with self.slots:
|
||
try:
|
||
await self.process(payload)
|
||
except Exception:
|
||
# Инфраструктура (БД/S3) недоступна настолько, что даже состояние не записать:
|
||
# вернуть сообщение в очередь с паузой; delivery-limit страхует от бесконечного цикла.
|
||
log.exception('job %s: infrastructure error, requeue', payload['job_id'])
|
||
await asyncio.sleep(2)
|
||
await message.nack(requeue=True)
|
||
return
|
||
await message.ack() # только после устойчивого эффекта (или no-op)
|
||
finally:
|
||
self.inflight.discard(task)
|
||
|
||
# ---------- обработка ----------
|
||
|
||
async def claim(self, job_id: str, attachment_id: str):
|
||
async with self.db.connection() as conn:
|
||
async with conn.transaction():
|
||
j = await conn.fetchrow(
|
||
'SELECT j.status, j.attempt, j.max_attempts, '
|
||
' coalesce(j.lease_until > clock_timestamp(), false) AS leased, '
|
||
' j.next_attempt_at > clock_timestamp() AS not_yet, '
|
||
' a.staging_key, a.object_key, a.size_bytes, a.sha256, a.pipeline_version '
|
||
'FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id '
|
||
'WHERE j.job_id = $1 AND j.attachment_id = $2 FOR UPDATE OF j', job_id, attachment_id)
|
||
if j is None:
|
||
return None, 'unknown job'
|
||
if j['status'] in ('done', 'failed'):
|
||
return None, f"already {j['status']} (duplicate delivery)"
|
||
if j['status'] == 'processing' and j['leased']:
|
||
return None, 'leased by another live worker (duplicate delivery)'
|
||
if j['status'] == 'queued' and j['not_yet']:
|
||
return None, 'retry is scheduled later (stale message)'
|
||
if j['attempt'] >= j['max_attempts']:
|
||
await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', 'no attempts left')
|
||
return None, 'attempts exhausted'
|
||
attempt = await conn.fetchval(
|
||
"UPDATE attachment_jobs SET status = 'processing', attempt = attempt + 1, worker_id = $2, "
|
||
"lease_until = clock_timestamp() + make_interval(secs => $3), started_at = clock_timestamp(), "
|
||
"updated_at = clock_timestamp() WHERE job_id = $1 RETURNING attempt",
|
||
job_id, self.worker_id, self.settings.job_lease)
|
||
await conn.execute(
|
||
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() "
|
||
"WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id)
|
||
return {**dict(j), 'attempt': attempt, 'max_attempts': j['max_attempts']}, None
|
||
|
||
async def heartbeat(self, job_id: str, attempt: int) -> None:
|
||
interval = max(1.0, self.settings.job_lease / 3)
|
||
while True:
|
||
await asyncio.sleep(interval)
|
||
try:
|
||
async with self.db.connection() as conn:
|
||
ok = await conn.fetchval(
|
||
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $3) "
|
||
"WHERE job_id = $1 AND worker_id = $2 AND attempt = $4 AND status = 'processing' RETURNING 1",
|
||
job_id, self.worker_id, self.settings.job_lease, attempt)
|
||
if not ok:
|
||
log.warning('job %s: lease lost', job_id)
|
||
return
|
||
except Exception:
|
||
log.warning('job %s: heartbeat failed', job_id)
|
||
|
||
async def process(self, payload: dict) -> None:
|
||
job_id, attachment_id = payload['job_id'], payload['attachment_id']
|
||
job, reason = await self.claim(job_id, attachment_id)
|
||
if job is None:
|
||
log.info('job %s: no-op, %s', job_id, reason)
|
||
return
|
||
started = time.monotonic()
|
||
beat = asyncio.create_task(self.heartbeat(job_id, job['attempt']))
|
||
try:
|
||
stages = await self.execute(job_id, attachment_id, job)
|
||
log.info('job %s: ready in %.2fs (attempt %d) %s', job_id, time.monotonic() - started, job['attempt'],
|
||
' '.join(f'{k}={v:.3f}' for k, v in stages.items()))
|
||
except imaging.PermanentImageError as exc:
|
||
await self.finish_failed(job_id, attachment_id, job['attempt'], exc.code, exc.code)
|
||
except LostLease:
|
||
log.warning('job %s: result not committed, lease taken over', job_id)
|
||
except (imaging.TransientImageError, ApiError) as exc:
|
||
code = getattr(exc, 'code', 'unavailable')
|
||
await self.retry_or_fail(job_id, attachment_id, job, f'transient: {code}')
|
||
finally:
|
||
beat.cancel()
|
||
|
||
async def execute(self, job_id: str, attachment_id: str, job) -> None:
|
||
version = job['pipeline_version']
|
||
stages, mark = {}, time.monotonic()
|
||
|
||
def lap(name):
|
||
nonlocal mark
|
||
now = time.monotonic()
|
||
stages[name] = now - mark
|
||
mark = now
|
||
try:
|
||
data = await self.storage.get_raw(job['staging_key'])
|
||
except ObjectMissing:
|
||
raise imaging.PermanentImageError('source_missing') from None
|
||
# Доверенные метаданные — из БД, а не из сообщения брокера.
|
||
if len(data) != job['size_bytes'] or hashlib.sha256(data).hexdigest() != job['sha256']:
|
||
raise imaging.PermanentImageError('source_corrupted')
|
||
lap('fetch')
|
||
variants = await imaging.run_isolated(data, self.settings.worker_max_pixels,
|
||
self.settings.worker_memory_mb, self.settings.job_timeout)
|
||
lap('build')
|
||
ext = {'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/png': 'png'}
|
||
for v in variants:
|
||
v['object_key'] = f"variants/{attachment_id}/v{version}/{v['name']}.{ext[v['content_type']]}"
|
||
await self.storage.put(v['object_key'], v.pop('data'), v['content_type'])
|
||
await self.storage.copy(job['staging_key'], job['object_key'])
|
||
lap('store')
|
||
faults.hit('worker_crash_after_s3')
|
||
async with self.db.connection() as conn:
|
||
async with conn.transaction():
|
||
owned = await conn.fetchval(
|
||
"UPDATE attachment_jobs SET status = 'done', lease_until = NULL, error_code = NULL, "
|
||
"last_error = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp() "
|
||
"WHERE job_id = $1 AND status = 'processing' AND worker_id = $2 AND attempt = $3 RETURNING 1",
|
||
job_id, self.worker_id, job['attempt'])
|
||
if not owned:
|
||
raise LostLease()
|
||
for v in variants:
|
||
await conn.execute(
|
||
'INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, '
|
||
'size_bytes, width, height, sha256, object_key) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) '
|
||
'ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING',
|
||
attachment_id, version, v['name'], v['content_type'], v['size_bytes'],
|
||
v['width'], v['height'], v['sha256'], v['object_key'])
|
||
await conn.execute(
|
||
"UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() "
|
||
"WHERE id = $1 AND status = 'processing'", attachment_id)
|
||
lap('commit')
|
||
faults.hit('worker_crash_after_commit')
|
||
try:
|
||
await self.storage.delete(job['staging_key']) # staging больше не нужен: результат зафиксирован
|
||
except ApiError:
|
||
log.warning('job %s: staging cleanup deferred to retention sweeper', job_id)
|
||
lap('cleanup')
|
||
return stages
|
||
|
||
async def finish_failed(self, job_id, attachment_id, attempt: int, code: str, reason: str) -> None:
|
||
async with self.db.connection() as conn:
|
||
async with conn.transaction():
|
||
await jobs.fail(conn, job_id, attachment_id, code, reason, worker_id=self.worker_id, attempt=attempt)
|
||
log.warning('job %s: failed (%s)', job_id, code)
|
||
|
||
async def retry_or_fail(self, job_id, attachment_id, job, reason: str) -> None:
|
||
async with self.db.connection() as conn:
|
||
async with conn.transaction():
|
||
if job['attempt'] >= job['max_attempts']:
|
||
await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', reason,
|
||
worker_id=self.worker_id, attempt=job['attempt'])
|
||
log.warning('job %s: failed, retry_exhausted after %d attempts', job_id, job['attempt'])
|
||
return
|
||
delay = jobs.backoff_seconds(job['attempt'], self.settings.retry_base_ms, self.settings.retry_max_ms)
|
||
await jobs.requeue(conn, job_id, attachment_id, job['attempt'] + 1, delay, reason,
|
||
worker_id=self.worker_id, attempt=job['attempt'])
|
||
log.warning('job %s: %s, retry in %.1fs (attempt %d/%d)', job_id, reason, delay,
|
||
job['attempt'], job['max_attempts'])
|
||
|
||
# ---------- обслуживание ----------
|
||
|
||
async def maintenance(self) -> None:
|
||
"""Пульс для healthcheck + retention staging (одна реплика за раз через advisory lock)."""
|
||
last_sweep = 0.0
|
||
while not self.stopping.is_set():
|
||
if self.connection is not None and not self.connection.is_closed:
|
||
ALIVE_FILE.touch()
|
||
if time.monotonic() - last_sweep > 300:
|
||
last_sweep = time.monotonic()
|
||
try:
|
||
await self.sweep_staging()
|
||
except Exception:
|
||
log.warning('staging sweep failed', exc_info=True)
|
||
try:
|
||
await asyncio.wait_for(self.stopping.wait(), 5)
|
||
except asyncio.TimeoutError:
|
||
pass
|
||
|
||
async def sweep_staging(self) -> None:
|
||
old = await self.storage.list_older('staging/', self.settings.staging_retention)
|
||
if not old:
|
||
return
|
||
async with self.db.connection() as conn:
|
||
async with conn.transaction():
|
||
if not await conn.fetchval('SELECT pg_try_advisory_xact_lock(20260006)'):
|
||
return
|
||
# Удаляем только staging, по которому НЕТ текущей работы (queued/processing).
|
||
active = {r['staging_key'] for r in await conn.fetch(
|
||
"SELECT staging_key FROM attachments WHERE staging_key = ANY($1::text[]) "
|
||
"AND status IN ('queued', 'processing')", old)}
|
||
removed = 0
|
||
for key in old:
|
||
if key not in active:
|
||
await self.storage.delete(key)
|
||
removed += 1
|
||
log.info('staging retention: removed %d objects older than %ds', removed, self.settings.staging_retention)
|
||
|
||
# ---------- жизненный цикл ----------
|
||
|
||
async def run(self) -> None:
|
||
await self.db.start()
|
||
loop = asyncio.get_running_loop()
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
loop.add_signal_handler(sig, self.stopping.set)
|
||
housekeeping = asyncio.create_task(self.maintenance())
|
||
while True:
|
||
try:
|
||
self.connection = await broker.connect(self.settings.broker_url)
|
||
break
|
||
except Exception as exc:
|
||
if self.stopping.is_set():
|
||
return
|
||
log.warning('broker not reachable (%s), retrying', type(exc).__name__)
|
||
await asyncio.sleep(2)
|
||
channel = await self.connection.channel()
|
||
# prefetch = сколько неподтверждённых сообщений брокер отдаёт этому worker (ограниченная конкуренция).
|
||
await channel.set_qos(prefetch_count=self.settings.worker_prefetch)
|
||
_exchange, queue = await broker.declare(channel)
|
||
tag = await queue.consume(self.on_message, no_ack=False)
|
||
log.info('worker %s consuming %s, concurrency=%d prefetch=%d', self.worker_id, broker.QUEUE,
|
||
self.settings.worker_concurrency, self.settings.worker_prefetch)
|
||
await self.stopping.wait()
|
||
# Graceful: перестать брать новые, дождаться текущих (неподтверждённые prefetch вернутся в очередь).
|
||
await queue.cancel(tag)
|
||
if self.inflight:
|
||
log.info('draining %d in-flight jobs', len(self.inflight))
|
||
await asyncio.wait(self.inflight, timeout=self.settings.job_timeout)
|
||
housekeeping.cancel()
|
||
await self.connection.close()
|
||
await self.db.close()
|
||
|
||
|
||
def main() -> None:
|
||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s')
|
||
asyncio.run(Worker().run())
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|