Files
mtusi2026lab5/app/worker.py
T
RainakandClaude Opus 5.5 0a221b0675 ЛР5 БВТ2304 Запорожец на 5
RabbitMQ (quorum queue, publisher confirms, manual ack, DLX) + отдельный worker:
staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued; relay outbox,
lease с heartbeat и fencing (worker_id, attempt), retry с backoff+jitter, failed-реестр,
безопасный replay с audit; thumbnail/crop/WebP в изолированном процессе с лимитами.
Тесты окон отказа (fault injection), трек 1 «Очередь под перегрузкой» с backpressure.
Перенесено исправление замечания lab3 PR #99 (U+0000 в тексте).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 18:33:52 +03:00

308 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Worker обработки вложений: python -m app.worker (отдельный сервис, не API).
Одна доставка сообщения:
1. проверить сообщение по contracts/jobs.schema.json (иначе reject без requeue -> DLQ);
2. claim в PostgreSQL: queued (или processing с истёкшей арендой) -> processing, attempt+1, lease;
done/failed/чужая живая аренда/ещё не время retry -> no-op + ack;
3. heartbeat продлевает lease, пока идёт работа (отличает долгую обработку от зависшей/мёртвой);
4. оригинал из staging, сверка size/sha256 с доверенными метаданными из БД;
5. варианты в дочернем процессе (лимиты памяти/времени), запись в S3 по ДЕТЕРМИНИРОВАННЫМ ключам
variants/<id>/v<pipeline>/<name> + server-side copy staging -> originals/<id>;
6. COMMIT результата с fencing по (worker_id, attempt): variants (ON CONFLICT DO NOTHING) + ready;
7. удалить staging (после безопасной фиксации), затем ack.
Сбой на любом шаге до ack -> redelivery/reaper -> повтор шагов 4–6 перезаписывает те же ключи тем же
содержимым и не создаёт второй набор вариантов (ключ эффекта (attachment_id, pipeline_version, variant)).
"""
import asyncio
import hashlib
import json
import logging
import os
import secrets
import signal
import socket
import time
from pathlib import Path
import jsonschema
from . import broker, config, faults, imaging, jobs
from .db import Database
from .errors import ApiError
from .storage import ObjectMissing, Storage
log = logging.getLogger('messenger.worker')
SCHEMA = json.loads((Path(__file__).resolve().parent.parent / 'contracts/jobs.schema.json').read_text(encoding='utf-8'))
VALIDATOR = jsonschema.Draft202012Validator(SCHEMA, format_checker=jsonschema.FormatChecker())
ALIVE_FILE = Path('/tmp/worker-alive')
class LostLease(Exception):
"""Аренду перехватили (истекла): результат фиксирует новый владелец, этот — только ack."""
class Worker:
def __init__(self):
self.settings = config.load()
self.db = Database(self.settings)
self.storage = Storage(self.settings)
self.worker_id = f'{socket.gethostname()}-{secrets.token_hex(3)}'
self.slots = asyncio.Semaphore(self.settings.worker_concurrency)
self.inflight: set[asyncio.Task] = set()
self.stopping = asyncio.Event()
self.connection = None
# ---------- приём сообщения ----------
async def on_message(self, message) -> None:
task = asyncio.current_task()
self.inflight.add(task)
try:
try:
payload = json.loads(message.body)
VALIDATOR.validate(payload)
except (ValueError, jsonschema.ValidationError):
log.error('invalid job message %s -> DLQ', message.message_id)
await message.reject(requeue=False)
return
async with self.slots:
try:
await self.process(payload)
except Exception:
# Инфраструктура (БД/S3) недоступна настолько, что даже состояние не записать:
# вернуть сообщение в очередь с паузой; delivery-limit страхует от бесконечного цикла.
log.exception('job %s: infrastructure error, requeue', payload['job_id'])
await asyncio.sleep(2)
await message.nack(requeue=True)
return
await message.ack() # только после устойчивого эффекта (или no-op)
finally:
self.inflight.discard(task)
# ---------- обработка ----------
async def claim(self, job_id: str, attachment_id: str):
async with self.db.connection() as conn:
async with conn.transaction():
j = await conn.fetchrow(
'SELECT j.status, j.attempt, j.max_attempts, '
' coalesce(j.lease_until > clock_timestamp(), false) AS leased, '
' j.next_attempt_at > clock_timestamp() AS not_yet, '
' a.staging_key, a.object_key, a.size_bytes, a.sha256, a.pipeline_version '
'FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id '
'WHERE j.job_id = $1 AND j.attachment_id = $2 FOR UPDATE OF j', job_id, attachment_id)
if j is None:
return None, 'unknown job'
if j['status'] in ('done', 'failed'):
return None, f"already {j['status']} (duplicate delivery)"
if j['status'] == 'processing' and j['leased']:
return None, 'leased by another live worker (duplicate delivery)'
if j['status'] == 'queued' and j['not_yet']:
return None, 'retry is scheduled later (stale message)'
if j['attempt'] >= j['max_attempts']:
await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', 'no attempts left')
return None, 'attempts exhausted'
attempt = await conn.fetchval(
"UPDATE attachment_jobs SET status = 'processing', attempt = attempt + 1, worker_id = $2, "
"lease_until = clock_timestamp() + make_interval(secs => $3), started_at = clock_timestamp(), "
"updated_at = clock_timestamp() WHERE job_id = $1 RETURNING attempt",
job_id, self.worker_id, self.settings.job_lease)
await conn.execute(
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() "
"WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id)
return {**dict(j), 'attempt': attempt, 'max_attempts': j['max_attempts']}, None
async def heartbeat(self, job_id: str, attempt: int) -> None:
interval = max(1.0, self.settings.job_lease / 3)
while True:
await asyncio.sleep(interval)
try:
async with self.db.connection() as conn:
ok = await conn.fetchval(
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $3) "
"WHERE job_id = $1 AND worker_id = $2 AND attempt = $4 AND status = 'processing' RETURNING 1",
job_id, self.worker_id, self.settings.job_lease, attempt)
if not ok:
log.warning('job %s: lease lost', job_id)
return
except Exception:
log.warning('job %s: heartbeat failed', job_id)
async def process(self, payload: dict) -> None:
job_id, attachment_id = payload['job_id'], payload['attachment_id']
job, reason = await self.claim(job_id, attachment_id)
if job is None:
log.info('job %s: no-op, %s', job_id, reason)
return
started = time.monotonic()
beat = asyncio.create_task(self.heartbeat(job_id, job['attempt']))
try:
stages = await self.execute(job_id, attachment_id, job)
log.info('job %s: ready in %.2fs (attempt %d) %s', job_id, time.monotonic() - started, job['attempt'],
' '.join(f'{k}={v:.3f}' for k, v in stages.items()))
except imaging.PermanentImageError as exc:
await self.finish_failed(job_id, attachment_id, job['attempt'], exc.code, exc.code)
except LostLease:
log.warning('job %s: result not committed, lease taken over', job_id)
except (imaging.TransientImageError, ApiError) as exc:
code = getattr(exc, 'code', 'unavailable')
await self.retry_or_fail(job_id, attachment_id, job, f'transient: {code}')
finally:
beat.cancel()
async def execute(self, job_id: str, attachment_id: str, job) -> None:
version = job['pipeline_version']
stages, mark = {}, time.monotonic()
def lap(name):
nonlocal mark
now = time.monotonic()
stages[name] = now - mark
mark = now
try:
data = await self.storage.get_raw(job['staging_key'])
except ObjectMissing:
raise imaging.PermanentImageError('source_missing') from None
# Доверенные метаданные — из БД, а не из сообщения брокера.
if len(data) != job['size_bytes'] or hashlib.sha256(data).hexdigest() != job['sha256']:
raise imaging.PermanentImageError('source_corrupted')
lap('fetch')
variants = await imaging.run_isolated(data, self.settings.worker_max_pixels,
self.settings.worker_memory_mb, self.settings.job_timeout)
lap('build')
ext = {'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/png': 'png'}
for v in variants:
v['object_key'] = f"variants/{attachment_id}/v{version}/{v['name']}.{ext[v['content_type']]}"
await self.storage.put(v['object_key'], v.pop('data'), v['content_type'])
await self.storage.copy(job['staging_key'], job['object_key'])
lap('store')
faults.hit('worker_crash_after_s3')
async with self.db.connection() as conn:
async with conn.transaction():
owned = await conn.fetchval(
"UPDATE attachment_jobs SET status = 'done', lease_until = NULL, error_code = NULL, "
"last_error = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp() "
"WHERE job_id = $1 AND status = 'processing' AND worker_id = $2 AND attempt = $3 RETURNING 1",
job_id, self.worker_id, job['attempt'])
if not owned:
raise LostLease()
for v in variants:
await conn.execute(
'INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, '
'size_bytes, width, height, sha256, object_key) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) '
'ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING',
attachment_id, version, v['name'], v['content_type'], v['size_bytes'],
v['width'], v['height'], v['sha256'], v['object_key'])
await conn.execute(
"UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() "
"WHERE id = $1 AND status = 'processing'", attachment_id)
lap('commit')
faults.hit('worker_crash_after_commit')
try:
await self.storage.delete(job['staging_key']) # staging больше не нужен: результат зафиксирован
except ApiError:
log.warning('job %s: staging cleanup deferred to retention sweeper', job_id)
lap('cleanup')
return stages
async def finish_failed(self, job_id, attachment_id, attempt: int, code: str, reason: str) -> None:
async with self.db.connection() as conn:
async with conn.transaction():
await jobs.fail(conn, job_id, attachment_id, code, reason, worker_id=self.worker_id, attempt=attempt)
log.warning('job %s: failed (%s)', job_id, code)
async def retry_or_fail(self, job_id, attachment_id, job, reason: str) -> None:
async with self.db.connection() as conn:
async with conn.transaction():
if job['attempt'] >= job['max_attempts']:
await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', reason,
worker_id=self.worker_id, attempt=job['attempt'])
log.warning('job %s: failed, retry_exhausted after %d attempts', job_id, job['attempt'])
return
delay = jobs.backoff_seconds(job['attempt'], self.settings.retry_base_ms, self.settings.retry_max_ms)
await jobs.requeue(conn, job_id, attachment_id, job['attempt'] + 1, delay, reason,
worker_id=self.worker_id, attempt=job['attempt'])
log.warning('job %s: %s, retry in %.1fs (attempt %d/%d)', job_id, reason, delay,
job['attempt'], job['max_attempts'])
# ---------- обслуживание ----------
async def maintenance(self) -> None:
"""Пульс для healthcheck + retention staging (одна реплика за раз через advisory lock)."""
last_sweep = 0.0
while not self.stopping.is_set():
if self.connection is not None and not self.connection.is_closed:
ALIVE_FILE.touch()
if time.monotonic() - last_sweep > 300:
last_sweep = time.monotonic()
try:
await self.sweep_staging()
except Exception:
log.warning('staging sweep failed', exc_info=True)
try:
await asyncio.wait_for(self.stopping.wait(), 5)
except asyncio.TimeoutError:
pass
async def sweep_staging(self) -> None:
old = await self.storage.list_older('staging/', self.settings.staging_retention)
if not old:
return
async with self.db.connection() as conn:
async with conn.transaction():
if not await conn.fetchval('SELECT pg_try_advisory_xact_lock(20260006)'):
return
# Удаляем только staging, по которому НЕТ текущей работы (queued/processing).
active = {r['staging_key'] for r in await conn.fetch(
"SELECT staging_key FROM attachments WHERE staging_key = ANY($1::text[]) "
"AND status IN ('queued', 'processing')", old)}
removed = 0
for key in old:
if key not in active:
await self.storage.delete(key)
removed += 1
log.info('staging retention: removed %d objects older than %ds', removed, self.settings.staging_retention)
# ---------- жизненный цикл ----------
async def run(self) -> None:
await self.db.start()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, self.stopping.set)
housekeeping = asyncio.create_task(self.maintenance())
while True:
try:
self.connection = await broker.connect(self.settings.broker_url)
break
except Exception as exc:
if self.stopping.is_set():
return
log.warning('broker not reachable (%s), retrying', type(exc).__name__)
await asyncio.sleep(2)
channel = await self.connection.channel()
# prefetch = сколько неподтверждённых сообщений брокер отдаёт этому worker (ограниченная конкуренция).
await channel.set_qos(prefetch_count=self.settings.worker_prefetch)
_exchange, queue = await broker.declare(channel)
tag = await queue.consume(self.on_message, no_ack=False)
log.info('worker %s consuming %s, concurrency=%d prefetch=%d', self.worker_id, broker.QUEUE,
self.settings.worker_concurrency, self.settings.worker_prefetch)
await self.stopping.wait()
# Graceful: перестать брать новые, дождаться текущих (неподтверждённые prefetch вернутся в очередь).
await queue.cancel(tag)
if self.inflight:
log.info('draining %d in-flight jobs', len(self.inflight))
await asyncio.wait(self.inflight, timeout=self.settings.job_timeout)
housekeeping.cancel()
await self.connection.close()
await self.db.close()
def main() -> None:
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s')
asyncio.run(Worker().run())
if __name__ == '__main__':
main()