"""Worker supervisors: concurrency процессов, prefetch=1, дочерний resize с timeout.""" from datetime import datetime import hashlib import json import logging import multiprocessing import os from pathlib import Path import re import signal import subprocess import sys from tempfile import TemporaryDirectory import time from uuid import UUID, uuid4 from psycopg.types.json import Jsonb from src import broker, db, faults, jobs, object_store, task_supervisor from src.errors import ApiError class ProcessingError(Exception): def __init__(self, code, permanent=False): self.code, self.permanent = code, permanent def verify_ready(job): # Старые ready из lab4 не имеют job; у новой job всегда все три варианта. if {v['name'] for v in job['variants']} != {'thumbnail', 'crop', 'optimized'}: raise ProcessingError('incomplete_result') for key, size in [(job['object_key'], job['size_bytes'])] + [ (job['object_key'].rsplit('/', 1)[0] + '/' + v['name'], v['size_bytes']) for v in job['variants'] ]: with object_store.available(): if object_store.client.stat_object(object_store.BUCKET, key).size != size: raise ProcessingError('incomplete_result') def process(job, workspace=None): faults.hit(job['id'], 'before_download') with TemporaryDirectory(prefix='image-job-', dir=workspace) as directory: output = Path(directory) source = output / 'original' response = object_store.get(job['staging_key']) size, digest = 0, hashlib.sha256() try: with source.open('wb') as file: while chunk := response.read(65536): size += len(chunk) if size > job['size_bytes'] or size > 10485760: raise ProcessingError('invalid_image', True) digest.update(chunk) file.write(chunk) finally: response.close() response.release_conn() if size != job['size_bytes'] or digest.hexdigest() != job['sha256']: raise ProcessingError('invalid_image', True) try: result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', str(source), str(output), str(job['id'])], timeout=jobs.TIMEOUT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.TimeoutExpired: raise ProcessingError('processing_timeout') from None if result.returncode: raise ProcessingError('invalid_image' if result.returncode == 65 else 'resource_limit', result.returncode == 65) variants = json.loads((output / 'variants.json').read_text()) # Короткая транзакция fence перед side effects и публикацией результата. with db.connection() as conn: conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'") current = jobs.locked(conn, job['id']) active = conn.execute('SELECT lease_until>clock_timestamp() AS active FROM attachment_jobs WHERE id=%s', (job['id'],)).fetchone()['active'] if current['status'] != 'processing' or current['lease_token'] != job['lease_token'] or not active: return False with source.open('rb') as file: object_store.put(job['object_key'], file, job['size_bytes'], job['content_type']) prefix = job['object_key'].rsplit('/', 1)[0] + '/' for variant in variants: with (output / variant['name']).open('rb') as file: object_store.put(prefix + variant['name'], file, variant['size_bytes'], variant['content_type']) faults.hit(job['id'], 'after_objects') conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (Jsonb(variants), job['attachment_id'])) conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=NULL WHERE id=%s', (job['id'],)) jobs.event(conn, job, 'ready') faults.hit(job['id'], 'before_ack') return True def handle(client, message_id, fields): try: payload = json.loads(fields['job']) required = {'schema_version', 'type', 'job_id', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} if not isinstance(payload, dict) or not required <= payload.keys() or payload.keys() - required - {'traceparent', 'tracestate'}: raise ValueError('invalid job') if type(payload['schema_version']) is not int or payload['schema_version'] != 1 or payload['type'] != 'attachment.process.v1': raise ValueError('invalid job') payload['job_id'] = str(UUID(payload['job_id'])) payload['attachment_id'] = str(UUID(payload['attachment_id'])) if type(payload['attempt']) is not int or payload['attempt'] < 1 or type(payload['pipeline_version']) is not int or payload['pipeline_version'] != 1: raise ValueError('invalid job') if not isinstance(payload['created_at'], str) or datetime.fromisoformat(payload['created_at'].replace('Z', '+00:00')).tzinfo is None: raise ValueError('invalid job') if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None): raise ValueError('invalid trace context') if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512): raise ValueError('invalid trace context') except (ValueError, TypeError, KeyError): logging.error('Invalid broker message rejected') broker.acknowledge(client, message_id) return state, job = jobs.claim(payload) if state == 'busy': return if state == 'ready': verify_ready(job) if state == 'claimed': try: faults.hit(job['id'], 'before_process') # Временные файлы удаляет consumer даже после SIGKILL runner. with TemporaryDirectory(prefix='image-attempt-') as workspace: result = task_supervisor.run( [sys.executable, '-m', 'src.job_runner', str(job['id']), str(job['lease_token']), str(os.getpid()), workspace], renew=lambda: jobs.renew(job), timeout=jobs.TASK_TIMEOUT, interval=jobs.HEARTBEAT) if result != 0: # Crash/неизвестный COMMIT: не ACK, восстановление через lease. return except task_supervisor.LeaseLost: return except subprocess.TimeoutExpired: jobs.fail(job, 'processing_timeout') except ProcessingError as error: jobs.fail(job, error.code, error.permanent) except Exception: jobs.fail(job, 'storage_unavailable') # ready/failed/obsolete или записанный retry; при потере COMMIT исключение не даёт ACK. broker.acknowledge(client, message_id) def consume(): db.pool.open(wait=True, timeout=30) client = broker.connect() consumer = 'worker-' + uuid4().hex cursor = '0-0' try: while True: try: broker.bootstrap(client) claimed = client.xautoclaim(broker.STREAM, broker.GROUP, consumer, min_idle_time=jobs.LEASE * 1000, start_id=cursor, count=1) cursor = claimed[0] entries = claimed[1] if not entries: batches = client.xreadgroup(broker.GROUP, consumer, {broker.STREAM: '>'}, count=1, block=1000) entries = batches[0][1] if batches else [] for message_id, fields in entries: handle(client, message_id, fields) except Exception as error: logging.error('Worker postponed: %s', type(error).__name__) time.sleep(1) finally: client.close() db.pool.close() def main(): concurrency = int(os.environ.get('WORKER_CONCURRENCY', '2')) if not 1 <= concurrency <= 2: raise ValueError('WORKER_CONCURRENCY: 1 или 2 для заданного memory limit') context = multiprocessing.get_context('spawn') processes = [] stopping = False def stop(signum, frame): nonlocal stopping stopping = True signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) try: while not stopping: alive = [] for child in processes: if child.is_alive(): alive.append(child) else: child.join() processes = alive while len(processes) < concurrency: child = context.Process(target=consume) child.start() processes.append(child) time.sleep(0.25) finally: for child in processes: child.terminate() for child in processes: child.join(timeout=5) if child.is_alive(): child.kill() child.join() if __name__ == '__main__': main()