- Добавлен heartbeat для продления lease долгих задач. - Добавлен общий таймаут с остановкой дерева процессов. - Сохранена защита от устаревших исполнителей. - Добавлены тесты и обновлена документация.
204 lines
9.3 KiB
Python
204 lines
9.3 KiB
Python
"""Worker supervisors: concurrency процессов, prefetch=1, дочерний resize с timeout."""
|
||
from datetime import datetime
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import multiprocessing
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
from tempfile import TemporaryDirectory
|
||
import time
|
||
from uuid import UUID, uuid4
|
||
|
||
from psycopg.types.json import Jsonb
|
||
|
||
from src import broker, db, faults, jobs, object_store, task_supervisor
|
||
from src.errors import ApiError
|
||
|
||
|
||
class ProcessingError(Exception):
|
||
def __init__(self, code, permanent=False):
|
||
self.code, self.permanent = code, permanent
|
||
|
||
|
||
def verify_ready(job):
|
||
# Старые ready из lab4 не имеют job; у новой job всегда все три варианта.
|
||
if {v['name'] for v in job['variants']} != {'thumbnail', 'crop', 'optimized'}:
|
||
raise ProcessingError('incomplete_result')
|
||
for key, size in [(job['object_key'], job['size_bytes'])] + [
|
||
(job['object_key'].rsplit('/', 1)[0] + '/' + v['name'], v['size_bytes']) for v in job['variants']
|
||
]:
|
||
with object_store.available():
|
||
if object_store.client.stat_object(object_store.BUCKET, key).size != size:
|
||
raise ProcessingError('incomplete_result')
|
||
|
||
|
||
def process(job, workspace=None):
|
||
faults.hit(job['id'], 'before_download')
|
||
with TemporaryDirectory(prefix='image-job-', dir=workspace) as directory:
|
||
output = Path(directory)
|
||
source = output / 'original'
|
||
response = object_store.get(job['staging_key'])
|
||
size, digest = 0, hashlib.sha256()
|
||
try:
|
||
with source.open('wb') as file:
|
||
while chunk := response.read(65536):
|
||
size += len(chunk)
|
||
if size > job['size_bytes'] or size > 10485760:
|
||
raise ProcessingError('invalid_image', True)
|
||
digest.update(chunk)
|
||
file.write(chunk)
|
||
finally:
|
||
response.close()
|
||
response.release_conn()
|
||
if size != job['size_bytes'] or digest.hexdigest() != job['sha256']:
|
||
raise ProcessingError('invalid_image', True)
|
||
try:
|
||
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', str(source), str(output), str(job['id'])],
|
||
timeout=jobs.TIMEOUT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
except subprocess.TimeoutExpired:
|
||
raise ProcessingError('processing_timeout') from None
|
||
if result.returncode:
|
||
raise ProcessingError('invalid_image' if result.returncode == 65 else 'resource_limit', result.returncode == 65)
|
||
variants = json.loads((output / 'variants.json').read_text())
|
||
# Короткая транзакция fence перед side effects и публикацией результата.
|
||
with db.connection() as conn:
|
||
conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'")
|
||
current = jobs.locked(conn, job['id'])
|
||
active = conn.execute('SELECT lease_until>clock_timestamp() AS active FROM attachment_jobs WHERE id=%s', (job['id'],)).fetchone()['active']
|
||
if current['status'] != 'processing' or current['lease_token'] != job['lease_token'] or not active:
|
||
return False
|
||
with source.open('rb') as file:
|
||
object_store.put(job['object_key'], file, job['size_bytes'], job['content_type'])
|
||
prefix = job['object_key'].rsplit('/', 1)[0] + '/'
|
||
for variant in variants:
|
||
with (output / variant['name']).open('rb') as file:
|
||
object_store.put(prefix + variant['name'], file, variant['size_bytes'], variant['content_type'])
|
||
faults.hit(job['id'], 'after_objects')
|
||
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (Jsonb(variants), job['attachment_id']))
|
||
conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=NULL WHERE id=%s', (job['id'],))
|
||
jobs.event(conn, job, 'ready')
|
||
faults.hit(job['id'], 'before_ack')
|
||
return True
|
||
|
||
|
||
def handle(client, message_id, fields):
|
||
try:
|
||
payload = json.loads(fields['job'])
|
||
required = {'schema_version', 'type', 'job_id', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
|
||
if not isinstance(payload, dict) or not required <= payload.keys() or payload.keys() - required - {'traceparent', 'tracestate'}:
|
||
raise ValueError('invalid job')
|
||
if type(payload['schema_version']) is not int or payload['schema_version'] != 1 or payload['type'] != 'attachment.process.v1':
|
||
raise ValueError('invalid job')
|
||
payload['job_id'] = str(UUID(payload['job_id']))
|
||
payload['attachment_id'] = str(UUID(payload['attachment_id']))
|
||
if type(payload['attempt']) is not int or payload['attempt'] < 1 or type(payload['pipeline_version']) is not int or payload['pipeline_version'] != 1:
|
||
raise ValueError('invalid job')
|
||
if not isinstance(payload['created_at'], str) or datetime.fromisoformat(payload['created_at'].replace('Z', '+00:00')).tzinfo is None:
|
||
raise ValueError('invalid job')
|
||
if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None):
|
||
raise ValueError('invalid trace context')
|
||
if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512):
|
||
raise ValueError('invalid trace context')
|
||
except (ValueError, TypeError, KeyError):
|
||
logging.error('Invalid broker message rejected')
|
||
broker.acknowledge(client, message_id)
|
||
return
|
||
state, job = jobs.claim(payload)
|
||
if state == 'busy':
|
||
return
|
||
if state == 'ready':
|
||
verify_ready(job)
|
||
if state == 'claimed':
|
||
try:
|
||
faults.hit(job['id'], 'before_process')
|
||
# Временные файлы удаляет consumer даже после SIGKILL runner.
|
||
with TemporaryDirectory(prefix='image-attempt-') as workspace:
|
||
result = task_supervisor.run(
|
||
[sys.executable, '-m', 'src.job_runner', str(job['id']), str(job['lease_token']),
|
||
str(os.getpid()), workspace],
|
||
renew=lambda: jobs.renew(job), timeout=jobs.TASK_TIMEOUT, interval=jobs.HEARTBEAT)
|
||
if result != 0:
|
||
# Crash/неизвестный COMMIT: не ACK, восстановление через lease.
|
||
return
|
||
except task_supervisor.LeaseLost:
|
||
return
|
||
except subprocess.TimeoutExpired:
|
||
jobs.fail(job, 'processing_timeout')
|
||
except ProcessingError as error:
|
||
jobs.fail(job, error.code, error.permanent)
|
||
except Exception:
|
||
jobs.fail(job, 'storage_unavailable')
|
||
# ready/failed/obsolete или записанный retry; при потере COMMIT исключение не даёт ACK.
|
||
broker.acknowledge(client, message_id)
|
||
|
||
|
||
def consume():
|
||
db.pool.open(wait=True, timeout=30)
|
||
client = broker.connect()
|
||
consumer = 'worker-' + uuid4().hex
|
||
cursor = '0-0'
|
||
try:
|
||
while True:
|
||
try:
|
||
broker.bootstrap(client)
|
||
claimed = client.xautoclaim(broker.STREAM, broker.GROUP, consumer,
|
||
min_idle_time=jobs.LEASE * 1000, start_id=cursor, count=1)
|
||
cursor = claimed[0]
|
||
entries = claimed[1]
|
||
if not entries:
|
||
batches = client.xreadgroup(broker.GROUP, consumer, {broker.STREAM: '>'}, count=1, block=1000)
|
||
entries = batches[0][1] if batches else []
|
||
for message_id, fields in entries:
|
||
handle(client, message_id, fields)
|
||
except Exception as error:
|
||
logging.error('Worker postponed: %s', type(error).__name__)
|
||
time.sleep(1)
|
||
finally:
|
||
client.close()
|
||
db.pool.close()
|
||
|
||
|
||
def main():
|
||
concurrency = int(os.environ.get('WORKER_CONCURRENCY', '2'))
|
||
if not 1 <= concurrency <= 2:
|
||
raise ValueError('WORKER_CONCURRENCY: 1 или 2 для заданного memory limit')
|
||
context = multiprocessing.get_context('spawn')
|
||
processes = []
|
||
stopping = False
|
||
def stop(signum, frame):
|
||
nonlocal stopping
|
||
stopping = True
|
||
signal.signal(signal.SIGTERM, stop)
|
||
signal.signal(signal.SIGINT, stop)
|
||
try:
|
||
while not stopping:
|
||
alive = []
|
||
for child in processes:
|
||
if child.is_alive():
|
||
alive.append(child)
|
||
else:
|
||
child.join()
|
||
processes = alive
|
||
while len(processes) < concurrency:
|
||
child = context.Process(target=consume)
|
||
child.start()
|
||
processes.append(child)
|
||
time.sleep(0.25)
|
||
finally:
|
||
for child in processes:
|
||
child.terminate()
|
||
for child in processes:
|
||
child.join(timeout=5)
|
||
if child.is_alive():
|
||
child.kill()
|
||
child.join()
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|