Files
mtusi2026lab5/src/worker.py
T
Savelev_Ivan fd501cbf68 fix: продление аренды задач и контроль зависшей обработки
- Добавлен heartbeat для продления lease долгих задач.
- Добавлен общий таймаут с остановкой дерева процессов.
- Сохранена защита от устаревших исполнителей.
- Добавлены тесты и обновлена документация.
2026-11-20 14:00:53 +03:00

204 lines
9.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Worker supervisors: concurrency процессов, prefetch=1, дочерний resize с timeout."""
from datetime import datetime
import hashlib
import json
import logging
import multiprocessing
import os
from pathlib import Path
import re
import signal
import subprocess
import sys
from tempfile import TemporaryDirectory
import time
from uuid import UUID, uuid4
from psycopg.types.json import Jsonb
from src import broker, db, faults, jobs, object_store, task_supervisor
from src.errors import ApiError
class ProcessingError(Exception):
def __init__(self, code, permanent=False):
self.code, self.permanent = code, permanent
def verify_ready(job):
# Старые ready из lab4 не имеют job; у новой job всегда все три варианта.
if {v['name'] for v in job['variants']} != {'thumbnail', 'crop', 'optimized'}:
raise ProcessingError('incomplete_result')
for key, size in [(job['object_key'], job['size_bytes'])] + [
(job['object_key'].rsplit('/', 1)[0] + '/' + v['name'], v['size_bytes']) for v in job['variants']
]:
with object_store.available():
if object_store.client.stat_object(object_store.BUCKET, key).size != size:
raise ProcessingError('incomplete_result')
def process(job, workspace=None):
faults.hit(job['id'], 'before_download')
with TemporaryDirectory(prefix='image-job-', dir=workspace) as directory:
output = Path(directory)
source = output / 'original'
response = object_store.get(job['staging_key'])
size, digest = 0, hashlib.sha256()
try:
with source.open('wb') as file:
while chunk := response.read(65536):
size += len(chunk)
if size > job['size_bytes'] or size > 10485760:
raise ProcessingError('invalid_image', True)
digest.update(chunk)
file.write(chunk)
finally:
response.close()
response.release_conn()
if size != job['size_bytes'] or digest.hexdigest() != job['sha256']:
raise ProcessingError('invalid_image', True)
try:
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', str(source), str(output), str(job['id'])],
timeout=jobs.TIMEOUT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except subprocess.TimeoutExpired:
raise ProcessingError('processing_timeout') from None
if result.returncode:
raise ProcessingError('invalid_image' if result.returncode == 65 else 'resource_limit', result.returncode == 65)
variants = json.loads((output / 'variants.json').read_text())
# Короткая транзакция fence перед side effects и публикацией результата.
with db.connection() as conn:
conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'")
current = jobs.locked(conn, job['id'])
active = conn.execute('SELECT lease_until>clock_timestamp() AS active FROM attachment_jobs WHERE id=%s', (job['id'],)).fetchone()['active']
if current['status'] != 'processing' or current['lease_token'] != job['lease_token'] or not active:
return False
with source.open('rb') as file:
object_store.put(job['object_key'], file, job['size_bytes'], job['content_type'])
prefix = job['object_key'].rsplit('/', 1)[0] + '/'
for variant in variants:
with (output / variant['name']).open('rb') as file:
object_store.put(prefix + variant['name'], file, variant['size_bytes'], variant['content_type'])
faults.hit(job['id'], 'after_objects')
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (Jsonb(variants), job['attachment_id']))
conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=NULL WHERE id=%s', (job['id'],))
jobs.event(conn, job, 'ready')
faults.hit(job['id'], 'before_ack')
return True
def handle(client, message_id, fields):
try:
payload = json.loads(fields['job'])
required = {'schema_version', 'type', 'job_id', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
if not isinstance(payload, dict) or not required <= payload.keys() or payload.keys() - required - {'traceparent', 'tracestate'}:
raise ValueError('invalid job')
if type(payload['schema_version']) is not int or payload['schema_version'] != 1 or payload['type'] != 'attachment.process.v1':
raise ValueError('invalid job')
payload['job_id'] = str(UUID(payload['job_id']))
payload['attachment_id'] = str(UUID(payload['attachment_id']))
if type(payload['attempt']) is not int or payload['attempt'] < 1 or type(payload['pipeline_version']) is not int or payload['pipeline_version'] != 1:
raise ValueError('invalid job')
if not isinstance(payload['created_at'], str) or datetime.fromisoformat(payload['created_at'].replace('Z', '+00:00')).tzinfo is None:
raise ValueError('invalid job')
if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None):
raise ValueError('invalid trace context')
if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512):
raise ValueError('invalid trace context')
except (ValueError, TypeError, KeyError):
logging.error('Invalid broker message rejected')
broker.acknowledge(client, message_id)
return
state, job = jobs.claim(payload)
if state == 'busy':
return
if state == 'ready':
verify_ready(job)
if state == 'claimed':
try:
faults.hit(job['id'], 'before_process')
# Временные файлы удаляет consumer даже после SIGKILL runner.
with TemporaryDirectory(prefix='image-attempt-') as workspace:
result = task_supervisor.run(
[sys.executable, '-m', 'src.job_runner', str(job['id']), str(job['lease_token']),
str(os.getpid()), workspace],
renew=lambda: jobs.renew(job), timeout=jobs.TASK_TIMEOUT, interval=jobs.HEARTBEAT)
if result != 0:
# Crash/неизвестный COMMIT: не ACK, восстановление через lease.
return
except task_supervisor.LeaseLost:
return
except subprocess.TimeoutExpired:
jobs.fail(job, 'processing_timeout')
except ProcessingError as error:
jobs.fail(job, error.code, error.permanent)
except Exception:
jobs.fail(job, 'storage_unavailable')
# ready/failed/obsolete или записанный retry; при потере COMMIT исключение не даёт ACK.
broker.acknowledge(client, message_id)
def consume():
db.pool.open(wait=True, timeout=30)
client = broker.connect()
consumer = 'worker-' + uuid4().hex
cursor = '0-0'
try:
while True:
try:
broker.bootstrap(client)
claimed = client.xautoclaim(broker.STREAM, broker.GROUP, consumer,
min_idle_time=jobs.LEASE * 1000, start_id=cursor, count=1)
cursor = claimed[0]
entries = claimed[1]
if not entries:
batches = client.xreadgroup(broker.GROUP, consumer, {broker.STREAM: '>'}, count=1, block=1000)
entries = batches[0][1] if batches else []
for message_id, fields in entries:
handle(client, message_id, fields)
except Exception as error:
logging.error('Worker postponed: %s', type(error).__name__)
time.sleep(1)
finally:
client.close()
db.pool.close()
def main():
concurrency = int(os.environ.get('WORKER_CONCURRENCY', '2'))
if not 1 <= concurrency <= 2:
raise ValueError('WORKER_CONCURRENCY: 1 или 2 для заданного memory limit')
context = multiprocessing.get_context('spawn')
processes = []
stopping = False
def stop(signum, frame):
nonlocal stopping
stopping = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while not stopping:
alive = []
for child in processes:
if child.is_alive():
alive.append(child)
else:
child.join()
processes = alive
while len(processes) < concurrency:
child = context.Process(target=consume)
child.start()
processes.append(child)
time.sleep(0.25)
finally:
for child in processes:
child.terminate()
for child in processes:
child.join(timeout=5)
if child.is_alive():
child.kill()
child.join()
if __name__ == '__main__':
main()