Исправить обработку крупных результатов worker

This commit is contained in:
2026-11-17 19:24:40 +03:00
parent e8becaf761
commit 3ed3046df1
4 changed files with 64 additions and 5 deletions
+13 -5
View File
@@ -53,15 +53,23 @@ def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[Ge
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
process.start()
sender.close()
process.join(timeout)
if not receiver.poll(timeout):
process.terminate()
process.join(5)
receiver.close()
raise ProcessingTimeout('processing_timeout')
try:
kind, result = receiver.recv()
except EOFError as error:
process.join(5)
receiver.close()
raise ProcessingTimeout('processing_failed') from error
receiver.close()
process.join(5)
if process.is_alive():
process.terminate()
process.join(5)
raise ProcessingTimeout('processing_timeout')
if not receiver.poll():
raise ProcessingTimeout('processing_failed')
kind, result = receiver.recv()
receiver.close()
if kind == 'ok':
return result
if kind == 'permanent':
+8
View File
@@ -84,6 +84,14 @@ class JobRepository:
)
)
async def failed_staging_key(self, job_id: UUID) -> str | None:
async with self._sessions() as session:
return await session.scalar(
select(AttachmentRow.staging_key)
.join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id)
.where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed')
)
async def ready_effect_complete(self, job_id: UUID) -> bool:
async with self._sessions() as session:
job = await session.get(AttachmentJobRow, job_id)
+15
View File
@@ -8,6 +8,7 @@ from uuid import UUID
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..repositories import JobRepository
from ..storage import S3Storage
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
@@ -21,6 +22,20 @@ async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool)
if job_id is None:
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
return 2
staging_key = await repository.failed_staging_key(job_id)
if staging_key is None:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
return 2
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
if not await storage.exists(staging_key):
print(f'REJECTED job_id={job_id} reason=staging_object_missing')
return 2
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
if not accepted:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
+28
View File
@@ -2,11 +2,14 @@ from __future__ import annotations
import hashlib
from io import BytesIO
import random
from types import SimpleNamespace
import pytest
from PIL import Image
from app.jobs.pipeline import PermanentImageError, process_image
from app.jobs.worker import Worker, _run_isolated
def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes:
@@ -87,3 +90,28 @@ def test_exif_orientation_is_applied_and_removed() -> None:
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256)
with Image.open(BytesIO(variants['thumbnail'].content)) as transformed:
assert not transformed.getexif()
def test_isolated_pipeline_returns_results_larger_than_pipe_buffer() -> None:
"""Drain child output before join so a large valid result cannot deadlock the pipe."""
pixels = random.Random(0).randbytes(512 * 512 * 3)
image = Image.frombytes('RGB', (512, 512), pixels)
output = BytesIO()
image.save(output, 'PNG')
source = output.getvalue()
variants = _run_isolated(
(source, 'image/png', hashlib.sha256(source).hexdigest(), 40_000_000, 85, 85, 82),
timeout=10,
memory_bytes=512 * 1024 * 1024,
)
assert {item.name for item in variants} == {'thumbnail', 'crop', 'optimized'}
assert sum(len(item.content) for item in variants) > 64 * 1024
def test_retry_delay_uses_capped_exponential_backoff_and_jitter(monkeypatch) -> None:
"""Calculate capped exponential backoff and add the configured random jitter."""
settings = SimpleNamespace(worker_concurrency=1, retry_base_seconds=2, retry_max_seconds=8)
worker = Worker(settings, jobs=None, storage=None)
monkeypatch.setattr('app.jobs.worker.random.uniform', lambda start, end: end / 2)
assert worker._delay(1) == 3
assert worker._delay(4) == 12