Исправить обработку крупных результатов worker
This commit is contained in:
+13
-5
@@ -53,15 +53,23 @@ def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[Ge
|
||||
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
|
||||
process.start()
|
||||
sender.close()
|
||||
process.join(timeout)
|
||||
if not receiver.poll(timeout):
|
||||
process.terminate()
|
||||
process.join(5)
|
||||
receiver.close()
|
||||
raise ProcessingTimeout('processing_timeout')
|
||||
try:
|
||||
kind, result = receiver.recv()
|
||||
except EOFError as error:
|
||||
process.join(5)
|
||||
receiver.close()
|
||||
raise ProcessingTimeout('processing_failed') from error
|
||||
receiver.close()
|
||||
process.join(5)
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join(5)
|
||||
raise ProcessingTimeout('processing_timeout')
|
||||
if not receiver.poll():
|
||||
raise ProcessingTimeout('processing_failed')
|
||||
kind, result = receiver.recv()
|
||||
receiver.close()
|
||||
if kind == 'ok':
|
||||
return result
|
||||
if kind == 'permanent':
|
||||
|
||||
@@ -84,6 +84,14 @@ class JobRepository:
|
||||
)
|
||||
)
|
||||
|
||||
async def failed_staging_key(self, job_id: UUID) -> str | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
select(AttachmentRow.staging_key)
|
||||
.join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id)
|
||||
.where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed')
|
||||
)
|
||||
|
||||
async def ready_effect_complete(self, job_id: UUID) -> bool:
|
||||
async with self._sessions() as session:
|
||||
job = await session.get(AttachmentJobRow, job_id)
|
||||
|
||||
@@ -8,6 +8,7 @@ from uuid import UUID
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..repositories import JobRepository
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
|
||||
@@ -21,6 +22,20 @@ async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool)
|
||||
if job_id is None:
|
||||
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
|
||||
return 2
|
||||
staging_key = await repository.failed_staging_key(job_id)
|
||||
if staging_key is None:
|
||||
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||
return 2
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
if not await storage.exists(staging_key):
|
||||
print(f'REJECTED job_id={job_id} reason=staging_object_missing')
|
||||
return 2
|
||||
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
|
||||
if not accepted:
|
||||
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||
|
||||
@@ -2,11 +2,14 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import random
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from app.jobs.pipeline import PermanentImageError, process_image
|
||||
from app.jobs.worker import Worker, _run_isolated
|
||||
|
||||
|
||||
def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes:
|
||||
@@ -87,3 +90,28 @@ def test_exif_orientation_is_applied_and_removed() -> None:
|
||||
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256)
|
||||
with Image.open(BytesIO(variants['thumbnail'].content)) as transformed:
|
||||
assert not transformed.getexif()
|
||||
|
||||
|
||||
def test_isolated_pipeline_returns_results_larger_than_pipe_buffer() -> None:
|
||||
"""Drain child output before join so a large valid result cannot deadlock the pipe."""
|
||||
pixels = random.Random(0).randbytes(512 * 512 * 3)
|
||||
image = Image.frombytes('RGB', (512, 512), pixels)
|
||||
output = BytesIO()
|
||||
image.save(output, 'PNG')
|
||||
source = output.getvalue()
|
||||
variants = _run_isolated(
|
||||
(source, 'image/png', hashlib.sha256(source).hexdigest(), 40_000_000, 85, 85, 82),
|
||||
timeout=10,
|
||||
memory_bytes=512 * 1024 * 1024,
|
||||
)
|
||||
assert {item.name for item in variants} == {'thumbnail', 'crop', 'optimized'}
|
||||
assert sum(len(item.content) for item in variants) > 64 * 1024
|
||||
|
||||
|
||||
def test_retry_delay_uses_capped_exponential_backoff_and_jitter(monkeypatch) -> None:
|
||||
"""Calculate capped exponential backoff and add the configured random jitter."""
|
||||
settings = SimpleNamespace(worker_concurrency=1, retry_base_seconds=2, retry_max_seconds=8)
|
||||
worker = Worker(settings, jobs=None, storage=None)
|
||||
monkeypatch.setattr('app.jobs.worker.random.uniform', lambda start, end: end / 2)
|
||||
assert worker._delay(1) == 3
|
||||
assert worker._delay(4) == 12
|
||||
|
||||
Reference in New Issue
Block a user