diff --git a/app/jobs/worker.py b/app/jobs/worker.py index 2aa94332..900f94d7 100644 --- a/app/jobs/worker.py +++ b/app/jobs/worker.py @@ -53,15 +53,23 @@ def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[Ge process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes)) process.start() sender.close() - process.join(timeout) + if not receiver.poll(timeout): + process.terminate() + process.join(5) + receiver.close() + raise ProcessingTimeout('processing_timeout') + try: + kind, result = receiver.recv() + except EOFError as error: + process.join(5) + receiver.close() + raise ProcessingTimeout('processing_failed') from error + receiver.close() + process.join(5) if process.is_alive(): process.terminate() process.join(5) - raise ProcessingTimeout('processing_timeout') - if not receiver.poll(): raise ProcessingTimeout('processing_failed') - kind, result = receiver.recv() - receiver.close() if kind == 'ok': return result if kind == 'permanent': diff --git a/app/repositories/jobs.py b/app/repositories/jobs.py index 5148678a..f40acef6 100644 --- a/app/repositories/jobs.py +++ b/app/repositories/jobs.py @@ -84,6 +84,14 @@ class JobRepository: ) ) + async def failed_staging_key(self, job_id: UUID) -> str | None: + async with self._sessions() as session: + return await session.scalar( + select(AttachmentRow.staging_key) + .join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id) + .where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed') + ) + async def ready_effect_complete(self, job_id: UUID) -> bool: async with self._sessions() as session: job = await session.get(AttachmentJobRow, job_id) diff --git a/app/tools/replay_failed.py b/app/tools/replay_failed.py index e6bd814b..03fcde40 100644 --- a/app/tools/replay_failed.py +++ b/app/tools/replay_failed.py @@ -8,6 +8,7 @@ from uuid import UUID from ..config import load_settings from ..db import create_engine, create_session_factory from ..repositories import JobRepository +from ..storage import S3Storage async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int: @@ -21,6 +22,20 @@ async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) if job_id is None: print(f'REJECTED attachment_id={attachment_id} reason=job_not_found') return 2 + staging_key = await repository.failed_staging_key(job_id) + if staging_key is None: + print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing') + return 2 + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + if not await storage.exists(staging_key): + print(f'REJECTED job_id={job_id} reason=staging_object_missing') + return 2 accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run) if not accepted: print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing') diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ade9492b..b4500d61 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,11 +2,14 @@ from __future__ import annotations import hashlib from io import BytesIO +import random +from types import SimpleNamespace import pytest from PIL import Image from app.jobs.pipeline import PermanentImageError, process_image +from app.jobs.worker import Worker, _run_isolated def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes: @@ -87,3 +90,28 @@ def test_exif_orientation_is_applied_and_removed() -> None: assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256) with Image.open(BytesIO(variants['thumbnail'].content)) as transformed: assert not transformed.getexif() + + +def test_isolated_pipeline_returns_results_larger_than_pipe_buffer() -> None: + """Drain child output before join so a large valid result cannot deadlock the pipe.""" + pixels = random.Random(0).randbytes(512 * 512 * 3) + image = Image.frombytes('RGB', (512, 512), pixels) + output = BytesIO() + image.save(output, 'PNG') + source = output.getvalue() + variants = _run_isolated( + (source, 'image/png', hashlib.sha256(source).hexdigest(), 40_000_000, 85, 85, 82), + timeout=10, + memory_bytes=512 * 1024 * 1024, + ) + assert {item.name for item in variants} == {'thumbnail', 'crop', 'optimized'} + assert sum(len(item.content) for item in variants) > 64 * 1024 + + +def test_retry_delay_uses_capped_exponential_backoff_and_jitter(monkeypatch) -> None: + """Calculate capped exponential backoff and add the configured random jitter.""" + settings = SimpleNamespace(worker_concurrency=1, retry_base_seconds=2, retry_max_seconds=8) + worker = Worker(settings, jobs=None, storage=None) + monkeypatch.setattr('app.jobs.worker.random.uniform', lambda start, end: end / 2) + assert worker._delay(1) == 3 + assert worker._delay(4) == 12