From f2242eb39d4e42cbd15d4ab3178d75095b9be245 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 29 Oct 2026 03:02:16 +0300 Subject: [PATCH] Render variants in killable processes with time and memory limits The renderer adapter keeps one single-process pool per WORKER_CONCURRENCY slot. A render that exceeds RENDER_TIMEOUT_SECONDS is killed and its slot is respawned, the render process has RLIMIT_AS = RENDER_MEMORY_MB on Linux, so both limits end in ImageTooDemanding, a permanent processing_failed. A render process killed from outside is a transient error. A cancelled render (lost lease, shutdown) kills its process too. The worker container limit grows to 1g: two 448 MiB render processes plus the worker itself. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 8 +- compose.yaml | 5 +- .../add-lab5-variants-and-recovery/tasks.md | 6 +- pyproject.toml | 2 +- src/messenger/attachments/config.py | 3 + src/messenger/attachments/entrypoints/ioc.py | 17 +++- .../attachments/infrastructure/images.py | 72 +++++++++++++-- student-tests/api/test_pipeline.py | 26 ++++++ student-tests/pipeline.py | 18 +++- student-tests/unit/test_render_pool.py | 91 +++++++++++++++++++ 10 files changed, 228 insertions(+), 20 deletions(-) create mode 100644 student-tests/unit/test_render_pool.py diff --git a/.env.example b/.env.example index 2d08f1e..ffb029c 100644 --- a/.env.example +++ b/.env.example @@ -63,5 +63,11 @@ JOB_BACKOFF_BASE_SECONDS=2 OUTBOX_POLL_SECONDS=1 SWEEP_INTERVAL_SECONDS=5 WORKER_CPUS=2 -WORKER_MEMORY=768m +WORKER_MEMORY=1g PIPELINE_DB_POOL_MAX_SIZE=4 +# Solution-specific for lab5 grade 4: render time per job (the render process is killed after it), +# address space of one render process (WORKER_MEMORY >= WORKER_CONCURRENCY * RENDER_MEMORY_MB plus +# ~120 MiB of the worker itself), jobs per render process before it is replaced. +RENDER_TIMEOUT_SECONDS=30 +RENDER_MEMORY_MB=448 +RENDER_TASKS_PER_CHILD=100 diff --git a/compose.yaml b/compose.yaml index 2372d91..14d8705 100644 --- a/compose.yaml +++ b/compose.yaml @@ -330,6 +330,9 @@ services: environment: <<: *pipeline-env WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + RENDER_TIMEOUT_SECONDS: ${RENDER_TIMEOUT_SECONDS:-30} + RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-448} + RENDER_TASKS_PER_CHILD: ${RENDER_TASKS_PER_CHILD:-100} secrets: - database_url - source: broker_url_worker @@ -340,7 +343,7 @@ services: target: s3_secret_access_key deploy: resources: - limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-768m}"} + limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-1g}"} k6: image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 diff --git a/openspec/changes/add-lab5-variants-and-recovery/tasks.md b/openspec/changes/add-lab5-variants-and-recovery/tasks.md index 554230c..77029b0 100644 --- a/openspec/changes/add-lab5-variants-and-recovery/tasks.md +++ b/openspec/changes/add-lab5-variants-and-recovery/tasks.md @@ -6,9 +6,9 @@ ## 2. Пределы worker -- [ ] 2.1 Адаптер `ProcessVariantRenderer`: слоты-процессы по `WORKER_CONCURRENCY`, таймаут с `kill_workers()`, `RLIMIT_AS` в процессе рендера, `BrokenProcessPool` → временная ошибка; настройки `RENDER_TIMEOUT_SECONDS`, `RENDER_MEMORY_MB`, `RENDER_TASKS_PER_CHILD`; закрытие пула при остановке worker -- [ ] 2.2 Классификация в `ProcessJobHandler`: `ImageTooDemanding` и отсутствующий оригинал → `processing_failed` без повторов -- [ ] 2.3 Проверка группы: тесты пула — результат, таймаут с восстановлением слота, падение процесса рендера → `Unavailable`, `MemoryError` → `ImageTooDemanding`, предел памяти применяется (Linux) +- [x] 2.1 Адаптер `ProcessVariantRenderer`: слоты-процессы по `WORKER_CONCURRENCY`, таймаут с `kill_workers()`, `RLIMIT_AS` в процессе рендера, `BrokenProcessPool` → временная ошибка; настройки `RENDER_TIMEOUT_SECONDS`, `RENDER_MEMORY_MB`, `RENDER_TASKS_PER_CHILD`; закрытие пула при остановке worker +- [x] 2.2 Классификация в `ProcessJobHandler`: `ImageTooDemanding` и отсутствующий оригинал → `processing_failed` без повторов +- [x] 2.3 Проверка группы: тесты пула — результат, таймаут с восстановлением слота, падение процесса рендера → `Unavailable`, `MemoryError` → `ImageTooDemanding`, предел памяти применяется (Linux) ## 3. Повторы и аренда diff --git a/pyproject.toml b/pyproject.toml index a035f0b..a518e85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"] +"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"] "scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"] "scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"] "scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"] diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py index 458408e..7d72de4 100644 --- a/src/messenger/attachments/config.py +++ b/src/messenger/attachments/config.py @@ -42,4 +42,7 @@ class PipelineSettings(AttachmentsSettings): shutdown_grace_seconds: float = Field(default=5.0, gt=0, le=9) heartbeat_file: Path | None = None heartbeat_interval_seconds: float = Field(default=5.0, gt=0, le=10) + render_timeout_seconds: float = Field(default=30.0, gt=0, le=120) + render_memory_mb: int = Field(default=448, ge=0, le=8192) + render_tasks_per_child: int = Field(default=100, ge=1, le=10_000) fault_point: str = "" diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index ef8d51e..4585dc5 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterator from contextlib import asynccontextmanager from datetime import timedelta @@ -41,7 +41,7 @@ from messenger.attachments.domain.jobs import RetryPolicy from messenger.attachments.infrastructure.faults import EnvironmentFaults from messenger.attachments.infrastructure.images import ( PillowImageInspector, - PillowVariantRenderer, + ProcessVariantRenderer, RenderLimits, ) from messenger.attachments.infrastructure.postgres import ( @@ -173,9 +173,16 @@ class PipelineProvider(Provider): class WorkerProvider(Provider): @provide(scope=Scope.APP) - def renderer(self, settings: PipelineSettings) -> VariantRenderer: - limits = RenderLimits(settings.max_upload_bytes, settings.max_image_pixels) - return PillowVariantRenderer(limits, settings.worker_concurrency) + def renderer(self, settings: PipelineSettings) -> Iterator[VariantRenderer]: + renderer = ProcessVariantRenderer( + RenderLimits(settings.max_upload_bytes, settings.max_image_pixels), + concurrency=settings.worker_concurrency, + timeout=settings.render_timeout_seconds, + memory_bytes=settings.render_memory_mb << 20, + tasks_per_child=settings.render_tasks_per_child, + ) + yield renderer + renderer.close() @provide(scope=Scope.APP) def faults(self, settings: PipelineSettings, logger: Logger) -> Faults: diff --git a/src/messenger/attachments/infrastructure/images.py b/src/messenger/attachments/infrastructure/images.py index 0877987..b4d0205 100644 --- a/src/messenger/attachments/infrastructure/images.py +++ b/src/messenger/attachments/infrastructure/images.py @@ -1,5 +1,10 @@ import asyncio +import resource +import sys import warnings +from collections.abc import Callable +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool from dataclasses import dataclass from io import BytesIO @@ -22,6 +27,7 @@ from messenger.attachments.domain.variants import ( THUMBNAIL_BOX, fit, ) +from messenger.shared.application.errors import Unavailable KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG} BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning) @@ -134,13 +140,67 @@ def render_variants(data: bytes, limits: RenderLimits) -> list[Rendition]: ] -class PillowVariantRenderer: - __slots__ = ("_limits", "_slots") +def limit_memory(limit_bytes: int) -> None: + if limit_bytes and sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes)) - def __init__(self, limits: RenderLimits, concurrency: int) -> None: + +type Render = Callable[[bytes, RenderLimits], list[Rendition]] + + +class ProcessVariantRenderer: + __slots__ = ("_idle", "_limits", "_memory", "_render", "_tasks", "_timeout") + + def __init__( + self, + limits: RenderLimits, + *, + concurrency: int, + timeout: float, + memory_bytes: int, + tasks_per_child: int, + render: Render = render_variants, + ) -> None: self._limits = limits - self._slots = asyncio.Semaphore(concurrency) + self._timeout = timeout + self._memory = memory_bytes + self._tasks = tasks_per_child + self._render = render + self._idle: asyncio.Queue[ProcessPoolExecutor] = asyncio.Queue() + for _ in range(concurrency): + self._idle.put_nowait(self._spawn()) async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: - async with self._slots: - return await asyncio.to_thread(render_variants, data, self._limits) + pool = await self._idle.get() + try: + future = asyncio.wrap_future(pool.submit(self._render, data, self._limits)) + return await asyncio.wait_for(future, self._timeout) + except TimeoutError as error: + pool = self._replace(pool) + raise ImageTooDemanding("рендер дольше допустимого времени") from error + except MemoryError as error: + raise ImageTooDemanding("рендер больше допустимой памяти") from error + except BrokenProcessPool as error: + pool = self._replace(pool) + raise Unavailable("процесс рендера завершился аварийно") from error + except asyncio.CancelledError: + pool = self._replace(pool) + raise + finally: + self._idle.put_nowait(pool) + + def close(self) -> None: + while not self._idle.empty(): + self._idle.get_nowait().kill_workers() + + def _replace(self, pool: ProcessPoolExecutor) -> ProcessPoolExecutor: + pool.kill_workers() + return self._spawn() + + def _spawn(self) -> ProcessPoolExecutor: + return ProcessPoolExecutor( + 1, + max_tasks_per_child=self._tasks, + initializer=limit_memory, + initargs=(self._memory,), + ) diff --git a/student-tests/api/test_pipeline.py b/student-tests/api/test_pipeline.py index 083d7f7..58e1cb3 100644 --- a/student-tests/api/test_pipeline.py +++ b/student-tests/api/test_pipeline.py @@ -10,6 +10,8 @@ from httpx import AsyncClient from test_attachments import storage, trio, upload from conftest import DATABASE, actor +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ImageTooDemanding from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy from messenger.attachments.entrypoints.schemas import JobMessage from messenger.shared.application.errors import Unavailable @@ -205,6 +207,30 @@ async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> N ) +class DemandingRenderer: + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + raise ImageTooDemanding("рендер дольше допустимого времени") + + +@pytest.mark.parametrize("cause", ["limit", "missing"]) +async def test_limits_and_lost_original_fail_without_retries( + client: AsyncClient, cause: str +) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + if cause == "limit": + pipeline.renderer = DemandingRenderer() + else: + del (await storage(client)).objects[f"staging/{attachment_id}"] + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["attempt"]) == ( + "failed", + "processing_failed", + 1, + ) + + async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py index 402fee6..718d0c6 100644 --- a/student-tests/pipeline.py +++ b/student-tests/pipeline.py @@ -18,11 +18,16 @@ from messenger.attachments.application.pipeline import ( PublishOutboxHandler, RecoverExpiredLeasesHandler, ) -from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer +from messenger.attachments.application.ports import ( + Faults, + ObjectStorage, + Rendition, + VariantRenderer, +) from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy from messenger.attachments.entrypoints.schemas import JobMessage from messenger.attachments.infrastructure.faults import EnvironmentFaults -from messenger.attachments.infrastructure.images import PillowVariantRenderer, RenderLimits +from messenger.attachments.infrastructure.images import RenderLimits, render_variants from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.clock import MonotonicUtcClock @@ -34,6 +39,13 @@ LIMITS = RenderLimits(10_485_760, 40_000_000) FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001) +class InlineRenderer: + __slots__ = () + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + return render_variants(data, LIMITS) + + class CapturingPublisher: __slots__ = ("bodies", "failing", "job_ids") @@ -65,7 +77,7 @@ class InlinePipeline: self.policy = policy self.lease = lease self.publisher = CapturingPublisher() - self.renderer: VariantRenderer = PillowVariantRenderer(LIMITS, 2) + self.renderer: VariantRenderer = InlineRenderer() self.faults: Faults = EnvironmentFaults("", NullLogger()) def uow(self) -> PostgresAttachmentsUnitOfWork: diff --git a/student-tests/unit/test_render_pool.py b/student-tests/unit/test_render_pool.py new file mode 100644 index 0000000..a22ec6d --- /dev/null +++ b/student-tests/unit/test_render_pool.py @@ -0,0 +1,91 @@ +import asyncio +import os +import sys +import time +from collections.abc import AsyncIterator + +import pytest + +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import ImageTooDemanding +from messenger.attachments.infrastructure.images import ( + ProcessVariantRenderer, + RenderLimits, + render_variants, +) +from messenger.shared.application.errors import Unavailable +from pictures import SAMPLE + +LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000) +HOG_BYTES = 512 << 20 + + +def misbehaving(data: bytes, limits: RenderLimits) -> list[Rendition]: + if data == b"slow": + time.sleep(30) + if data == b"crash": + os._exit(1) + if data == b"oom": + raise MemoryError + if data == b"hog": + bytearray(HOG_BYTES) + return render_variants(data, limits) + + +@pytest.fixture +async def renderer() -> AsyncIterator[ProcessVariantRenderer]: + pool = ProcessVariantRenderer( + LIMITS, + concurrency=1, + timeout=3.0, + memory_bytes=256 << 20, + tasks_per_child=10, + render=misbehaving, + ) + yield pool + pool.close() + + +async def test_renders_in_a_separate_process(renderer: ProcessVariantRenderer) -> None: + assert await renderer.render(SAMPLE, 1) == render_variants(SAMPLE, LIMITS) + + +async def test_timeout_kills_the_render_and_frees_the_slot( + renderer: ProcessVariantRenderer, +) -> None: + started = time.monotonic() + with pytest.raises(ImageTooDemanding): + await renderer.render(b"slow", 1) + assert time.monotonic() - started < 10 + assert len(await renderer.render(SAMPLE, 1)) == 3 + + +async def test_crashed_render_process_is_a_transient_error( + renderer: ProcessVariantRenderer, +) -> None: + with pytest.raises(Unavailable): + await renderer.render(b"crash", 1) + assert len(await renderer.render(SAMPLE, 1)) == 3 + + +async def test_memory_error_is_a_permanent_error(renderer: ProcessVariantRenderer) -> None: + with pytest.raises(ImageTooDemanding): + await renderer.render(b"oom", 1) + assert len(await renderer.render(SAMPLE, 1)) == 3 + + +async def test_cancelled_render_does_not_block_the_slot(renderer: ProcessVariantRenderer) -> None: + slow = asyncio.create_task(renderer.render(b"slow", 1)) + await asyncio.sleep(1) + slow.cancel() + with pytest.raises(asyncio.CancelledError): + await slow + assert len(await asyncio.wait_for(renderer.render(SAMPLE, 1), 10)) == 3 + + +@pytest.mark.skipif(sys.platform != "linux", reason="RLIMIT_AS применяется только в Linux") +async def test_address_space_limit_applies_to_the_render_process( + renderer: ProcessVariantRenderer, +) -> None: + with pytest.raises(ImageTooDemanding): + await renderer.render(b"hog", 1)