Render variants in killable processes with time and memory limits

The renderer adapter keeps one single-process pool per WORKER_CONCURRENCY
slot. A render that exceeds RENDER_TIMEOUT_SECONDS is killed and its slot
is respawned, the render process has RLIMIT_AS = RENDER_MEMORY_MB on
Linux, so both limits end in ImageTooDemanding, a permanent
processing_failed. A render process killed from outside is a transient
error. A cancelled render (lost lease, shutdown) kills its process too.
The worker container limit grows to 1g: two 448 MiB render processes
plus the worker itself.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 03:02:16 +03:00
co-authored by Claude Opus 5.5
parent 431b82fd43
commit f2242eb39d
10 changed files with 228 additions and 20 deletions
+7 -1
View File
@@ -63,5 +63,11 @@ JOB_BACKOFF_BASE_SECONDS=2
OUTBOX_POLL_SECONDS=1
SWEEP_INTERVAL_SECONDS=5
WORKER_CPUS=2
WORKER_MEMORY=768m
WORKER_MEMORY=1g
PIPELINE_DB_POOL_MAX_SIZE=4
# Solution-specific for lab5 grade 4: render time per job (the render process is killed after it),
# address space of one render process (WORKER_MEMORY >= WORKER_CONCURRENCY * RENDER_MEMORY_MB plus
# ~120 MiB of the worker itself), jobs per render process before it is replaced.
RENDER_TIMEOUT_SECONDS=30
RENDER_MEMORY_MB=448
RENDER_TASKS_PER_CHILD=100
+4 -1
View File
@@ -330,6 +330,9 @@ services:
environment:
<<: *pipeline-env
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
RENDER_TIMEOUT_SECONDS: ${RENDER_TIMEOUT_SECONDS:-30}
RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-448}
RENDER_TASKS_PER_CHILD: ${RENDER_TASKS_PER_CHILD:-100}
secrets:
- database_url
- source: broker_url_worker
@@ -340,7 +343,7 @@ services:
target: s3_secret_access_key
deploy:
resources:
limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-768m}"}
limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-1g}"}
k6:
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
@@ -6,9 +6,9 @@
## 2. Пределы worker
- [ ] 2.1 Адаптер `ProcessVariantRenderer`: слоты-процессы по `WORKER_CONCURRENCY`, таймаут с `kill_workers()`, `RLIMIT_AS` в процессе рендера, `BrokenProcessPool` → временная ошибка; настройки `RENDER_TIMEOUT_SECONDS`, `RENDER_MEMORY_MB`, `RENDER_TASKS_PER_CHILD`; закрытие пула при остановке worker
- [ ] 2.2 Классификация в `ProcessJobHandler`: `ImageTooDemanding` и отсутствующий оригинал → `processing_failed` без повторов
- [ ] 2.3 Проверка группы: тесты пула — результат, таймаут с восстановлением слота, падение процесса рендера → `Unavailable`, `MemoryError` → `ImageTooDemanding`, предел памяти применяется (Linux)
- [x] 2.1 Адаптер `ProcessVariantRenderer`: слоты-процессы по `WORKER_CONCURRENCY`, таймаут с `kill_workers()`, `RLIMIT_AS` в процессе рендера, `BrokenProcessPool` → временная ошибка; настройки `RENDER_TIMEOUT_SECONDS`, `RENDER_MEMORY_MB`, `RENDER_TASKS_PER_CHILD`; закрытие пула при остановке worker
- [x] 2.2 Классификация в `ProcessJobHandler`: `ImageTooDemanding` и отсутствующий оригинал → `processing_failed` без повторов
- [x] 2.3 Проверка группы: тесты пула — результат, таймаут с восстановлением слота, падение процесса рендера → `Unavailable`, `MemoryError` → `ImageTooDemanding`, предел памяти применяется (Linux)
## 3. Повторы и аренда
+1 -1
View File
@@ -81,7 +81,7 @@ ignore = [
]
[tool.ruff.lint.per-file-ignores]
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"]
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"]
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
+3
View File
@@ -42,4 +42,7 @@ class PipelineSettings(AttachmentsSettings):
shutdown_grace_seconds: float = Field(default=5.0, gt=0, le=9)
heartbeat_file: Path | None = None
heartbeat_interval_seconds: float = Field(default=5.0, gt=0, le=10)
render_timeout_seconds: float = Field(default=30.0, gt=0, le=120)
render_memory_mb: int = Field(default=448, ge=0, le=8192)
render_tasks_per_child: int = Field(default=100, ge=1, le=10_000)
fault_point: str = ""
+12 -5
View File
@@ -1,4 +1,4 @@
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Iterator
from contextlib import asynccontextmanager
from datetime import timedelta
@@ -41,7 +41,7 @@ from messenger.attachments.domain.jobs import RetryPolicy
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import (
PillowImageInspector,
PillowVariantRenderer,
ProcessVariantRenderer,
RenderLimits,
)
from messenger.attachments.infrastructure.postgres import (
@@ -173,9 +173,16 @@ class PipelineProvider(Provider):
class WorkerProvider(Provider):
@provide(scope=Scope.APP)
def renderer(self, settings: PipelineSettings) -> VariantRenderer:
limits = RenderLimits(settings.max_upload_bytes, settings.max_image_pixels)
return PillowVariantRenderer(limits, settings.worker_concurrency)
def renderer(self, settings: PipelineSettings) -> Iterator[VariantRenderer]:
renderer = ProcessVariantRenderer(
RenderLimits(settings.max_upload_bytes, settings.max_image_pixels),
concurrency=settings.worker_concurrency,
timeout=settings.render_timeout_seconds,
memory_bytes=settings.render_memory_mb << 20,
tasks_per_child=settings.render_tasks_per_child,
)
yield renderer
renderer.close()
@provide(scope=Scope.APP)
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
@@ -1,5 +1,10 @@
import asyncio
import resource
import sys
import warnings
from collections.abc import Callable
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures.process import BrokenProcessPool
from dataclasses import dataclass
from io import BytesIO
@@ -22,6 +27,7 @@ from messenger.attachments.domain.variants import (
THUMBNAIL_BOX,
fit,
)
from messenger.shared.application.errors import Unavailable
KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG}
BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning)
@@ -134,13 +140,67 @@ def render_variants(data: bytes, limits: RenderLimits) -> list[Rendition]:
]
class PillowVariantRenderer:
__slots__ = ("_limits", "_slots")
def limit_memory(limit_bytes: int) -> None:
if limit_bytes and sys.platform == "linux":
resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
def __init__(self, limits: RenderLimits, concurrency: int) -> None:
type Render = Callable[[bytes, RenderLimits], list[Rendition]]
class ProcessVariantRenderer:
__slots__ = ("_idle", "_limits", "_memory", "_render", "_tasks", "_timeout")
def __init__(
self,
limits: RenderLimits,
*,
concurrency: int,
timeout: float,
memory_bytes: int,
tasks_per_child: int,
render: Render = render_variants,
) -> None:
self._limits = limits
self._slots = asyncio.Semaphore(concurrency)
self._timeout = timeout
self._memory = memory_bytes
self._tasks = tasks_per_child
self._render = render
self._idle: asyncio.Queue[ProcessPoolExecutor] = asyncio.Queue()
for _ in range(concurrency):
self._idle.put_nowait(self._spawn())
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
async with self._slots:
return await asyncio.to_thread(render_variants, data, self._limits)
pool = await self._idle.get()
try:
future = asyncio.wrap_future(pool.submit(self._render, data, self._limits))
return await asyncio.wait_for(future, self._timeout)
except TimeoutError as error:
pool = self._replace(pool)
raise ImageTooDemanding("рендер дольше допустимого времени") from error
except MemoryError as error:
raise ImageTooDemanding("рендер больше допустимой памяти") from error
except BrokenProcessPool as error:
pool = self._replace(pool)
raise Unavailable("процесс рендера завершился аварийно") from error
except asyncio.CancelledError:
pool = self._replace(pool)
raise
finally:
self._idle.put_nowait(pool)
def close(self) -> None:
while not self._idle.empty():
self._idle.get_nowait().kill_workers()
def _replace(self, pool: ProcessPoolExecutor) -> ProcessPoolExecutor:
pool.kill_workers()
return self._spawn()
def _spawn(self) -> ProcessPoolExecutor:
return ProcessPoolExecutor(
1,
max_tasks_per_child=self._tasks,
initializer=limit_memory,
initargs=(self._memory,),
)
+26
View File
@@ -10,6 +10,8 @@ from httpx import AsyncClient
from test_attachments import storage, trio, upload
from conftest import DATABASE, actor
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.errors import ImageTooDemanding
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.shared.application.errors import Unavailable
@@ -205,6 +207,30 @@ async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> N
)
class DemandingRenderer:
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
raise ImageTooDemanding("рендер дольше допустимого времени")
@pytest.mark.parametrize("cause", ["limit", "missing"])
async def test_limits_and_lost_original_fail_without_retries(
client: AsyncClient, cause: str
) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
if cause == "limit":
pipeline.renderer = DemandingRenderer()
else:
del (await storage(client)).objects[f"staging/{attachment_id}"]
await pipeline.drain()
failed = await state(attachment_id)
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
"failed",
"processing_failed",
1,
)
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
+15 -3
View File
@@ -18,11 +18,16 @@ from messenger.attachments.application.pipeline import (
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
)
from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer
from messenger.attachments.application.ports import (
Faults,
ObjectStorage,
Rendition,
VariantRenderer,
)
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import PillowVariantRenderer, RenderLimits
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.clock import MonotonicUtcClock
@@ -34,6 +39,13 @@ LIMITS = RenderLimits(10_485_760, 40_000_000)
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
class InlineRenderer:
__slots__ = ()
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
return render_variants(data, LIMITS)
class CapturingPublisher:
__slots__ = ("bodies", "failing", "job_ids")
@@ -65,7 +77,7 @@ class InlinePipeline:
self.policy = policy
self.lease = lease
self.publisher = CapturingPublisher()
self.renderer: VariantRenderer = PillowVariantRenderer(LIMITS, 2)
self.renderer: VariantRenderer = InlineRenderer()
self.faults: Faults = EnvironmentFaults("", NullLogger())
def uow(self) -> PostgresAttachmentsUnitOfWork:
+91
View File
@@ -0,0 +1,91 @@
import asyncio
import os
import sys
import time
from collections.abc import AsyncIterator
import pytest
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.errors import ImageTooDemanding
from messenger.attachments.infrastructure.images import (
ProcessVariantRenderer,
RenderLimits,
render_variants,
)
from messenger.shared.application.errors import Unavailable
from pictures import SAMPLE
LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000)
HOG_BYTES = 512 << 20
def misbehaving(data: bytes, limits: RenderLimits) -> list[Rendition]:
if data == b"slow":
time.sleep(30)
if data == b"crash":
os._exit(1)
if data == b"oom":
raise MemoryError
if data == b"hog":
bytearray(HOG_BYTES)
return render_variants(data, limits)
@pytest.fixture
async def renderer() -> AsyncIterator[ProcessVariantRenderer]:
pool = ProcessVariantRenderer(
LIMITS,
concurrency=1,
timeout=3.0,
memory_bytes=256 << 20,
tasks_per_child=10,
render=misbehaving,
)
yield pool
pool.close()
async def test_renders_in_a_separate_process(renderer: ProcessVariantRenderer) -> None:
assert await renderer.render(SAMPLE, 1) == render_variants(SAMPLE, LIMITS)
async def test_timeout_kills_the_render_and_frees_the_slot(
renderer: ProcessVariantRenderer,
) -> None:
started = time.monotonic()
with pytest.raises(ImageTooDemanding):
await renderer.render(b"slow", 1)
assert time.monotonic() - started < 10
assert len(await renderer.render(SAMPLE, 1)) == 3
async def test_crashed_render_process_is_a_transient_error(
renderer: ProcessVariantRenderer,
) -> None:
with pytest.raises(Unavailable):
await renderer.render(b"crash", 1)
assert len(await renderer.render(SAMPLE, 1)) == 3
async def test_memory_error_is_a_permanent_error(renderer: ProcessVariantRenderer) -> None:
with pytest.raises(ImageTooDemanding):
await renderer.render(b"oom", 1)
assert len(await renderer.render(SAMPLE, 1)) == 3
async def test_cancelled_render_does_not_block_the_slot(renderer: ProcessVariantRenderer) -> None:
slow = asyncio.create_task(renderer.render(b"slow", 1))
await asyncio.sleep(1)
slow.cancel()
with pytest.raises(asyncio.CancelledError):
await slow
assert len(await asyncio.wait_for(renderer.render(SAMPLE, 1), 10)) == 3
@pytest.mark.skipif(sys.platform != "linux", reason="RLIMIT_AS применяется только в Linux")
async def test_address_space_limit_applies_to_the_render_process(
renderer: ProcessVariantRenderer,
) -> None:
with pytest.raises(ImageTooDemanding):
await renderer.render(b"hog", 1)