Render variants in killable processes with time and memory limits
The renderer adapter keeps one single-process pool per WORKER_CONCURRENCY slot. A render that exceeds RENDER_TIMEOUT_SECONDS is killed and its slot is respawned, the render process has RLIMIT_AS = RENDER_MEMORY_MB on Linux, so both limits end in ImageTooDemanding, a permanent processing_failed. A render process killed from outside is a transient error. A cancelled render (lost lease, shutdown) kills its process too. The worker container limit grows to 1g: two 448 MiB render processes plus the worker itself. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
431b82fd43
commit
f2242eb39d
+7
-1
@@ -63,5 +63,11 @@ JOB_BACKOFF_BASE_SECONDS=2
|
||||
OUTBOX_POLL_SECONDS=1
|
||||
SWEEP_INTERVAL_SECONDS=5
|
||||
WORKER_CPUS=2
|
||||
WORKER_MEMORY=768m
|
||||
WORKER_MEMORY=1g
|
||||
PIPELINE_DB_POOL_MAX_SIZE=4
|
||||
# Solution-specific for lab5 grade 4: render time per job (the render process is killed after it),
|
||||
# address space of one render process (WORKER_MEMORY >= WORKER_CONCURRENCY * RENDER_MEMORY_MB plus
|
||||
# ~120 MiB of the worker itself), jobs per render process before it is replaced.
|
||||
RENDER_TIMEOUT_SECONDS=30
|
||||
RENDER_MEMORY_MB=448
|
||||
RENDER_TASKS_PER_CHILD=100
|
||||
|
||||
+4
-1
@@ -330,6 +330,9 @@ services:
|
||||
environment:
|
||||
<<: *pipeline-env
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
RENDER_TIMEOUT_SECONDS: ${RENDER_TIMEOUT_SECONDS:-30}
|
||||
RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-448}
|
||||
RENDER_TASKS_PER_CHILD: ${RENDER_TASKS_PER_CHILD:-100}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: broker_url_worker
|
||||
@@ -340,7 +343,7 @@ services:
|
||||
target: s3_secret_access_key
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-768m}"}
|
||||
limits: {cpus: "${WORKER_CPUS:-2}", memory: "${WORKER_MEMORY:-1g}"}
|
||||
|
||||
k6:
|
||||
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
|
||||
|
||||
@@ -6,9 +6,9 @@
|
||||
|
||||
## 2. Пределы worker
|
||||
|
||||
- [ ] 2.1 Адаптер `ProcessVariantRenderer`: слоты-процессы по `WORKER_CONCURRENCY`, таймаут с `kill_workers()`, `RLIMIT_AS` в процессе рендера, `BrokenProcessPool` → временная ошибка; настройки `RENDER_TIMEOUT_SECONDS`, `RENDER_MEMORY_MB`, `RENDER_TASKS_PER_CHILD`; закрытие пула при остановке worker
|
||||
- [ ] 2.2 Классификация в `ProcessJobHandler`: `ImageTooDemanding` и отсутствующий оригинал → `processing_failed` без повторов
|
||||
- [ ] 2.3 Проверка группы: тесты пула — результат, таймаут с восстановлением слота, падение процесса рендера → `Unavailable`, `MemoryError` → `ImageTooDemanding`, предел памяти применяется (Linux)
|
||||
- [x] 2.1 Адаптер `ProcessVariantRenderer`: слоты-процессы по `WORKER_CONCURRENCY`, таймаут с `kill_workers()`, `RLIMIT_AS` в процессе рендера, `BrokenProcessPool` → временная ошибка; настройки `RENDER_TIMEOUT_SECONDS`, `RENDER_MEMORY_MB`, `RENDER_TASKS_PER_CHILD`; закрытие пула при остановке worker
|
||||
- [x] 2.2 Классификация в `ProcessJobHandler`: `ImageTooDemanding` и отсутствующий оригинал → `processing_failed` без повторов
|
||||
- [x] 2.3 Проверка группы: тесты пула — результат, таймаут с восстановлением слота, падение процесса рендера → `Unavailable`, `MemoryError` → `ImageTooDemanding`, предел памяти применяется (Linux)
|
||||
|
||||
## 3. Повторы и аренда
|
||||
|
||||
|
||||
+1
-1
@@ -81,7 +81,7 @@ ignore = [
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"]
|
||||
"student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "ARG002", "T20", "TID251"]
|
||||
"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"]
|
||||
"scripts/pipeline_suite.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402", "S608"]
|
||||
"scripts/pipeline_evidence.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251", "E402"]
|
||||
|
||||
@@ -42,4 +42,7 @@ class PipelineSettings(AttachmentsSettings):
|
||||
shutdown_grace_seconds: float = Field(default=5.0, gt=0, le=9)
|
||||
heartbeat_file: Path | None = None
|
||||
heartbeat_interval_seconds: float = Field(default=5.0, gt=0, le=10)
|
||||
render_timeout_seconds: float = Field(default=30.0, gt=0, le=120)
|
||||
render_memory_mb: int = Field(default=448, ge=0, le=8192)
|
||||
render_tasks_per_child: int = Field(default=100, ge=1, le=10_000)
|
||||
fault_point: str = ""
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import timedelta
|
||||
|
||||
@@ -41,7 +41,7 @@ from messenger.attachments.domain.jobs import RetryPolicy
|
||||
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
||||
from messenger.attachments.infrastructure.images import (
|
||||
PillowImageInspector,
|
||||
PillowVariantRenderer,
|
||||
ProcessVariantRenderer,
|
||||
RenderLimits,
|
||||
)
|
||||
from messenger.attachments.infrastructure.postgres import (
|
||||
@@ -173,9 +173,16 @@ class PipelineProvider(Provider):
|
||||
|
||||
class WorkerProvider(Provider):
|
||||
@provide(scope=Scope.APP)
|
||||
def renderer(self, settings: PipelineSettings) -> VariantRenderer:
|
||||
limits = RenderLimits(settings.max_upload_bytes, settings.max_image_pixels)
|
||||
return PillowVariantRenderer(limits, settings.worker_concurrency)
|
||||
def renderer(self, settings: PipelineSettings) -> Iterator[VariantRenderer]:
|
||||
renderer = ProcessVariantRenderer(
|
||||
RenderLimits(settings.max_upload_bytes, settings.max_image_pixels),
|
||||
concurrency=settings.worker_concurrency,
|
||||
timeout=settings.render_timeout_seconds,
|
||||
memory_bytes=settings.render_memory_mb << 20,
|
||||
tasks_per_child=settings.render_tasks_per_child,
|
||||
)
|
||||
yield renderer
|
||||
renderer.close()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import asyncio
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
|
||||
@@ -22,6 +27,7 @@ from messenger.attachments.domain.variants import (
|
||||
THUMBNAIL_BOX,
|
||||
fit,
|
||||
)
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
|
||||
KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG}
|
||||
BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning)
|
||||
@@ -134,13 +140,67 @@ def render_variants(data: bytes, limits: RenderLimits) -> list[Rendition]:
|
||||
]
|
||||
|
||||
|
||||
class PillowVariantRenderer:
|
||||
__slots__ = ("_limits", "_slots")
|
||||
def limit_memory(limit_bytes: int) -> None:
|
||||
if limit_bytes and sys.platform == "linux":
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
|
||||
|
||||
def __init__(self, limits: RenderLimits, concurrency: int) -> None:
|
||||
|
||||
type Render = Callable[[bytes, RenderLimits], list[Rendition]]
|
||||
|
||||
|
||||
class ProcessVariantRenderer:
|
||||
__slots__ = ("_idle", "_limits", "_memory", "_render", "_tasks", "_timeout")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
limits: RenderLimits,
|
||||
*,
|
||||
concurrency: int,
|
||||
timeout: float,
|
||||
memory_bytes: int,
|
||||
tasks_per_child: int,
|
||||
render: Render = render_variants,
|
||||
) -> None:
|
||||
self._limits = limits
|
||||
self._slots = asyncio.Semaphore(concurrency)
|
||||
self._timeout = timeout
|
||||
self._memory = memory_bytes
|
||||
self._tasks = tasks_per_child
|
||||
self._render = render
|
||||
self._idle: asyncio.Queue[ProcessPoolExecutor] = asyncio.Queue()
|
||||
for _ in range(concurrency):
|
||||
self._idle.put_nowait(self._spawn())
|
||||
|
||||
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
||||
async with self._slots:
|
||||
return await asyncio.to_thread(render_variants, data, self._limits)
|
||||
pool = await self._idle.get()
|
||||
try:
|
||||
future = asyncio.wrap_future(pool.submit(self._render, data, self._limits))
|
||||
return await asyncio.wait_for(future, self._timeout)
|
||||
except TimeoutError as error:
|
||||
pool = self._replace(pool)
|
||||
raise ImageTooDemanding("рендер дольше допустимого времени") from error
|
||||
except MemoryError as error:
|
||||
raise ImageTooDemanding("рендер больше допустимой памяти") from error
|
||||
except BrokenProcessPool as error:
|
||||
pool = self._replace(pool)
|
||||
raise Unavailable("процесс рендера завершился аварийно") from error
|
||||
except asyncio.CancelledError:
|
||||
pool = self._replace(pool)
|
||||
raise
|
||||
finally:
|
||||
self._idle.put_nowait(pool)
|
||||
|
||||
def close(self) -> None:
|
||||
while not self._idle.empty():
|
||||
self._idle.get_nowait().kill_workers()
|
||||
|
||||
def _replace(self, pool: ProcessPoolExecutor) -> ProcessPoolExecutor:
|
||||
pool.kill_workers()
|
||||
return self._spawn()
|
||||
|
||||
def _spawn(self) -> ProcessPoolExecutor:
|
||||
return ProcessPoolExecutor(
|
||||
1,
|
||||
max_tasks_per_child=self._tasks,
|
||||
initializer=limit_memory,
|
||||
initargs=(self._memory,),
|
||||
)
|
||||
|
||||
@@ -10,6 +10,8 @@ from httpx import AsyncClient
|
||||
from test_attachments import storage, trio, upload
|
||||
|
||||
from conftest import DATABASE, actor
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.errors import ImageTooDemanding
|
||||
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
||||
from messenger.attachments.entrypoints.schemas import JobMessage
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
@@ -205,6 +207,30 @@ async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> N
|
||||
)
|
||||
|
||||
|
||||
class DemandingRenderer:
|
||||
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
||||
raise ImageTooDemanding("рендер дольше допустимого времени")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cause", ["limit", "missing"])
|
||||
async def test_limits_and_lost_original_fail_without_retries(
|
||||
client: AsyncClient, cause: str
|
||||
) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client)
|
||||
if cause == "limit":
|
||||
pipeline.renderer = DemandingRenderer()
|
||||
else:
|
||||
del (await storage(client)).objects[f"staging/{attachment_id}"]
|
||||
await pipeline.drain()
|
||||
failed = await state(attachment_id)
|
||||
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
|
||||
"failed",
|
||||
"processing_failed",
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client)
|
||||
|
||||
@@ -18,11 +18,16 @@ from messenger.attachments.application.pipeline import (
|
||||
PublishOutboxHandler,
|
||||
RecoverExpiredLeasesHandler,
|
||||
)
|
||||
from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer
|
||||
from messenger.attachments.application.ports import (
|
||||
Faults,
|
||||
ObjectStorage,
|
||||
Rendition,
|
||||
VariantRenderer,
|
||||
)
|
||||
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
||||
from messenger.attachments.entrypoints.schemas import JobMessage
|
||||
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
||||
from messenger.attachments.infrastructure.images import PillowVariantRenderer, RenderLimits
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
||||
@@ -34,6 +39,13 @@ LIMITS = RenderLimits(10_485_760, 40_000_000)
|
||||
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
|
||||
|
||||
|
||||
class InlineRenderer:
|
||||
__slots__ = ()
|
||||
|
||||
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
||||
return render_variants(data, LIMITS)
|
||||
|
||||
|
||||
class CapturingPublisher:
|
||||
__slots__ = ("bodies", "failing", "job_ids")
|
||||
|
||||
@@ -65,7 +77,7 @@ class InlinePipeline:
|
||||
self.policy = policy
|
||||
self.lease = lease
|
||||
self.publisher = CapturingPublisher()
|
||||
self.renderer: VariantRenderer = PillowVariantRenderer(LIMITS, 2)
|
||||
self.renderer: VariantRenderer = InlineRenderer()
|
||||
self.faults: Faults = EnvironmentFaults("", NullLogger())
|
||||
|
||||
def uow(self) -> PostgresAttachmentsUnitOfWork:
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.errors import ImageTooDemanding
|
||||
from messenger.attachments.infrastructure.images import (
|
||||
ProcessVariantRenderer,
|
||||
RenderLimits,
|
||||
render_variants,
|
||||
)
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from pictures import SAMPLE
|
||||
|
||||
LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000)
|
||||
HOG_BYTES = 512 << 20
|
||||
|
||||
|
||||
def misbehaving(data: bytes, limits: RenderLimits) -> list[Rendition]:
|
||||
if data == b"slow":
|
||||
time.sleep(30)
|
||||
if data == b"crash":
|
||||
os._exit(1)
|
||||
if data == b"oom":
|
||||
raise MemoryError
|
||||
if data == b"hog":
|
||||
bytearray(HOG_BYTES)
|
||||
return render_variants(data, limits)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def renderer() -> AsyncIterator[ProcessVariantRenderer]:
|
||||
pool = ProcessVariantRenderer(
|
||||
LIMITS,
|
||||
concurrency=1,
|
||||
timeout=3.0,
|
||||
memory_bytes=256 << 20,
|
||||
tasks_per_child=10,
|
||||
render=misbehaving,
|
||||
)
|
||||
yield pool
|
||||
pool.close()
|
||||
|
||||
|
||||
async def test_renders_in_a_separate_process(renderer: ProcessVariantRenderer) -> None:
|
||||
assert await renderer.render(SAMPLE, 1) == render_variants(SAMPLE, LIMITS)
|
||||
|
||||
|
||||
async def test_timeout_kills_the_render_and_frees_the_slot(
|
||||
renderer: ProcessVariantRenderer,
|
||||
) -> None:
|
||||
started = time.monotonic()
|
||||
with pytest.raises(ImageTooDemanding):
|
||||
await renderer.render(b"slow", 1)
|
||||
assert time.monotonic() - started < 10
|
||||
assert len(await renderer.render(SAMPLE, 1)) == 3
|
||||
|
||||
|
||||
async def test_crashed_render_process_is_a_transient_error(
|
||||
renderer: ProcessVariantRenderer,
|
||||
) -> None:
|
||||
with pytest.raises(Unavailable):
|
||||
await renderer.render(b"crash", 1)
|
||||
assert len(await renderer.render(SAMPLE, 1)) == 3
|
||||
|
||||
|
||||
async def test_memory_error_is_a_permanent_error(renderer: ProcessVariantRenderer) -> None:
|
||||
with pytest.raises(ImageTooDemanding):
|
||||
await renderer.render(b"oom", 1)
|
||||
assert len(await renderer.render(SAMPLE, 1)) == 3
|
||||
|
||||
|
||||
async def test_cancelled_render_does_not_block_the_slot(renderer: ProcessVariantRenderer) -> None:
|
||||
slow = asyncio.create_task(renderer.render(b"slow", 1))
|
||||
await asyncio.sleep(1)
|
||||
slow.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await slow
|
||||
assert len(await asyncio.wait_for(renderer.render(SAMPLE, 1), 10)) == 3
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="RLIMIT_AS применяется только в Linux")
|
||||
async def test_address_space_limit_applies_to_the_render_process(
|
||||
renderer: ProcessVariantRenderer,
|
||||
) -> None:
|
||||
with pytest.raises(ImageTooDemanding):
|
||||
await renderer.render(b"hog", 1)
|
||||
Reference in New Issue
Block a user