forked from mtusi2026/mtusi2026lab2
make level4 runs the level 3 and level 4 scenarios of the pipeline suite and the invariants into evidence/level4.json. New scenarios: worker_limits (a 40 Mpx render under a 0.5 s timeout and a 200 MiB address space fails with processing_failed while the worker keeps running, then succeeds with the default limits under the container limit), retry_backoff_jitter (delays from the job journal stay within [d/2, d] and differ) and failed_retention (failed staging is kept, removed after the retention, then replay answers expired). The stalled holder must be fenced and the republished copy must be a no-op. The faults override passes the stall, render limits and retention through; the admin CLI is python -m messenger.admin. make failed, make replay ID=... and make dlq are the operator entry points; the lab 4 target becomes lab4-level4. CI runs level4 instead of level3. The registry prints ISO timestamps. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
308 lines
12 KiB
Python
308 lines
12 KiB
Python
import asyncio
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timedelta
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage
|
|
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
|
|
|
|
from conftest import DATABASE
|
|
from fakes import NullLogger
|
|
from messenger.admin import serve
|
|
from messenger.attachments.application.pipeline import (
|
|
AFTER_CLAIM,
|
|
AFTER_COMMIT_BEFORE_ACK,
|
|
AFTER_COMMIT_BEFORE_PUBLISH,
|
|
AFTER_PUBLISH_BEFORE_CONFIRM,
|
|
AFTER_S3_WRITE,
|
|
STALL_AFTER_CLAIM,
|
|
TRANSIENT_ERROR,
|
|
)
|
|
from messenger.attachments.application.ports import Rendition
|
|
from messenger.attachments.domain.jobs import Outcome, ReplayResult
|
|
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
|
from messenger.shared.application.errors import Unavailable
|
|
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
|
|
|
|
WORKER_POINTS = [
|
|
AFTER_CLAIM,
|
|
STALL_AFTER_CLAIM,
|
|
TRANSIENT_ERROR,
|
|
AFTER_S3_WRITE,
|
|
AFTER_COMMIT_BEFORE_ACK,
|
|
]
|
|
|
|
|
|
class Crash(BaseException):
|
|
pass
|
|
|
|
|
|
class RecordingFaults:
|
|
__slots__ = ("crash_at", "reached")
|
|
|
|
def __init__(self, crash_at: str = "") -> None:
|
|
self.crash_at = crash_at
|
|
self.reached: list[str] = []
|
|
|
|
async def reach(self, point: str) -> None:
|
|
self.reached.append(point)
|
|
if point == self.crash_at:
|
|
raise Crash(point)
|
|
|
|
|
|
class SlowRenderer:
|
|
__slots__ = ("seconds",)
|
|
|
|
def __init__(self, seconds: float) -> None:
|
|
self.seconds = seconds
|
|
|
|
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
|
await asyncio.sleep(self.seconds)
|
|
return await InlineRenderer().render(data, pipeline_version)
|
|
|
|
|
|
class ExpiringFaults:
|
|
__slots__ = ("attachment_id", "pipeline")
|
|
|
|
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
|
|
self.pipeline = pipeline
|
|
self.attachment_id = attachment_id
|
|
|
|
async def reach(self, point: str) -> None:
|
|
if point == STALL_AFTER_CLAIM:
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' "
|
|
"WHERE attachment_id = $1",
|
|
self.attachment_id,
|
|
)
|
|
await self.pipeline.recover()
|
|
|
|
|
|
async def outbox(attachment_id: str) -> list[bool]:
|
|
found = await rows(
|
|
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = $1 ORDER BY o.id",
|
|
attachment_id,
|
|
)
|
|
return [row["published"] for row in found]
|
|
|
|
|
|
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
|
|
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
|
|
|
|
|
|
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
faults = RecordingFaults()
|
|
pipeline.faults = faults
|
|
published = await pipeline.publish()
|
|
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
|
|
faults.reached.clear()
|
|
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
|
|
assert await pipeline.process(job) is Outcome.DONE
|
|
assert faults.reached == WORKER_POINTS
|
|
|
|
|
|
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
|
|
async def test_relay_crash_leaves_the_row_for_the_next_relay(
|
|
client: AsyncClient, point: str
|
|
) -> None:
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline.faults = RecordingFaults(point)
|
|
with pytest.raises(Crash):
|
|
await pipeline.publish()
|
|
assert await outbox(attachment_id) == [False]
|
|
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
|
|
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
|
|
pipeline.faults = RecordingFaults()
|
|
outcomes = await pipeline.drain()
|
|
assert Outcome.DONE in outcomes
|
|
assert await outbox(attachment_id) == [True]
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
|
|
with pytest.raises(Crash):
|
|
await pipeline.process(job)
|
|
objects = (await storage(client)).objects
|
|
written = stored(objects, attachment_id)
|
|
assert len(written) == 3
|
|
assert await variants(attachment_id) == []
|
|
assert (await state(attachment_id))["status"] == "processing"
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
pipeline.faults = RecordingFaults()
|
|
assert await pipeline.recover() >= 1
|
|
await pipeline.drain()
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["attempt"]) == ("ready", 2)
|
|
assert stored(objects, attachment_id) == written
|
|
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
|
|
|
|
|
|
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
|
|
with pytest.raises(Crash):
|
|
await pipeline.process(job)
|
|
before = await variants(attachment_id)
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
pipeline.faults = RecordingFaults()
|
|
assert await pipeline.process(job) is Outcome.DUPLICATE
|
|
assert await variants(attachment_id) == before
|
|
|
|
|
|
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
|
|
pipeline.renderer = SlowRenderer(2.5)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
|
|
async def sweep() -> None:
|
|
for _ in range(4):
|
|
await asyncio.sleep(0.6)
|
|
await pipeline.recover()
|
|
|
|
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
|
|
assert outcome is Outcome.DONE
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
|
|
assert "lease_expired" not in await events(attachment_id)
|
|
|
|
|
|
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
|
|
assert await pipeline.process(job) is Outcome.FENCED
|
|
objects = (await storage(client)).objects
|
|
assert stored(objects, attachment_id) == []
|
|
assert f"originals/{attachment_id}" not in objects
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
|
|
pipeline.faults = RecordingFaults()
|
|
await pipeline.drain()
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
|
|
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
|
|
started = asyncio.get_running_loop().time()
|
|
await stall.reach(AFTER_CLAIM)
|
|
await stall.reach(STALL_AFTER_CLAIM)
|
|
await stall.reach(STALL_AFTER_CLAIM)
|
|
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
|
|
with pytest.raises(Unavailable):
|
|
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
|
|
|
|
|
|
def test_environment_fault_crashes_the_process_with_137() -> None:
|
|
script = (
|
|
"import asyncio\n"
|
|
"from fakes import NullLogger\n"
|
|
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
|
|
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
|
|
".reach('after_commit_before_ack'))\n"
|
|
"print('survived')\n"
|
|
)
|
|
done = subprocess.run(
|
|
[sys.executable, "-c", script],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
env={"PYTHONPATH": ":".join(sys.path)},
|
|
)
|
|
assert (done.returncode, done.stdout) == (137, "")
|
|
|
|
|
|
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
pipeline.faults = RecordingFaults()
|
|
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
|
|
return pipeline, attachment_id
|
|
|
|
|
|
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
|
|
pipeline, attachment_id = await failed(client)
|
|
job = await job_of(attachment_id)
|
|
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
|
|
"queued",
|
|
"queued",
|
|
1,
|
|
None,
|
|
)
|
|
assert await outbox(attachment_id) == [True] * 5 + [False]
|
|
await pipeline.drain()
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["attempt"]) == ("ready", 1)
|
|
assert (await job_of(attachment_id)).id == job.id
|
|
before = await variants(attachment_id)
|
|
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
|
|
assert await outbox(attachment_id) == [True] * 6
|
|
assert await variants(attachment_id) == before
|
|
assert (await events(attachment_id))[-1] == "replay_noop"
|
|
assert "replayed" in await events(attachment_id)
|
|
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
|
|
|
|
|
|
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
|
|
pipeline, attachment_id = await failed(client)
|
|
_, _, waiting = await accepted(client)
|
|
objects = (await storage(client)).objects
|
|
await pipeline.clean(timedelta(days=1))
|
|
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
|
|
assert f"staging/{attachment_id}" in objects
|
|
await pipeline.clean(timedelta(0))
|
|
assert (await state(attachment_id))["staging_key"] is None
|
|
assert f"staging/{attachment_id}" not in objects
|
|
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
|
|
assert f"staging/{waiting}" in objects
|
|
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
|
|
assert (await state(attachment_id))["status"] == "failed"
|
|
|
|
|
|
async def test_operator_cli_lists_failed_jobs_and_replays(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
_, attachment_id = await failed(client)
|
|
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
|
|
monkeypatch.setenv("LOG_LEVEL", "WARNING")
|
|
assert await serve(["failed", "--limit", "500"]) == 0
|
|
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
|
|
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
|
|
assert datetime.fromisoformat(mine["failed_at"]).utcoffset() == timedelta(0)
|
|
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
|
|
"retry_exhausted",
|
|
5,
|
|
"Unavailable",
|
|
)
|
|
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
|
|
report = json.loads(capsys.readouterr().out.splitlines()[-1])
|
|
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
|
|
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1
|