Files
mtusi2026labs/student-tests/api/test_recovery.py
T
ArtyomandClaude Opus 5.5 b9d28d190e Run the level 4 scenarios with one command
make level4 runs the level 3 and level 4 scenarios of the pipeline suite
and the invariants into evidence/level4.json. New scenarios: worker_limits
(a 40 Mpx render under a 0.5 s timeout and a 200 MiB address space fails
with processing_failed while the worker keeps running, then succeeds with
the default limits under the container limit), retry_backoff_jitter
(delays from the job journal stay within [d/2, d] and differ) and
failed_retention (failed staging is kept, removed after the retention,
then replay answers expired). The stalled holder must be fenced and the
republished copy must be a no-op. The faults override passes the stall,
render limits and retention through; the admin CLI is python -m
messenger.admin. make failed, make replay ID=... and make dlq are the
operator entry points; the lab 4 target becomes lab4-level4. CI runs
level4 instead of level3. The registry prints ISO timestamps.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:19:28 +03:00

308 lines
12 KiB
Python

import asyncio
import json
import subprocess
import sys
from datetime import datetime, timedelta
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from test_attachments import storage
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
from conftest import DATABASE
from fakes import NullLogger
from messenger.admin import serve
from messenger.attachments.application.pipeline import (
AFTER_CLAIM,
AFTER_COMMIT_BEFORE_ACK,
AFTER_COMMIT_BEFORE_PUBLISH,
AFTER_PUBLISH_BEFORE_CONFIRM,
AFTER_S3_WRITE,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
)
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.jobs import Outcome, ReplayResult
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.shared.application.errors import Unavailable
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
WORKER_POINTS = [
AFTER_CLAIM,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
AFTER_S3_WRITE,
AFTER_COMMIT_BEFORE_ACK,
]
class Crash(BaseException):
pass
class RecordingFaults:
__slots__ = ("crash_at", "reached")
def __init__(self, crash_at: str = "") -> None:
self.crash_at = crash_at
self.reached: list[str] = []
async def reach(self, point: str) -> None:
self.reached.append(point)
if point == self.crash_at:
raise Crash(point)
class SlowRenderer:
__slots__ = ("seconds",)
def __init__(self, seconds: float) -> None:
self.seconds = seconds
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
await asyncio.sleep(self.seconds)
return await InlineRenderer().render(data, pipeline_version)
class ExpiringFaults:
__slots__ = ("attachment_id", "pipeline")
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
self.pipeline = pipeline
self.attachment_id = attachment_id
async def reach(self, point: str) -> None:
if point == STALL_AFTER_CLAIM:
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' "
"WHERE attachment_id = $1",
self.attachment_id,
)
await self.pipeline.recover()
async def outbox(attachment_id: str) -> list[bool]:
found = await rows(
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
"WHERE j.attachment_id = $1 ORDER BY o.id",
attachment_id,
)
return [row["published"] for row in found]
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
faults = RecordingFaults()
pipeline.faults = faults
published = await pipeline.publish()
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
faults.reached.clear()
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
assert await pipeline.process(job) is Outcome.DONE
assert faults.reached == WORKER_POINTS
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
async def test_relay_crash_leaves_the_row_for_the_next_relay(
client: AsyncClient, point: str
) -> None:
pipeline = await pipeline_of(client)
await pipeline.drain()
_, _, attachment_id = await accepted(client)
pipeline.faults = RecordingFaults(point)
with pytest.raises(Crash):
await pipeline.publish()
assert await outbox(attachment_id) == [False]
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
pipeline.faults = RecordingFaults()
outcomes = await pipeline.drain()
assert Outcome.DONE in outcomes
assert await outbox(attachment_id) == [True]
assert (await state(attachment_id))["status"] == "ready"
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
with pytest.raises(Crash):
await pipeline.process(job)
objects = (await storage(client)).objects
written = stored(objects, attachment_id)
assert len(written) == 3
assert await variants(attachment_id) == []
assert (await state(attachment_id))["status"] == "processing"
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
attachment_id,
)
pipeline.faults = RecordingFaults()
assert await pipeline.recover() >= 1
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 2)
assert stored(objects, attachment_id) == written
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
with pytest.raises(Crash):
await pipeline.process(job)
before = await variants(attachment_id)
assert (await state(attachment_id))["status"] == "ready"
pipeline.faults = RecordingFaults()
assert await pipeline.process(job) is Outcome.DUPLICATE
assert await variants(attachment_id) == before
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
pipeline.renderer = SlowRenderer(2.5)
await pipeline.publish()
job = await job_of(attachment_id)
async def sweep() -> None:
for _ in range(4):
await asyncio.sleep(0.6)
await pipeline.recover()
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
assert outcome is Outcome.DONE
done = await state(attachment_id)
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
assert "lease_expired" not in await events(attachment_id)
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
assert await pipeline.process(job) is Outcome.FENCED
objects = (await storage(client)).objects
assert stored(objects, attachment_id) == []
assert f"originals/{attachment_id}" not in objects
assert (await state(attachment_id))["status"] == "queued"
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
pipeline.faults = RecordingFaults()
await pipeline.drain()
assert (await state(attachment_id))["status"] == "ready"
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
started = asyncio.get_running_loop().time()
await stall.reach(AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
with pytest.raises(Unavailable):
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
def test_environment_fault_crashes_the_process_with_137() -> None:
script = (
"import asyncio\n"
"from fakes import NullLogger\n"
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
".reach('after_commit_before_ack'))\n"
"print('survived')\n"
)
done = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
env={"PYTHONPATH": ":".join(sys.path)},
)
assert (done.returncode, done.stdout) == (137, "")
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.faults = FailingFaults()
await pipeline.drain()
pipeline.faults = RecordingFaults()
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
return pipeline, attachment_id
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
job = await job_of(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
queued = await state(attachment_id)
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
"queued",
"queued",
1,
None,
)
assert await outbox(attachment_id) == [True] * 5 + [False]
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 1)
assert (await job_of(attachment_id)).id == job.id
before = await variants(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
assert await outbox(attachment_id) == [True] * 6
assert await variants(attachment_id) == before
assert (await events(attachment_id))[-1] == "replay_noop"
assert "replayed" in await events(attachment_id)
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
_, _, waiting = await accepted(client)
objects = (await storage(client)).objects
await pipeline.clean(timedelta(days=1))
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
assert f"staging/{attachment_id}" in objects
await pipeline.clean(timedelta(0))
assert (await state(attachment_id))["staging_key"] is None
assert f"staging/{attachment_id}" not in objects
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
assert f"staging/{waiting}" in objects
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
assert (await state(attachment_id))["status"] == "failed"
async def test_operator_cli_lists_failed_jobs_and_replays(
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
_, attachment_id = await failed(client)
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
monkeypatch.setenv("LOG_LEVEL", "WARNING")
assert await serve(["failed", "--limit", "500"]) == 0
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
assert datetime.fromisoformat(mine["failed_at"]).utcoffset() == timedelta(0)
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
"retry_exhausted",
5,
"Unavailable",
)
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
report = json.loads(capsys.readouterr().out.splitlines()[-1])
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1