Files
mtusi2026lab2/student-tests/api/test_recovery.py
T
ArtyomandClaude Opus 5.5 5ae3d77152 Carry over the lab 5 solution
Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks,
scenario scripts and the CI job from the lab 5 solution commit 64d5c67,
keep the lab 5 Makefile targets on top of the unchanged skeleton
targets, and append the lab 5 solution variables to the skeleton
.env.example. The package and the application follow the contract
version 1.6.0, and the image, the throwaway test containers and the
broker vhost description are tagged lab6 so they do not collide with a
lab 5 stand on the same host.

This is the starting point of the lab 6 cycle: lint, own tests and the
offline skeleton check pass before any lab 6 change.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:51:58 +03:00

308 lines
12 KiB
Python

import asyncio
import json
import subprocess
import sys
from datetime import datetime, timedelta
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from test_attachments import storage
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
from conftest import DATABASE
from fakes import NullLogger
from messenger.admin import serve
from messenger.attachments.application.pipeline import (
AFTER_CLAIM,
AFTER_COMMIT_BEFORE_ACK,
AFTER_COMMIT_BEFORE_PUBLISH,
AFTER_PUBLISH_BEFORE_CONFIRM,
AFTER_S3_WRITE,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
)
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.jobs import Outcome, ReplayResult
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.shared.application.errors import Unavailable
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
WORKER_POINTS = [
AFTER_CLAIM,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
AFTER_S3_WRITE,
AFTER_COMMIT_BEFORE_ACK,
]
class Crash(BaseException):
pass
class RecordingFaults:
__slots__ = ("crash_at", "reached")
def __init__(self, crash_at: str = "") -> None:
self.crash_at = crash_at
self.reached: list[str] = []
async def reach(self, point: str) -> None:
self.reached.append(point)
if point == self.crash_at:
raise Crash(point)
class SlowRenderer:
__slots__ = ("seconds",)
def __init__(self, seconds: float) -> None:
self.seconds = seconds
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
await asyncio.sleep(self.seconds)
return await InlineRenderer().render(data, pipeline_version)
class ExpiringFaults:
__slots__ = ("attachment_id", "pipeline")
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
self.pipeline = pipeline
self.attachment_id = attachment_id
async def reach(self, point: str) -> None:
if point == STALL_AFTER_CLAIM:
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' "
"WHERE attachment_id = $1",
self.attachment_id,
)
await self.pipeline.recover()
async def outbox(attachment_id: str) -> list[bool]:
found = await rows(
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
"WHERE j.attachment_id = $1 ORDER BY o.id",
attachment_id,
)
return [row["published"] for row in found]
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
faults = RecordingFaults()
pipeline.faults = faults
published = await pipeline.publish()
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
faults.reached.clear()
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
assert await pipeline.process(job) is Outcome.DONE
assert faults.reached == WORKER_POINTS
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
async def test_relay_crash_leaves_the_row_for_the_next_relay(
client: AsyncClient, point: str
) -> None:
pipeline = await pipeline_of(client)
await pipeline.drain()
_, _, attachment_id = await accepted(client)
pipeline.faults = RecordingFaults(point)
with pytest.raises(Crash):
await pipeline.publish()
assert await outbox(attachment_id) == [False]
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
pipeline.faults = RecordingFaults()
outcomes = await pipeline.drain()
assert Outcome.DONE in outcomes
assert await outbox(attachment_id) == [True]
assert (await state(attachment_id))["status"] == "ready"
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
with pytest.raises(Crash):
await pipeline.process(job)
objects = (await storage(client)).objects
written = stored(objects, attachment_id)
assert len(written) == 3
assert await variants(attachment_id) == []
assert (await state(attachment_id))["status"] == "processing"
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
attachment_id,
)
pipeline.faults = RecordingFaults()
assert await pipeline.recover() >= 1
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 2)
assert stored(objects, attachment_id) == written
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
with pytest.raises(Crash):
await pipeline.process(job)
before = await variants(attachment_id)
assert (await state(attachment_id))["status"] == "ready"
pipeline.faults = RecordingFaults()
assert await pipeline.process(job) is Outcome.DUPLICATE
assert await variants(attachment_id) == before
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
pipeline.renderer = SlowRenderer(2.5)
await pipeline.publish()
job = await job_of(attachment_id)
async def sweep() -> None:
for _ in range(4):
await asyncio.sleep(0.6)
await pipeline.recover()
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
assert outcome is Outcome.DONE
done = await state(attachment_id)
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
assert "lease_expired" not in await events(attachment_id)
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
assert await pipeline.process(job) is Outcome.FENCED
objects = (await storage(client)).objects
assert stored(objects, attachment_id) == []
assert f"originals/{attachment_id}" not in objects
assert (await state(attachment_id))["status"] == "queued"
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
pipeline.faults = RecordingFaults()
await pipeline.drain()
assert (await state(attachment_id))["status"] == "ready"
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
started = asyncio.get_running_loop().time()
await stall.reach(AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
with pytest.raises(Unavailable):
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
def test_environment_fault_crashes_the_process_with_137() -> None:
script = (
"import asyncio\n"
"from fakes import NullLogger\n"
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
".reach('after_commit_before_ack'))\n"
"print('survived')\n"
)
done = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
env={"PYTHONPATH": ":".join(sys.path)},
)
assert (done.returncode, done.stdout) == (137, "")
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.faults = FailingFaults()
await pipeline.drain()
pipeline.faults = RecordingFaults()
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
return pipeline, attachment_id
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
job = await job_of(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
queued = await state(attachment_id)
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
"queued",
"queued",
1,
None,
)
assert await outbox(attachment_id) == [True] * 5 + [False]
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 1)
assert (await job_of(attachment_id)).id == job.id
before = await variants(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
assert await outbox(attachment_id) == [True] * 6
assert await variants(attachment_id) == before
assert (await events(attachment_id))[-1] == "replay_noop"
assert "replayed" in await events(attachment_id)
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
_, _, waiting = await accepted(client)
objects = (await storage(client)).objects
await pipeline.clean(timedelta(days=1))
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
assert f"staging/{attachment_id}" in objects
await pipeline.clean(timedelta(0))
assert (await state(attachment_id))["staging_key"] is None
assert f"staging/{attachment_id}" not in objects
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
assert f"staging/{waiting}" in objects
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
assert (await state(attachment_id))["status"] == "failed"
async def test_operator_cli_lists_failed_jobs_and_replays(
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
_, attachment_id = await failed(client)
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
monkeypatch.setenv("LOG_LEVEL", "WARNING")
assert await serve(["failed", "--limit", "500"]) == 0
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
assert datetime.fromisoformat(mine["failed_at"]).utcoffset() == timedelta(0)
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
"retry_exhausted",
5,
"Unavailable",
)
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
report = json.loads(capsys.readouterr().out.splitlines()[-1])
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1