Files
mtusi2026lab2/student-tests/api/test_recovery.py
T
ArtyomandClaude Opus 5.5 b568fcf9c9 Add the failed registry, an explicit replay and failed staging retention
python -m messenger.admin is a new process assembly over the attachments
zone with only a database: "failed" lists failed jobs (served by the new
partial index jobs_failed_idx), "replay --attachment-id" moves a failed
attachment back to queued with the same job_id, attempt 1 and a fresh
outbox row, writing a replayed event with the operator. A ready, queued
or processing attachment is a noop (replay_noop event), a failed one
whose staging is already gone is expired. Every request is audited.

The relay now also removes the staging object of a failed attachment
once STAGING_RETENTION_SECONDS have passed since it failed; replay and
the cleanup take the same row lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:12:19 +03:00

307 lines
12 KiB
Python

import asyncio
import json
import subprocess
import sys
from datetime import timedelta
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from test_attachments import storage
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
from conftest import DATABASE
from fakes import NullLogger
from messenger.admin import serve
from messenger.attachments.application.pipeline import (
AFTER_CLAIM,
AFTER_COMMIT_BEFORE_ACK,
AFTER_COMMIT_BEFORE_PUBLISH,
AFTER_PUBLISH_BEFORE_CONFIRM,
AFTER_S3_WRITE,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
)
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.jobs import Outcome, ReplayResult
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.shared.application.errors import Unavailable
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
WORKER_POINTS = [
AFTER_CLAIM,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
AFTER_S3_WRITE,
AFTER_COMMIT_BEFORE_ACK,
]
class Crash(BaseException):
pass
class RecordingFaults:
__slots__ = ("crash_at", "reached")
def __init__(self, crash_at: str = "") -> None:
self.crash_at = crash_at
self.reached: list[str] = []
async def reach(self, point: str) -> None:
self.reached.append(point)
if point == self.crash_at:
raise Crash(point)
class SlowRenderer:
__slots__ = ("seconds",)
def __init__(self, seconds: float) -> None:
self.seconds = seconds
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
await asyncio.sleep(self.seconds)
return await InlineRenderer().render(data, pipeline_version)
class ExpiringFaults:
__slots__ = ("attachment_id", "pipeline")
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
self.pipeline = pipeline
self.attachment_id = attachment_id
async def reach(self, point: str) -> None:
if point == STALL_AFTER_CLAIM:
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' "
"WHERE attachment_id = $1",
self.attachment_id,
)
await self.pipeline.recover()
async def outbox(attachment_id: str) -> list[bool]:
found = await rows(
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
"WHERE j.attachment_id = $1 ORDER BY o.id",
attachment_id,
)
return [row["published"] for row in found]
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
faults = RecordingFaults()
pipeline.faults = faults
published = await pipeline.publish()
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
faults.reached.clear()
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
assert await pipeline.process(job) is Outcome.DONE
assert faults.reached == WORKER_POINTS
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
async def test_relay_crash_leaves_the_row_for_the_next_relay(
client: AsyncClient, point: str
) -> None:
pipeline = await pipeline_of(client)
await pipeline.drain()
_, _, attachment_id = await accepted(client)
pipeline.faults = RecordingFaults(point)
with pytest.raises(Crash):
await pipeline.publish()
assert await outbox(attachment_id) == [False]
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
pipeline.faults = RecordingFaults()
outcomes = await pipeline.drain()
assert Outcome.DONE in outcomes
assert await outbox(attachment_id) == [True]
assert (await state(attachment_id))["status"] == "ready"
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
with pytest.raises(Crash):
await pipeline.process(job)
objects = (await storage(client)).objects
written = stored(objects, attachment_id)
assert len(written) == 3
assert await variants(attachment_id) == []
assert (await state(attachment_id))["status"] == "processing"
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
attachment_id,
)
pipeline.faults = RecordingFaults()
assert await pipeline.recover() >= 1
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 2)
assert stored(objects, attachment_id) == written
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
with pytest.raises(Crash):
await pipeline.process(job)
before = await variants(attachment_id)
assert (await state(attachment_id))["status"] == "ready"
pipeline.faults = RecordingFaults()
assert await pipeline.process(job) is Outcome.DUPLICATE
assert await variants(attachment_id) == before
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
pipeline.renderer = SlowRenderer(2.5)
await pipeline.publish()
job = await job_of(attachment_id)
async def sweep() -> None:
for _ in range(4):
await asyncio.sleep(0.6)
await pipeline.recover()
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
assert outcome is Outcome.DONE
done = await state(attachment_id)
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
assert "lease_expired" not in await events(attachment_id)
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
assert await pipeline.process(job) is Outcome.FENCED
objects = (await storage(client)).objects
assert stored(objects, attachment_id) == []
assert f"originals/{attachment_id}" not in objects
assert (await state(attachment_id))["status"] == "queued"
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
pipeline.faults = RecordingFaults()
await pipeline.drain()
assert (await state(attachment_id))["status"] == "ready"
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
started = asyncio.get_running_loop().time()
await stall.reach(AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
with pytest.raises(Unavailable):
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
def test_environment_fault_crashes_the_process_with_137() -> None:
script = (
"import asyncio\n"
"from fakes import NullLogger\n"
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
".reach('after_commit_before_ack'))\n"
"print('survived')\n"
)
done = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
env={"PYTHONPATH": ":".join(sys.path)},
)
assert (done.returncode, done.stdout) == (137, "")
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.faults = FailingFaults()
await pipeline.drain()
pipeline.faults = RecordingFaults()
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
return pipeline, attachment_id
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
job = await job_of(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
queued = await state(attachment_id)
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
"queued",
"queued",
1,
None,
)
assert await outbox(attachment_id) == [True] * 5 + [False]
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 1)
assert (await job_of(attachment_id)).id == job.id
before = await variants(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
assert await outbox(attachment_id) == [True] * 6
assert await variants(attachment_id) == before
assert (await events(attachment_id))[-1] == "replay_noop"
assert "replayed" in await events(attachment_id)
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
_, _, waiting = await accepted(client)
objects = (await storage(client)).objects
await pipeline.clean(timedelta(days=1))
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
assert f"staging/{attachment_id}" in objects
await pipeline.clean(timedelta(0))
assert (await state(attachment_id))["staging_key"] is None
assert f"staging/{attachment_id}" not in objects
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
assert f"staging/{waiting}" in objects
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
assert (await state(attachment_id))["status"] == "failed"
async def test_operator_cli_lists_failed_jobs_and_replays(
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
_, attachment_id = await failed(client)
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
monkeypatch.setenv("LOG_LEVEL", "WARNING")
assert await serve(["failed", "--limit", "500"]) == 0
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
"retry_exhausted",
5,
"Unavailable",
)
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
report = json.loads(capsys.readouterr().out.splitlines()[-1])
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1