Bring the application, own tests, Dockerfile, toolchain, compose stack, HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks, scenario scripts and the CI job from the lab 5 solution commit 64d5c67, keep the lab 5 Makefile targets on top of the unchanged skeleton targets, and append the lab 5 solution variables to the skeleton .env.example. The package and the application follow the contract version 1.6.0, and the image, the throwaway test containers and the broker vhost description are tagged lab6 so they do not collide with a lab 5 stand on the same host. This is the starting point of the lab 6 cycle: lint, own tests and the offline skeleton check pass before any lab 6 change. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
308 lines
12 KiB
Python
308 lines
12 KiB
Python
import asyncio
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timedelta
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage
|
|
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
|
|
|
|
from conftest import DATABASE
|
|
from fakes import NullLogger
|
|
from messenger.admin import serve
|
|
from messenger.attachments.application.pipeline import (
|
|
AFTER_CLAIM,
|
|
AFTER_COMMIT_BEFORE_ACK,
|
|
AFTER_COMMIT_BEFORE_PUBLISH,
|
|
AFTER_PUBLISH_BEFORE_CONFIRM,
|
|
AFTER_S3_WRITE,
|
|
STALL_AFTER_CLAIM,
|
|
TRANSIENT_ERROR,
|
|
)
|
|
from messenger.attachments.application.ports import Rendition
|
|
from messenger.attachments.domain.jobs import Outcome, ReplayResult
|
|
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
|
from messenger.shared.application.errors import Unavailable
|
|
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
|
|
|
|
WORKER_POINTS = [
|
|
AFTER_CLAIM,
|
|
STALL_AFTER_CLAIM,
|
|
TRANSIENT_ERROR,
|
|
AFTER_S3_WRITE,
|
|
AFTER_COMMIT_BEFORE_ACK,
|
|
]
|
|
|
|
|
|
class Crash(BaseException):
|
|
pass
|
|
|
|
|
|
class RecordingFaults:
|
|
__slots__ = ("crash_at", "reached")
|
|
|
|
def __init__(self, crash_at: str = "") -> None:
|
|
self.crash_at = crash_at
|
|
self.reached: list[str] = []
|
|
|
|
async def reach(self, point: str) -> None:
|
|
self.reached.append(point)
|
|
if point == self.crash_at:
|
|
raise Crash(point)
|
|
|
|
|
|
class SlowRenderer:
|
|
__slots__ = ("seconds",)
|
|
|
|
def __init__(self, seconds: float) -> None:
|
|
self.seconds = seconds
|
|
|
|
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
|
await asyncio.sleep(self.seconds)
|
|
return await InlineRenderer().render(data, pipeline_version)
|
|
|
|
|
|
class ExpiringFaults:
|
|
__slots__ = ("attachment_id", "pipeline")
|
|
|
|
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
|
|
self.pipeline = pipeline
|
|
self.attachment_id = attachment_id
|
|
|
|
async def reach(self, point: str) -> None:
|
|
if point == STALL_AFTER_CLAIM:
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' "
|
|
"WHERE attachment_id = $1",
|
|
self.attachment_id,
|
|
)
|
|
await self.pipeline.recover()
|
|
|
|
|
|
async def outbox(attachment_id: str) -> list[bool]:
|
|
found = await rows(
|
|
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = $1 ORDER BY o.id",
|
|
attachment_id,
|
|
)
|
|
return [row["published"] for row in found]
|
|
|
|
|
|
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
|
|
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
|
|
|
|
|
|
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
faults = RecordingFaults()
|
|
pipeline.faults = faults
|
|
published = await pipeline.publish()
|
|
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
|
|
faults.reached.clear()
|
|
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
|
|
assert await pipeline.process(job) is Outcome.DONE
|
|
assert faults.reached == WORKER_POINTS
|
|
|
|
|
|
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
|
|
async def test_relay_crash_leaves_the_row_for_the_next_relay(
|
|
client: AsyncClient, point: str
|
|
) -> None:
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline.faults = RecordingFaults(point)
|
|
with pytest.raises(Crash):
|
|
await pipeline.publish()
|
|
assert await outbox(attachment_id) == [False]
|
|
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
|
|
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
|
|
pipeline.faults = RecordingFaults()
|
|
outcomes = await pipeline.drain()
|
|
assert Outcome.DONE in outcomes
|
|
assert await outbox(attachment_id) == [True]
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
|
|
with pytest.raises(Crash):
|
|
await pipeline.process(job)
|
|
objects = (await storage(client)).objects
|
|
written = stored(objects, attachment_id)
|
|
assert len(written) == 3
|
|
assert await variants(attachment_id) == []
|
|
assert (await state(attachment_id))["status"] == "processing"
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
pipeline.faults = RecordingFaults()
|
|
assert await pipeline.recover() >= 1
|
|
await pipeline.drain()
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["attempt"]) == ("ready", 2)
|
|
assert stored(objects, attachment_id) == written
|
|
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
|
|
|
|
|
|
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
|
|
with pytest.raises(Crash):
|
|
await pipeline.process(job)
|
|
before = await variants(attachment_id)
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
pipeline.faults = RecordingFaults()
|
|
assert await pipeline.process(job) is Outcome.DUPLICATE
|
|
assert await variants(attachment_id) == before
|
|
|
|
|
|
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
|
|
pipeline.renderer = SlowRenderer(2.5)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
|
|
async def sweep() -> None:
|
|
for _ in range(4):
|
|
await asyncio.sleep(0.6)
|
|
await pipeline.recover()
|
|
|
|
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
|
|
assert outcome is Outcome.DONE
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
|
|
assert "lease_expired" not in await events(attachment_id)
|
|
|
|
|
|
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
|
|
assert await pipeline.process(job) is Outcome.FENCED
|
|
objects = (await storage(client)).objects
|
|
assert stored(objects, attachment_id) == []
|
|
assert f"originals/{attachment_id}" not in objects
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
|
|
pipeline.faults = RecordingFaults()
|
|
await pipeline.drain()
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
|
|
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
|
|
started = asyncio.get_running_loop().time()
|
|
await stall.reach(AFTER_CLAIM)
|
|
await stall.reach(STALL_AFTER_CLAIM)
|
|
await stall.reach(STALL_AFTER_CLAIM)
|
|
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
|
|
with pytest.raises(Unavailable):
|
|
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
|
|
|
|
|
|
def test_environment_fault_crashes_the_process_with_137() -> None:
|
|
script = (
|
|
"import asyncio\n"
|
|
"from fakes import NullLogger\n"
|
|
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
|
|
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
|
|
".reach('after_commit_before_ack'))\n"
|
|
"print('survived')\n"
|
|
)
|
|
done = subprocess.run(
|
|
[sys.executable, "-c", script],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
env={"PYTHONPATH": ":".join(sys.path)},
|
|
)
|
|
assert (done.returncode, done.stdout) == (137, "")
|
|
|
|
|
|
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
pipeline.faults = RecordingFaults()
|
|
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
|
|
return pipeline, attachment_id
|
|
|
|
|
|
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
|
|
pipeline, attachment_id = await failed(client)
|
|
job = await job_of(attachment_id)
|
|
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
|
|
"queued",
|
|
"queued",
|
|
1,
|
|
None,
|
|
)
|
|
assert await outbox(attachment_id) == [True] * 5 + [False]
|
|
await pipeline.drain()
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["attempt"]) == ("ready", 1)
|
|
assert (await job_of(attachment_id)).id == job.id
|
|
before = await variants(attachment_id)
|
|
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
|
|
assert await outbox(attachment_id) == [True] * 6
|
|
assert await variants(attachment_id) == before
|
|
assert (await events(attachment_id))[-1] == "replay_noop"
|
|
assert "replayed" in await events(attachment_id)
|
|
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
|
|
|
|
|
|
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
|
|
pipeline, attachment_id = await failed(client)
|
|
_, _, waiting = await accepted(client)
|
|
objects = (await storage(client)).objects
|
|
await pipeline.clean(timedelta(days=1))
|
|
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
|
|
assert f"staging/{attachment_id}" in objects
|
|
await pipeline.clean(timedelta(0))
|
|
assert (await state(attachment_id))["staging_key"] is None
|
|
assert f"staging/{attachment_id}" not in objects
|
|
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
|
|
assert f"staging/{waiting}" in objects
|
|
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
|
|
assert (await state(attachment_id))["status"] == "failed"
|
|
|
|
|
|
async def test_operator_cli_lists_failed_jobs_and_replays(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
_, attachment_id = await failed(client)
|
|
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
|
|
monkeypatch.setenv("LOG_LEVEL", "WARNING")
|
|
assert await serve(["failed", "--limit", "500"]) == 0
|
|
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
|
|
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
|
|
assert datetime.fromisoformat(mine["failed_at"]).utcoffset() == timedelta(0)
|
|
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
|
|
"retry_exhausted",
|
|
5,
|
|
"Unavailable",
|
|
)
|
|
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
|
|
report = json.loads(capsys.readouterr().out.splitlines()[-1])
|
|
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
|
|
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1
|