Files
mtusi2026lab2/student-tests/api/test_pipeline.py
T
ArtyomandClaude Opus 5.5 b152b6e07e Process attachment jobs through an outbox, a relay and a worker
Upload now stages the checked original under staging/, and one
transaction moves the intent to queued, inserts the job (pipeline
version 1, attempt 1), its outbox row and an audit event; the 202 is
the in-memory snapshot, never a reread. Content answers 409 until the
attachment is ready, the thumbnail variant is served from the
published version and absent variants are 404.

The relay publishes outbox rows through FastStream with publisher
confirms, mandatory routing and persistent delivery, marks a row only
after the broker acked it, returns jobs whose lease expired and deletes
the staging object of ready attachments. The worker consumes the
quorum queue through a FastStream subscriber that declares nothing,
takes prefetch from WORKER_CONCURRENCY and acks manually after the
result commit, rejects messages off jobs.schema.json to the dead letter
exchange and requeues after an infrastructure error. A job is claimed
under a lease token; the original is copied to originals/ and a
256 px JPEG thumbnail is written under a content-addressed key; the
commit is fenced by the token, inserts variants with ON CONFLICT DO
NOTHING and publishes only a newer pipeline version. A transient error
reschedules the same job with an exponential delay and a new outbox
row, the last attempt fails with retry_exhausted, an undecodable image
fails at once with invalid_image, and a redelivered finished job is a
recorded no-op.

The worker and the relay are assembled in worker.py and relay.py with
the dishka FastStream integration and the platform providers of the
API; the shared core imports neither FastStream nor aio-pika. Tests
cover the thumbnail geometry and rendering, the job message against
the contract, the pipeline against PostgreSQL (duplicates, parallel
copies, retries, the attempt bound, leases and fencing) and the broker
adapters against a throwaway RabbitMQ with the stand's topology and
permissions.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 01:08:06 +03:00

245 lines
9.3 KiB
Python

import asyncio
import json
from typing import Any
from uuid import uuid4
import asyncpg
import pytest
from httpx import AsyncClient
from test_attachments import storage, trio, upload
from conftest import DATABASE, actor
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.shared.application.errors import Unavailable
from pictures import SAMPLE
from pipeline import pipeline_of
async def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
connection = await asyncpg.connect(DATABASE["url"])
try:
return [dict(row) for row in await connection.fetch(sql, *args)]
finally:
await connection.close()
async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]:
alice, bob, _, chat = await trio(http)
response = await upload(http, alice, chat)
assert response.status_code == 202, response.text
return alice, bob, response.json()["id"]
async def state(attachment_id: str) -> dict[str, Any]:
(row,) = await rows(
"SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, "
"j.status AS job_status, j.attempt, j.claims FROM attachments a "
"JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1",
attachment_id,
)
return row
async def variants(attachment_id: str) -> list[dict[str, Any]]:
return await rows(
"SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants "
"WHERE attachment_id = $1 ORDER BY name",
attachment_id,
)
async def events(attachment_id: str) -> list[str]:
found = await rows(
"SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id
)
return [row["kind"] for row in found]
async def job_of(attachment_id: str) -> Job:
(row,) = await rows(
"SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) "
"WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1",
attachment_id,
)
return JobMessage.model_validate_json(row["body"]).job()
async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None:
_, bob, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
outcomes = await pipeline.drain()
assert Outcome.DONE in outcomes
done = await state(attachment_id)
assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1)
assert done["original_key"] == f"originals/{attachment_id}"
assert done["staging_key"] == f"staging/{attachment_id}"
(thumbnail,) = await variants(attachment_id)
assert thumbnail["object_key"].startswith(f"variants/{attachment_id}/1/thumbnail-")
assert await pipeline.clean() >= 1
assert (await state(attachment_id))["staging_key"] is None
objects = (await storage(client)).objects
assert f"staging/{attachment_id}" not in objects
assert objects[f"originals/{attachment_id}"][0] == SAMPLE
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob))
assert content.content == SAMPLE
assert await events(attachment_id) == ["enqueued", "claimed", "done"]
async def test_message_is_a_contract_job(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
mine = [
json.loads(body)
for body in pipeline.publisher.bodies
if json.loads(body)["attachment_id"] == attachment_id
]
assert len(mine) == 1
assert set(mine[0]) == {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1)
assert await pipeline.publish() == []
async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.drain()
before, job = await variants(attachment_id), await job_of(attachment_id)
assert await pipeline.process(job) is Outcome.DUPLICATE
assert await variants(attachment_id) == before
assert (await state(attachment_id))["status"] == "ready"
assert (await events(attachment_id))[-1] == "duplicate_noop"
async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job))
assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE])
assert len(await variants(attachment_id)) == 1
assert (await state(attachment_id))["claims"] == 1
async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
store = await storage(client)
store.failing = True
try:
assert await pipeline.process(job) is Outcome.RESCHEDULED
finally:
store.failing = False
queued = await state(attachment_id)
assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2)
(retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id]
assert (retry.id, retry.attempt) == (job.id, 2)
assert await pipeline.process(retry) is Outcome.DONE
assert (await state(attachment_id))["status"] == "ready"
async def test_attempts_are_bounded(client: AsyncClient) -> None:
alice, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.faults = FailingFaults()
await pipeline.drain()
failed = await state(attachment_id)
assert (failed["status"], failed["error_code"], failed["job_status"]) == (
"failed",
"retry_exhausted",
"failed",
)
assert failed["attempt"] == pipeline.policy.max_attempts
assert (await events(attachment_id)).count("retry_scheduled") == 4
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice))
assert content.status_code == 409
async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
(await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png")
pipeline = await pipeline_of(client)
await pipeline.drain()
failed = await state(attachment_id)
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
"failed",
"invalid_image",
1,
)
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
uow = pipeline.uow()
async with uow.transaction():
claim = await uow.jobs.claim(job, uuid4(), pipeline.lease)
await uow.commit()
assert claim is not None
assert (await state(attachment_id))["status"] == "processing"
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
attachment_id,
)
assert await pipeline.recover() >= 1
assert (await state(attachment_id))["status"] == "queued"
late = pipeline.uow()
async with late.transaction():
outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}")
await late.commit()
assert outcome is Outcome.FENCED
assert await variants(attachment_id) == []
await pipeline.drain()
recovered = await state(attachment_id)
assert (recovered["status"], recovered["attempt"]) == ("ready", 2)
assert {"lease_expired", "fenced"} <= set(await events(attachment_id))
async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.publisher.failing = True
with pytest.raises(Unavailable):
await pipeline.publish()
pending = await rows(
"SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1",
attachment_id,
)
assert pending == [{"published_at": None}]
pipeline.publisher.failing = False
assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish())
async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60))
pipeline.faults = FailingFaults()
await pipeline.drain()
queued = await state(attachment_id)
assert (queued["status"], queued["attempt"]) == ("queued", 2)
(row,) = await rows(
"SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs "
"WHERE attachment_id = $1",
attachment_id,
)
assert 25 < row["wait"] <= 30
class FailingFaults:
def reach(self, point: str) -> None:
raise Unavailable(point)