forked from mtusi2026/mtusi2026lab2
Upload now stages the checked original under staging/, and one transaction moves the intent to queued, inserts the job (pipeline version 1, attempt 1), its outbox row and an audit event; the 202 is the in-memory snapshot, never a reread. Content answers 409 until the attachment is ready, the thumbnail variant is served from the published version and absent variants are 404. The relay publishes outbox rows through FastStream with publisher confirms, mandatory routing and persistent delivery, marks a row only after the broker acked it, returns jobs whose lease expired and deletes the staging object of ready attachments. The worker consumes the quorum queue through a FastStream subscriber that declares nothing, takes prefetch from WORKER_CONCURRENCY and acks manually after the result commit, rejects messages off jobs.schema.json to the dead letter exchange and requeues after an infrastructure error. A job is claimed under a lease token; the original is copied to originals/ and a 256 px JPEG thumbnail is written under a content-addressed key; the commit is fenced by the token, inserts variants with ON CONFLICT DO NOTHING and publishes only a newer pipeline version. A transient error reschedules the same job with an exponential delay and a new outbox row, the last attempt fails with retry_exhausted, an undecodable image fails at once with invalid_image, and a redelivered finished job is a recorded no-op. The worker and the relay are assembled in worker.py and relay.py with the dishka FastStream integration and the platform providers of the API; the shared core imports neither FastStream nor aio-pika. Tests cover the thumbnail geometry and rendering, the job message against the contract, the pipeline against PostgreSQL (duplicates, parallel copies, retries, the attempt bound, leases and fencing) and the broker adapters against a throwaway RabbitMQ with the stand's topology and permissions. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
245 lines
9.3 KiB
Python
245 lines
9.3 KiB
Python
import asyncio
|
|
import json
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import DATABASE, actor
|
|
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
|
from messenger.attachments.entrypoints.schemas import JobMessage
|
|
from messenger.shared.application.errors import Unavailable
|
|
from pictures import SAMPLE
|
|
from pipeline import pipeline_of
|
|
|
|
|
|
async def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
|
|
connection = await asyncpg.connect(DATABASE["url"])
|
|
try:
|
|
return [dict(row) for row in await connection.fetch(sql, *args)]
|
|
finally:
|
|
await connection.close()
|
|
|
|
|
|
async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]:
|
|
alice, bob, _, chat = await trio(http)
|
|
response = await upload(http, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
return alice, bob, response.json()["id"]
|
|
|
|
|
|
async def state(attachment_id: str) -> dict[str, Any]:
|
|
(row,) = await rows(
|
|
"SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, "
|
|
"j.status AS job_status, j.attempt, j.claims FROM attachments a "
|
|
"JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1",
|
|
attachment_id,
|
|
)
|
|
return row
|
|
|
|
|
|
async def variants(attachment_id: str) -> list[dict[str, Any]]:
|
|
return await rows(
|
|
"SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants "
|
|
"WHERE attachment_id = $1 ORDER BY name",
|
|
attachment_id,
|
|
)
|
|
|
|
|
|
async def events(attachment_id: str) -> list[str]:
|
|
found = await rows(
|
|
"SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id
|
|
)
|
|
return [row["kind"] for row in found]
|
|
|
|
|
|
async def job_of(attachment_id: str) -> Job:
|
|
(row,) = await rows(
|
|
"SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1",
|
|
attachment_id,
|
|
)
|
|
return JobMessage.model_validate_json(row["body"]).job()
|
|
|
|
|
|
async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None:
|
|
_, bob, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
outcomes = await pipeline.drain()
|
|
assert Outcome.DONE in outcomes
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1)
|
|
assert done["original_key"] == f"originals/{attachment_id}"
|
|
assert done["staging_key"] == f"staging/{attachment_id}"
|
|
(thumbnail,) = await variants(attachment_id)
|
|
assert thumbnail["object_key"].startswith(f"variants/{attachment_id}/1/thumbnail-")
|
|
assert await pipeline.clean() >= 1
|
|
assert (await state(attachment_id))["staging_key"] is None
|
|
objects = (await storage(client)).objects
|
|
assert f"staging/{attachment_id}" not in objects
|
|
assert objects[f"originals/{attachment_id}"][0] == SAMPLE
|
|
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob))
|
|
assert content.content == SAMPLE
|
|
assert await events(attachment_id) == ["enqueued", "claimed", "done"]
|
|
|
|
|
|
async def test_message_is_a_contract_job(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
mine = [
|
|
json.loads(body)
|
|
for body in pipeline.publisher.bodies
|
|
if json.loads(body)["attachment_id"] == attachment_id
|
|
]
|
|
assert len(mine) == 1
|
|
assert set(mine[0]) == {
|
|
"schema_version",
|
|
"job_id",
|
|
"type",
|
|
"attachment_id",
|
|
"pipeline_version",
|
|
"attempt",
|
|
"created_at",
|
|
}
|
|
assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1)
|
|
assert await pipeline.publish() == []
|
|
|
|
|
|
async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
before, job = await variants(attachment_id), await job_of(attachment_id)
|
|
assert await pipeline.process(job) is Outcome.DUPLICATE
|
|
assert await variants(attachment_id) == before
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
assert (await events(attachment_id))[-1] == "duplicate_noop"
|
|
|
|
|
|
async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job))
|
|
assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE])
|
|
assert len(await variants(attachment_id)) == 1
|
|
assert (await state(attachment_id))["claims"] == 1
|
|
|
|
|
|
async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
store = await storage(client)
|
|
store.failing = True
|
|
try:
|
|
assert await pipeline.process(job) is Outcome.RESCHEDULED
|
|
finally:
|
|
store.failing = False
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2)
|
|
(retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id]
|
|
assert (retry.id, retry.attempt) == (job.id, 2)
|
|
assert await pipeline.process(retry) is Outcome.DONE
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_attempts_are_bounded(client: AsyncClient) -> None:
|
|
alice, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["job_status"]) == (
|
|
"failed",
|
|
"retry_exhausted",
|
|
"failed",
|
|
)
|
|
assert failed["attempt"] == pipeline.policy.max_attempts
|
|
assert (await events(attachment_id)).count("retry_scheduled") == 4
|
|
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice))
|
|
assert content.status_code == 409
|
|
|
|
|
|
async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
(await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png")
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
|
|
"failed",
|
|
"invalid_image",
|
|
1,
|
|
)
|
|
|
|
|
|
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
uow = pipeline.uow()
|
|
async with uow.transaction():
|
|
claim = await uow.jobs.claim(job, uuid4(), pipeline.lease)
|
|
await uow.commit()
|
|
assert claim is not None
|
|
assert (await state(attachment_id))["status"] == "processing"
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert await pipeline.recover() >= 1
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
late = pipeline.uow()
|
|
async with late.transaction():
|
|
outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}")
|
|
await late.commit()
|
|
assert outcome is Outcome.FENCED
|
|
assert await variants(attachment_id) == []
|
|
await pipeline.drain()
|
|
recovered = await state(attachment_id)
|
|
assert (recovered["status"], recovered["attempt"]) == ("ready", 2)
|
|
assert {"lease_expired", "fenced"} <= set(await events(attachment_id))
|
|
|
|
|
|
async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.publisher.failing = True
|
|
with pytest.raises(Unavailable):
|
|
await pipeline.publish()
|
|
pending = await rows(
|
|
"SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert pending == [{"published_at": None}]
|
|
pipeline.publisher.failing = False
|
|
assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish())
|
|
|
|
|
|
async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60))
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["attempt"]) == ("queued", 2)
|
|
(row,) = await rows(
|
|
"SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs "
|
|
"WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert 25 < row["wait"] <= 30
|
|
|
|
|
|
class FailingFaults:
|
|
def reach(self, point: str) -> None:
|
|
raise Unavailable(point)
|