Files
mtusi2026lab2/student-tests/api/test_pipeline.py
T

301 lines
11 KiB
Python

import asyncio
import json
import re
from typing import Any
from uuid import uuid4
import pytest
from httpx import AsyncClient
from test_attachments import storage, trio, upload
from conftest import actor, sql_connection
from messenger.attachments.application.pipeline import TRANSIENT_ERROR
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.errors import ImageTooDemanding
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.shared.application.errors import Unavailable
from pictures import SAMPLE
from pipeline import pipeline_of
async def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
async with sql_connection() as connection:
result = await connection.exec_driver_sql(sql, args)
return [dict(row) for row in result.mappings()] if result.returns_rows else []
async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]:
alice, bob, _, chat = await trio(http)
response = await upload(http, alice, chat)
assert response.status_code == 202, response.text
return alice, bob, response.json()["id"]
async def state(attachment_id: str) -> dict[str, Any]:
(row,) = await rows(
"SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, "
"j.status AS job_status, j.attempt, j.claims FROM attachments a "
"JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1",
attachment_id,
)
return row
async def variants(attachment_id: str) -> list[dict[str, Any]]:
return await rows(
"SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants "
"WHERE attachment_id = $1 ORDER BY name",
attachment_id,
)
async def events(attachment_id: str) -> list[str]:
found = await rows(
"SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id
)
return [row["kind"] for row in found]
async def job_of(attachment_id: str) -> Job:
(row,) = await rows(
"SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) "
"WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1",
attachment_id,
)
return JobMessage.model_validate_json(row["body"]).job()
async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None:
_, bob, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
outcomes = await pipeline.drain()
assert Outcome.DONE in outcomes
done = await state(attachment_id)
assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1)
assert done["original_key"] == f"originals/{attachment_id}"
assert done["staging_key"] == f"staging/{attachment_id}"
pattern = rf"variants/{attachment_id}/1/(crop|optimized|thumbnail)-[0-9a-f]{{16}}\.(jpg|webp)"
found = [re.fullmatch(pattern, row["object_key"]) for row in await variants(attachment_id)]
assert [match.groups() for match in found if match] == [
("crop", "jpg"),
("optimized", "webp"),
("thumbnail", "jpg"),
]
assert await pipeline.clean() >= 1
assert (await state(attachment_id))["staging_key"] is None
objects = (await storage(client)).objects
assert f"staging/{attachment_id}" not in objects
assert objects[f"originals/{attachment_id}"][0] == SAMPLE
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob))
assert content.content == SAMPLE
assert await events(attachment_id) == ["enqueued", "claimed", "done"]
async def test_message_is_a_contract_job(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
mine = [
json.loads(body)
for body in pipeline.publisher.bodies
if json.loads(body)["attachment_id"] == attachment_id
]
assert len(mine) == 1
assert set(mine[0]) == {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1)
assert await pipeline.publish() == []
async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.drain()
before, job = await variants(attachment_id), await job_of(attachment_id)
assert await pipeline.process(job) is Outcome.DUPLICATE
assert await variants(attachment_id) == before
assert (await state(attachment_id))["status"] == "ready"
assert (await events(attachment_id))[-1] == "duplicate_noop"
async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job))
assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE])
assert len(await variants(attachment_id)) == 3
assert (await state(attachment_id))["claims"] == 1
async def test_partial_variant_set_is_never_published(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
store = await storage(client)
async def refuse(key: str) -> None:
if "/optimized-" in key:
raise Unavailable("S3 недоступно")
store.after_put = refuse
try:
assert await pipeline.process(await job_of(attachment_id)) is Outcome.RESCHEDULED
finally:
store.after_put = None
assert (await state(attachment_id))["status"] == "queued"
assert await variants(attachment_id) == []
async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
store = await storage(client)
store.failing = True
try:
assert await pipeline.process(job) is Outcome.RESCHEDULED
finally:
store.failing = False
queued = await state(attachment_id)
assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2)
(retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id]
assert (retry.id, retry.attempt) == (job.id, 2)
assert await pipeline.process(retry) is Outcome.DONE
assert (await state(attachment_id))["status"] == "ready"
async def test_attempts_are_bounded(client: AsyncClient) -> None:
alice, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.faults = FailingFaults()
await pipeline.drain()
failed = await state(attachment_id)
assert (failed["status"], failed["error_code"], failed["job_status"]) == (
"failed",
"retry_exhausted",
"failed",
)
assert failed["attempt"] == pipeline.policy.max_attempts
assert (await events(attachment_id)).count("retry_scheduled") == 4
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice))
assert content.status_code == 409
async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
(await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png")
pipeline = await pipeline_of(client)
await pipeline.drain()
failed = await state(attachment_id)
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
"failed",
"invalid_image",
1,
)
class DemandingRenderer:
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
raise ImageTooDemanding("рендер дольше допустимого времени")
@pytest.mark.parametrize("cause", ["limit", "missing"])
async def test_limits_and_lost_original_fail_without_retries(
client: AsyncClient, cause: str
) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
if cause == "limit":
pipeline.renderer = DemandingRenderer()
else:
del (await storage(client)).objects[f"staging/{attachment_id}"]
await pipeline.drain()
failed = await state(attachment_id)
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
"failed",
"processing_failed",
1,
)
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
uow = pipeline.uow()
async with uow.transaction():
claim = await uow.jobs.claim(job, uuid4(), pipeline.lease)
await uow.commit()
assert claim is not None
assert (await state(attachment_id))["status"] == "processing"
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
attachment_id,
)
assert await pipeline.recover() >= 1
assert (await state(attachment_id))["status"] == "queued"
late = pipeline.uow()
async with late.transaction():
outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}")
await late.commit()
assert outcome is Outcome.FENCED
assert await variants(attachment_id) == []
await pipeline.drain()
recovered = await state(attachment_id)
assert (recovered["status"], recovered["attempt"]) == ("ready", 2)
assert {"lease_expired", "fenced"} <= set(await events(attachment_id))
async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.publisher.failing = True
with pytest.raises(Unavailable):
await pipeline.publish()
pending = await rows(
"SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1",
attachment_id,
)
assert pending == [{"published_at": None}]
pipeline.publisher.failing = False
assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish())
async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60))
pipeline.faults = FailingFaults()
await pipeline.drain()
queued = await state(attachment_id)
assert (queued["status"], queued["attempt"]) == ("queued", 2)
(row,) = await rows(
"SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs "
"WHERE attachment_id = $1",
attachment_id,
)
assert 20 < row["wait"] <= 22.5
(event,) = await rows(
"SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events "
"WHERE attachment_id = $1 AND kind = 'retry_scheduled'",
attachment_id,
)
assert event["delay"] == 22.5
class FailingFaults:
async def reach(self, point: str) -> None:
if point == TRANSIENT_ERROR:
raise Unavailable(point)