import asyncio import json import re from typing import Any from uuid import uuid4 import pytest from httpx import AsyncClient from test_attachments import storage, trio, upload from conftest import actor, sql_connection from messenger.attachments.application.pipeline import TRANSIENT_ERROR from messenger.attachments.application.ports import Rendition from messenger.attachments.domain.errors import ImageTooDemanding from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy from messenger.attachments.entrypoints.schemas import JobMessage from messenger.shared.application.errors import Unavailable from pictures import SAMPLE from pipeline import pipeline_of async def rows(sql: str, *args: Any) -> list[dict[str, Any]]: async with sql_connection() as connection: result = await connection.exec_driver_sql(sql, args) return [dict(row) for row in result.mappings()] if result.returns_rows else [] async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]: alice, bob, _, chat = await trio(http) response = await upload(http, alice, chat) assert response.status_code == 202, response.text return alice, bob, response.json()["id"] async def state(attachment_id: str) -> dict[str, Any]: (row,) = await rows( "SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, " "j.status AS job_status, j.attempt, j.claims FROM attachments a " "JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1", attachment_id, ) return row async def variants(attachment_id: str) -> list[dict[str, Any]]: return await rows( "SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants " "WHERE attachment_id = $1 ORDER BY name", attachment_id, ) async def events(attachment_id: str) -> list[str]: found = await rows( "SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id ) return [row["kind"] for row in found] async def job_of(attachment_id: str) -> Job: (row,) = await rows( "SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) " "WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1", attachment_id, ) return JobMessage.model_validate_json(row["body"]).job() async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None: _, bob, attachment_id = await accepted(client) pipeline = await pipeline_of(client) outcomes = await pipeline.drain() assert Outcome.DONE in outcomes done = await state(attachment_id) assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1) assert done["original_key"] == f"originals/{attachment_id}" assert done["staging_key"] == f"staging/{attachment_id}" pattern = rf"variants/{attachment_id}/1/(crop|optimized|thumbnail)-[0-9a-f]{{16}}\.(jpg|webp)" found = [re.fullmatch(pattern, row["object_key"]) for row in await variants(attachment_id)] assert [match.groups() for match in found if match] == [ ("crop", "jpg"), ("optimized", "webp"), ("thumbnail", "jpg"), ] assert await pipeline.clean() >= 1 assert (await state(attachment_id))["staging_key"] is None objects = (await storage(client)).objects assert f"staging/{attachment_id}" not in objects assert objects[f"originals/{attachment_id}"][0] == SAMPLE content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob)) assert content.content == SAMPLE assert await events(attachment_id) == ["enqueued", "claimed", "done"] async def test_message_is_a_contract_job(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) await pipeline.publish() mine = [ json.loads(body) for body in pipeline.publisher.bodies if json.loads(body)["attachment_id"] == attachment_id ] assert len(mine) == 1 assert set(mine[0]) == { "schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", } assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1) assert await pipeline.publish() == [] async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) await pipeline.drain() before, job = await variants(attachment_id), await job_of(attachment_id) assert await pipeline.process(job) is Outcome.DUPLICATE assert await variants(attachment_id) == before assert (await state(attachment_id))["status"] == "ready" assert (await events(attachment_id))[-1] == "duplicate_noop" async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) await pipeline.publish() job = await job_of(attachment_id) outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job)) assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE]) assert len(await variants(attachment_id)) == 3 assert (await state(attachment_id))["claims"] == 1 async def test_partial_variant_set_is_never_published(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) await pipeline.publish() store = await storage(client) async def refuse(key: str) -> None: if "/optimized-" in key: raise Unavailable("S3 недоступно") store.after_put = refuse try: assert await pipeline.process(await job_of(attachment_id)) is Outcome.RESCHEDULED finally: store.after_put = None assert (await state(attachment_id))["status"] == "queued" assert await variants(attachment_id) == [] async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) await pipeline.publish() job = await job_of(attachment_id) store = await storage(client) store.failing = True try: assert await pipeline.process(job) is Outcome.RESCHEDULED finally: store.failing = False queued = await state(attachment_id) assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2) (retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id] assert (retry.id, retry.attempt) == (job.id, 2) assert await pipeline.process(retry) is Outcome.DONE assert (await state(attachment_id))["status"] == "ready" async def test_attempts_are_bounded(client: AsyncClient) -> None: alice, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) pipeline.faults = FailingFaults() await pipeline.drain() failed = await state(attachment_id) assert (failed["status"], failed["error_code"], failed["job_status"]) == ( "failed", "retry_exhausted", "failed", ) assert failed["attempt"] == pipeline.policy.max_attempts assert (await events(attachment_id)).count("retry_scheduled") == 4 content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice)) assert content.status_code == 409 async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) (await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png") pipeline = await pipeline_of(client) await pipeline.drain() failed = await state(attachment_id) assert (failed["status"], failed["error_code"], failed["attempt"]) == ( "failed", "invalid_image", 1, ) class DemandingRenderer: async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: raise ImageTooDemanding("рендер дольше допустимого времени") @pytest.mark.parametrize("cause", ["limit", "missing"]) async def test_limits_and_lost_original_fail_without_retries( client: AsyncClient, cause: str ) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) if cause == "limit": pipeline.renderer = DemandingRenderer() else: del (await storage(client)).objects[f"staging/{attachment_id}"] await pipeline.drain() failed = await state(attachment_id) assert (failed["status"], failed["error_code"], failed["attempt"]) == ( "failed", "processing_failed", 1, ) async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) await pipeline.publish() job = await job_of(attachment_id) uow = pipeline.uow() async with uow.transaction(): claim = await uow.jobs.claim(job, uuid4(), pipeline.lease) await uow.commit() assert claim is not None assert (await state(attachment_id))["status"] == "processing" await rows( "UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1", attachment_id, ) assert await pipeline.recover() >= 1 assert (await state(attachment_id))["status"] == "queued" late = pipeline.uow() async with late.transaction(): outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}") await late.commit() assert outcome is Outcome.FENCED assert await variants(attachment_id) == [] await pipeline.drain() recovered = await state(attachment_id) assert (recovered["status"], recovered["attempt"]) == ("ready", 2) assert {"lease_expired", "fenced"} <= set(await events(attachment_id)) async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client) pipeline.publisher.failing = True with pytest.raises(Unavailable): await pipeline.publish() pending = await rows( "SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1", attachment_id, ) assert pending == [{"published_at": None}] pipeline.publisher.failing = False assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish()) async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None: _, _, attachment_id = await accepted(client) pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60)) pipeline.faults = FailingFaults() await pipeline.drain() queued = await state(attachment_id) assert (queued["status"], queued["attempt"]) == ("queued", 2) (row,) = await rows( "SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs " "WHERE attachment_id = $1", attachment_id, ) assert 20 < row["wait"] <= 22.5 (event,) = await rows( "SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events " "WHERE attachment_id = $1 AND kind = 'retry_scheduled'", attachment_id, ) assert event["delay"] == 22.5 class FailingFaults: async def reach(self, point: str) -> None: if point == TRANSIENT_ERROR: raise Unavailable(point)