import asyncio import json from uuid import UUID, uuid4 import pytest from httpx import AsyncClient from test_attachments import storage from test_pipeline import accepted, job_of, rows from conftest import DATABASE, actor from messenger.admin import serve from messenger.attachments.application.commands import ReprocessAttachment from messenger.attachments.application.pipeline import ( AFTER_CLAIM, AFTER_S3_WRITE, ReprocessAttachmentHandler, ) from messenger.attachments.application.ports import Rendition from messenger.attachments.domain.errors import ImageTooDemanding from messenger.attachments.domain.jobs import Outcome, ReprocessResult from messenger.shared.infrastructure.clock import MonotonicUtcClock from messenger.shared.infrastructure.ids import Uuid7Generator from pipeline import InlinePipeline, InlineRenderer, pipeline_of async def reprocess(pipeline: InlinePipeline, ident: str, version: int = 2) -> ReprocessResult: handler = ReprocessAttachmentHandler(pipeline.uow(), Uuid7Generator(), MonotonicUtcClock()) return await handler(ReprocessAttachment(UUID(ident), version, "test")) async def attachment(ident: str) -> dict: (row,) = await rows( "SELECT status, published_version, error_code FROM attachments WHERE id = $1", ident ) return row async def versions(pipeline: InlinePipeline, ident: str) -> dict: return { job.pipeline_version: job for job in await pipeline.publish() if str(job.attachment_id) == ident } async def test_concurrent_reprocess_creates_one_job_and_outbox(client: AsyncClient) -> None: _, _, ident = await accepted(client) pipeline = await pipeline_of(client) results = await asyncio.gather(*(reprocess(pipeline, ident) for _ in range(6))) assert results.count(ReprocessResult.QUEUED) == 1 assert results.count(ReprocessResult.NOOP) == 5 jobs = await rows( "SELECT pipeline_version, count(*) FROM jobs WHERE attachment_id = $1 GROUP BY 1", ident ) assert {r["pipeline_version"]: r["count"] for r in jobs} == {1: 1, 2: 1} assert len(await versions(pipeline, ident)) == 2 @pytest.mark.parametrize("order", [(1, 2), (2, 1)]) async def test_version_order_never_reverts_published_bytes( client: AsyncClient, order: tuple[int, int] ) -> None: _, bob, ident = await accepted(client) pipeline = await pipeline_of(client) assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED jobs = await versions(pipeline, ident) for version in order: outcome = await pipeline.process(jobs[version]) assert outcome is (Outcome.SUPERSEDED if order == (2, 1) and version == 1 else Outcome.DONE) assert (await attachment(ident))["status"] == "ready" await pipeline.clean() assert (await attachment(ident))["published_version"] == 2 public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() stored = await rows( "SELECT pipeline_version, name, sha256 FROM attachment_variants WHERE attachment_id = $1", ident, ) current = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 2} older = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 1} assert current != older assert {r["name"]: r["sha256"] for r in public["variants"]} == current assert await pipeline.process(jobs[1]) is Outcome.DUPLICATE assert await reprocess(pipeline, ident) is ReprocessResult.NOOP assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public class PausedEffect: def __init__(self, point: str) -> None: self.point = point self.reached = asyncio.Event() self.resume = asyncio.Event() async def reach(self, point: str) -> None: if point == self.point: self.reached.set() await self.resume.wait() @pytest.mark.parametrize("point", [AFTER_CLAIM, AFTER_S3_WRITE]) async def test_old_holder_finishes_after_new_publication(client: AsyncClient, point: str) -> None: _, bob, ident = await accepted(client) old, new = await pipeline_of(client), await pipeline_of(client) old.faults = pause = PausedEffect(point) job = await job_of(ident) running = asyncio.create_task(old.process(job)) try: await asyncio.wait_for(pause.reached.wait(), 5) assert await reprocess(new, ident) is ReprocessResult.QUEUED jobs = await versions(new, ident) assert await new.process(jobs[2]) is Outcome.DONE public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() assert await new.clean() == 0 pause.resume.set() assert await asyncio.wait_for(running, 5) is Outcome.SUPERSEDED assert await new.clean() == 1 assert (await attachment(ident))["published_version"] == 2 assert ( await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob)) ).json() == public finally: pause.resume.set() await running class RejectingRenderer: async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: raise ImageTooDemanding("test limit") async def test_failed_old_version_keeps_other_version_active(client: AsyncClient) -> None: _, _, ident = await accepted(client) pipeline = await pipeline_of(client) assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED jobs = await versions(pipeline, ident) pipeline.renderer = RejectingRenderer() assert await pipeline.process(jobs[1]) is Outcome.FAILED assert (await attachment(ident))["status"] in ("queued", "processing") pipeline.renderer = InlineRenderer() assert await pipeline.process(jobs[2]) is Outcome.DONE assert (await attachment(ident))["published_version"] == 2 async def test_concurrent_failures_leave_no_stuck_attachment(client: AsyncClient) -> None: _, _, ident = await accepted(client) pipeline = await pipeline_of(client) assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED jobs = await versions(pipeline, ident) pipeline.renderer = RejectingRenderer() assert await asyncio.gather(*(pipeline.process(job) for job in jobs.values())) == [ Outcome.FAILED, Outcome.FAILED, ] assert (await attachment(ident))["status"] == "failed" assert await reprocess(pipeline, ident) is ReprocessResult.CONFLICT assert await pipeline.publish() == [] async def test_failed_new_version_keeps_ready_original(client: AsyncClient) -> None: _, bob, ident = await accepted(client) pipeline = await pipeline_of(client) await pipeline.drain() public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED assert (await attachment(ident))["status"] == "ready" jobs = await versions(pipeline, ident) pipeline.renderer = RejectingRenderer() assert await pipeline.process(jobs[2]) is Outcome.FAILED assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public async def test_reprocess_rejects_unknown_id_and_version(client: AsyncClient) -> None: pipeline = await pipeline_of(client) assert await reprocess(pipeline, str(uuid4())) is ReprocessResult.NOT_FOUND with pytest.raises(ValueError, match="версия"): await reprocess(pipeline, str(uuid4()), 3) async def test_operator_cli_schedules_version_once( client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: _, _, ident = await accepted(client) monkeypatch.setenv("DATABASE_URL", DATABASE["url"]) monkeypatch.setenv("LOG_LEVEL", "WARNING") args = ["reprocess", "--attachment-id", ident, "--pipeline-version", "2"] for expected in ("queued", "noop"): assert await serve(args) == 0 report = json.loads(capsys.readouterr().out.splitlines()[-1]) assert report["result"] == expected assert report["pipeline_version"] == 2 async def test_ready_reprocess_survives_cleanup_before_metadata_commit(client: AsyncClient) -> None: _, _, ident = await accepted(client) pipeline = await pipeline_of(client) await pipeline.drain() await (await storage(client)).delete(f"staging/{ident}") assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED jobs = await versions(pipeline, ident) assert await pipeline.process(jobs[2]) is Outcome.DONE assert (await attachment(ident))["published_version"] == 2