Files
mtusi2026lab2/student-tests/api/test_versioned_pipeline.py
T

201 lines
8.4 KiB
Python

import asyncio
import json
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from test_attachments import storage
from test_pipeline import accepted, job_of, rows
from conftest import DATABASE, actor
from messenger.admin import serve
from messenger.attachments.application.commands import ReprocessAttachment
from messenger.attachments.application.pipeline import (
AFTER_CLAIM,
AFTER_S3_WRITE,
ReprocessAttachmentHandler,
)
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.errors import ImageTooDemanding
from messenger.attachments.domain.jobs import Outcome, ReprocessResult
from messenger.shared.infrastructure.clock import MonotonicUtcClock
from messenger.shared.infrastructure.ids import Uuid7Generator
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
async def reprocess(pipeline: InlinePipeline, ident: str, version: int = 2) -> ReprocessResult:
handler = ReprocessAttachmentHandler(pipeline.uow(), Uuid7Generator(), MonotonicUtcClock())
return await handler(ReprocessAttachment(UUID(ident), version, "test"))
async def attachment(ident: str) -> dict:
(row,) = await rows(
"SELECT status, published_version, error_code FROM attachments WHERE id = $1", ident
)
return row
async def versions(pipeline: InlinePipeline, ident: str) -> dict:
return {
job.pipeline_version: job
for job in await pipeline.publish()
if str(job.attachment_id) == ident
}
async def test_concurrent_reprocess_creates_one_job_and_outbox(client: AsyncClient) -> None:
_, _, ident = await accepted(client)
pipeline = await pipeline_of(client)
results = await asyncio.gather(*(reprocess(pipeline, ident) for _ in range(6)))
assert results.count(ReprocessResult.QUEUED) == 1
assert results.count(ReprocessResult.NOOP) == 5
jobs = await rows(
"SELECT pipeline_version, count(*) FROM jobs WHERE attachment_id = $1 GROUP BY 1", ident
)
assert {r["pipeline_version"]: r["count"] for r in jobs} == {1: 1, 2: 1}
assert len(await versions(pipeline, ident)) == 2
@pytest.mark.parametrize("order", [(1, 2), (2, 1)])
async def test_version_order_never_reverts_published_bytes(
client: AsyncClient, order: tuple[int, int]
) -> None:
_, bob, ident = await accepted(client)
pipeline = await pipeline_of(client)
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
jobs = await versions(pipeline, ident)
for version in order:
outcome = await pipeline.process(jobs[version])
assert outcome is (Outcome.SUPERSEDED if order == (2, 1) and version == 1 else Outcome.DONE)
assert (await attachment(ident))["status"] == "ready"
await pipeline.clean()
assert (await attachment(ident))["published_version"] == 2
public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json()
stored = await rows(
"SELECT pipeline_version, name, sha256 FROM attachment_variants WHERE attachment_id = $1",
ident,
)
current = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 2}
older = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 1}
assert current != older
assert {r["name"]: r["sha256"] for r in public["variants"]} == current
assert await pipeline.process(jobs[1]) is Outcome.DUPLICATE
assert await reprocess(pipeline, ident) is ReprocessResult.NOOP
assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public
class PausedEffect:
def __init__(self, point: str) -> None:
self.point = point
self.reached = asyncio.Event()
self.resume = asyncio.Event()
async def reach(self, point: str) -> None:
if point == self.point:
self.reached.set()
await self.resume.wait()
@pytest.mark.parametrize("point", [AFTER_CLAIM, AFTER_S3_WRITE])
async def test_old_holder_finishes_after_new_publication(client: AsyncClient, point: str) -> None:
_, bob, ident = await accepted(client)
old, new = await pipeline_of(client), await pipeline_of(client)
old.faults = pause = PausedEffect(point)
job = await job_of(ident)
running = asyncio.create_task(old.process(job))
try:
await asyncio.wait_for(pause.reached.wait(), 5)
assert await reprocess(new, ident) is ReprocessResult.QUEUED
jobs = await versions(new, ident)
assert await new.process(jobs[2]) is Outcome.DONE
public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json()
assert await new.clean() == 0
pause.resume.set()
assert await asyncio.wait_for(running, 5) is Outcome.SUPERSEDED
assert await new.clean() == 1
assert (await attachment(ident))["published_version"] == 2
assert (
await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))
).json() == public
finally:
pause.resume.set()
await running
class RejectingRenderer:
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
raise ImageTooDemanding("test limit")
async def test_failed_old_version_keeps_other_version_active(client: AsyncClient) -> None:
_, _, ident = await accepted(client)
pipeline = await pipeline_of(client)
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
jobs = await versions(pipeline, ident)
pipeline.renderer = RejectingRenderer()
assert await pipeline.process(jobs[1]) is Outcome.FAILED
assert (await attachment(ident))["status"] in ("queued", "processing")
pipeline.renderer = InlineRenderer()
assert await pipeline.process(jobs[2]) is Outcome.DONE
assert (await attachment(ident))["published_version"] == 2
async def test_concurrent_failures_leave_no_stuck_attachment(client: AsyncClient) -> None:
_, _, ident = await accepted(client)
pipeline = await pipeline_of(client)
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
jobs = await versions(pipeline, ident)
pipeline.renderer = RejectingRenderer()
assert await asyncio.gather(*(pipeline.process(job) for job in jobs.values())) == [
Outcome.FAILED,
Outcome.FAILED,
]
assert (await attachment(ident))["status"] == "failed"
assert await reprocess(pipeline, ident) is ReprocessResult.CONFLICT
assert await pipeline.publish() == []
async def test_failed_new_version_keeps_ready_original(client: AsyncClient) -> None:
_, bob, ident = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.drain()
public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json()
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
assert (await attachment(ident))["status"] == "ready"
jobs = await versions(pipeline, ident)
pipeline.renderer = RejectingRenderer()
assert await pipeline.process(jobs[2]) is Outcome.FAILED
assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public
async def test_reprocess_rejects_unknown_id_and_version(client: AsyncClient) -> None:
pipeline = await pipeline_of(client)
assert await reprocess(pipeline, str(uuid4())) is ReprocessResult.NOT_FOUND
with pytest.raises(ValueError, match="версия"):
await reprocess(pipeline, str(uuid4()), 3)
async def test_operator_cli_schedules_version_once(
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
_, _, ident = await accepted(client)
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
monkeypatch.setenv("LOG_LEVEL", "WARNING")
args = ["reprocess", "--attachment-id", ident, "--pipeline-version", "2"]
for expected in ("queued", "noop"):
assert await serve(args) == 0
report = json.loads(capsys.readouterr().out.splitlines()[-1])
assert report["result"] == expected
assert report["pipeline_version"] == 2
async def test_ready_reprocess_survives_cleanup_before_metadata_commit(client: AsyncClient) -> None:
_, _, ident = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.drain()
await (await storage(client)).delete(f"staging/{ident}")
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
jobs = await versions(pipeline, ident)
assert await pipeline.process(jobs[2]) is Outcome.DONE
assert (await attachment(ident))["published_version"] == 2