201 lines
8.4 KiB
Python
201 lines
8.4 KiB
Python
import asyncio
|
|
import json
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage
|
|
from test_pipeline import accepted, job_of, rows
|
|
|
|
from conftest import DATABASE, actor
|
|
from messenger.admin import serve
|
|
from messenger.attachments.application.commands import ReprocessAttachment
|
|
from messenger.attachments.application.pipeline import (
|
|
AFTER_CLAIM,
|
|
AFTER_S3_WRITE,
|
|
ReprocessAttachmentHandler,
|
|
)
|
|
from messenger.attachments.application.ports import Rendition
|
|
from messenger.attachments.domain.errors import ImageTooDemanding
|
|
from messenger.attachments.domain.jobs import Outcome, ReprocessResult
|
|
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
|
from messenger.shared.infrastructure.ids import Uuid7Generator
|
|
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
|
|
|
|
|
|
async def reprocess(pipeline: InlinePipeline, ident: str, version: int = 2) -> ReprocessResult:
|
|
handler = ReprocessAttachmentHandler(pipeline.uow(), Uuid7Generator(), MonotonicUtcClock())
|
|
return await handler(ReprocessAttachment(UUID(ident), version, "test"))
|
|
|
|
|
|
async def attachment(ident: str) -> dict:
|
|
(row,) = await rows(
|
|
"SELECT status, published_version, error_code FROM attachments WHERE id = $1", ident
|
|
)
|
|
return row
|
|
|
|
|
|
async def versions(pipeline: InlinePipeline, ident: str) -> dict:
|
|
return {
|
|
job.pipeline_version: job
|
|
for job in await pipeline.publish()
|
|
if str(job.attachment_id) == ident
|
|
}
|
|
|
|
|
|
async def test_concurrent_reprocess_creates_one_job_and_outbox(client: AsyncClient) -> None:
|
|
_, _, ident = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
results = await asyncio.gather(*(reprocess(pipeline, ident) for _ in range(6)))
|
|
assert results.count(ReprocessResult.QUEUED) == 1
|
|
assert results.count(ReprocessResult.NOOP) == 5
|
|
jobs = await rows(
|
|
"SELECT pipeline_version, count(*) FROM jobs WHERE attachment_id = $1 GROUP BY 1", ident
|
|
)
|
|
assert {r["pipeline_version"]: r["count"] for r in jobs} == {1: 1, 2: 1}
|
|
assert len(await versions(pipeline, ident)) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("order", [(1, 2), (2, 1)])
|
|
async def test_version_order_never_reverts_published_bytes(
|
|
client: AsyncClient, order: tuple[int, int]
|
|
) -> None:
|
|
_, bob, ident = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
|
|
jobs = await versions(pipeline, ident)
|
|
for version in order:
|
|
outcome = await pipeline.process(jobs[version])
|
|
assert outcome is (Outcome.SUPERSEDED if order == (2, 1) and version == 1 else Outcome.DONE)
|
|
assert (await attachment(ident))["status"] == "ready"
|
|
await pipeline.clean()
|
|
assert (await attachment(ident))["published_version"] == 2
|
|
public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json()
|
|
stored = await rows(
|
|
"SELECT pipeline_version, name, sha256 FROM attachment_variants WHERE attachment_id = $1",
|
|
ident,
|
|
)
|
|
current = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 2}
|
|
older = {r["name"]: r["sha256"] for r in stored if r["pipeline_version"] == 1}
|
|
assert current != older
|
|
assert {r["name"]: r["sha256"] for r in public["variants"]} == current
|
|
assert await pipeline.process(jobs[1]) is Outcome.DUPLICATE
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.NOOP
|
|
assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public
|
|
|
|
|
|
class PausedEffect:
|
|
def __init__(self, point: str) -> None:
|
|
self.point = point
|
|
self.reached = asyncio.Event()
|
|
self.resume = asyncio.Event()
|
|
|
|
async def reach(self, point: str) -> None:
|
|
if point == self.point:
|
|
self.reached.set()
|
|
await self.resume.wait()
|
|
|
|
|
|
@pytest.mark.parametrize("point", [AFTER_CLAIM, AFTER_S3_WRITE])
|
|
async def test_old_holder_finishes_after_new_publication(client: AsyncClient, point: str) -> None:
|
|
_, bob, ident = await accepted(client)
|
|
old, new = await pipeline_of(client), await pipeline_of(client)
|
|
old.faults = pause = PausedEffect(point)
|
|
job = await job_of(ident)
|
|
running = asyncio.create_task(old.process(job))
|
|
try:
|
|
await asyncio.wait_for(pause.reached.wait(), 5)
|
|
assert await reprocess(new, ident) is ReprocessResult.QUEUED
|
|
jobs = await versions(new, ident)
|
|
assert await new.process(jobs[2]) is Outcome.DONE
|
|
public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json()
|
|
assert await new.clean() == 0
|
|
pause.resume.set()
|
|
assert await asyncio.wait_for(running, 5) is Outcome.SUPERSEDED
|
|
assert await new.clean() == 1
|
|
assert (await attachment(ident))["published_version"] == 2
|
|
assert (
|
|
await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))
|
|
).json() == public
|
|
finally:
|
|
pause.resume.set()
|
|
await running
|
|
|
|
|
|
class RejectingRenderer:
|
|
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
|
raise ImageTooDemanding("test limit")
|
|
|
|
|
|
async def test_failed_old_version_keeps_other_version_active(client: AsyncClient) -> None:
|
|
_, _, ident = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
|
|
jobs = await versions(pipeline, ident)
|
|
pipeline.renderer = RejectingRenderer()
|
|
assert await pipeline.process(jobs[1]) is Outcome.FAILED
|
|
assert (await attachment(ident))["status"] in ("queued", "processing")
|
|
pipeline.renderer = InlineRenderer()
|
|
assert await pipeline.process(jobs[2]) is Outcome.DONE
|
|
assert (await attachment(ident))["published_version"] == 2
|
|
|
|
|
|
async def test_concurrent_failures_leave_no_stuck_attachment(client: AsyncClient) -> None:
|
|
_, _, ident = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
|
|
jobs = await versions(pipeline, ident)
|
|
pipeline.renderer = RejectingRenderer()
|
|
assert await asyncio.gather(*(pipeline.process(job) for job in jobs.values())) == [
|
|
Outcome.FAILED,
|
|
Outcome.FAILED,
|
|
]
|
|
assert (await attachment(ident))["status"] == "failed"
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.CONFLICT
|
|
assert await pipeline.publish() == []
|
|
|
|
|
|
async def test_failed_new_version_keeps_ready_original(client: AsyncClient) -> None:
|
|
_, bob, ident = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
public = (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json()
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
|
|
assert (await attachment(ident))["status"] == "ready"
|
|
jobs = await versions(pipeline, ident)
|
|
pipeline.renderer = RejectingRenderer()
|
|
assert await pipeline.process(jobs[2]) is Outcome.FAILED
|
|
assert (await client.get(f"/api/v1/attachments/{ident}", headers=actor(bob))).json() == public
|
|
|
|
|
|
async def test_reprocess_rejects_unknown_id_and_version(client: AsyncClient) -> None:
|
|
pipeline = await pipeline_of(client)
|
|
assert await reprocess(pipeline, str(uuid4())) is ReprocessResult.NOT_FOUND
|
|
with pytest.raises(ValueError, match="версия"):
|
|
await reprocess(pipeline, str(uuid4()), 3)
|
|
|
|
|
|
async def test_operator_cli_schedules_version_once(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
_, _, ident = await accepted(client)
|
|
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
|
|
monkeypatch.setenv("LOG_LEVEL", "WARNING")
|
|
args = ["reprocess", "--attachment-id", ident, "--pipeline-version", "2"]
|
|
for expected in ("queued", "noop"):
|
|
assert await serve(args) == 0
|
|
report = json.loads(capsys.readouterr().out.splitlines()[-1])
|
|
assert report["result"] == expected
|
|
assert report["pipeline_version"] == 2
|
|
|
|
|
|
async def test_ready_reprocess_survives_cleanup_before_metadata_commit(client: AsyncClient) -> None:
|
|
_, _, ident = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
await (await storage(client)).delete(f"staging/{ident}")
|
|
assert await reprocess(pipeline, ident) is ReprocessResult.QUEUED
|
|
jobs = await versions(pipeline, ident)
|
|
assert await pipeline.process(jobs[2]) is Outcome.DONE
|
|
assert (await attachment(ident))["published_version"] == 2
|