301 lines
11 KiB
Python
301 lines
11 KiB
Python
import asyncio
|
|
import json
|
|
import re
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import actor, sql_connection
|
|
from messenger.attachments.application.pipeline import TRANSIENT_ERROR
|
|
from messenger.attachments.application.ports import Rendition
|
|
from messenger.attachments.domain.errors import ImageTooDemanding
|
|
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
|
from messenger.attachments.entrypoints.schemas import JobMessage
|
|
from messenger.shared.application.errors import Unavailable
|
|
from pictures import SAMPLE
|
|
from pipeline import pipeline_of
|
|
|
|
|
|
async def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
|
|
async with sql_connection() as connection:
|
|
result = await connection.exec_driver_sql(sql, args)
|
|
return [dict(row) for row in result.mappings()] if result.returns_rows else []
|
|
|
|
|
|
async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]:
|
|
alice, bob, _, chat = await trio(http)
|
|
response = await upload(http, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
return alice, bob, response.json()["id"]
|
|
|
|
|
|
async def state(attachment_id: str) -> dict[str, Any]:
|
|
(row,) = await rows(
|
|
"SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, "
|
|
"j.status AS job_status, j.attempt, j.claims FROM attachments a "
|
|
"JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1",
|
|
attachment_id,
|
|
)
|
|
return row
|
|
|
|
|
|
async def variants(attachment_id: str) -> list[dict[str, Any]]:
|
|
return await rows(
|
|
"SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants "
|
|
"WHERE attachment_id = $1 ORDER BY name",
|
|
attachment_id,
|
|
)
|
|
|
|
|
|
async def events(attachment_id: str) -> list[str]:
|
|
found = await rows(
|
|
"SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id
|
|
)
|
|
return [row["kind"] for row in found]
|
|
|
|
|
|
async def job_of(attachment_id: str) -> Job:
|
|
(row,) = await rows(
|
|
"SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1",
|
|
attachment_id,
|
|
)
|
|
return JobMessage.model_validate_json(row["body"]).job()
|
|
|
|
|
|
async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None:
|
|
_, bob, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
outcomes = await pipeline.drain()
|
|
assert Outcome.DONE in outcomes
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1)
|
|
assert done["original_key"] == f"originals/{attachment_id}"
|
|
assert done["staging_key"] == f"staging/{attachment_id}"
|
|
pattern = rf"variants/{attachment_id}/1/(crop|optimized|thumbnail)-[0-9a-f]{{16}}\.(jpg|webp)"
|
|
found = [re.fullmatch(pattern, row["object_key"]) for row in await variants(attachment_id)]
|
|
assert [match.groups() for match in found if match] == [
|
|
("crop", "jpg"),
|
|
("optimized", "webp"),
|
|
("thumbnail", "jpg"),
|
|
]
|
|
assert await pipeline.clean() >= 1
|
|
assert (await state(attachment_id))["staging_key"] is None
|
|
objects = (await storage(client)).objects
|
|
assert f"staging/{attachment_id}" not in objects
|
|
assert objects[f"originals/{attachment_id}"][0] == SAMPLE
|
|
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob))
|
|
assert content.content == SAMPLE
|
|
assert await events(attachment_id) == ["enqueued", "claimed", "done"]
|
|
|
|
|
|
async def test_message_is_a_contract_job(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
mine = [
|
|
json.loads(body)
|
|
for body in pipeline.publisher.bodies
|
|
if json.loads(body)["attachment_id"] == attachment_id
|
|
]
|
|
assert len(mine) == 1
|
|
assert set(mine[0]) == {
|
|
"schema_version",
|
|
"job_id",
|
|
"type",
|
|
"attachment_id",
|
|
"pipeline_version",
|
|
"attempt",
|
|
"created_at",
|
|
}
|
|
assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1)
|
|
assert await pipeline.publish() == []
|
|
|
|
|
|
async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
before, job = await variants(attachment_id), await job_of(attachment_id)
|
|
assert await pipeline.process(job) is Outcome.DUPLICATE
|
|
assert await variants(attachment_id) == before
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
assert (await events(attachment_id))[-1] == "duplicate_noop"
|
|
|
|
|
|
async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job))
|
|
assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE])
|
|
assert len(await variants(attachment_id)) == 3
|
|
assert (await state(attachment_id))["claims"] == 1
|
|
|
|
|
|
async def test_partial_variant_set_is_never_published(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
store = await storage(client)
|
|
|
|
async def refuse(key: str) -> None:
|
|
if "/optimized-" in key:
|
|
raise Unavailable("S3 недоступно")
|
|
|
|
store.after_put = refuse
|
|
try:
|
|
assert await pipeline.process(await job_of(attachment_id)) is Outcome.RESCHEDULED
|
|
finally:
|
|
store.after_put = None
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
assert await variants(attachment_id) == []
|
|
|
|
|
|
async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
store = await storage(client)
|
|
store.failing = True
|
|
try:
|
|
assert await pipeline.process(job) is Outcome.RESCHEDULED
|
|
finally:
|
|
store.failing = False
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2)
|
|
(retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id]
|
|
assert (retry.id, retry.attempt) == (job.id, 2)
|
|
assert await pipeline.process(retry) is Outcome.DONE
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_attempts_are_bounded(client: AsyncClient) -> None:
|
|
alice, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["job_status"]) == (
|
|
"failed",
|
|
"retry_exhausted",
|
|
"failed",
|
|
)
|
|
assert failed["attempt"] == pipeline.policy.max_attempts
|
|
assert (await events(attachment_id)).count("retry_scheduled") == 4
|
|
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice))
|
|
assert content.status_code == 409
|
|
|
|
|
|
async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
(await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png")
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
|
|
"failed",
|
|
"invalid_image",
|
|
1,
|
|
)
|
|
|
|
|
|
class DemandingRenderer:
|
|
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
|
raise ImageTooDemanding("рендер дольше допустимого времени")
|
|
|
|
|
|
@pytest.mark.parametrize("cause", ["limit", "missing"])
|
|
async def test_limits_and_lost_original_fail_without_retries(
|
|
client: AsyncClient, cause: str
|
|
) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
if cause == "limit":
|
|
pipeline.renderer = DemandingRenderer()
|
|
else:
|
|
del (await storage(client)).objects[f"staging/{attachment_id}"]
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
|
|
"failed",
|
|
"processing_failed",
|
|
1,
|
|
)
|
|
|
|
|
|
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
uow = pipeline.uow()
|
|
async with uow.transaction():
|
|
claim = await uow.jobs.claim(job, uuid4(), pipeline.lease)
|
|
await uow.commit()
|
|
assert claim is not None
|
|
assert (await state(attachment_id))["status"] == "processing"
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert await pipeline.recover() >= 1
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
late = pipeline.uow()
|
|
async with late.transaction():
|
|
outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}")
|
|
await late.commit()
|
|
assert outcome is Outcome.FENCED
|
|
assert await variants(attachment_id) == []
|
|
await pipeline.drain()
|
|
recovered = await state(attachment_id)
|
|
assert (recovered["status"], recovered["attempt"]) == ("ready", 2)
|
|
assert {"lease_expired", "fenced"} <= set(await events(attachment_id))
|
|
|
|
|
|
async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.publisher.failing = True
|
|
with pytest.raises(Unavailable):
|
|
await pipeline.publish()
|
|
pending = await rows(
|
|
"SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert pending == [{"published_at": None}]
|
|
pipeline.publisher.failing = False
|
|
assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish())
|
|
|
|
|
|
async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60))
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["attempt"]) == ("queued", 2)
|
|
(row,) = await rows(
|
|
"SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs "
|
|
"WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert 20 < row["wait"] <= 22.5
|
|
(event,) = await rows(
|
|
"SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events "
|
|
"WHERE attachment_id = $1 AND kind = 'retry_scheduled'",
|
|
attachment_id,
|
|
)
|
|
assert event["delay"] == 22.5
|
|
|
|
|
|
class FailingFaults:
|
|
async def reach(self, point: str) -> None:
|
|
if point == TRANSIENT_ERROR:
|
|
raise Unavailable(point)
|