forked from mtusi2026/mtusi2026lab6
Bring the application, own tests, Dockerfile, toolchain, compose stack, HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks, scenario scripts and the CI job from the lab 5 solution commit 64d5c67, keep the lab 5 Makefile targets on top of the unchanged skeleton targets, and append the lab 5 solution variables to the skeleton .env.example. The package and the application follow the contract version 1.6.0, and the image, the throwaway test containers and the broker vhost description are tagged lab6 so they do not collide with a lab 5 stand on the same host. This is the starting point of the lab 6 cycle: lint, own tests and the offline skeleton check pass before any lab 6 change. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
304 lines
11 KiB
Python
304 lines
11 KiB
Python
import asyncio
|
|
import json
|
|
import re
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import DATABASE, actor
|
|
from messenger.attachments.application.pipeline import TRANSIENT_ERROR
|
|
from messenger.attachments.application.ports import Rendition
|
|
from messenger.attachments.domain.errors import ImageTooDemanding
|
|
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
|
from messenger.attachments.entrypoints.schemas import JobMessage
|
|
from messenger.shared.application.errors import Unavailable
|
|
from pictures import SAMPLE
|
|
from pipeline import pipeline_of
|
|
|
|
|
|
async def rows(sql: str, *args: Any) -> list[dict[str, Any]]:
|
|
connection = await asyncpg.connect(DATABASE["url"])
|
|
try:
|
|
return [dict(row) for row in await connection.fetch(sql, *args)]
|
|
finally:
|
|
await connection.close()
|
|
|
|
|
|
async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]:
|
|
alice, bob, _, chat = await trio(http)
|
|
response = await upload(http, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
return alice, bob, response.json()["id"]
|
|
|
|
|
|
async def state(attachment_id: str) -> dict[str, Any]:
|
|
(row,) = await rows(
|
|
"SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, "
|
|
"j.status AS job_status, j.attempt, j.claims FROM attachments a "
|
|
"JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1",
|
|
attachment_id,
|
|
)
|
|
return row
|
|
|
|
|
|
async def variants(attachment_id: str) -> list[dict[str, Any]]:
|
|
return await rows(
|
|
"SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants "
|
|
"WHERE attachment_id = $1 ORDER BY name",
|
|
attachment_id,
|
|
)
|
|
|
|
|
|
async def events(attachment_id: str) -> list[str]:
|
|
found = await rows(
|
|
"SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id
|
|
)
|
|
return [row["kind"] for row in found]
|
|
|
|
|
|
async def job_of(attachment_id: str) -> Job:
|
|
(row,) = await rows(
|
|
"SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1",
|
|
attachment_id,
|
|
)
|
|
return JobMessage.model_validate_json(row["body"]).job()
|
|
|
|
|
|
async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None:
|
|
_, bob, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
outcomes = await pipeline.drain()
|
|
assert Outcome.DONE in outcomes
|
|
done = await state(attachment_id)
|
|
assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1)
|
|
assert done["original_key"] == f"originals/{attachment_id}"
|
|
assert done["staging_key"] == f"staging/{attachment_id}"
|
|
pattern = rf"variants/{attachment_id}/1/(crop|optimized|thumbnail)-[0-9a-f]{{16}}\.(jpg|webp)"
|
|
found = [re.fullmatch(pattern, row["object_key"]) for row in await variants(attachment_id)]
|
|
assert [match.groups() for match in found if match] == [
|
|
("crop", "jpg"),
|
|
("optimized", "webp"),
|
|
("thumbnail", "jpg"),
|
|
]
|
|
assert await pipeline.clean() >= 1
|
|
assert (await state(attachment_id))["staging_key"] is None
|
|
objects = (await storage(client)).objects
|
|
assert f"staging/{attachment_id}" not in objects
|
|
assert objects[f"originals/{attachment_id}"][0] == SAMPLE
|
|
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob))
|
|
assert content.content == SAMPLE
|
|
assert await events(attachment_id) == ["enqueued", "claimed", "done"]
|
|
|
|
|
|
async def test_message_is_a_contract_job(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
mine = [
|
|
json.loads(body)
|
|
for body in pipeline.publisher.bodies
|
|
if json.loads(body)["attachment_id"] == attachment_id
|
|
]
|
|
assert len(mine) == 1
|
|
assert set(mine[0]) == {
|
|
"schema_version",
|
|
"job_id",
|
|
"type",
|
|
"attachment_id",
|
|
"pipeline_version",
|
|
"attempt",
|
|
"created_at",
|
|
}
|
|
assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1)
|
|
assert await pipeline.publish() == []
|
|
|
|
|
|
async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
before, job = await variants(attachment_id), await job_of(attachment_id)
|
|
assert await pipeline.process(job) is Outcome.DUPLICATE
|
|
assert await variants(attachment_id) == before
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
assert (await events(attachment_id))[-1] == "duplicate_noop"
|
|
|
|
|
|
async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job))
|
|
assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE])
|
|
assert len(await variants(attachment_id)) == 3
|
|
assert (await state(attachment_id))["claims"] == 1
|
|
|
|
|
|
async def test_partial_variant_set_is_never_published(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
store = await storage(client)
|
|
|
|
async def refuse(key: str) -> None:
|
|
if "/optimized-" in key:
|
|
raise Unavailable("S3 недоступно")
|
|
|
|
store.after_put = refuse
|
|
try:
|
|
assert await pipeline.process(await job_of(attachment_id)) is Outcome.RESCHEDULED
|
|
finally:
|
|
store.after_put = None
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
assert await variants(attachment_id) == []
|
|
|
|
|
|
async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
store = await storage(client)
|
|
store.failing = True
|
|
try:
|
|
assert await pipeline.process(job) is Outcome.RESCHEDULED
|
|
finally:
|
|
store.failing = False
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2)
|
|
(retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id]
|
|
assert (retry.id, retry.attempt) == (job.id, 2)
|
|
assert await pipeline.process(retry) is Outcome.DONE
|
|
assert (await state(attachment_id))["status"] == "ready"
|
|
|
|
|
|
async def test_attempts_are_bounded(client: AsyncClient) -> None:
|
|
alice, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["job_status"]) == (
|
|
"failed",
|
|
"retry_exhausted",
|
|
"failed",
|
|
)
|
|
assert failed["attempt"] == pipeline.policy.max_attempts
|
|
assert (await events(attachment_id)).count("retry_scheduled") == 4
|
|
content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice))
|
|
assert content.status_code == 409
|
|
|
|
|
|
async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
(await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png")
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
|
|
"failed",
|
|
"invalid_image",
|
|
1,
|
|
)
|
|
|
|
|
|
class DemandingRenderer:
|
|
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
|
raise ImageTooDemanding("рендер дольше допустимого времени")
|
|
|
|
|
|
@pytest.mark.parametrize("cause", ["limit", "missing"])
|
|
async def test_limits_and_lost_original_fail_without_retries(
|
|
client: AsyncClient, cause: str
|
|
) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
if cause == "limit":
|
|
pipeline.renderer = DemandingRenderer()
|
|
else:
|
|
del (await storage(client)).objects[f"staging/{attachment_id}"]
|
|
await pipeline.drain()
|
|
failed = await state(attachment_id)
|
|
assert (failed["status"], failed["error_code"], failed["attempt"]) == (
|
|
"failed",
|
|
"processing_failed",
|
|
1,
|
|
)
|
|
|
|
|
|
async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
await pipeline.publish()
|
|
job = await job_of(attachment_id)
|
|
uow = pipeline.uow()
|
|
async with uow.transaction():
|
|
claim = await uow.jobs.claim(job, uuid4(), pipeline.lease)
|
|
await uow.commit()
|
|
assert claim is not None
|
|
assert (await state(attachment_id))["status"] == "processing"
|
|
await rows(
|
|
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert await pipeline.recover() >= 1
|
|
assert (await state(attachment_id))["status"] == "queued"
|
|
late = pipeline.uow()
|
|
async with late.transaction():
|
|
outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}")
|
|
await late.commit()
|
|
assert outcome is Outcome.FENCED
|
|
assert await variants(attachment_id) == []
|
|
await pipeline.drain()
|
|
recovered = await state(attachment_id)
|
|
assert (recovered["status"], recovered["attempt"]) == ("ready", 2)
|
|
assert {"lease_expired", "fenced"} <= set(await events(attachment_id))
|
|
|
|
|
|
async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client)
|
|
pipeline.publisher.failing = True
|
|
with pytest.raises(Unavailable):
|
|
await pipeline.publish()
|
|
pending = await rows(
|
|
"SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert pending == [{"published_at": None}]
|
|
pipeline.publisher.failing = False
|
|
assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish())
|
|
|
|
|
|
async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
|
|
_, _, attachment_id = await accepted(client)
|
|
pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60))
|
|
pipeline.faults = FailingFaults()
|
|
await pipeline.drain()
|
|
queued = await state(attachment_id)
|
|
assert (queued["status"], queued["attempt"]) == ("queued", 2)
|
|
(row,) = await rows(
|
|
"SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs "
|
|
"WHERE attachment_id = $1",
|
|
attachment_id,
|
|
)
|
|
assert 20 < row["wait"] <= 22.5
|
|
(event,) = await rows(
|
|
"SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events "
|
|
"WHERE attachment_id = $1 AND kind = 'retry_scheduled'",
|
|
attachment_id,
|
|
)
|
|
assert event["delay"] == 22.5
|
|
|
|
|
|
class FailingFaults:
|
|
async def reach(self, point: str) -> None:
|
|
if point == TRANSIENT_ERROR:
|
|
raise Unavailable(point)
|