Files
mtusi2026lab6/service-tests/unit/test_attachment_processing.py
T

659 lines
21 KiB
Python

"""`AttachmentProcessingService`: claim, fencing, итоги попытки, блоки."""
import asyncio
from dataclasses import replace
from datetime import timedelta
from uuid import UUID, uuid4
import pytest
from builders import attachment_dto, chat_dto, job_dto, user_dto
from fakes import (
FakeDatabase,
FakeImageRenderer,
FakeObjectStorage,
FakeUnitOfWork,
)
from src.config import settings
from src.core import exceptions
from src.core.services.attachment_processing import (
AttachmentProcessingService,
retry_delay,
)
from src.schemas import dataclasses
from src.schemas.enums import (
AttachmentErrorCode,
AttachmentJobReason,
AttachmentJobState,
AttachmentStatus,
AttachmentVariantName,
JobDeliveryOutcome,
JobOutcome,
)
from src.schemas.jobs import build_job_body
from src.utils.utc import utcnow
pytestmark = pytest.mark.anyio
Outcome = JobDeliveryOutcome
SOURCE = b"source-bytes"
def _rendered() -> dataclasses.RenderedVariantsDTO:
return dataclasses.RenderedVariantsDTO(
variants=tuple(
dataclasses.RenderedVariantDTO(
name=name,
content_type="image/jpeg",
width=10,
height=10,
data=name.value.encode(),
)
for name in AttachmentVariantName
),
peak_kib=100,
memory_limit_applied=True,
render_ms=5,
)
@pytest.fixture
def renderer(database: FakeDatabase) -> FakeImageRenderer:
return FakeImageRenderer(database, _rendered())
@pytest.fixture
def service(
uow: FakeUnitOfWork,
storage: FakeObjectStorage,
renderer: FakeImageRenderer,
) -> AttachmentProcessingService:
return AttachmentProcessingService(uow, storage, renderer, asyncio.Event())
@pytest.fixture
async def queued(
uow: FakeUnitOfWork,
database: FakeDatabase,
storage: FakeObjectStorage,
) -> dataclasses.AttachmentJobDTO:
owner = user_dto()
chat = chat_dto(member_ids=(owner.id,))
now = utcnow()
row = attachment_dto(
chat_id=chat.id,
owner_id=owner.id,
status=AttachmentStatus.QUEUED,
created_at=now,
data=SOURCE,
)
job = job_dto(attachment_id=row.id, created_at=now)
async with uow:
await uow.attachments.insert_attachment(attachment=row)
await uow.jobs.insert_job(job=job)
await uow.commit()
storage.seed(row.staging_key, SOURCE)
database.events.clear()
return job
def _body(job: dataclasses.AttachmentJobDTO) -> bytes:
return build_job_body(job, attempt=job.attempt + 1)
async def _result(
service: AttachmentProcessingService,
job: dataclasses.AttachmentJobDTO,
) -> dataclasses.JobDeliveryResultDTO:
return await service.handle(body=_body(job), redelivered=False)
async def _handle(
service: AttachmentProcessingService,
job: dataclasses.AttachmentJobDTO,
) -> JobDeliveryOutcome:
return (await _result(service, job)).outcome
@pytest.mark.parametrize(
("outcome", "action"),
[
(Outcome.READY, JobOutcome.ACK),
(Outcome.FAILED, JobOutcome.ACK),
(Outcome.RETRY, JobOutcome.ACK),
(Outcome.SKIPPED, JobOutcome.ACK),
(Outcome.REQUEUED, JobOutcome.REQUEUE),
(Outcome.DROPPED, JobOutcome.DROP),
(Outcome.ERROR, JobOutcome.REQUEUE),
],
)
def test_outcome_action(outcome: Outcome, action: JobOutcome) -> None:
"""Исход доставки → действие с сообщением: прежние ack, requeue, drop."""
assert outcome.action == action
def _set_job(database: FakeDatabase, job_id: UUID, **fields) -> None:
database.jobs[job_id] = replace(database.jobs[job_id], **fields)
async def test_success_commits_ready_with_variants(
service: AttachmentProcessingService,
database: FakeDatabase,
storage: FakeObjectStorage,
queued: dataclasses.AttachmentJobDTO,
) -> None:
outcome = await _handle(service, queued)
assert outcome == Outcome.READY
row = database.attachments[queued.attachment_id]
job = database.jobs[queued.id]
assert row.status == AttachmentStatus.READY
assert job.state == AttachmentJobState.SUCCEEDED
assert job.attempt == 1
assert len(database.variants) == 3
# оригинал в конечном ключе и объекты вариантов
assert storage.objects[row.object_key][0] == SOURCE
assert len(storage.objects) == 5
async def test_put_is_not_inside_a_block(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
"""Вызовов S3 в транзакциях worker нет."""
await _handle(service, queued)
assert "storage.put" in database.events
assert not database.inside_block("storage.put")
assert not database.inside_block("storage.get")
assert not database.inside_block("renderer.render")
async def test_ready_and_variants_one_block(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
await _handle(service, queued)
events = database.events
start = events.index("attachments.commit_ready")
assert events[start - 1] == "jobs.lock_job"
assert events[start:] == [
"attachments.commit_ready",
"variants.insert_variant",
"variants.insert_variant",
"variants.insert_variant",
"jobs.finish_job",
"commit",
"exit",
]
async def test_malformed_message_dropped(
service: AttachmentProcessingService,
database: FakeDatabase,
) -> None:
result = await service.handle(body=b"not json", redelivered=False)
assert result == dataclasses.JobDeliveryResultDTO(outcome=Outcome.DROPPED)
assert database.events == []
# --- итог доставки (design change job-trace-propagation, решение 8) -------
async def test_result_of_ready_has_time_to_ready(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
"""Время до готовности — от приёма (`created_at` задания) до фиксации
`ready`: ожидание в очереди входит."""
_set_job(
database,
queued.id,
created_at=queued.created_at - timedelta(seconds=30),
)
result = await _result(service, queued)
assert result.outcome == Outcome.READY
assert result.attempt == 1
assert result.reason is None
assert result.error_code is None
assert result.time_to_ready_seconds is not None
assert 30 <= result.time_to_ready_seconds < 40
async def test_result_after_replay_has_no_time_to_ready(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
"""Задание после replay (`attempt_base > 0`) во время до готовности не
входит: его приём — действие оператора."""
_set_job(database, queued.id, attempt=2, attempt_base=2)
result = await _result(service, queued)
assert result.outcome == Outcome.READY
assert result.attempt == 3
assert result.time_to_ready_seconds is None
async def test_result_of_retry_names_reason(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
database.fail_on["storage.get"] = exceptions.ObjectStorageUnavailable()
result = await _result(service, queued)
assert result == dataclasses.JobDeliveryResultDTO(
outcome=Outcome.RETRY,
attempt=1,
reason=AttachmentJobReason.STORAGE_UNAVAILABLE,
)
async def test_result_of_failed_names_reason_and_code(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
database.fail_on["renderer.render"] = exceptions.ImageRejected(
reason=AttachmentJobReason.UNDECODABLE,
)
result = await _result(service, queued)
assert result == dataclasses.JobDeliveryResultDTO(
outcome=Outcome.FAILED,
attempt=1,
reason=AttachmentJobReason.UNDECODABLE,
error_code=AttachmentErrorCode.INVALID_IMAGE,
)
async def test_result_of_skipped_has_message_attempt(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
"""Повторная доставка готового: без попытки, номер — из сообщения, без
времени до готовности (нового наблюдения нет)."""
await _handle(service, queued)
result = await _result(service, queued)
assert result == dataclasses.JobDeliveryResultDTO(
outcome=Outcome.SKIPPED,
attempt=1,
)
async def test_unknown_job_acked_without_attempt(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
other = replace(queued, id=uuid4())
assert await _handle(service, other) == Outcome.SKIPPED
assert database.jobs[queued.id].attempt == 0
async def test_foreign_attachment_acked_without_attempt(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
other = replace(queued, attachment_id=uuid4())
assert await _handle(service, other) == Outcome.SKIPPED
assert database.jobs[queued.id].attempt == 0
async def test_finished_jobs_acked_without_write(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
await _handle(service, queued)
database.events.clear()
# повторная доставка завершённого
assert await _handle(service, queued) == Outcome.SKIPPED
assert database.jobs[queued.id].attempt == 1
assert "commit" not in database.events
_set_job(database, queued.id, state=AttachmentJobState.FAILED)
database.events.clear()
assert await _handle(service, queued) == Outcome.SKIPPED
assert "commit" not in database.events
async def test_not_due_acked_without_attempt(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
_set_job(
database,
queued.id,
next_attempt_at=utcnow() + timedelta(seconds=30),
)
assert await _handle(service, queued) == Outcome.SKIPPED
assert database.jobs[queued.id].attempt == 0
async def test_published_retry_is_taken_before_its_time(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
"""Отметка публикации есть: срок уже проверил relay."""
_set_job(
database,
queued.id,
next_attempt_at=utcnow() + timedelta(seconds=30),
published_at=utcnow(),
)
await _handle(service, queued)
assert database.jobs[queued.id].state == AttachmentJobState.SUCCEEDED
async def test_live_lease_acked_without_attempt(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
_set_job(
database,
queued.id,
state=AttachmentJobState.PROCESSING,
attempt=1,
lease_expires_at=utcnow() + timedelta(seconds=30),
)
assert await _handle(service, queued) == Outcome.SKIPPED
assert database.jobs[queued.id].attempt == 1
assert "renderer.render" not in database.events
async def test_expired_lease_is_taken(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
_set_job(
database,
queued.id,
state=AttachmentJobState.PROCESSING,
attempt=1,
lease_expires_at=utcnow() - timedelta(seconds=1),
)
await _handle(service, queued)
job = database.jobs[queued.id]
assert job.attempt == 2
assert job.state == AttachmentJobState.SUCCEEDED
async def test_budget_spent_fails_without_attempt(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
attempts = settings.worker.max_attempts
_set_job(database, queued.id, attempt=attempts)
assert await _handle(service, queued) == Outcome.FAILED
job = database.jobs[queued.id]
row = database.attachments[queued.attachment_id]
assert job.state == AttachmentJobState.FAILED
assert job.attempt == attempts
assert row.error_code == AttachmentErrorCode.RETRY_EXHAUSTED
assert "renderer.render" not in database.events
async def test_budget_counts_from_attempt_base(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
attempts = settings.worker.max_attempts
_set_job(database, queued.id, attempt=attempts, attempt_base=attempts)
await _handle(service, queued)
assert database.jobs[queued.id].state == AttachmentJobState.SUCCEEDED
async def test_attempt_takes_lease_and_processing(
service: AttachmentProcessingService,
database: FakeDatabase,
renderer: FakeImageRenderer,
queued: dataclasses.AttachmentJobDTO,
) -> None:
"""Взятие: аренда и `processing` зафиксированы до работы с байтами."""
seen: dict[str, object] = {}
async def render(data: bytes) -> dataclasses.RenderedVariantsDTO:
seen["job"] = database.jobs[queued.id]
seen["status"] = database.attachments[queued.attachment_id].status
return _rendered()
renderer.render = render
before = utcnow()
await _handle(service, queued)
job = seen["job"]
worker = settings.worker
margin = worker.job_timeout_seconds + worker.lease_margin_seconds
assert seen["status"] == AttachmentStatus.PROCESSING
assert job.state == AttachmentJobState.PROCESSING
assert job.lease_expires_at >= before + timedelta(seconds=margin)
async def test_permanent_error_fails_at_once(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
database.fail_on["renderer.render"] = exceptions.ImageRejected(
reason=AttachmentJobReason.UNDECODABLE,
)
assert await _handle(service, queued) == Outcome.FAILED
row = database.attachments[queued.attachment_id]
job = database.jobs[queued.id]
assert row.status == AttachmentStatus.FAILED
assert row.error_code == AttachmentErrorCode.INVALID_IMAGE
assert job.state == AttachmentJobState.FAILED
assert job.last_error_code == AttachmentJobReason.UNDECODABLE.value
@pytest.mark.parametrize(
("change", "reason"),
[
("missing", AttachmentJobReason.SOURCE_MISSING),
("size", AttachmentJobReason.SOURCE_CORRUPTED),
("checksum", AttachmentJobReason.SOURCE_CORRUPTED),
],
)
async def test_damaged_source_fails_at_once(
service: AttachmentProcessingService,
database: FakeDatabase,
storage: FakeObjectStorage,
queued: dataclasses.AttachmentJobDTO,
change: str,
reason: AttachmentJobReason,
) -> None:
key = database.attachments[queued.attachment_id].staging_key
if change == "missing":
del storage.objects[key]
elif change == "size":
storage.seed(key, b"other size bytes!")
else:
storage.seed(key, b"X" * len(SOURCE))
assert await _handle(service, queued) == Outcome.FAILED
row = database.attachments[queued.attachment_id]
assert row.status == AttachmentStatus.FAILED
assert row.error_code == AttachmentErrorCode.PROCESSING_FAILED
assert database.jobs[queued.id].last_error_code == reason.value
@pytest.mark.parametrize(
("event", "failure", "reason"),
[
(
"storage.get",
exceptions.ObjectStorageUnavailable(),
AttachmentJobReason.STORAGE_UNAVAILABLE,
),
(
"storage.put",
exceptions.ObjectStorageUnavailable(),
AttachmentJobReason.STORAGE_UNAVAILABLE,
),
(
"renderer.render",
exceptions.RenderInterrupted(
reason=AttachmentJobReason.RENDER_KILLED,
),
AttachmentJobReason.RENDER_KILLED,
),
(
"renderer.render",
RuntimeError("boom"),
AttachmentJobReason.UNEXPECTED,
),
],
ids=["get", "put", "render_killed", "unexpected"],
)
async def test_temporary_error_returns_to_queue(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
event: str,
failure: BaseException,
reason: AttachmentJobReason,
) -> None:
database.fail_on[event] = failure
before = utcnow()
assert await _handle(service, queued) == Outcome.RETRY
job = database.jobs[queued.id]
row = database.attachments[queued.attachment_id]
assert job.state == AttachmentJobState.QUEUED
assert job.attempt == 1
assert job.last_error_code == reason.value
assert job.published_at is None
assert job.next_attempt_at >= before + timedelta(seconds=2.5)
assert row.status == AttachmentStatus.QUEUED
async def test_temporary_error_after_budget_is_failed(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
_set_job(database, queued.id, attempt=settings.worker.max_attempts - 1)
database.fail_on["storage.get"] = exceptions.ObjectStorageUnavailable()
assert await _handle(service, queued) == Outcome.FAILED
row = database.attachments[queued.attachment_id]
assert row.error_code == AttachmentErrorCode.RETRY_EXHAUSTED
assert database.jobs[queued.id].state == AttachmentJobState.FAILED
async def _with_supersede(
service: AttachmentProcessingService,
database: FakeDatabase,
renderer: FakeImageRenderer,
queued: dataclasses.AttachmentJobDTO,
*,
failure: BaseException | None = None,
) -> tuple[JobDeliveryOutcome, dataclasses.AttachmentJobDTO]:
"""Пока попытка работает, задание взяла следующая попытка."""
async def render(data: bytes) -> dataclasses.RenderedVariantsDTO:
job = database.jobs[queued.id]
_set_job(database, queued.id, attempt=job.attempt + 1)
if failure is not None:
raise failure
return _rendered()
renderer.render = render
outcome = await _handle(service, queued)
return outcome, database.jobs[queued.id]
async def test_superseded_on_ready_no_effect(
service: AttachmentProcessingService,
database: FakeDatabase,
renderer: FakeImageRenderer,
queued: dataclasses.AttachmentJobDTO,
) -> None:
outcome, job = await _with_supersede(service, database, renderer, queued)
assert outcome == Outcome.SKIPPED
assert job.state == AttachmentJobState.PROCESSING
assert database.attachments[queued.attachment_id].status == (
AttachmentStatus.PROCESSING
)
assert database.variants == {}
async def test_superseded_on_failed_no_effect(
service: AttachmentProcessingService,
database: FakeDatabase,
renderer: FakeImageRenderer,
queued: dataclasses.AttachmentJobDTO,
) -> None:
outcome, job = await _with_supersede(
service,
database,
renderer,
queued,
failure=exceptions.ImageRejected(
reason=AttachmentJobReason.UNDECODABLE,
),
)
assert outcome == Outcome.SKIPPED
assert job.state == AttachmentJobState.PROCESSING
assert database.attachments[queued.attachment_id].error_code is None
async def test_superseded_on_return_no_effect(
service: AttachmentProcessingService,
database: FakeDatabase,
renderer: FakeImageRenderer,
queued: dataclasses.AttachmentJobDTO,
) -> None:
outcome, job = await _with_supersede(
service,
database,
renderer,
queued,
failure=exceptions.RenderInterrupted(
reason=AttachmentJobReason.RENDER_KILLED,
),
)
assert outcome == Outcome.SKIPPED
assert job.state == AttachmentJobState.PROCESSING
assert database.attachments[queued.attachment_id].status == (
AttachmentStatus.PROCESSING
)
async def test_database_failure_requeues_and_pauses(
service: AttachmentProcessingService,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
) -> None:
database.fail_on["jobs.lock_job"] = exceptions.ServiceUnavailable()
assert await _handle(service, queued) == Outcome.REQUEUED
assert database.jobs[queued.id].attempt == 0
async def test_stop_interrupts_pause(
uow: FakeUnitOfWork,
storage: FakeObjectStorage,
renderer: FakeImageRenderer,
database: FakeDatabase,
queued: dataclasses.AttachmentJobDTO,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings.worker, "reconnect_seconds", 30)
stop = asyncio.Event()
stop.set()
service = AttachmentProcessingService(uow, storage, renderer, stop)
database.fail_on["jobs.lock_job"] = exceptions.ServiceUnavailable()
async with asyncio.timeout(1):
assert await _handle(service, queued) == Outcome.REQUEUED
def test_retry_delay_bounds() -> None:
for attempt in range(1, 8):
delay = retry_delay(attempt=attempt, base=5, maximum=60)
top = min(60, 5 * 2 ** (attempt - 1))
assert top / 2 <= delay <= top