"""`AttachmentProcessingService`: claim, fencing, итоги попытки, блоки.""" import asyncio from dataclasses import replace from datetime import timedelta from uuid import UUID, uuid4 import pytest from builders import attachment_dto, chat_dto, job_dto, user_dto from fakes import ( FakeDatabase, FakeImageRenderer, FakeObjectStorage, FakeUnitOfWork, ) from src.config import settings from src.core import exceptions from src.core.services.attachment_processing import ( AttachmentProcessingService, retry_delay, ) from src.schemas import dataclasses from src.schemas.enums import ( AttachmentErrorCode, AttachmentJobReason, AttachmentJobState, AttachmentStatus, AttachmentVariantName, JobDeliveryOutcome, JobOutcome, ) from src.schemas.jobs import build_job_body from src.utils.utc import utcnow pytestmark = pytest.mark.anyio Outcome = JobDeliveryOutcome SOURCE = b"source-bytes" def _rendered() -> dataclasses.RenderedVariantsDTO: return dataclasses.RenderedVariantsDTO( variants=tuple( dataclasses.RenderedVariantDTO( name=name, content_type="image/jpeg", width=10, height=10, data=name.value.encode(), ) for name in AttachmentVariantName ), peak_kib=100, memory_limit_applied=True, render_ms=5, ) @pytest.fixture def renderer(database: FakeDatabase) -> FakeImageRenderer: return FakeImageRenderer(database, _rendered()) @pytest.fixture def service( uow: FakeUnitOfWork, storage: FakeObjectStorage, renderer: FakeImageRenderer, ) -> AttachmentProcessingService: return AttachmentProcessingService(uow, storage, renderer, asyncio.Event()) @pytest.fixture async def queued( uow: FakeUnitOfWork, database: FakeDatabase, storage: FakeObjectStorage, ) -> dataclasses.AttachmentJobDTO: owner = user_dto() chat = chat_dto(member_ids=(owner.id,)) now = utcnow() row = attachment_dto( chat_id=chat.id, owner_id=owner.id, status=AttachmentStatus.QUEUED, created_at=now, data=SOURCE, ) job = job_dto(attachment_id=row.id, created_at=now) async with uow: await uow.attachments.insert_attachment(attachment=row) await uow.jobs.insert_job(job=job) await uow.commit() storage.seed(row.staging_key, SOURCE) database.events.clear() return job def _body(job: dataclasses.AttachmentJobDTO) -> bytes: return build_job_body(job, attempt=job.attempt + 1) async def _result( service: AttachmentProcessingService, job: dataclasses.AttachmentJobDTO, ) -> dataclasses.JobDeliveryResultDTO: return await service.handle(body=_body(job), redelivered=False) async def _handle( service: AttachmentProcessingService, job: dataclasses.AttachmentJobDTO, ) -> JobDeliveryOutcome: return (await _result(service, job)).outcome @pytest.mark.parametrize( ("outcome", "action"), [ (Outcome.READY, JobOutcome.ACK), (Outcome.FAILED, JobOutcome.ACK), (Outcome.RETRY, JobOutcome.ACK), (Outcome.SKIPPED, JobOutcome.ACK), (Outcome.REQUEUED, JobOutcome.REQUEUE), (Outcome.DROPPED, JobOutcome.DROP), (Outcome.ERROR, JobOutcome.REQUEUE), ], ) def test_outcome_action(outcome: Outcome, action: JobOutcome) -> None: """Исход доставки → действие с сообщением: прежние ack, requeue, drop.""" assert outcome.action == action def _set_job(database: FakeDatabase, job_id: UUID, **fields) -> None: database.jobs[job_id] = replace(database.jobs[job_id], **fields) async def test_success_commits_ready_with_variants( service: AttachmentProcessingService, database: FakeDatabase, storage: FakeObjectStorage, queued: dataclasses.AttachmentJobDTO, ) -> None: outcome = await _handle(service, queued) assert outcome == Outcome.READY row = database.attachments[queued.attachment_id] job = database.jobs[queued.id] assert row.status == AttachmentStatus.READY assert job.state == AttachmentJobState.SUCCEEDED assert job.attempt == 1 assert len(database.variants) == 3 # оригинал в конечном ключе и объекты вариантов assert storage.objects[row.object_key][0] == SOURCE assert len(storage.objects) == 5 async def test_put_is_not_inside_a_block( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: """Вызовов S3 в транзакциях worker нет.""" await _handle(service, queued) assert "storage.put" in database.events assert not database.inside_block("storage.put") assert not database.inside_block("storage.get") assert not database.inside_block("renderer.render") async def test_ready_and_variants_one_block( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: await _handle(service, queued) events = database.events start = events.index("attachments.commit_ready") assert events[start - 1] == "jobs.lock_job" assert events[start:] == [ "attachments.commit_ready", "variants.insert_variant", "variants.insert_variant", "variants.insert_variant", "jobs.finish_job", "commit", "exit", ] async def test_malformed_message_dropped( service: AttachmentProcessingService, database: FakeDatabase, ) -> None: result = await service.handle(body=b"not json", redelivered=False) assert result == dataclasses.JobDeliveryResultDTO(outcome=Outcome.DROPPED) assert database.events == [] # --- итог доставки (design change job-trace-propagation, решение 8) ------- async def test_result_of_ready_has_time_to_ready( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: """Время до готовности — от приёма (`created_at` задания) до фиксации `ready`: ожидание в очереди входит.""" _set_job( database, queued.id, created_at=queued.created_at - timedelta(seconds=30), ) result = await _result(service, queued) assert result.outcome == Outcome.READY assert result.attempt == 1 assert result.reason is None assert result.error_code is None assert result.time_to_ready_seconds is not None assert 30 <= result.time_to_ready_seconds < 40 async def test_result_after_replay_has_no_time_to_ready( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: """Задание после replay (`attempt_base > 0`) во время до готовности не входит: его приём — действие оператора.""" _set_job(database, queued.id, attempt=2, attempt_base=2) result = await _result(service, queued) assert result.outcome == Outcome.READY assert result.attempt == 3 assert result.time_to_ready_seconds is None async def test_result_of_retry_names_reason( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: database.fail_on["storage.get"] = exceptions.ObjectStorageUnavailable() result = await _result(service, queued) assert result == dataclasses.JobDeliveryResultDTO( outcome=Outcome.RETRY, attempt=1, reason=AttachmentJobReason.STORAGE_UNAVAILABLE, ) async def test_result_of_failed_names_reason_and_code( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: database.fail_on["renderer.render"] = exceptions.ImageRejected( reason=AttachmentJobReason.UNDECODABLE, ) result = await _result(service, queued) assert result == dataclasses.JobDeliveryResultDTO( outcome=Outcome.FAILED, attempt=1, reason=AttachmentJobReason.UNDECODABLE, error_code=AttachmentErrorCode.INVALID_IMAGE, ) async def test_result_of_skipped_has_message_attempt( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: """Повторная доставка готового: без попытки, номер — из сообщения, без времени до готовности (нового наблюдения нет).""" await _handle(service, queued) result = await _result(service, queued) assert result == dataclasses.JobDeliveryResultDTO( outcome=Outcome.SKIPPED, attempt=1, ) async def test_unknown_job_acked_without_attempt( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: other = replace(queued, id=uuid4()) assert await _handle(service, other) == Outcome.SKIPPED assert database.jobs[queued.id].attempt == 0 async def test_foreign_attachment_acked_without_attempt( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: other = replace(queued, attachment_id=uuid4()) assert await _handle(service, other) == Outcome.SKIPPED assert database.jobs[queued.id].attempt == 0 async def test_finished_jobs_acked_without_write( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: await _handle(service, queued) database.events.clear() # повторная доставка завершённого assert await _handle(service, queued) == Outcome.SKIPPED assert database.jobs[queued.id].attempt == 1 assert "commit" not in database.events _set_job(database, queued.id, state=AttachmentJobState.FAILED) database.events.clear() assert await _handle(service, queued) == Outcome.SKIPPED assert "commit" not in database.events async def test_not_due_acked_without_attempt( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: _set_job( database, queued.id, next_attempt_at=utcnow() + timedelta(seconds=30), ) assert await _handle(service, queued) == Outcome.SKIPPED assert database.jobs[queued.id].attempt == 0 async def test_published_retry_is_taken_before_its_time( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: """Отметка публикации есть: срок уже проверил relay.""" _set_job( database, queued.id, next_attempt_at=utcnow() + timedelta(seconds=30), published_at=utcnow(), ) await _handle(service, queued) assert database.jobs[queued.id].state == AttachmentJobState.SUCCEEDED async def test_live_lease_acked_without_attempt( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: _set_job( database, queued.id, state=AttachmentJobState.PROCESSING, attempt=1, lease_expires_at=utcnow() + timedelta(seconds=30), ) assert await _handle(service, queued) == Outcome.SKIPPED assert database.jobs[queued.id].attempt == 1 assert "renderer.render" not in database.events async def test_expired_lease_is_taken( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: _set_job( database, queued.id, state=AttachmentJobState.PROCESSING, attempt=1, lease_expires_at=utcnow() - timedelta(seconds=1), ) await _handle(service, queued) job = database.jobs[queued.id] assert job.attempt == 2 assert job.state == AttachmentJobState.SUCCEEDED async def test_budget_spent_fails_without_attempt( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: attempts = settings.worker.max_attempts _set_job(database, queued.id, attempt=attempts) assert await _handle(service, queued) == Outcome.FAILED job = database.jobs[queued.id] row = database.attachments[queued.attachment_id] assert job.state == AttachmentJobState.FAILED assert job.attempt == attempts assert row.error_code == AttachmentErrorCode.RETRY_EXHAUSTED assert "renderer.render" not in database.events async def test_budget_counts_from_attempt_base( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: attempts = settings.worker.max_attempts _set_job(database, queued.id, attempt=attempts, attempt_base=attempts) await _handle(service, queued) assert database.jobs[queued.id].state == AttachmentJobState.SUCCEEDED async def test_attempt_takes_lease_and_processing( service: AttachmentProcessingService, database: FakeDatabase, renderer: FakeImageRenderer, queued: dataclasses.AttachmentJobDTO, ) -> None: """Взятие: аренда и `processing` зафиксированы до работы с байтами.""" seen: dict[str, object] = {} async def render(data: bytes) -> dataclasses.RenderedVariantsDTO: seen["job"] = database.jobs[queued.id] seen["status"] = database.attachments[queued.attachment_id].status return _rendered() renderer.render = render before = utcnow() await _handle(service, queued) job = seen["job"] worker = settings.worker margin = worker.job_timeout_seconds + worker.lease_margin_seconds assert seen["status"] == AttachmentStatus.PROCESSING assert job.state == AttachmentJobState.PROCESSING assert job.lease_expires_at >= before + timedelta(seconds=margin) async def test_permanent_error_fails_at_once( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: database.fail_on["renderer.render"] = exceptions.ImageRejected( reason=AttachmentJobReason.UNDECODABLE, ) assert await _handle(service, queued) == Outcome.FAILED row = database.attachments[queued.attachment_id] job = database.jobs[queued.id] assert row.status == AttachmentStatus.FAILED assert row.error_code == AttachmentErrorCode.INVALID_IMAGE assert job.state == AttachmentJobState.FAILED assert job.last_error_code == AttachmentJobReason.UNDECODABLE.value @pytest.mark.parametrize( ("change", "reason"), [ ("missing", AttachmentJobReason.SOURCE_MISSING), ("size", AttachmentJobReason.SOURCE_CORRUPTED), ("checksum", AttachmentJobReason.SOURCE_CORRUPTED), ], ) async def test_damaged_source_fails_at_once( service: AttachmentProcessingService, database: FakeDatabase, storage: FakeObjectStorage, queued: dataclasses.AttachmentJobDTO, change: str, reason: AttachmentJobReason, ) -> None: key = database.attachments[queued.attachment_id].staging_key if change == "missing": del storage.objects[key] elif change == "size": storage.seed(key, b"other size bytes!") else: storage.seed(key, b"X" * len(SOURCE)) assert await _handle(service, queued) == Outcome.FAILED row = database.attachments[queued.attachment_id] assert row.status == AttachmentStatus.FAILED assert row.error_code == AttachmentErrorCode.PROCESSING_FAILED assert database.jobs[queued.id].last_error_code == reason.value @pytest.mark.parametrize( ("event", "failure", "reason"), [ ( "storage.get", exceptions.ObjectStorageUnavailable(), AttachmentJobReason.STORAGE_UNAVAILABLE, ), ( "storage.put", exceptions.ObjectStorageUnavailable(), AttachmentJobReason.STORAGE_UNAVAILABLE, ), ( "renderer.render", exceptions.RenderInterrupted( reason=AttachmentJobReason.RENDER_KILLED, ), AttachmentJobReason.RENDER_KILLED, ), ( "renderer.render", RuntimeError("boom"), AttachmentJobReason.UNEXPECTED, ), ], ids=["get", "put", "render_killed", "unexpected"], ) async def test_temporary_error_returns_to_queue( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, event: str, failure: BaseException, reason: AttachmentJobReason, ) -> None: database.fail_on[event] = failure before = utcnow() assert await _handle(service, queued) == Outcome.RETRY job = database.jobs[queued.id] row = database.attachments[queued.attachment_id] assert job.state == AttachmentJobState.QUEUED assert job.attempt == 1 assert job.last_error_code == reason.value assert job.published_at is None assert job.next_attempt_at >= before + timedelta(seconds=2.5) assert row.status == AttachmentStatus.QUEUED async def test_temporary_error_after_budget_is_failed( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: _set_job(database, queued.id, attempt=settings.worker.max_attempts - 1) database.fail_on["storage.get"] = exceptions.ObjectStorageUnavailable() assert await _handle(service, queued) == Outcome.FAILED row = database.attachments[queued.attachment_id] assert row.error_code == AttachmentErrorCode.RETRY_EXHAUSTED assert database.jobs[queued.id].state == AttachmentJobState.FAILED async def _with_supersede( service: AttachmentProcessingService, database: FakeDatabase, renderer: FakeImageRenderer, queued: dataclasses.AttachmentJobDTO, *, failure: BaseException | None = None, ) -> tuple[JobDeliveryOutcome, dataclasses.AttachmentJobDTO]: """Пока попытка работает, задание взяла следующая попытка.""" async def render(data: bytes) -> dataclasses.RenderedVariantsDTO: job = database.jobs[queued.id] _set_job(database, queued.id, attempt=job.attempt + 1) if failure is not None: raise failure return _rendered() renderer.render = render outcome = await _handle(service, queued) return outcome, database.jobs[queued.id] async def test_superseded_on_ready_no_effect( service: AttachmentProcessingService, database: FakeDatabase, renderer: FakeImageRenderer, queued: dataclasses.AttachmentJobDTO, ) -> None: outcome, job = await _with_supersede(service, database, renderer, queued) assert outcome == Outcome.SKIPPED assert job.state == AttachmentJobState.PROCESSING assert database.attachments[queued.attachment_id].status == ( AttachmentStatus.PROCESSING ) assert database.variants == {} async def test_superseded_on_failed_no_effect( service: AttachmentProcessingService, database: FakeDatabase, renderer: FakeImageRenderer, queued: dataclasses.AttachmentJobDTO, ) -> None: outcome, job = await _with_supersede( service, database, renderer, queued, failure=exceptions.ImageRejected( reason=AttachmentJobReason.UNDECODABLE, ), ) assert outcome == Outcome.SKIPPED assert job.state == AttachmentJobState.PROCESSING assert database.attachments[queued.attachment_id].error_code is None async def test_superseded_on_return_no_effect( service: AttachmentProcessingService, database: FakeDatabase, renderer: FakeImageRenderer, queued: dataclasses.AttachmentJobDTO, ) -> None: outcome, job = await _with_supersede( service, database, renderer, queued, failure=exceptions.RenderInterrupted( reason=AttachmentJobReason.RENDER_KILLED, ), ) assert outcome == Outcome.SKIPPED assert job.state == AttachmentJobState.PROCESSING assert database.attachments[queued.attachment_id].status == ( AttachmentStatus.PROCESSING ) async def test_database_failure_requeues_and_pauses( service: AttachmentProcessingService, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, ) -> None: database.fail_on["jobs.lock_job"] = exceptions.ServiceUnavailable() assert await _handle(service, queued) == Outcome.REQUEUED assert database.jobs[queued.id].attempt == 0 async def test_stop_interrupts_pause( uow: FakeUnitOfWork, storage: FakeObjectStorage, renderer: FakeImageRenderer, database: FakeDatabase, queued: dataclasses.AttachmentJobDTO, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings.worker, "reconnect_seconds", 30) stop = asyncio.Event() stop.set() service = AttachmentProcessingService(uow, storage, renderer, stop) database.fail_on["jobs.lock_job"] = exceptions.ServiceUnavailable() async with asyncio.timeout(1): assert await _handle(service, queued) == Outcome.REQUEUED def test_retry_delay_bounds() -> None: for attempt in range(1, 8): delay = retry_delay(attempt=attempt, base=5, maximum=60) top = min(60, 5 * 2 ** (attempt - 1)) assert top / 2 <= delay <= top