659 lines
21 KiB
Python
659 lines
21 KiB
Python
"""`AttachmentProcessingService`: claim, fencing, итоги попытки, блоки."""
|
|
|
|
import asyncio
|
|
from dataclasses import replace
|
|
from datetime import timedelta
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
|
|
from builders import attachment_dto, chat_dto, job_dto, user_dto
|
|
from fakes import (
|
|
FakeDatabase,
|
|
FakeImageRenderer,
|
|
FakeObjectStorage,
|
|
FakeUnitOfWork,
|
|
)
|
|
from src.config import settings
|
|
from src.core import exceptions
|
|
from src.core.services.attachment_processing import (
|
|
AttachmentProcessingService,
|
|
retry_delay,
|
|
)
|
|
from src.schemas import dataclasses
|
|
from src.schemas.enums import (
|
|
AttachmentErrorCode,
|
|
AttachmentJobReason,
|
|
AttachmentJobState,
|
|
AttachmentStatus,
|
|
AttachmentVariantName,
|
|
JobDeliveryOutcome,
|
|
JobOutcome,
|
|
)
|
|
from src.schemas.jobs import build_job_body
|
|
from src.utils.utc import utcnow
|
|
|
|
pytestmark = pytest.mark.anyio
|
|
|
|
Outcome = JobDeliveryOutcome
|
|
|
|
SOURCE = b"source-bytes"
|
|
|
|
|
|
def _rendered() -> dataclasses.RenderedVariantsDTO:
|
|
return dataclasses.RenderedVariantsDTO(
|
|
variants=tuple(
|
|
dataclasses.RenderedVariantDTO(
|
|
name=name,
|
|
content_type="image/jpeg",
|
|
width=10,
|
|
height=10,
|
|
data=name.value.encode(),
|
|
)
|
|
for name in AttachmentVariantName
|
|
),
|
|
peak_kib=100,
|
|
memory_limit_applied=True,
|
|
render_ms=5,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def renderer(database: FakeDatabase) -> FakeImageRenderer:
|
|
return FakeImageRenderer(database, _rendered())
|
|
|
|
|
|
@pytest.fixture
|
|
def service(
|
|
uow: FakeUnitOfWork,
|
|
storage: FakeObjectStorage,
|
|
renderer: FakeImageRenderer,
|
|
) -> AttachmentProcessingService:
|
|
return AttachmentProcessingService(uow, storage, renderer, asyncio.Event())
|
|
|
|
|
|
@pytest.fixture
|
|
async def queued(
|
|
uow: FakeUnitOfWork,
|
|
database: FakeDatabase,
|
|
storage: FakeObjectStorage,
|
|
) -> dataclasses.AttachmentJobDTO:
|
|
owner = user_dto()
|
|
chat = chat_dto(member_ids=(owner.id,))
|
|
now = utcnow()
|
|
row = attachment_dto(
|
|
chat_id=chat.id,
|
|
owner_id=owner.id,
|
|
status=AttachmentStatus.QUEUED,
|
|
created_at=now,
|
|
data=SOURCE,
|
|
)
|
|
job = job_dto(attachment_id=row.id, created_at=now)
|
|
async with uow:
|
|
await uow.attachments.insert_attachment(attachment=row)
|
|
await uow.jobs.insert_job(job=job)
|
|
await uow.commit()
|
|
storage.seed(row.staging_key, SOURCE)
|
|
database.events.clear()
|
|
return job
|
|
|
|
|
|
def _body(job: dataclasses.AttachmentJobDTO) -> bytes:
|
|
return build_job_body(job, attempt=job.attempt + 1)
|
|
|
|
|
|
async def _result(
|
|
service: AttachmentProcessingService,
|
|
job: dataclasses.AttachmentJobDTO,
|
|
) -> dataclasses.JobDeliveryResultDTO:
|
|
return await service.handle(body=_body(job), redelivered=False)
|
|
|
|
|
|
async def _handle(
|
|
service: AttachmentProcessingService,
|
|
job: dataclasses.AttachmentJobDTO,
|
|
) -> JobDeliveryOutcome:
|
|
return (await _result(service, job)).outcome
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("outcome", "action"),
|
|
[
|
|
(Outcome.READY, JobOutcome.ACK),
|
|
(Outcome.FAILED, JobOutcome.ACK),
|
|
(Outcome.RETRY, JobOutcome.ACK),
|
|
(Outcome.SKIPPED, JobOutcome.ACK),
|
|
(Outcome.REQUEUED, JobOutcome.REQUEUE),
|
|
(Outcome.DROPPED, JobOutcome.DROP),
|
|
(Outcome.ERROR, JobOutcome.REQUEUE),
|
|
],
|
|
)
|
|
def test_outcome_action(outcome: Outcome, action: JobOutcome) -> None:
|
|
"""Исход доставки → действие с сообщением: прежние ack, requeue, drop."""
|
|
assert outcome.action == action
|
|
|
|
|
|
def _set_job(database: FakeDatabase, job_id: UUID, **fields) -> None:
|
|
database.jobs[job_id] = replace(database.jobs[job_id], **fields)
|
|
|
|
|
|
async def test_success_commits_ready_with_variants(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
storage: FakeObjectStorage,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
outcome = await _handle(service, queued)
|
|
assert outcome == Outcome.READY
|
|
row = database.attachments[queued.attachment_id]
|
|
job = database.jobs[queued.id]
|
|
assert row.status == AttachmentStatus.READY
|
|
assert job.state == AttachmentJobState.SUCCEEDED
|
|
assert job.attempt == 1
|
|
assert len(database.variants) == 3
|
|
# оригинал в конечном ключе и объекты вариантов
|
|
assert storage.objects[row.object_key][0] == SOURCE
|
|
assert len(storage.objects) == 5
|
|
|
|
|
|
async def test_put_is_not_inside_a_block(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
"""Вызовов S3 в транзакциях worker нет."""
|
|
await _handle(service, queued)
|
|
assert "storage.put" in database.events
|
|
assert not database.inside_block("storage.put")
|
|
assert not database.inside_block("storage.get")
|
|
assert not database.inside_block("renderer.render")
|
|
|
|
|
|
async def test_ready_and_variants_one_block(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
await _handle(service, queued)
|
|
events = database.events
|
|
start = events.index("attachments.commit_ready")
|
|
assert events[start - 1] == "jobs.lock_job"
|
|
assert events[start:] == [
|
|
"attachments.commit_ready",
|
|
"variants.insert_variant",
|
|
"variants.insert_variant",
|
|
"variants.insert_variant",
|
|
"jobs.finish_job",
|
|
"commit",
|
|
"exit",
|
|
]
|
|
|
|
|
|
async def test_malformed_message_dropped(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
) -> None:
|
|
result = await service.handle(body=b"not json", redelivered=False)
|
|
assert result == dataclasses.JobDeliveryResultDTO(outcome=Outcome.DROPPED)
|
|
assert database.events == []
|
|
|
|
|
|
# --- итог доставки (design change job-trace-propagation, решение 8) -------
|
|
|
|
|
|
async def test_result_of_ready_has_time_to_ready(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
"""Время до готовности — от приёма (`created_at` задания) до фиксации
|
|
`ready`: ожидание в очереди входит."""
|
|
_set_job(
|
|
database,
|
|
queued.id,
|
|
created_at=queued.created_at - timedelta(seconds=30),
|
|
)
|
|
result = await _result(service, queued)
|
|
assert result.outcome == Outcome.READY
|
|
assert result.attempt == 1
|
|
assert result.reason is None
|
|
assert result.error_code is None
|
|
assert result.time_to_ready_seconds is not None
|
|
assert 30 <= result.time_to_ready_seconds < 40
|
|
|
|
|
|
async def test_result_after_replay_has_no_time_to_ready(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
"""Задание после replay (`attempt_base > 0`) во время до готовности не
|
|
входит: его приём — действие оператора."""
|
|
_set_job(database, queued.id, attempt=2, attempt_base=2)
|
|
result = await _result(service, queued)
|
|
assert result.outcome == Outcome.READY
|
|
assert result.attempt == 3
|
|
assert result.time_to_ready_seconds is None
|
|
|
|
|
|
async def test_result_of_retry_names_reason(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
database.fail_on["storage.get"] = exceptions.ObjectStorageUnavailable()
|
|
result = await _result(service, queued)
|
|
assert result == dataclasses.JobDeliveryResultDTO(
|
|
outcome=Outcome.RETRY,
|
|
attempt=1,
|
|
reason=AttachmentJobReason.STORAGE_UNAVAILABLE,
|
|
)
|
|
|
|
|
|
async def test_result_of_failed_names_reason_and_code(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
database.fail_on["renderer.render"] = exceptions.ImageRejected(
|
|
reason=AttachmentJobReason.UNDECODABLE,
|
|
)
|
|
result = await _result(service, queued)
|
|
assert result == dataclasses.JobDeliveryResultDTO(
|
|
outcome=Outcome.FAILED,
|
|
attempt=1,
|
|
reason=AttachmentJobReason.UNDECODABLE,
|
|
error_code=AttachmentErrorCode.INVALID_IMAGE,
|
|
)
|
|
|
|
|
|
async def test_result_of_skipped_has_message_attempt(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
"""Повторная доставка готового: без попытки, номер — из сообщения, без
|
|
времени до готовности (нового наблюдения нет)."""
|
|
await _handle(service, queued)
|
|
result = await _result(service, queued)
|
|
assert result == dataclasses.JobDeliveryResultDTO(
|
|
outcome=Outcome.SKIPPED,
|
|
attempt=1,
|
|
)
|
|
|
|
|
|
async def test_unknown_job_acked_without_attempt(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
other = replace(queued, id=uuid4())
|
|
assert await _handle(service, other) == Outcome.SKIPPED
|
|
assert database.jobs[queued.id].attempt == 0
|
|
|
|
|
|
async def test_foreign_attachment_acked_without_attempt(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
other = replace(queued, attachment_id=uuid4())
|
|
assert await _handle(service, other) == Outcome.SKIPPED
|
|
assert database.jobs[queued.id].attempt == 0
|
|
|
|
|
|
async def test_finished_jobs_acked_without_write(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
await _handle(service, queued)
|
|
database.events.clear()
|
|
# повторная доставка завершённого
|
|
assert await _handle(service, queued) == Outcome.SKIPPED
|
|
assert database.jobs[queued.id].attempt == 1
|
|
assert "commit" not in database.events
|
|
_set_job(database, queued.id, state=AttachmentJobState.FAILED)
|
|
database.events.clear()
|
|
assert await _handle(service, queued) == Outcome.SKIPPED
|
|
assert "commit" not in database.events
|
|
|
|
|
|
async def test_not_due_acked_without_attempt(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
_set_job(
|
|
database,
|
|
queued.id,
|
|
next_attempt_at=utcnow() + timedelta(seconds=30),
|
|
)
|
|
assert await _handle(service, queued) == Outcome.SKIPPED
|
|
assert database.jobs[queued.id].attempt == 0
|
|
|
|
|
|
async def test_published_retry_is_taken_before_its_time(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
"""Отметка публикации есть: срок уже проверил relay."""
|
|
_set_job(
|
|
database,
|
|
queued.id,
|
|
next_attempt_at=utcnow() + timedelta(seconds=30),
|
|
published_at=utcnow(),
|
|
)
|
|
await _handle(service, queued)
|
|
assert database.jobs[queued.id].state == AttachmentJobState.SUCCEEDED
|
|
|
|
|
|
async def test_live_lease_acked_without_attempt(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
_set_job(
|
|
database,
|
|
queued.id,
|
|
state=AttachmentJobState.PROCESSING,
|
|
attempt=1,
|
|
lease_expires_at=utcnow() + timedelta(seconds=30),
|
|
)
|
|
assert await _handle(service, queued) == Outcome.SKIPPED
|
|
assert database.jobs[queued.id].attempt == 1
|
|
assert "renderer.render" not in database.events
|
|
|
|
|
|
async def test_expired_lease_is_taken(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
_set_job(
|
|
database,
|
|
queued.id,
|
|
state=AttachmentJobState.PROCESSING,
|
|
attempt=1,
|
|
lease_expires_at=utcnow() - timedelta(seconds=1),
|
|
)
|
|
await _handle(service, queued)
|
|
job = database.jobs[queued.id]
|
|
assert job.attempt == 2
|
|
assert job.state == AttachmentJobState.SUCCEEDED
|
|
|
|
|
|
async def test_budget_spent_fails_without_attempt(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
attempts = settings.worker.max_attempts
|
|
_set_job(database, queued.id, attempt=attempts)
|
|
assert await _handle(service, queued) == Outcome.FAILED
|
|
job = database.jobs[queued.id]
|
|
row = database.attachments[queued.attachment_id]
|
|
assert job.state == AttachmentJobState.FAILED
|
|
assert job.attempt == attempts
|
|
assert row.error_code == AttachmentErrorCode.RETRY_EXHAUSTED
|
|
assert "renderer.render" not in database.events
|
|
|
|
|
|
async def test_budget_counts_from_attempt_base(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
attempts = settings.worker.max_attempts
|
|
_set_job(database, queued.id, attempt=attempts, attempt_base=attempts)
|
|
await _handle(service, queued)
|
|
assert database.jobs[queued.id].state == AttachmentJobState.SUCCEEDED
|
|
|
|
|
|
async def test_attempt_takes_lease_and_processing(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
renderer: FakeImageRenderer,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
"""Взятие: аренда и `processing` зафиксированы до работы с байтами."""
|
|
seen: dict[str, object] = {}
|
|
|
|
async def render(data: bytes) -> dataclasses.RenderedVariantsDTO:
|
|
seen["job"] = database.jobs[queued.id]
|
|
seen["status"] = database.attachments[queued.attachment_id].status
|
|
return _rendered()
|
|
|
|
renderer.render = render
|
|
before = utcnow()
|
|
await _handle(service, queued)
|
|
job = seen["job"]
|
|
worker = settings.worker
|
|
margin = worker.job_timeout_seconds + worker.lease_margin_seconds
|
|
assert seen["status"] == AttachmentStatus.PROCESSING
|
|
assert job.state == AttachmentJobState.PROCESSING
|
|
assert job.lease_expires_at >= before + timedelta(seconds=margin)
|
|
|
|
|
|
async def test_permanent_error_fails_at_once(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
database.fail_on["renderer.render"] = exceptions.ImageRejected(
|
|
reason=AttachmentJobReason.UNDECODABLE,
|
|
)
|
|
assert await _handle(service, queued) == Outcome.FAILED
|
|
row = database.attachments[queued.attachment_id]
|
|
job = database.jobs[queued.id]
|
|
assert row.status == AttachmentStatus.FAILED
|
|
assert row.error_code == AttachmentErrorCode.INVALID_IMAGE
|
|
assert job.state == AttachmentJobState.FAILED
|
|
assert job.last_error_code == AttachmentJobReason.UNDECODABLE.value
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("change", "reason"),
|
|
[
|
|
("missing", AttachmentJobReason.SOURCE_MISSING),
|
|
("size", AttachmentJobReason.SOURCE_CORRUPTED),
|
|
("checksum", AttachmentJobReason.SOURCE_CORRUPTED),
|
|
],
|
|
)
|
|
async def test_damaged_source_fails_at_once(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
storage: FakeObjectStorage,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
change: str,
|
|
reason: AttachmentJobReason,
|
|
) -> None:
|
|
key = database.attachments[queued.attachment_id].staging_key
|
|
if change == "missing":
|
|
del storage.objects[key]
|
|
elif change == "size":
|
|
storage.seed(key, b"other size bytes!")
|
|
else:
|
|
storage.seed(key, b"X" * len(SOURCE))
|
|
assert await _handle(service, queued) == Outcome.FAILED
|
|
row = database.attachments[queued.attachment_id]
|
|
assert row.status == AttachmentStatus.FAILED
|
|
assert row.error_code == AttachmentErrorCode.PROCESSING_FAILED
|
|
assert database.jobs[queued.id].last_error_code == reason.value
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("event", "failure", "reason"),
|
|
[
|
|
(
|
|
"storage.get",
|
|
exceptions.ObjectStorageUnavailable(),
|
|
AttachmentJobReason.STORAGE_UNAVAILABLE,
|
|
),
|
|
(
|
|
"storage.put",
|
|
exceptions.ObjectStorageUnavailable(),
|
|
AttachmentJobReason.STORAGE_UNAVAILABLE,
|
|
),
|
|
(
|
|
"renderer.render",
|
|
exceptions.RenderInterrupted(
|
|
reason=AttachmentJobReason.RENDER_KILLED,
|
|
),
|
|
AttachmentJobReason.RENDER_KILLED,
|
|
),
|
|
(
|
|
"renderer.render",
|
|
RuntimeError("boom"),
|
|
AttachmentJobReason.UNEXPECTED,
|
|
),
|
|
],
|
|
ids=["get", "put", "render_killed", "unexpected"],
|
|
)
|
|
async def test_temporary_error_returns_to_queue(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
event: str,
|
|
failure: BaseException,
|
|
reason: AttachmentJobReason,
|
|
) -> None:
|
|
database.fail_on[event] = failure
|
|
before = utcnow()
|
|
assert await _handle(service, queued) == Outcome.RETRY
|
|
job = database.jobs[queued.id]
|
|
row = database.attachments[queued.attachment_id]
|
|
assert job.state == AttachmentJobState.QUEUED
|
|
assert job.attempt == 1
|
|
assert job.last_error_code == reason.value
|
|
assert job.published_at is None
|
|
assert job.next_attempt_at >= before + timedelta(seconds=2.5)
|
|
assert row.status == AttachmentStatus.QUEUED
|
|
|
|
|
|
async def test_temporary_error_after_budget_is_failed(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
_set_job(database, queued.id, attempt=settings.worker.max_attempts - 1)
|
|
database.fail_on["storage.get"] = exceptions.ObjectStorageUnavailable()
|
|
assert await _handle(service, queued) == Outcome.FAILED
|
|
row = database.attachments[queued.attachment_id]
|
|
assert row.error_code == AttachmentErrorCode.RETRY_EXHAUSTED
|
|
assert database.jobs[queued.id].state == AttachmentJobState.FAILED
|
|
|
|
|
|
async def _with_supersede(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
renderer: FakeImageRenderer,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
*,
|
|
failure: BaseException | None = None,
|
|
) -> tuple[JobDeliveryOutcome, dataclasses.AttachmentJobDTO]:
|
|
"""Пока попытка работает, задание взяла следующая попытка."""
|
|
|
|
async def render(data: bytes) -> dataclasses.RenderedVariantsDTO:
|
|
job = database.jobs[queued.id]
|
|
_set_job(database, queued.id, attempt=job.attempt + 1)
|
|
if failure is not None:
|
|
raise failure
|
|
return _rendered()
|
|
|
|
renderer.render = render
|
|
outcome = await _handle(service, queued)
|
|
return outcome, database.jobs[queued.id]
|
|
|
|
|
|
async def test_superseded_on_ready_no_effect(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
renderer: FakeImageRenderer,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
outcome, job = await _with_supersede(service, database, renderer, queued)
|
|
assert outcome == Outcome.SKIPPED
|
|
assert job.state == AttachmentJobState.PROCESSING
|
|
assert database.attachments[queued.attachment_id].status == (
|
|
AttachmentStatus.PROCESSING
|
|
)
|
|
assert database.variants == {}
|
|
|
|
|
|
async def test_superseded_on_failed_no_effect(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
renderer: FakeImageRenderer,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
outcome, job = await _with_supersede(
|
|
service,
|
|
database,
|
|
renderer,
|
|
queued,
|
|
failure=exceptions.ImageRejected(
|
|
reason=AttachmentJobReason.UNDECODABLE,
|
|
),
|
|
)
|
|
assert outcome == Outcome.SKIPPED
|
|
assert job.state == AttachmentJobState.PROCESSING
|
|
assert database.attachments[queued.attachment_id].error_code is None
|
|
|
|
|
|
async def test_superseded_on_return_no_effect(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
renderer: FakeImageRenderer,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
outcome, job = await _with_supersede(
|
|
service,
|
|
database,
|
|
renderer,
|
|
queued,
|
|
failure=exceptions.RenderInterrupted(
|
|
reason=AttachmentJobReason.RENDER_KILLED,
|
|
),
|
|
)
|
|
assert outcome == Outcome.SKIPPED
|
|
assert job.state == AttachmentJobState.PROCESSING
|
|
assert database.attachments[queued.attachment_id].status == (
|
|
AttachmentStatus.PROCESSING
|
|
)
|
|
|
|
|
|
async def test_database_failure_requeues_and_pauses(
|
|
service: AttachmentProcessingService,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
) -> None:
|
|
database.fail_on["jobs.lock_job"] = exceptions.ServiceUnavailable()
|
|
assert await _handle(service, queued) == Outcome.REQUEUED
|
|
assert database.jobs[queued.id].attempt == 0
|
|
|
|
|
|
async def test_stop_interrupts_pause(
|
|
uow: FakeUnitOfWork,
|
|
storage: FakeObjectStorage,
|
|
renderer: FakeImageRenderer,
|
|
database: FakeDatabase,
|
|
queued: dataclasses.AttachmentJobDTO,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(settings.worker, "reconnect_seconds", 30)
|
|
stop = asyncio.Event()
|
|
stop.set()
|
|
service = AttachmentProcessingService(uow, storage, renderer, stop)
|
|
database.fail_on["jobs.lock_job"] = exceptions.ServiceUnavailable()
|
|
async with asyncio.timeout(1):
|
|
assert await _handle(service, queued) == Outcome.REQUEUED
|
|
|
|
|
|
def test_retry_delay_bounds() -> None:
|
|
for attempt in range(1, 8):
|
|
delay = retry_delay(attempt=attempt, base=5, maximum=60)
|
|
top = min(60, 5 * 2 ** (attempt - 1))
|
|
assert top / 2 <= delay <= top
|