468 lines
15 KiB
Python
468 lines
15 KiB
Python
"""`AttachmentRelayService`: порядок публикации, предел, аренды, потерянные."""
|
||
|
||
import asyncio
|
||
import json
|
||
from dataclasses import replace
|
||
from datetime import timedelta
|
||
from uuid import uuid4
|
||
|
||
import pytest
|
||
|
||
from builders import attachment_dto, job_dto
|
||
from fakes import FakeDatabase, FakeJobPublisher, FakeUnitOfWork
|
||
from src.config import settings
|
||
from src.core import exceptions
|
||
from src.core.services.attachment_relay import AttachmentRelayService
|
||
from src.relay import main as relay_main
|
||
from src.schemas import dataclasses
|
||
from src.schemas.enums import (
|
||
AttachmentJobReason,
|
||
AttachmentJobState,
|
||
AttachmentStatus,
|
||
)
|
||
from src.utils.utc import utcnow
|
||
|
||
pytestmark = pytest.mark.anyio
|
||
|
||
|
||
@pytest.fixture
|
||
def service(
|
||
uow: FakeUnitOfWork,
|
||
publisher: FakeJobPublisher,
|
||
) -> AttachmentRelayService:
|
||
return AttachmentRelayService(uow, publisher)
|
||
|
||
|
||
async def _jobs(
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
count: int = 1,
|
||
) -> list[dataclasses.AttachmentJobDTO]:
|
||
now = utcnow() - timedelta(seconds=1)
|
||
jobs = []
|
||
async with uow:
|
||
for index in range(count):
|
||
row = attachment_dto(
|
||
chat_id=uuid4(),
|
||
owner_id=uuid4(),
|
||
status=AttachmentStatus.QUEUED,
|
||
created_at=now,
|
||
)
|
||
job = job_dto(
|
||
attachment_id=row.id,
|
||
created_at=now + timedelta(milliseconds=index),
|
||
)
|
||
await uow.attachments.insert_attachment(attachment=row)
|
||
await uow.jobs.insert_job(job=job)
|
||
jobs.append(job)
|
||
await uow.commit()
|
||
database.events.clear()
|
||
return jobs
|
||
|
||
|
||
async def _pass(
|
||
service: AttachmentRelayService,
|
||
*,
|
||
batch_size: int = 10,
|
||
max_backlog: int = 100,
|
||
stop_requested=None,
|
||
):
|
||
return await service.publish_pass(
|
||
batch_size=batch_size,
|
||
max_published_backlog=max_backlog,
|
||
republish_after_seconds=60,
|
||
**(
|
||
{}
|
||
if stop_requested is None
|
||
else {"stop_requested": stop_requested}
|
||
),
|
||
)
|
||
|
||
|
||
async def test_publish_order_in_one_block(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
"""Сценарий «Порядок публикации»."""
|
||
jobs = await _jobs(uow, database, 2)
|
||
result = await _pass(service)
|
||
assert result.published == 2
|
||
start = database.events.index("jobs.lock_due_jobs")
|
||
assert database.events[start:] == [
|
||
"jobs.lock_due_jobs",
|
||
"publisher.publish_many",
|
||
"jobs.mark_published",
|
||
"commit",
|
||
"exit",
|
||
]
|
||
assert database.inside_block("publisher.publish_many")
|
||
assert all(database.jobs[job.id].published_at for job in jobs)
|
||
|
||
|
||
async def test_confirmed_subset_marked(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
publisher: FakeJobPublisher,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
first, second = await _jobs(uow, database, 2)
|
||
publisher.unconfirmed = {second.id}
|
||
result = await _pass(service)
|
||
assert result.published == 1
|
||
assert result.unconfirmed == 1
|
||
assert database.jobs[first.id].published_at is not None
|
||
assert database.jobs[second.id].published_at is None
|
||
|
||
|
||
async def test_nothing_confirmed_no_mark(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
publisher: FakeJobPublisher,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
(first,) = await _jobs(uow, database, 1)
|
||
publisher.unconfirmed = {first.id}
|
||
result = await _pass(service)
|
||
assert result.unconfirmed == 1
|
||
assert "jobs.mark_published" not in database.events
|
||
|
||
|
||
async def test_backlog_limit(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
publisher: FakeJobPublisher,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
jobs = await _jobs(uow, database, 3)
|
||
result = await _pass(service, max_backlog=2)
|
||
assert result.published == 2
|
||
assert result.published_backlog == 2
|
||
assert not result.backlog_full
|
||
publisher.published.clear()
|
||
result = await _pass(service, max_backlog=2)
|
||
assert result.published == 0
|
||
assert result.backlog_full
|
||
assert publisher.published == []
|
||
assert database.jobs[jobs[2].id].published_at is None
|
||
|
||
|
||
async def test_held_job_skipped(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
publisher: FakeJobPublisher,
|
||
) -> None:
|
||
first, second = await _jobs(uow, database, 2)
|
||
database.held.add(first.id)
|
||
await _pass(service)
|
||
assert publisher.published == [second.id]
|
||
|
||
|
||
async def test_expired_leases_one_block(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
(job,) = await _jobs(uow, database, 1)
|
||
async with uow:
|
||
await uow.jobs.start_attempt(
|
||
job_id=job.id,
|
||
lease_expires_at=utcnow() - timedelta(seconds=1),
|
||
now=utcnow(),
|
||
)
|
||
await uow.attachments.mark_processing(
|
||
attachment_id=job.attachment_id,
|
||
updated_at=utcnow(),
|
||
)
|
||
await uow.commit()
|
||
database.events.clear()
|
||
result = await _pass(service)
|
||
assert result.recovered == 1
|
||
assert database.events[:4] == [
|
||
"enter",
|
||
"jobs.recover_expired_leases",
|
||
"attachments.return_group_to_queued",
|
||
"commit",
|
||
]
|
||
stored = database.jobs[job.id]
|
||
assert stored.last_error_code == AttachmentJobReason.LEASE_EXPIRED.value
|
||
assert database.attachments[job.attachment_id].status == (
|
||
AttachmentStatus.QUEUED
|
||
)
|
||
|
||
|
||
async def _published_long_ago(
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
) -> dataclasses.AttachmentJobDTO:
|
||
(job,) = await _jobs(uow, database, 1)
|
||
database.jobs[job.id] = replace(
|
||
database.jobs[job.id],
|
||
published_at=utcnow() - timedelta(seconds=600),
|
||
)
|
||
database.events.clear()
|
||
return job
|
||
|
||
|
||
async def test_lost_messages_reset_at_zero_ready(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
publisher: FakeJobPublisher,
|
||
) -> None:
|
||
job = await _published_long_ago(uow, database)
|
||
publisher.ready = 0
|
||
result = await _pass(service)
|
||
assert result.republished == 1
|
||
# тем же проходом опубликовано заново
|
||
assert result.published == 1
|
||
assert database.jobs[job.id].published_at is not None
|
||
assert not database.inside_block("publisher.ready_messages")
|
||
|
||
|
||
async def test_lost_messages_kept_when_queue_has_ready(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
publisher: FakeJobPublisher,
|
||
) -> None:
|
||
await _published_long_ago(uow, database)
|
||
publisher.ready = 5
|
||
result = await _pass(service)
|
||
assert result.republished == 0
|
||
assert "jobs.reset_published_before" not in database.events
|
||
|
||
|
||
async def test_broker_not_asked_without_candidates(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
await _jobs(uow, database, 1)
|
||
await _pass(service)
|
||
assert "publisher.ready_messages" not in database.events
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("ready_calls", "expected_absent"),
|
||
[
|
||
(1, ["jobs.has_published_before", "jobs.lock_due_jobs"]),
|
||
(2, ["jobs.lock_due_jobs"]),
|
||
],
|
||
ids=["after_recover", "after_lost"],
|
||
)
|
||
async def test_stop_between_steps(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
ready_calls: int,
|
||
expected_absent: list[str],
|
||
) -> None:
|
||
await _jobs(uow, database, 1)
|
||
calls = 0
|
||
|
||
def stop_requested() -> bool:
|
||
nonlocal calls
|
||
calls += 1
|
||
return calls >= ready_calls
|
||
|
||
result = await _pass(service, stop_requested=stop_requested)
|
||
assert result.published == 0
|
||
for name in expected_absent:
|
||
assert name not in database.events
|
||
|
||
|
||
async def test_broker_failure_rolls_back_block(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
(job,) = await _jobs(uow, database, 1)
|
||
database.fail_on["publisher.publish_many"] = exceptions.BrokerUnavailable()
|
||
with pytest.raises(exceptions.BrokerUnavailable):
|
||
await _pass(service)
|
||
stored = database.jobs[job.id]
|
||
assert stored.published_at is None
|
||
assert stored.state == AttachmentJobState.QUEUED
|
||
|
||
|
||
# --- публикации и снимок очереди (design change job-trace-propagation) ---
|
||
|
||
TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"
|
||
|
||
|
||
async def test_publication_items_carry_context_and_attempt(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
publisher: FakeJobPublisher,
|
||
) -> None:
|
||
"""Элемент публикации: номер следующей попытки, признак replay, причина
|
||
повтора, контекст из колонок задания; без контекста — `None`."""
|
||
traced, plain, replayed = await _jobs(uow, database, 3)
|
||
database.jobs[traced.id] = replace(
|
||
database.jobs[traced.id],
|
||
attempt=1,
|
||
last_error_code=AttachmentJobReason.STORAGE_UNAVAILABLE.value,
|
||
traceparent=TRACEPARENT,
|
||
tracestate="vendor=value",
|
||
)
|
||
database.jobs[replayed.id] = replace(
|
||
database.jobs[replayed.id],
|
||
attempt=4,
|
||
attempt_base=4,
|
||
traceparent=TRACEPARENT,
|
||
)
|
||
await _pass(service)
|
||
items = {item.job_id: item for item in publisher.items}
|
||
first = items[traced.id]
|
||
assert first.attempt == 2
|
||
assert first.replayed is False
|
||
assert first.retry_reason == "storage_unavailable"
|
||
assert first.trace_context == dataclasses.TraceContextDTO(
|
||
traceparent=TRACEPARENT,
|
||
tracestate="vendor=value",
|
||
)
|
||
assert json.loads(first.body)["attempt"] == 2
|
||
# тело — по схеме, без полей контекста: carrier — заголовки
|
||
assert "traceparent" not in json.loads(first.body)
|
||
second = items[plain.id]
|
||
assert second.attempt == 1
|
||
assert second.retry_reason is None
|
||
assert second.trace_context is None
|
||
third = items[replayed.id]
|
||
assert third.replayed is True
|
||
assert third.attempt == 5
|
||
assert third.trace_context == dataclasses.TraceContextDTO(
|
||
traceparent=TRACEPARENT,
|
||
)
|
||
|
||
|
||
async def test_queue_snapshot_one_block(
|
||
service: AttachmentRelayService,
|
||
uow: FakeUnitOfWork,
|
||
database: FakeDatabase,
|
||
) -> None:
|
||
jobs = await _jobs(uow, database, 2)
|
||
before = utcnow()
|
||
snapshot = await service.queue_snapshot()
|
||
assert database.events == ["enter", "jobs.queue_snapshot", "exit"]
|
||
assert (snapshot.outbox, snapshot.published, snapshot.processing) == (
|
||
2,
|
||
0,
|
||
0,
|
||
)
|
||
assert snapshot.taken_at >= before
|
||
assert snapshot.oldest_created_at == min(job.created_at for job in jobs)
|
||
|
||
|
||
class _LoopService:
|
||
"""Проходы relay для цикла `_serve`: после `passes` проходов — сигнал."""
|
||
|
||
def __init__(self, stop: asyncio.Event, *, passes: int) -> None:
|
||
self.stop = stop
|
||
self.passes = passes
|
||
self.snapshots = 0
|
||
self.fail_snapshot = False
|
||
|
||
async def publish_pass(self, **_: object) -> dataclasses.RelayPassDTO:
|
||
self.passes -= 1
|
||
if self.passes <= 0:
|
||
self.stop.set()
|
||
return dataclasses.RelayPassDTO(
|
||
published=0,
|
||
unconfirmed=0,
|
||
recovered=0,
|
||
republished=0,
|
||
published_backlog=0,
|
||
backlog_full=False,
|
||
selected=0,
|
||
duration_ms=0,
|
||
)
|
||
|
||
async def queue_snapshot(self) -> dataclasses.QueueSnapshotDTO:
|
||
self.snapshots += 1
|
||
if self.fail_snapshot:
|
||
raise exceptions.ServiceUnavailable
|
||
return dataclasses.QueueSnapshotDTO(
|
||
taken_at=utcnow(),
|
||
scheduled=0,
|
||
outbox=1,
|
||
published=0,
|
||
processing=0,
|
||
)
|
||
|
||
|
||
async def test_relay_loop_takes_snapshot_not_more_often(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Цикл relay: снимок после успешного прохода, не чаще 5 с; без
|
||
телеметрии — снимков нет; отказ PostgreSQL — пропуск."""
|
||
monkeypatch.setattr(settings.relay, "poll_interval_seconds", 0)
|
||
stop = asyncio.Event()
|
||
loop_service = _LoopService(stop, passes=3)
|
||
snapshots = relay_main.QueueSnapshots()
|
||
await relay_main._serve(loop_service, stop=stop, snapshots=None)
|
||
assert loop_service.snapshots == 0
|
||
stop.clear()
|
||
loop_service.passes = 3
|
||
await relay_main._serve(loop_service, stop=stop, snapshots=snapshots)
|
||
# первый проход — снимок, следующие в пределах 5 с — нет; последний
|
||
# проход видел сигнал
|
||
assert loop_service.snapshots == 1
|
||
assert snapshots.latest is not None
|
||
assert snapshots.latest.outbox == 1
|
||
stop.clear()
|
||
loop_service.passes = 2
|
||
loop_service.fail_snapshot = True
|
||
previous = snapshots.latest
|
||
failing = relay_main.QueueSnapshots()
|
||
await relay_main._serve(loop_service, stop=stop, snapshots=failing)
|
||
assert failing.latest is None
|
||
assert snapshots.latest is previous
|
||
|
||
|
||
class _RecordingLogger:
|
||
"""Записи цикла relay: событие, уровень и поля."""
|
||
|
||
def __init__(self) -> None:
|
||
self.records: list[tuple[str, str, dict[str, object]]] = []
|
||
|
||
def __getattr__(self, level: str):
|
||
def record(event: str, **fields: object) -> None:
|
||
self.records.append((level, event, fields))
|
||
|
||
return record
|
||
|
||
|
||
class _BrokenSnapshotService(_LoopService):
|
||
"""Снимок падает не отказом PostgreSQL, а непредвиденной ошибкой."""
|
||
|
||
async def queue_snapshot(self) -> dataclasses.QueueSnapshotDTO:
|
||
self.snapshots += 1
|
||
raise RuntimeError("probe")
|
||
|
||
|
||
async def test_relay_loop_survives_snapshot_error(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Непредвиденная ошибка снимка не останавливает relay: запись `error`
|
||
«Queue snapshot failed» с `exc_info`, проходы идут дальше, снимка нет
|
||
(verify W4)."""
|
||
monkeypatch.setattr(settings.relay, "poll_interval_seconds", 0)
|
||
log = _RecordingLogger()
|
||
monkeypatch.setattr(relay_main, "logger", log)
|
||
stop = asyncio.Event()
|
||
loop_service = _BrokenSnapshotService(stop, passes=3)
|
||
snapshots = relay_main.QueueSnapshots()
|
||
await relay_main._serve(loop_service, stop=stop, snapshots=snapshots)
|
||
# все три прохода выполнены: сигнал поставил последний
|
||
assert loop_service.passes == 0
|
||
assert loop_service.snapshots == 1
|
||
assert snapshots.latest is None
|
||
errors = [r for r in log.records if r[0] == "error"]
|
||
assert [event for _, event, _ in errors] == ["Queue snapshot failed"]
|
||
assert isinstance(errors[0][2]["exc_info"], RuntimeError)
|
||
assert "Relay pass failed" not in [event for _, event, _ in log.records]
|