Files
mtusi2026lab6/service-tests/unit/test_attachment_relay.py
T
2026-11-11 20:53:13 +03:00

468 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""`AttachmentRelayService`: порядок публикации, предел, аренды, потерянные."""
import asyncio
import json
from dataclasses import replace
from datetime import timedelta
from uuid import uuid4
import pytest
from builders import attachment_dto, job_dto
from fakes import FakeDatabase, FakeJobPublisher, FakeUnitOfWork
from src.config import settings
from src.core import exceptions
from src.core.services.attachment_relay import AttachmentRelayService
from src.relay import main as relay_main
from src.schemas import dataclasses
from src.schemas.enums import (
AttachmentJobReason,
AttachmentJobState,
AttachmentStatus,
)
from src.utils.utc import utcnow
pytestmark = pytest.mark.anyio
@pytest.fixture
def service(
uow: FakeUnitOfWork,
publisher: FakeJobPublisher,
) -> AttachmentRelayService:
return AttachmentRelayService(uow, publisher)
async def _jobs(
uow: FakeUnitOfWork,
database: FakeDatabase,
count: int = 1,
) -> list[dataclasses.AttachmentJobDTO]:
now = utcnow() - timedelta(seconds=1)
jobs = []
async with uow:
for index in range(count):
row = attachment_dto(
chat_id=uuid4(),
owner_id=uuid4(),
status=AttachmentStatus.QUEUED,
created_at=now,
)
job = job_dto(
attachment_id=row.id,
created_at=now + timedelta(milliseconds=index),
)
await uow.attachments.insert_attachment(attachment=row)
await uow.jobs.insert_job(job=job)
jobs.append(job)
await uow.commit()
database.events.clear()
return jobs
async def _pass(
service: AttachmentRelayService,
*,
batch_size: int = 10,
max_backlog: int = 100,
stop_requested=None,
):
return await service.publish_pass(
batch_size=batch_size,
max_published_backlog=max_backlog,
republish_after_seconds=60,
**(
{}
if stop_requested is None
else {"stop_requested": stop_requested}
),
)
async def test_publish_order_in_one_block(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
) -> None:
"""Сценарий «Порядок публикации»."""
jobs = await _jobs(uow, database, 2)
result = await _pass(service)
assert result.published == 2
start = database.events.index("jobs.lock_due_jobs")
assert database.events[start:] == [
"jobs.lock_due_jobs",
"publisher.publish_many",
"jobs.mark_published",
"commit",
"exit",
]
assert database.inside_block("publisher.publish_many")
assert all(database.jobs[job.id].published_at for job in jobs)
async def test_confirmed_subset_marked(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
publisher: FakeJobPublisher,
database: FakeDatabase,
) -> None:
first, second = await _jobs(uow, database, 2)
publisher.unconfirmed = {second.id}
result = await _pass(service)
assert result.published == 1
assert result.unconfirmed == 1
assert database.jobs[first.id].published_at is not None
assert database.jobs[second.id].published_at is None
async def test_nothing_confirmed_no_mark(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
publisher: FakeJobPublisher,
database: FakeDatabase,
) -> None:
(first,) = await _jobs(uow, database, 1)
publisher.unconfirmed = {first.id}
result = await _pass(service)
assert result.unconfirmed == 1
assert "jobs.mark_published" not in database.events
async def test_backlog_limit(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
publisher: FakeJobPublisher,
database: FakeDatabase,
) -> None:
jobs = await _jobs(uow, database, 3)
result = await _pass(service, max_backlog=2)
assert result.published == 2
assert result.published_backlog == 2
assert not result.backlog_full
publisher.published.clear()
result = await _pass(service, max_backlog=2)
assert result.published == 0
assert result.backlog_full
assert publisher.published == []
assert database.jobs[jobs[2].id].published_at is None
async def test_held_job_skipped(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
publisher: FakeJobPublisher,
) -> None:
first, second = await _jobs(uow, database, 2)
database.held.add(first.id)
await _pass(service)
assert publisher.published == [second.id]
async def test_expired_leases_one_block(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
) -> None:
(job,) = await _jobs(uow, database, 1)
async with uow:
await uow.jobs.start_attempt(
job_id=job.id,
lease_expires_at=utcnow() - timedelta(seconds=1),
now=utcnow(),
)
await uow.attachments.mark_processing(
attachment_id=job.attachment_id,
updated_at=utcnow(),
)
await uow.commit()
database.events.clear()
result = await _pass(service)
assert result.recovered == 1
assert database.events[:4] == [
"enter",
"jobs.recover_expired_leases",
"attachments.return_group_to_queued",
"commit",
]
stored = database.jobs[job.id]
assert stored.last_error_code == AttachmentJobReason.LEASE_EXPIRED.value
assert database.attachments[job.attachment_id].status == (
AttachmentStatus.QUEUED
)
async def _published_long_ago(
uow: FakeUnitOfWork,
database: FakeDatabase,
) -> dataclasses.AttachmentJobDTO:
(job,) = await _jobs(uow, database, 1)
database.jobs[job.id] = replace(
database.jobs[job.id],
published_at=utcnow() - timedelta(seconds=600),
)
database.events.clear()
return job
async def test_lost_messages_reset_at_zero_ready(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
publisher: FakeJobPublisher,
) -> None:
job = await _published_long_ago(uow, database)
publisher.ready = 0
result = await _pass(service)
assert result.republished == 1
# тем же проходом опубликовано заново
assert result.published == 1
assert database.jobs[job.id].published_at is not None
assert not database.inside_block("publisher.ready_messages")
async def test_lost_messages_kept_when_queue_has_ready(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
publisher: FakeJobPublisher,
) -> None:
await _published_long_ago(uow, database)
publisher.ready = 5
result = await _pass(service)
assert result.republished == 0
assert "jobs.reset_published_before" not in database.events
async def test_broker_not_asked_without_candidates(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
) -> None:
await _jobs(uow, database, 1)
await _pass(service)
assert "publisher.ready_messages" not in database.events
@pytest.mark.parametrize(
("ready_calls", "expected_absent"),
[
(1, ["jobs.has_published_before", "jobs.lock_due_jobs"]),
(2, ["jobs.lock_due_jobs"]),
],
ids=["after_recover", "after_lost"],
)
async def test_stop_between_steps(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
ready_calls: int,
expected_absent: list[str],
) -> None:
await _jobs(uow, database, 1)
calls = 0
def stop_requested() -> bool:
nonlocal calls
calls += 1
return calls >= ready_calls
result = await _pass(service, stop_requested=stop_requested)
assert result.published == 0
for name in expected_absent:
assert name not in database.events
async def test_broker_failure_rolls_back_block(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
) -> None:
(job,) = await _jobs(uow, database, 1)
database.fail_on["publisher.publish_many"] = exceptions.BrokerUnavailable()
with pytest.raises(exceptions.BrokerUnavailable):
await _pass(service)
stored = database.jobs[job.id]
assert stored.published_at is None
assert stored.state == AttachmentJobState.QUEUED
# --- публикации и снимок очереди (design change job-trace-propagation) ---
TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"
async def test_publication_items_carry_context_and_attempt(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
publisher: FakeJobPublisher,
) -> None:
"""Элемент публикации: номер следующей попытки, признак replay, причина
повтора, контекст из колонок задания; без контекста — `None`."""
traced, plain, replayed = await _jobs(uow, database, 3)
database.jobs[traced.id] = replace(
database.jobs[traced.id],
attempt=1,
last_error_code=AttachmentJobReason.STORAGE_UNAVAILABLE.value,
traceparent=TRACEPARENT,
tracestate="vendor=value",
)
database.jobs[replayed.id] = replace(
database.jobs[replayed.id],
attempt=4,
attempt_base=4,
traceparent=TRACEPARENT,
)
await _pass(service)
items = {item.job_id: item for item in publisher.items}
first = items[traced.id]
assert first.attempt == 2
assert first.replayed is False
assert first.retry_reason == "storage_unavailable"
assert first.trace_context == dataclasses.TraceContextDTO(
traceparent=TRACEPARENT,
tracestate="vendor=value",
)
assert json.loads(first.body)["attempt"] == 2
# тело — по схеме, без полей контекста: carrier — заголовки
assert "traceparent" not in json.loads(first.body)
second = items[plain.id]
assert second.attempt == 1
assert second.retry_reason is None
assert second.trace_context is None
third = items[replayed.id]
assert third.replayed is True
assert third.attempt == 5
assert third.trace_context == dataclasses.TraceContextDTO(
traceparent=TRACEPARENT,
)
async def test_queue_snapshot_one_block(
service: AttachmentRelayService,
uow: FakeUnitOfWork,
database: FakeDatabase,
) -> None:
jobs = await _jobs(uow, database, 2)
before = utcnow()
snapshot = await service.queue_snapshot()
assert database.events == ["enter", "jobs.queue_snapshot", "exit"]
assert (snapshot.outbox, snapshot.published, snapshot.processing) == (
2,
0,
0,
)
assert snapshot.taken_at >= before
assert snapshot.oldest_created_at == min(job.created_at for job in jobs)
class _LoopService:
"""Проходы relay для цикла `_serve`: после `passes` проходов — сигнал."""
def __init__(self, stop: asyncio.Event, *, passes: int) -> None:
self.stop = stop
self.passes = passes
self.snapshots = 0
self.fail_snapshot = False
async def publish_pass(self, **_: object) -> dataclasses.RelayPassDTO:
self.passes -= 1
if self.passes <= 0:
self.stop.set()
return dataclasses.RelayPassDTO(
published=0,
unconfirmed=0,
recovered=0,
republished=0,
published_backlog=0,
backlog_full=False,
selected=0,
duration_ms=0,
)
async def queue_snapshot(self) -> dataclasses.QueueSnapshotDTO:
self.snapshots += 1
if self.fail_snapshot:
raise exceptions.ServiceUnavailable
return dataclasses.QueueSnapshotDTO(
taken_at=utcnow(),
scheduled=0,
outbox=1,
published=0,
processing=0,
)
async def test_relay_loop_takes_snapshot_not_more_often(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Цикл relay: снимок после успешного прохода, не чаще 5 с; без
телеметрии — снимков нет; отказ PostgreSQL — пропуск."""
monkeypatch.setattr(settings.relay, "poll_interval_seconds", 0)
stop = asyncio.Event()
loop_service = _LoopService(stop, passes=3)
snapshots = relay_main.QueueSnapshots()
await relay_main._serve(loop_service, stop=stop, snapshots=None)
assert loop_service.snapshots == 0
stop.clear()
loop_service.passes = 3
await relay_main._serve(loop_service, stop=stop, snapshots=snapshots)
# первый проход — снимок, следующие в пределах 5 с — нет; последний
# проход видел сигнал
assert loop_service.snapshots == 1
assert snapshots.latest is not None
assert snapshots.latest.outbox == 1
stop.clear()
loop_service.passes = 2
loop_service.fail_snapshot = True
previous = snapshots.latest
failing = relay_main.QueueSnapshots()
await relay_main._serve(loop_service, stop=stop, snapshots=failing)
assert failing.latest is None
assert snapshots.latest is previous
class _RecordingLogger:
"""Записи цикла relay: событие, уровень и поля."""
def __init__(self) -> None:
self.records: list[tuple[str, str, dict[str, object]]] = []
def __getattr__(self, level: str):
def record(event: str, **fields: object) -> None:
self.records.append((level, event, fields))
return record
class _BrokenSnapshotService(_LoopService):
"""Снимок падает не отказом PostgreSQL, а непредвиденной ошибкой."""
async def queue_snapshot(self) -> dataclasses.QueueSnapshotDTO:
self.snapshots += 1
raise RuntimeError("probe")
async def test_relay_loop_survives_snapshot_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Непредвиденная ошибка снимка не останавливает relay: запись `error`
«Queue snapshot failed» с `exc_info`, проходы идут дальше, снимка нет
(verify W4)."""
monkeypatch.setattr(settings.relay, "poll_interval_seconds", 0)
log = _RecordingLogger()
monkeypatch.setattr(relay_main, "logger", log)
stop = asyncio.Event()
loop_service = _BrokenSnapshotService(stop, passes=3)
snapshots = relay_main.QueueSnapshots()
await relay_main._serve(loop_service, stop=stop, snapshots=snapshots)
# все три прохода выполнены: сигнал поставил последний
assert loop_service.passes == 0
assert loop_service.snapshots == 1
assert snapshots.latest is None
errors = [r for r in log.records if r[0] == "error"]
assert [event for _, event, _ in errors] == ["Queue snapshot failed"]
assert isinstance(errors[0][2]["exc_info"], RuntimeError)
assert "Relay pass failed" not in [event for _, event, _ in log.records]