diff --git a/openspec/changes/add-lab6-telemetry-pipeline/tasks.md b/openspec/changes/add-lab6-telemetry-pipeline/tasks.md index 9794fade..9f86be3b 100644 --- a/openspec/changes/add-lab6-telemetry-pipeline/tasks.md +++ b/openspec/changes/add-lab6-telemetry-pipeline/tasks.md @@ -8,9 +8,9 @@ ## 2. Зона `attachments` -- [ ] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider` -- [ ] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал -- [ ] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed` +- [x] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider` +- [x] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал +- [x] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed` ## 3. Сборка процессов diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index e32f42cc..32922e44 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -153,3 +153,8 @@ class AttachmentDirectory(Protocol): async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... async def failed_jobs(self, limit: int) -> list[FailedJob]: ... + + +@runtime_checkable +class JobObserver(Protocol): + def observe(self, job: Job, outcome: str, seconds: float) -> None: ... diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index c661dd1e..0e60b98b 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -34,6 +34,7 @@ from messenger.attachments.application.ports import ( Faults, ImageInspector, Jitter, + JobObserver, JobPublisher, ObjectStorage, VariantRenderer, @@ -54,6 +55,7 @@ from messenger.attachments.infrastructure.postgres import ( ) from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.attachments.infrastructure.telemetry import OtelJobObserver from messenger.shared.application.commands import Command from messenger.shared.application.messagebus import CommandHandler from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger @@ -196,6 +198,10 @@ class WorkerProvider(Provider): def jitter(self) -> Jitter: return RandomJitter() + @provide(scope=Scope.APP) + def observer(self) -> JobObserver: + return OtelJobObserver() + @provide(scope=Scope.REQUEST) def command_bus( self, diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py index 8de721bd..c5a317e2 100644 --- a/src/messenger/attachments/entrypoints/worker.py +++ b/src/messenger/attachments/entrypoints/worker.py @@ -1,4 +1,5 @@ import asyncio +import time from dishka import FromDishka from faststream import AckPolicy @@ -8,6 +9,8 @@ from pydantic import ValidationError from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.application.ports import JobObserver +from messenger.attachments.domain.jobs import Job from messenger.attachments.entrypoints.schemas import JobMessage from messenger.attachments.infrastructure.rabbitmq import QUEUE from messenger.shared.application.ports import Logger @@ -24,6 +27,7 @@ async def handle_job( message: RabbitMessage, bus: FromDishka[AttachmentsCommandBus], logger: FromDishka[Logger], + observer: FromDishka[JobObserver], ) -> None: try: job = JobMessage.model_validate_json(message.body).job() @@ -31,16 +35,32 @@ async def handle_job( logger.warning("job.rejected", reason="message does not match jobs.schema.json") await message.reject(requeue=False) return + started = time.perf_counter() try: - await bus.execute(ProcessJob(job)) + outcome = (await bus.execute(ProcessJob(job))).value except Exception as error: + observed(job, "error", started, observer, logger) logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__) await asyncio.sleep(REQUEUE_DELAY_SECONDS) await message.nack(requeue=True) return + observed(job, outcome, started, observer, logger) await message.ack() +def observed(job: Job, outcome: str, started: float, observer: JobObserver, logger: Logger) -> None: + seconds = time.perf_counter() - started + observer.observe(job, outcome, seconds) + logger.info( + "job.processed", + job_id=str(job.id), + attachment_id=str(job.attachment_id), + attempt=job.attempt, + outcome=outcome, + duration_ms=round(seconds * 1000, 3), + ) + + def subscribe(broker: RabbitBroker, concurrency: int) -> None: broker.subscriber( JOBS, diff --git a/src/messenger/attachments/infrastructure/telemetry.py b/src/messenger/attachments/infrastructure/telemetry.py new file mode 100644 index 00000000..7a6a2f2b --- /dev/null +++ b/src/messenger/attachments/infrastructure/telemetry.py @@ -0,0 +1,40 @@ +from opentelemetry import metrics, trace +from opentelemetry.metrics import MeterProvider +from opentelemetry.trace import StatusCode + +from messenger.attachments.domain.jobs import JOB_TYPE, Job + +DURATION_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0) +FAILURES = frozenset({"retry_scheduled", "failed", "error"}) + + +class OtelJobObserver: + __slots__ = ("_duration", "_processed") + + def __init__(self, provider: MeterProvider | None = None) -> None: + meter = metrics.get_meter("messenger.attachments", meter_provider=provider) + self._processed = meter.create_counter( + "messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу" + ) + self._duration = meter.create_histogram( + "messenger.job.duration", + unit="s", + description="Длительность обработки доставки задания", + explicit_bucket_boundaries_advisory=DURATION_BUCKETS, + ) + + def observe(self, job: Job, outcome: str, seconds: float) -> None: + labels = {"job.type": JOB_TYPE, "outcome": outcome} + self._processed.add(1, labels) + self._duration.record(seconds, labels) + span = trace.get_current_span() + span.set_attributes( + { + "messenger.job_id": str(job.id), + "messenger.attachment_id": str(job.attachment_id), + "messenger.job.attempt": job.attempt, + "messenger.job.outcome": outcome, + } + ) + if outcome in FAILURES: + span.set_status(StatusCode.ERROR, outcome) diff --git a/student-tests/adapters/test_rabbitmq.py b/student-tests/adapters/test_rabbitmq.py index fb62b845..c112e290 100644 --- a/student-tests/adapters/test_rabbitmq.py +++ b/student-tests/adapters/test_rabbitmq.py @@ -21,9 +21,11 @@ from conftest import ROOT, STARTUP_SECONDS, container, image, published from fakes import NullLogger from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.application.ports import JobObserver from messenger.attachments.domain.jobs import Job, Outcome from messenger.attachments.entrypoints.worker import subscribe from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher +from messenger.attachments.infrastructure.telemetry import OtelJobObserver from messenger.shared.application.errors import Unavailable from messenger.shared.application.ports import Logger @@ -175,6 +177,10 @@ class FakeBus(Provider): def logger(self) -> Logger: return NullLogger() + @provide(scope=Scope.APP) + def observer(self) -> JobObserver: + return OtelJobObserver() + @provide(scope=Scope.REQUEST) def bus(self) -> AttachmentsCommandBus: return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher()) diff --git a/student-tests/unit/test_job_observer.py b/student-tests/unit/test_job_observer.py new file mode 100644 index 00000000..a8cce472 --- /dev/null +++ b/student-tests/unit/test_job_observer.py @@ -0,0 +1,62 @@ +from datetime import UTC, datetime +from uuid import uuid4 + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import ( + HistogramDataPoint, + InMemoryMetricReader, + NumberDataPoint, +) +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind, StatusCode + +from messenger.attachments.domain.jobs import Job +from messenger.attachments.infrastructure.telemetry import OtelJobObserver + + +def job() -> Job: + return Job(uuid4(), uuid4(), 1, 2, datetime.now(UTC)) + + +def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None: + reader = InMemoryMetricReader() + observer = OtelJobObserver(MeterProvider(metric_readers=[reader], shutdown_on_exit=False)) + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer("test") + done, failed = job(), job() + for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")): + with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER): + observer.observe(item, outcome, 0.3) + data = reader.get_metrics_data() + assert data is not None + found = { + metric.name: metric + for resource in data.resource_metrics + for scope in resource.scope_metrics + for metric in scope.metrics + } + counted = { + (point.attributes or {})["outcome"]: point.value + for point in found["messenger.jobs.processed"].data.data_points + if isinstance(point, NumberDataPoint) + } + assert counted == {"done": 1, "failed": 1, "duplicate_noop": 1} + duration = found["messenger.job.duration"] + assert duration.unit == "s" + point = next(iter(duration.data.data_points)) + assert isinstance(point, HistogramDataPoint) + assert set(point.attributes or {}) == {"job.type", "outcome"} + assert point.explicit_bounds[0] == 0.05 + assert point.explicit_bounds[-1] == 60.0 + spans: tuple[ReadableSpan, ...] = exporter.get_finished_spans() + assert [span.status.status_code for span in spans] == [ + StatusCode.UNSET, + StatusCode.ERROR, + StatusCode.UNSET, + ] + assert (spans[1].attributes or {})["messenger.attachment_id"] == str(failed.attachment_id) + assert (spans[1].attributes or {})["messenger.job.attempt"] == 2