From e2ec3fe9af51acfdab9054b81d806ebc074263a3 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 29 Oct 2026 04:45:20 +0300 Subject: [PATCH] Count and trace attachment jobs through a job observer port Add the JobObserver port to the attachments zone and its adapter on the OpenTelemetry API: every delivery adds one to messenger.jobs.processed and records messenger.job.duration in seconds with buckets from 50 ms to 60 s, both labelled only by job.type and outcome, and puts the job id, the attachment id, the attempt and the outcome on the current consumer span, which turns to ERROR on retry_scheduled, failed and error. The worker entrypoint measures each delivery, reports the outcome of the handler or error when it raised, and writes a job.processed record. Without the SDK the adapter runs on the no-op providers of the API. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../add-lab6-telemetry-pipeline/tasks.md | 6 +- .../attachments/application/ports.py | 5 ++ src/messenger/attachments/entrypoints/ioc.py | 6 ++ .../attachments/entrypoints/worker.py | 22 ++++++- .../attachments/infrastructure/telemetry.py | 40 ++++++++++++ student-tests/adapters/test_rabbitmq.py | 6 ++ student-tests/unit/test_job_observer.py | 62 +++++++++++++++++++ 7 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 src/messenger/attachments/infrastructure/telemetry.py create mode 100644 student-tests/unit/test_job_observer.py diff --git a/openspec/changes/add-lab6-telemetry-pipeline/tasks.md b/openspec/changes/add-lab6-telemetry-pipeline/tasks.md index 9794fade..9f86be3b 100644 --- a/openspec/changes/add-lab6-telemetry-pipeline/tasks.md +++ b/openspec/changes/add-lab6-telemetry-pipeline/tasks.md @@ -8,9 +8,9 @@ ## 2. Зона `attachments` -- [ ] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider` -- [ ] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал -- [ ] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed` +- [x] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider` +- [x] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал +- [x] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed` ## 3. Сборка процессов diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index e32f42cc..32922e44 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -153,3 +153,8 @@ class AttachmentDirectory(Protocol): async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... async def failed_jobs(self, limit: int) -> list[FailedJob]: ... + + +@runtime_checkable +class JobObserver(Protocol): + def observe(self, job: Job, outcome: str, seconds: float) -> None: ... diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index c661dd1e..0e60b98b 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -34,6 +34,7 @@ from messenger.attachments.application.ports import ( Faults, ImageInspector, Jitter, + JobObserver, JobPublisher, ObjectStorage, VariantRenderer, @@ -54,6 +55,7 @@ from messenger.attachments.infrastructure.postgres import ( ) from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.attachments.infrastructure.telemetry import OtelJobObserver from messenger.shared.application.commands import Command from messenger.shared.application.messagebus import CommandHandler from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger @@ -196,6 +198,10 @@ class WorkerProvider(Provider): def jitter(self) -> Jitter: return RandomJitter() + @provide(scope=Scope.APP) + def observer(self) -> JobObserver: + return OtelJobObserver() + @provide(scope=Scope.REQUEST) def command_bus( self, diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py index 8de721bd..c5a317e2 100644 --- a/src/messenger/attachments/entrypoints/worker.py +++ b/src/messenger/attachments/entrypoints/worker.py @@ -1,4 +1,5 @@ import asyncio +import time from dishka import FromDishka from faststream import AckPolicy @@ -8,6 +9,8 @@ from pydantic import ValidationError from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.application.ports import JobObserver +from messenger.attachments.domain.jobs import Job from messenger.attachments.entrypoints.schemas import JobMessage from messenger.attachments.infrastructure.rabbitmq import QUEUE from messenger.shared.application.ports import Logger @@ -24,6 +27,7 @@ async def handle_job( message: RabbitMessage, bus: FromDishka[AttachmentsCommandBus], logger: FromDishka[Logger], + observer: FromDishka[JobObserver], ) -> None: try: job = JobMessage.model_validate_json(message.body).job() @@ -31,16 +35,32 @@ async def handle_job( logger.warning("job.rejected", reason="message does not match jobs.schema.json") await message.reject(requeue=False) return + started = time.perf_counter() try: - await bus.execute(ProcessJob(job)) + outcome = (await bus.execute(ProcessJob(job))).value except Exception as error: + observed(job, "error", started, observer, logger) logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__) await asyncio.sleep(REQUEUE_DELAY_SECONDS) await message.nack(requeue=True) return + observed(job, outcome, started, observer, logger) await message.ack() +def observed(job: Job, outcome: str, started: float, observer: JobObserver, logger: Logger) -> None: + seconds = time.perf_counter() - started + observer.observe(job, outcome, seconds) + logger.info( + "job.processed", + job_id=str(job.id), + attachment_id=str(job.attachment_id), + attempt=job.attempt, + outcome=outcome, + duration_ms=round(seconds * 1000, 3), + ) + + def subscribe(broker: RabbitBroker, concurrency: int) -> None: broker.subscriber( JOBS, diff --git a/src/messenger/attachments/infrastructure/telemetry.py b/src/messenger/attachments/infrastructure/telemetry.py new file mode 100644 index 00000000..7a6a2f2b --- /dev/null +++ b/src/messenger/attachments/infrastructure/telemetry.py @@ -0,0 +1,40 @@ +from opentelemetry import metrics, trace +from opentelemetry.metrics import MeterProvider +from opentelemetry.trace import StatusCode + +from messenger.attachments.domain.jobs import JOB_TYPE, Job + +DURATION_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0) +FAILURES = frozenset({"retry_scheduled", "failed", "error"}) + + +class OtelJobObserver: + __slots__ = ("_duration", "_processed") + + def __init__(self, provider: MeterProvider | None = None) -> None: + meter = metrics.get_meter("messenger.attachments", meter_provider=provider) + self._processed = meter.create_counter( + "messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу" + ) + self._duration = meter.create_histogram( + "messenger.job.duration", + unit="s", + description="Длительность обработки доставки задания", + explicit_bucket_boundaries_advisory=DURATION_BUCKETS, + ) + + def observe(self, job: Job, outcome: str, seconds: float) -> None: + labels = {"job.type": JOB_TYPE, "outcome": outcome} + self._processed.add(1, labels) + self._duration.record(seconds, labels) + span = trace.get_current_span() + span.set_attributes( + { + "messenger.job_id": str(job.id), + "messenger.attachment_id": str(job.attachment_id), + "messenger.job.attempt": job.attempt, + "messenger.job.outcome": outcome, + } + ) + if outcome in FAILURES: + span.set_status(StatusCode.ERROR, outcome) diff --git a/student-tests/adapters/test_rabbitmq.py b/student-tests/adapters/test_rabbitmq.py index fb62b845..c112e290 100644 --- a/student-tests/adapters/test_rabbitmq.py +++ b/student-tests/adapters/test_rabbitmq.py @@ -21,9 +21,11 @@ from conftest import ROOT, STARTUP_SECONDS, container, image, published from fakes import NullLogger from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.application.ports import JobObserver from messenger.attachments.domain.jobs import Job, Outcome from messenger.attachments.entrypoints.worker import subscribe from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher +from messenger.attachments.infrastructure.telemetry import OtelJobObserver from messenger.shared.application.errors import Unavailable from messenger.shared.application.ports import Logger @@ -175,6 +177,10 @@ class FakeBus(Provider): def logger(self) -> Logger: return NullLogger() + @provide(scope=Scope.APP) + def observer(self) -> JobObserver: + return OtelJobObserver() + @provide(scope=Scope.REQUEST) def bus(self) -> AttachmentsCommandBus: return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher()) diff --git a/student-tests/unit/test_job_observer.py b/student-tests/unit/test_job_observer.py new file mode 100644 index 00000000..a8cce472 --- /dev/null +++ b/student-tests/unit/test_job_observer.py @@ -0,0 +1,62 @@ +from datetime import UTC, datetime +from uuid import uuid4 + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import ( + HistogramDataPoint, + InMemoryMetricReader, + NumberDataPoint, +) +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind, StatusCode + +from messenger.attachments.domain.jobs import Job +from messenger.attachments.infrastructure.telemetry import OtelJobObserver + + +def job() -> Job: + return Job(uuid4(), uuid4(), 1, 2, datetime.now(UTC)) + + +def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None: + reader = InMemoryMetricReader() + observer = OtelJobObserver(MeterProvider(metric_readers=[reader], shutdown_on_exit=False)) + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer("test") + done, failed = job(), job() + for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")): + with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER): + observer.observe(item, outcome, 0.3) + data = reader.get_metrics_data() + assert data is not None + found = { + metric.name: metric + for resource in data.resource_metrics + for scope in resource.scope_metrics + for metric in scope.metrics + } + counted = { + (point.attributes or {})["outcome"]: point.value + for point in found["messenger.jobs.processed"].data.data_points + if isinstance(point, NumberDataPoint) + } + assert counted == {"done": 1, "failed": 1, "duplicate_noop": 1} + duration = found["messenger.job.duration"] + assert duration.unit == "s" + point = next(iter(duration.data.data_points)) + assert isinstance(point, HistogramDataPoint) + assert set(point.attributes or {}) == {"job.type", "outcome"} + assert point.explicit_bounds[0] == 0.05 + assert point.explicit_bounds[-1] == 60.0 + spans: tuple[ReadableSpan, ...] = exporter.get_finished_spans() + assert [span.status.status_code for span in spans] == [ + StatusCode.UNSET, + StatusCode.ERROR, + StatusCode.UNSET, + ] + assert (spans[1].attributes or {})["messenger.attachment_id"] == str(failed.attachment_id) + assert (spans[1].attributes or {})["messenger.job.attempt"] == 2