Count and trace attachment jobs through a job observer port

Add the JobObserver port to the attachments zone and its adapter on the
OpenTelemetry API: every delivery adds one to messenger.jobs.processed
and records messenger.job.duration in seconds with buckets from 50 ms to
60 s, both labelled only by job.type and outcome, and puts the job id,
the attachment id, the attempt and the outcome on the current consumer
span, which turns to ERROR on retry_scheduled, failed and error. The
worker entrypoint measures each delivery, reports the outcome of the
handler or error when it raised, and writes a job.processed record.
Without the SDK the adapter runs on the no-op providers of the API.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 04:45:20 +03:00
co-authored by Claude Opus 5.5
parent e60b4eac98
commit e2ec3fe9af
7 changed files with 143 additions and 4 deletions
@@ -8,9 +8,9 @@
## 2. Зона `attachments`
- [ ] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider`
- [ ] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал
- [ ] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed`
- [x] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider`
- [x] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал
- [x] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed`
## 3. Сборка процессов
@@ -153,3 +153,8 @@ class AttachmentDirectory(Protocol):
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ...
async def failed_jobs(self, limit: int) -> list[FailedJob]: ...
@runtime_checkable
class JobObserver(Protocol):
def observe(self, job: Job, outcome: str, seconds: float) -> None: ...
@@ -34,6 +34,7 @@ from messenger.attachments.application.ports import (
Faults,
ImageInspector,
Jitter,
JobObserver,
JobPublisher,
ObjectStorage,
VariantRenderer,
@@ -54,6 +55,7 @@ from messenger.attachments.infrastructure.postgres import (
)
from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.attachments.infrastructure.telemetry import OtelJobObserver
from messenger.shared.application.commands import Command
from messenger.shared.application.messagebus import CommandHandler
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
@@ -196,6 +198,10 @@ class WorkerProvider(Provider):
def jitter(self) -> Jitter:
return RandomJitter()
@provide(scope=Scope.APP)
def observer(self) -> JobObserver:
return OtelJobObserver()
@provide(scope=Scope.REQUEST)
def command_bus(
self,
@@ -1,4 +1,5 @@
import asyncio
import time
from dishka import FromDishka
from faststream import AckPolicy
@@ -8,6 +9,8 @@ from pydantic import ValidationError
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import ProcessJob
from messenger.attachments.application.ports import JobObserver
from messenger.attachments.domain.jobs import Job
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.attachments.infrastructure.rabbitmq import QUEUE
from messenger.shared.application.ports import Logger
@@ -24,6 +27,7 @@ async def handle_job(
message: RabbitMessage,
bus: FromDishka[AttachmentsCommandBus],
logger: FromDishka[Logger],
observer: FromDishka[JobObserver],
) -> None:
try:
job = JobMessage.model_validate_json(message.body).job()
@@ -31,16 +35,32 @@ async def handle_job(
logger.warning("job.rejected", reason="message does not match jobs.schema.json")
await message.reject(requeue=False)
return
started = time.perf_counter()
try:
await bus.execute(ProcessJob(job))
outcome = (await bus.execute(ProcessJob(job))).value
except Exception as error:
observed(job, "error", started, observer, logger)
logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__)
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
await message.nack(requeue=True)
return
observed(job, outcome, started, observer, logger)
await message.ack()
def observed(job: Job, outcome: str, started: float, observer: JobObserver, logger: Logger) -> None:
seconds = time.perf_counter() - started
observer.observe(job, outcome, seconds)
logger.info(
"job.processed",
job_id=str(job.id),
attachment_id=str(job.attachment_id),
attempt=job.attempt,
outcome=outcome,
duration_ms=round(seconds * 1000, 3),
)
def subscribe(broker: RabbitBroker, concurrency: int) -> None:
broker.subscriber(
JOBS,
@@ -0,0 +1,40 @@
from opentelemetry import metrics, trace
from opentelemetry.metrics import MeterProvider
from opentelemetry.trace import StatusCode
from messenger.attachments.domain.jobs import JOB_TYPE, Job
DURATION_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0)
FAILURES = frozenset({"retry_scheduled", "failed", "error"})
class OtelJobObserver:
__slots__ = ("_duration", "_processed")
def __init__(self, provider: MeterProvider | None = None) -> None:
meter = metrics.get_meter("messenger.attachments", meter_provider=provider)
self._processed = meter.create_counter(
"messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу"
)
self._duration = meter.create_histogram(
"messenger.job.duration",
unit="s",
description="Длительность обработки доставки задания",
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
)
def observe(self, job: Job, outcome: str, seconds: float) -> None:
labels = {"job.type": JOB_TYPE, "outcome": outcome}
self._processed.add(1, labels)
self._duration.record(seconds, labels)
span = trace.get_current_span()
span.set_attributes(
{
"messenger.job_id": str(job.id),
"messenger.attachment_id": str(job.attachment_id),
"messenger.job.attempt": job.attempt,
"messenger.job.outcome": outcome,
}
)
if outcome in FAILURES:
span.set_status(StatusCode.ERROR, outcome)
+6
View File
@@ -21,9 +21,11 @@ from conftest import ROOT, STARTUP_SECONDS, container, image, published
from fakes import NullLogger
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import ProcessJob
from messenger.attachments.application.ports import JobObserver
from messenger.attachments.domain.jobs import Job, Outcome
from messenger.attachments.entrypoints.worker import subscribe
from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher
from messenger.attachments.infrastructure.telemetry import OtelJobObserver
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Logger
@@ -175,6 +177,10 @@ class FakeBus(Provider):
def logger(self) -> Logger:
return NullLogger()
@provide(scope=Scope.APP)
def observer(self) -> JobObserver:
return OtelJobObserver()
@provide(scope=Scope.REQUEST)
def bus(self) -> AttachmentsCommandBus:
return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher())
+62
View File
@@ -0,0 +1,62 @@
from datetime import UTC, datetime
from uuid import uuid4
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import (
HistogramDataPoint,
InMemoryMetricReader,
NumberDataPoint,
)
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import SpanKind, StatusCode
from messenger.attachments.domain.jobs import Job
from messenger.attachments.infrastructure.telemetry import OtelJobObserver
def job() -> Job:
return Job(uuid4(), uuid4(), 1, 2, datetime.now(UTC))
def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None:
reader = InMemoryMetricReader()
observer = OtelJobObserver(MeterProvider(metric_readers=[reader], shutdown_on_exit=False))
exporter = InMemorySpanExporter()
provider = TracerProvider(shutdown_on_exit=False)
provider.add_span_processor(SimpleSpanProcessor(exporter))
tracer = provider.get_tracer("test")
done, failed = job(), job()
for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")):
with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER):
observer.observe(item, outcome, 0.3)
data = reader.get_metrics_data()
assert data is not None
found = {
metric.name: metric
for resource in data.resource_metrics
for scope in resource.scope_metrics
for metric in scope.metrics
}
counted = {
(point.attributes or {})["outcome"]: point.value
for point in found["messenger.jobs.processed"].data.data_points
if isinstance(point, NumberDataPoint)
}
assert counted == {"done": 1, "failed": 1, "duplicate_noop": 1}
duration = found["messenger.job.duration"]
assert duration.unit == "s"
point = next(iter(duration.data.data_points))
assert isinstance(point, HistogramDataPoint)
assert set(point.attributes or {}) == {"job.type", "outcome"}
assert point.explicit_bounds[0] == 0.05
assert point.explicit_bounds[-1] == 60.0
spans: tuple[ReadableSpan, ...] = exporter.get_finished_spans()
assert [span.status.status_code for span in spans] == [
StatusCode.UNSET,
StatusCode.ERROR,
StatusCode.UNSET,
]
assert (spans[1].attributes or {})["messenger.attachment_id"] == str(failed.attachment_id)
assert (spans[1].attributes or {})["messenger.job.attempt"] == 2