forked from mtusi2026/mtusi2026lab6
Count and trace attachment jobs through a job observer port
Add the JobObserver port to the attachments zone and its adapter on the OpenTelemetry API: every delivery adds one to messenger.jobs.processed and records messenger.job.duration in seconds with buckets from 50 ms to 60 s, both labelled only by job.type and outcome, and puts the job id, the attachment id, the attempt and the outcome on the current consumer span, which turns to ERROR on retry_scheduled, failed and error. The worker entrypoint measures each delivery, reports the outcome of the handler or error when it raised, and writes a job.processed record. Without the SDK the adapter runs on the no-op providers of the API. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e60b4eac98
commit
e2ec3fe9af
@@ -8,9 +8,9 @@
|
||||
|
||||
## 2. Зона `attachments`
|
||||
|
||||
- [ ] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider`
|
||||
- [ ] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал
|
||||
- [ ] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed`
|
||||
- [x] 2.1 Порт `JobObserver`, адаптер `OtelJobObserver` (counter, histogram с buckets, атрибуты и статус span), провайдер в `WorkerProvider`
|
||||
- [x] 2.2 Точка входа worker: длительность, исход, `job.processed` в журнал
|
||||
- [x] 2.3 Проверка группы: тест адаптера на `InMemoryMetricReader` и `InMemorySpanExporter` — labels `job.type`/`outcome`, атрибуты и `ERROR` для `failed`
|
||||
|
||||
## 3. Сборка процессов
|
||||
|
||||
|
||||
@@ -153,3 +153,8 @@ class AttachmentDirectory(Protocol):
|
||||
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ...
|
||||
|
||||
async def failed_jobs(self, limit: int) -> list[FailedJob]: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class JobObserver(Protocol):
|
||||
def observe(self, job: Job, outcome: str, seconds: float) -> None: ...
|
||||
|
||||
@@ -34,6 +34,7 @@ from messenger.attachments.application.ports import (
|
||||
Faults,
|
||||
ImageInspector,
|
||||
Jitter,
|
||||
JobObserver,
|
||||
JobPublisher,
|
||||
ObjectStorage,
|
||||
VariantRenderer,
|
||||
@@ -54,6 +55,7 @@ from messenger.attachments.infrastructure.postgres import (
|
||||
)
|
||||
from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher
|
||||
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
|
||||
from messenger.attachments.infrastructure.telemetry import OtelJobObserver
|
||||
from messenger.shared.application.commands import Command
|
||||
from messenger.shared.application.messagebus import CommandHandler
|
||||
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
|
||||
@@ -196,6 +198,10 @@ class WorkerProvider(Provider):
|
||||
def jitter(self) -> Jitter:
|
||||
return RandomJitter()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def observer(self) -> JobObserver:
|
||||
return OtelJobObserver()
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def command_bus(
|
||||
self,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from dishka import FromDishka
|
||||
from faststream import AckPolicy
|
||||
@@ -8,6 +9,8 @@ from pydantic import ValidationError
|
||||
|
||||
from messenger.attachments.application.bus import AttachmentsCommandBus
|
||||
from messenger.attachments.application.commands import ProcessJob
|
||||
from messenger.attachments.application.ports import JobObserver
|
||||
from messenger.attachments.domain.jobs import Job
|
||||
from messenger.attachments.entrypoints.schemas import JobMessage
|
||||
from messenger.attachments.infrastructure.rabbitmq import QUEUE
|
||||
from messenger.shared.application.ports import Logger
|
||||
@@ -24,6 +27,7 @@ async def handle_job(
|
||||
message: RabbitMessage,
|
||||
bus: FromDishka[AttachmentsCommandBus],
|
||||
logger: FromDishka[Logger],
|
||||
observer: FromDishka[JobObserver],
|
||||
) -> None:
|
||||
try:
|
||||
job = JobMessage.model_validate_json(message.body).job()
|
||||
@@ -31,16 +35,32 @@ async def handle_job(
|
||||
logger.warning("job.rejected", reason="message does not match jobs.schema.json")
|
||||
await message.reject(requeue=False)
|
||||
return
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
await bus.execute(ProcessJob(job))
|
||||
outcome = (await bus.execute(ProcessJob(job))).value
|
||||
except Exception as error:
|
||||
observed(job, "error", started, observer, logger)
|
||||
logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__)
|
||||
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
|
||||
await message.nack(requeue=True)
|
||||
return
|
||||
observed(job, outcome, started, observer, logger)
|
||||
await message.ack()
|
||||
|
||||
|
||||
def observed(job: Job, outcome: str, started: float, observer: JobObserver, logger: Logger) -> None:
|
||||
seconds = time.perf_counter() - started
|
||||
observer.observe(job, outcome, seconds)
|
||||
logger.info(
|
||||
"job.processed",
|
||||
job_id=str(job.id),
|
||||
attachment_id=str(job.attachment_id),
|
||||
attempt=job.attempt,
|
||||
outcome=outcome,
|
||||
duration_ms=round(seconds * 1000, 3),
|
||||
)
|
||||
|
||||
|
||||
def subscribe(broker: RabbitBroker, concurrency: int) -> None:
|
||||
broker.subscriber(
|
||||
JOBS,
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.metrics import MeterProvider
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from messenger.attachments.domain.jobs import JOB_TYPE, Job
|
||||
|
||||
DURATION_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0)
|
||||
FAILURES = frozenset({"retry_scheduled", "failed", "error"})
|
||||
|
||||
|
||||
class OtelJobObserver:
|
||||
__slots__ = ("_duration", "_processed")
|
||||
|
||||
def __init__(self, provider: MeterProvider | None = None) -> None:
|
||||
meter = metrics.get_meter("messenger.attachments", meter_provider=provider)
|
||||
self._processed = meter.create_counter(
|
||||
"messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу"
|
||||
)
|
||||
self._duration = meter.create_histogram(
|
||||
"messenger.job.duration",
|
||||
unit="s",
|
||||
description="Длительность обработки доставки задания",
|
||||
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
|
||||
)
|
||||
|
||||
def observe(self, job: Job, outcome: str, seconds: float) -> None:
|
||||
labels = {"job.type": JOB_TYPE, "outcome": outcome}
|
||||
self._processed.add(1, labels)
|
||||
self._duration.record(seconds, labels)
|
||||
span = trace.get_current_span()
|
||||
span.set_attributes(
|
||||
{
|
||||
"messenger.job_id": str(job.id),
|
||||
"messenger.attachment_id": str(job.attachment_id),
|
||||
"messenger.job.attempt": job.attempt,
|
||||
"messenger.job.outcome": outcome,
|
||||
}
|
||||
)
|
||||
if outcome in FAILURES:
|
||||
span.set_status(StatusCode.ERROR, outcome)
|
||||
@@ -21,9 +21,11 @@ from conftest import ROOT, STARTUP_SECONDS, container, image, published
|
||||
from fakes import NullLogger
|
||||
from messenger.attachments.application.bus import AttachmentsCommandBus
|
||||
from messenger.attachments.application.commands import ProcessJob
|
||||
from messenger.attachments.application.ports import JobObserver
|
||||
from messenger.attachments.domain.jobs import Job, Outcome
|
||||
from messenger.attachments.entrypoints.worker import subscribe
|
||||
from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher
|
||||
from messenger.attachments.infrastructure.telemetry import OtelJobObserver
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.ports import Logger
|
||||
|
||||
@@ -175,6 +177,10 @@ class FakeBus(Provider):
|
||||
def logger(self) -> Logger:
|
||||
return NullLogger()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def observer(self) -> JobObserver:
|
||||
return OtelJobObserver()
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def bus(self) -> AttachmentsCommandBus:
|
||||
return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher())
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import (
|
||||
HistogramDataPoint,
|
||||
InMemoryMetricReader,
|
||||
NumberDataPoint,
|
||||
)
|
||||
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import SpanKind, StatusCode
|
||||
|
||||
from messenger.attachments.domain.jobs import Job
|
||||
from messenger.attachments.infrastructure.telemetry import OtelJobObserver
|
||||
|
||||
|
||||
def job() -> Job:
|
||||
return Job(uuid4(), uuid4(), 1, 2, datetime.now(UTC))
|
||||
|
||||
|
||||
def test_job_observer_counts_by_type_and_outcome_and_marks_the_span() -> None:
|
||||
reader = InMemoryMetricReader()
|
||||
observer = OtelJobObserver(MeterProvider(metric_readers=[reader], shutdown_on_exit=False))
|
||||
exporter = InMemorySpanExporter()
|
||||
provider = TracerProvider(shutdown_on_exit=False)
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
tracer = provider.get_tracer("test")
|
||||
done, failed = job(), job()
|
||||
for item, outcome in ((done, "done"), (failed, "failed"), (done, "duplicate_noop")):
|
||||
with tracer.start_as_current_span("process", kind=SpanKind.CONSUMER):
|
||||
observer.observe(item, outcome, 0.3)
|
||||
data = reader.get_metrics_data()
|
||||
assert data is not None
|
||||
found = {
|
||||
metric.name: metric
|
||||
for resource in data.resource_metrics
|
||||
for scope in resource.scope_metrics
|
||||
for metric in scope.metrics
|
||||
}
|
||||
counted = {
|
||||
(point.attributes or {})["outcome"]: point.value
|
||||
for point in found["messenger.jobs.processed"].data.data_points
|
||||
if isinstance(point, NumberDataPoint)
|
||||
}
|
||||
assert counted == {"done": 1, "failed": 1, "duplicate_noop": 1}
|
||||
duration = found["messenger.job.duration"]
|
||||
assert duration.unit == "s"
|
||||
point = next(iter(duration.data.data_points))
|
||||
assert isinstance(point, HistogramDataPoint)
|
||||
assert set(point.attributes or {}) == {"job.type", "outcome"}
|
||||
assert point.explicit_bounds[0] == 0.05
|
||||
assert point.explicit_bounds[-1] == 60.0
|
||||
spans: tuple[ReadableSpan, ...] = exporter.get_finished_spans()
|
||||
assert [span.status.status_code for span in spans] == [
|
||||
StatusCode.UNSET,
|
||||
StatusCode.ERROR,
|
||||
StatusCode.UNSET,
|
||||
]
|
||||
assert (spans[1].attributes or {})["messenger.attachment_id"] == str(failed.attachment_id)
|
||||
assert (spans[1].attributes or {})["messenger.job.attempt"] == 2
|
||||
Reference in New Issue
Block a user