Preserve distributed job context and expose grade 4 operational signals

This commit is contained in:
Artyom
2026-11-06 12:57:10 +03:00
parent 1a171d0d0f
commit 44d81fd80e
32 changed files with 1098 additions and 35 deletions
@@ -223,7 +223,7 @@ class PublishOutboxHandler:
for entry in await self._uow.outbox.pending(command.limit):
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
try:
await self._publisher.publish(entry.job_id, entry.body)
await self._publisher.publish(entry.job_id, entry.body, entry.trace_context)
except Unavailable as error:
failure = error
break
@@ -1,6 +1,6 @@
from collections.abc import AsyncIterator, Sequence
from contextlib import AbstractAsyncContextManager
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from typing import Protocol, runtime_checkable
from uuid import UUID
@@ -33,6 +33,7 @@ class OutboxEntry:
id: int
job_id: UUID
body: bytes
trace_context: dict[str, str] = field(default_factory=dict)
@dataclass(frozen=True, slots=True)
@@ -77,7 +78,9 @@ class VariantRenderer(Protocol):
@runtime_checkable
class JobPublisher(Protocol):
async def publish(self, job_id: UUID, body: bytes) -> None: ...
async def publish(
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
) -> None: ...
@runtime_checkable
@@ -179,6 +182,8 @@ class AttachmentDirectory(Protocol):
@runtime_checkable
class JobObserver(Protocol):
def active(self, delta: int) -> None: ...
def observe(self, job: Job, outcome: str, seconds: float) -> None: ...
+2 -2
View File
@@ -203,8 +203,8 @@ class WorkerProvider(Provider):
return RandomJitter()
@provide(scope=Scope.APP)
def observer(self) -> JobObserver:
return OtelJobObserver()
def observer(self, settings: PipelineSettings) -> JobObserver:
return OtelJobObserver(capacity=settings.worker_concurrency)
@provide(scope=Scope.REQUEST)
def command_bus(
@@ -35,6 +35,7 @@ async def handle_job(
logger.warning("job.rejected", reason="message does not match jobs.schema.json")
await message.reject(requeue=False)
return
observer.active(1)
started = time.perf_counter()
try:
outcome = (await bus.execute(ProcessJob(job))).value
@@ -49,6 +50,8 @@ async def handle_job(
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
await message.nack(requeue=True)
return
finally:
observer.active(-1)
observed(job, outcome, started, observer, logger)
await message.ack()
@@ -45,6 +45,7 @@ from messenger.shared.infrastructure.postgres import (
EventSource,
PostgresUnitOfWork,
)
from messenger.shared.infrastructure.trace_context import capture_context
class AttachmentRow(Base):
@@ -153,6 +154,7 @@ class JobRow(Base):
pipeline_version: Mapped[int]
status: Mapped[str]
attempt: Mapped[int]
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
claims: Mapped[int] = mapped_column(server_default="0")
lease_token: Mapped[UUID | None]
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@@ -168,6 +170,7 @@ class OutboxRow(Base):
id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
job_id: Mapped[UUID] = mapped_column(ForeignKey("jobs.job_id"))
payload: Mapped[dict[str, Any]] = mapped_column(JSONB)
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
@@ -472,10 +475,18 @@ class PostgresJobRepository:
self._session = session
async def _enqueue(self, job: Job, available_at: datetime | None = None) -> None:
context = (
capture_context()
or await self._session.scalar(
select(JobRow.trace_context).where(JobRow.job_id == job.id)
)
or {}
)
await self._session.execute(
insert(OutboxRow).values(
job_id=job.id,
payload=job.payload(),
trace_context=context,
available_at=func.now() if available_at is None else available_at,
)
)
@@ -518,6 +529,7 @@ class PostgresJobRepository:
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
status="queued",
trace_context=capture_context(),
attempt=job.attempt,
created_at=job.created_at,
updated_at=job.created_at,
@@ -824,7 +836,10 @@ class PostgresOutboxRepository:
.limit(limit)
.with_for_update(skip_locked=True)
)
return [OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode()) for row in rows]
return [
OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode(), row.trace_context)
for row in rows
]
async def published(self, entry_ids: Sequence[int]) -> None:
await self._session.execute(
@@ -0,0 +1,62 @@
import asyncio
from contextlib import suppress
from datetime import UTC, datetime
from opentelemetry import metrics
from opentelemetry.metrics import Observation
from sqlalchemy import func, select
from messenger.attachments.infrastructure.postgres import JobRow
from messenger.shared.application.ports import Logger
from messenger.shared.infrastructure.postgres import Database
class QueueMetrics:
__slots__ = ("_database", "_depth", "_logger", "_oldest", "_task")
def __init__(self, database: Database, logger: Logger) -> None:
self._database = database
self._logger = logger
self._depth = 0
self._oldest: datetime | None = None
self._task: asyncio.Task[None] | None = None
meter = metrics.get_meter("messenger.queue")
labels = {"queue.name": "attachments.process"}
meter.create_observable_gauge(
"messenger.queue.depth",
unit="{job}",
callbacks=[lambda _: [Observation(self._depth, labels)]],
)
meter.create_observable_gauge(
"messenger.queue.oldest_age",
unit="s",
callbacks=[lambda _: [Observation(self.age(), labels)]],
)
def age(self) -> float:
return max(0, (datetime.now(UTC) - self._oldest).total_seconds()) if self._oldest else 0
def start(self) -> None:
self._task = asyncio.create_task(self._poll())
async def stop(self) -> None:
if self._task:
self._task.cancel()
with suppress(asyncio.CancelledError):
await self._task
async def _poll(self) -> None:
while True:
try:
async with self._database.session() as session:
row = (
await session.execute(
select(func.count(), func.min(JobRow.created_at)).where(
JobRow.status.in_(("queued", "processing"))
)
)
).one()
self._depth, self._oldest = row
except Exception as error:
self._logger.warning("queue.metrics_failed", error=type(error).__name__)
await asyncio.sleep(5)
@@ -5,6 +5,7 @@ from pamqp.commands import Basic
from messenger.attachments.domain.jobs import JOB_TYPE
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.trace_context import restored_context
EXCHANGE = RabbitExchange("attachments", declare=False)
QUEUE = "attachments.process"
@@ -18,19 +19,22 @@ class RabbitJobPublisher:
self._broker = broker
self._timeout = timeout
async def publish(self, job_id: UUID, body: bytes) -> None:
async def publish(
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
) -> None:
try:
confirmation = await self._broker.publish(
body,
exchange=EXCHANGE,
routing_key=JOB_TYPE,
mandatory=True,
persist=True,
message_id=str(job_id),
message_type=JOB_TYPE,
content_type="application/json",
timeout=self._timeout,
)
with restored_context(trace_context):
confirmation = await self._broker.publish(
body,
exchange=EXCHANGE,
routing_key=JOB_TYPE,
mandatory=True,
persist=True,
message_id=str(job_id),
message_type=JOB_TYPE,
content_type="application/json",
timeout=self._timeout,
)
except Exception as error:
raise Unavailable("брокер не принял задание") from error
if not isinstance(confirmation, Basic.Ack):
+26 -7
View File
@@ -6,6 +6,8 @@ from aiobotocore.session import get_session
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from botocore.handlers import add_expect_header
from opentelemetry import trace
from opentelemetry.trace import SpanKind, StatusCode
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import ObjectContent, StoredObject
@@ -60,17 +62,31 @@ class S3ObjectStorage:
self._client = client
self._bucket = bucket
async def _call(self, operation: str, **arguments: Any) -> Any:
with trace.get_tracer(__name__).start_as_current_span(
f"S3 {operation}",
kind=SpanKind.CLIENT,
attributes={"rpc.system": "aws-api", "rpc.service": "S3", "rpc.method": operation},
record_exception=False,
set_status_on_exception=False,
) as span:
try:
return await getattr(self._client, operation)(**arguments)
except Exception as error:
span.set_status(StatusCode.ERROR, type(error).__name__)
raise
async def put(self, key: str, data: bytes, content_type: str) -> None:
try:
await self._client.put_object(
Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
await self._call(
"put_object", Bucket=self._bucket, Key=key, Body=data, ContentType=content_type
)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def open(self, key: str) -> ObjectContent:
try:
response = await self._client.get_object(Bucket=self._bucket, Key=key)
response = await self._call("get_object", Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
raise ObjectMissingError(key) from error
@@ -86,15 +102,18 @@ class S3ObjectStorage:
async def copy(self, source: str, target: str) -> None:
try:
await self._client.copy_object(
Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source}
await self._call(
"copy_object",
Bucket=self._bucket,
Key=target,
CopySource={"Bucket": self._bucket, "Key": source},
)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
async def delete(self, key: str) -> None:
try:
await self._client.delete_object(Bucket=self._bucket, Key=key)
await self._call("delete_object", Bucket=self._bucket, Key=key)
except FAILURES as error:
raise Unavailable("S3 недоступно") from error
@@ -113,7 +132,7 @@ class S3ObjectStorage:
async def stat(self, key: str) -> StoredObject | None:
try:
head = await self._client.head_object(Bucket=self._bucket, Key=key)
head = await self._call("head_object", Bucket=self._bucket, Key=key)
except FAILURES as error:
if missing(error):
return None
@@ -1,5 +1,7 @@
from datetime import UTC, datetime
from opentelemetry import metrics, trace
from opentelemetry.metrics import MeterProvider
from opentelemetry.metrics import MeterProvider, Observation
from opentelemetry.trace import StatusCode
from messenger.attachments.domain.jobs import JOB_TYPE, Job
@@ -9,10 +11,22 @@ FAILURES = frozenset({"retry_scheduled", "failed", "error"})
class OtelJobObserver:
__slots__ = ("_duration", "_processed")
__slots__ = ("_busy", "_duration", "_processed", "_ready")
def __init__(self, provider: MeterProvider | None = None) -> None:
def __init__(self, provider: MeterProvider | None = None, capacity: int = 1) -> None:
meter = metrics.get_meter("messenger.attachments", meter_provider=provider)
self._busy = 0
meter.create_observable_gauge(
"messenger.worker.busy", callbacks=[lambda _: [Observation(self._busy)]], unit="{job}"
)
meter.create_observable_gauge(
"messenger.worker.capacity", callbacks=[lambda _: [Observation(capacity)]], unit="{job}"
)
self._ready = meter.create_histogram(
"messenger.job.time_to_ready",
unit="s",
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
)
self._processed = meter.create_counter(
"messenger.jobs.processed", unit="{job}", description="Доставки заданий по исходу"
)
@@ -23,7 +37,14 @@ class OtelJobObserver:
explicit_bucket_boundaries_advisory=DURATION_BUCKETS,
)
def active(self, delta: int) -> None:
self._busy += delta
def observe(self, job: Job, outcome: str, seconds: float) -> None:
if outcome == "done":
self._ready.record(
max(0, (datetime.now(UTC) - job.created_at).total_seconds()), {"job.type": JOB_TYPE}
)
labels = {"job.type": JOB_TYPE, "outcome": outcome}
self._processed.add(1, labels)
self._duration.record(seconds, labels)
+7
View File
@@ -3,16 +3,19 @@ import signal
from dishka import make_async_container
from faststream.rabbit import Channel, RabbitBroker
from faststream.rabbit.opentelemetry import RabbitTelemetryMiddleware
from messenger.attachments.config import PipelineSettings
from messenger.attachments.entrypoints.heartbeat import Heartbeat
from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal
from messenger.attachments.entrypoints.relay import Relay
from messenger.attachments.infrastructure.queue_metrics import QueueMetrics
from messenger.shared.application.messagebus import EventBus
from messenger.shared.config import PlatformSettings
from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
from messenger.shared.infrastructure.postgres import Database
from messenger.shared.infrastructure.telemetry import start_telemetry
PING_TIMEOUT_SECONDS = 2.0
@@ -28,6 +31,7 @@ async def serve() -> None:
pipeline.broker_url.get_secret_value(),
default_channel=Channel(publisher_confirms=True, on_return_raises=True),
logger=None,
middlewares=() if telemetry is None else (RabbitTelemetryMiddleware(),),
)
container = make_async_container(
PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker)
@@ -47,6 +51,8 @@ async def serve() -> None:
pipeline.heartbeat_interval_seconds,
logger,
)
queue_metrics = QueueMetrics(await container.get(Database), logger)
queue_metrics.start()
relay.start()
heartbeat.start()
logger.info("relay.started")
@@ -54,6 +60,7 @@ async def serve() -> None:
await stop.wait()
finally:
await heartbeat.stop()
await queue_metrics.stop()
await relay.stop()
await bus.stop()
await broker.stop()
@@ -1,13 +1,16 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import ClassVar, Protocol
from typing import ClassVar, Protocol, cast
from opentelemetry import metrics
from opentelemetry.metrics import Observation
from sqlalchemy import Text, make_url, select
from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeout
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.pool import AsyncAdaptedQueuePool, ConnectionPoolEntry
from messenger.shared.application.errors import Unavailable
from messenger.shared.domain.events import DomainEvent
@@ -38,6 +41,17 @@ def unavailable(error: BaseException) -> bool:
return isinstance(state, str) and (state.startswith(("08", "53", "57")) or state == "55P03")
class ObservedPool(AsyncAdaptedQueuePool):
waiting = 0
def _do_get(self) -> ConnectionPoolEntry:
self.waiting += 1
try:
return super()._do_get()
finally:
self.waiting -= 1
class Database:
__slots__ = ("_engine", "_sessions")
@@ -52,6 +66,7 @@ class Database:
) -> None:
self._engine = create_async_engine(
make_url(dsn).set(drivername="postgresql+asyncpg"),
poolclass=ObservedPool,
pool_size=max_size,
max_overflow=0,
pool_timeout=acquire_timeout,
@@ -68,6 +83,22 @@ class Database:
},
},
)
meter = metrics.get_meter("messenger.database")
meter.create_observable_gauge(
"messenger.db.pool.used",
callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).checkedout())]],
unit="{connection}",
)
meter.create_observable_gauge(
"messenger.db.pool.waiting",
callbacks=[lambda _: [Observation(cast(ObservedPool, self._engine.pool).waiting)]],
unit="{request}",
)
meter.create_observable_gauge(
"messenger.db.pool.capacity",
callbacks=[lambda _: [Observation(max_size)]],
unit="{connection}",
)
self._sessions = async_sessionmaker(self._engine, expire_on_commit=False)
async def open(self) -> None:
@@ -0,0 +1,26 @@
import os
import resource
from pathlib import Path
from opentelemetry.metrics import MeterProvider, Observation
def rss() -> int:
try:
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
except OSError:
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
def observe_process(provider: MeterProvider) -> None:
meter = provider.get_meter("messenger.process")
meter.create_observable_counter(
"process.cpu.time",
unit="s",
callbacks=[lambda _: [Observation(sum(os.times()[:2]))]],
)
meter.create_observable_gauge(
"process.memory.usage",
unit="By",
callbacks=[lambda _: [Observation(rss())]],
)
@@ -25,6 +25,7 @@ from opentelemetry.trace import Link, SpanKind, TraceState
from opentelemetry.util.types import Attributes
from messenger.shared.config import PlatformSettings
from messenger.shared.infrastructure.runtime_metrics import observe_process
SERVICE_VERSION = version("mtusi-messenger")
REQUEST_ID = "messenger.request_id"
@@ -127,6 +128,7 @@ def start_telemetry(
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
shutdown_on_exit=False,
)
observe_process(meter)
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
trace.set_tracer_provider(tracer)
@@ -0,0 +1,22 @@
from collections.abc import Iterator
from contextlib import contextmanager
from opentelemetry.context import attach, detach
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
PROPAGATOR = TraceContextTextMapPropagator()
def capture_context() -> dict[str, str]:
carrier: dict[str, str] = {}
PROPAGATOR.inject(carrier)
return carrier
@contextmanager
def restored_context(carrier: dict[str, str] | None) -> Iterator[None]:
token = attach(PROPAGATOR.extract(carrier or {}))
try:
yield
finally:
detach(token)