Files
mtusi2026lab6/service-tests/unit/test_telemetry.py
T
ricoromaandClaude Opus 5.5 509afca89e feat: telemetry cost profile with head sampling, coarse buckets and balanced tail sampling
- [telemetry] trace_sample_ratio (ParentBased(TraceIdRatioBased), 1 keeps
  ParentBased(ALWAYS_ON)) and metric_buckets (default|coarse with SLO bounds),
  export interval <= 30 s; otel.sdk.span.started, span.live dropped; startup
  records name ratio, buckets and interval
- bench: telemetry services on mtusi-bench, TELEMETRY=on switch for uploads
  and queue, make bench_telemetry[_quick] with off/full/full-aa/head/tail/
  metrics/balanced, rare events, lookup from Tempo/Loki/Prometheus, cost
  sampler, verdicts H1-H8 and B1-B5; collector layers and lean Prometheus in
  bench/otel with preflight checks
- REPORT: telemetry cost hypotheses and criteria before the label

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 19:17:25 +03:00

2027 lines
66 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Телеметрия на in-memory экспортёрах SDK, без сети (design change
`otel-telemetry`, решение 21): выключенная телеметрия, `OTEL_*`, ресурс,
server span и метрика HTTP, запись журнала, правило корня, span Valkey и S3,
закрытие со сроком, исход доставки worker → span и histogram."""
import asyncio
import io
import json
import logging
import threading
import time
from collections.abc import Iterator
from dataclasses import dataclass, field
from typing import Any
from uuid import uuid4
import httpx
import pytest
import requests
from botocore.exceptions import ClientError
from fastapi import APIRouter, FastAPI
from opentelemetry import trace
from opentelemetry._logs import SeverityNumber
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import (
InMemoryLogRecordExporter,
LogRecordExporter,
LogRecordExportResult,
SimpleLogRecordProcessor,
)
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import (
SimpleSpanProcessor,
SpanExporter,
SpanExportResult,
)
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from opentelemetry.trace import SpanKind, StatusCode
from redis.exceptions import RedisError
from src.api.middlewares import RequestContextMiddleware
from src.config import settings
from src.config.settings import Telemetry as TelemetrySettings
from src.infra import logger as app_logger
from src.infra import telemetry
from src.infra.broker import JobPublisher
from src.infra.cache.client import Cache
from src.infra.logger import get_logger
from src.infra.s3.client import ObjectStorage
from src.infra.telemetry import core, metrics, sdk
from src.schemas import dataclasses
from src.schemas.enums import (
AttachmentErrorCode,
AttachmentJobReason,
JobDeliveryOutcome,
MetricBucketsEnum,
)
from src.utils.constants import (
ATTACHMENT_JOB_TYPE,
HTTP_DURATION_BUCKETS,
HTTP_DURATION_BUCKETS_COARSE,
JOB_DURATION_BUCKETS_COARSE,
METRIC_HTTP_SERVER_DURATION,
METRIC_JOB_DURATION,
METRIC_JOB_TIME_TO_READY,
SERVICE_VERSION,
TIME_TO_READY_BUCKETS_COARSE,
)
from src.worker import main as worker_main
pytestmark = pytest.mark.anyio
SERVICE = "messenger-api"
SECRET_TEXT = "secret-text-in-exception"
COLLECTOR = "http://otel-collector:4318"
PROXY = "http://proxy.invalid:3128"
@dataclass
class Signals:
spans: InMemorySpanExporter
metrics: InMemoryMetricReader
logs: InMemoryLogRecordExporter
closed: bool = False
def finish(self) -> None:
"""Закрыть провайдеры: батчи отправлены в in-memory экспортёры."""
if self.closed:
return
self.closed = True
for thread in telemetry.begin_shutdown():
thread.join(5)
def finished_spans(self) -> list[ReadableSpan]:
self.finish()
return list(self.spans.get_finished_spans())
def finished_logs(self) -> list[Any]:
self.finish()
return [item.log_record for item in self.logs.get_finished_logs()]
def points(self, name: str) -> list[Any]:
data = self.metrics.get_metrics_data()
if data is None:
return []
return [
point
for resource in data.resource_metrics
for scope in resource.scope_metrics
for metric in scope.metrics
if metric.name == name
for point in metric.data.data_points
]
@pytest.fixture
def stdout() -> Iterator[io.StringIO]:
"""Вывод handler журнала (он пишет в sys.stdout, взятый при настройке)."""
get_logger()
handler = logging.getLogger("src").handlers[0]
buffer = io.StringIO()
previous = handler.setStream(buffer)
try:
yield buffer
finally:
handler.setStream(previous)
@pytest.fixture
def signals(monkeypatch: pytest.MonkeyPatch) -> Iterator[Signals]:
monkeypatch.setattr(settings.telemetry, "environment", "lab")
collected = Signals(
spans=InMemorySpanExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
)
started = telemetry.start_process_telemetry(
SERVICE,
exporters=telemetry.Exporters(
spans=collected.spans,
metrics=collected.metrics,
logs=collected.logs,
),
)
assert started
try:
yield collected
finally:
collected.finish()
telemetry.remove_log_hooks()
def _records(buffer: io.StringIO) -> list[dict[str, Any]]:
return [json.loads(line) for line in buffer.getvalue().splitlines()]
# --- выключенная телеметрия ---------------------------------------------
def test_disabled_has_no_providers_and_hooks(
monkeypatch: pytest.MonkeyPatch,
stdout: io.StringIO,
) -> None:
monkeypatch.setattr(settings.telemetry, "enabled", False)
monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
assert telemetry.start_process_telemetry(SERVICE) is False
assert not telemetry.is_enabled()
assert app_logger._export_hook is None
assert app_logger._trace_fields_hook is None
# выключенная телеметрия окружение не трогает
assert "OTEL_SDK_DISABLED" in __import__("os").environ
with telemetry.child_span("GET") as span:
assert span is None
get_logger().info("Plain record", answer=42)
(record,) = _records(stdout)
assert "trace_id" not in record
assert "span_id" not in record
assert record["event"] == "Plain record"
assert record["answer"] == 42
def test_disabled_record_fields_unchanged(stdout: io.StringIO) -> None:
"""Поля записи без телеметрии — прежние: полей трассы нет."""
get_logger().bind(module="probe").info("Plain record", value=1)
(record,) = _records(stdout)
assert list(record)[:2] == ["event", "instance_id"]
assert set(record) == {
"event",
"instance_id",
"module",
"value",
"level",
"timestamp",
"filename",
"func_name",
"lineno",
}
# --- запуск, OTEL_*, ресурс ----------------------------------------------
def test_otel_environment_removed_and_named(
monkeypatch: pytest.MonkeyPatch,
stdout: io.StringIO,
) -> None:
monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://elsewhere:9")
collected = Signals(
spans=InMemorySpanExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
)
try:
telemetry.start_process_telemetry(
SERVICE,
exporters=telemetry.Exporters(
spans=collected.spans,
metrics=collected.metrics,
logs=collected.logs,
),
)
import os
assert "OTEL_SDK_DISABLED" not in os.environ
assert "OTEL_EXPORTER_OTLP_ENDPOINT" not in os.environ
# SDK не выключен переменной: span пишется
with core.tracer().start_as_current_span("probe"):
pass
assert [span.name for span in collected.finished_spans()] == ["probe"]
finally:
collected.finish()
telemetry.remove_log_hooks()
output = stdout.getvalue()
assert "elsewhere" not in output
(warning,) = [
record
for record in _records(stdout)
if record["event"] == "Ignored OpenTelemetry environment variables"
]
assert warning["level"] == "warning"
assert warning["names"] == [
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_SDK_DISABLED",
]
def test_resource_has_four_attributes(signals: Signals) -> None:
with core.tracer().start_as_current_span("probe"):
pass
(span,) = signals.finished_spans()
assert dict(span.resource.attributes) == {
"service.name": SERVICE,
"service.version": SERVICE_VERSION,
"service.instance.id": "test-instance",
"deployment.environment.name": "lab",
}
# --- server span и метрика HTTP -------------------------------------------
def _app() -> FastAPI:
app = FastAPI()
v1 = APIRouter(prefix="/api/v1")
items = APIRouter(prefix="/items")
@items.get("/{item_id}")
async def get_item(item_id: str) -> dict[str, str]:
get_logger().info("Inside handler")
return {"id": item_id}
@items.get("/{item_id}/boom")
async def boom(item_id: str) -> None:
raise RuntimeError(SECRET_TEXT)
@app.get("/health/live")
async def live() -> dict[str, str]:
return {"status": "ok"}
v1.include_router(items)
app.include_router(v1)
app.add_middleware(RequestContextMiddleware)
return app
async def _request(
method: str,
path: str,
headers: dict[str, str] | None = None,
) -> httpx.Response:
transport = httpx.ASGITransport(app=_app(), raise_app_exceptions=False)
async with httpx.AsyncClient(
transport=transport,
base_url="http://test",
) as client:
return await client.request(method, path, headers=headers)
async def test_server_span_and_metric(
signals: Signals,
stdout: io.StringIO,
) -> None:
response = await _request("GET", "/api/v1/items/abc")
assert response.status_code == 200
request_id = response.headers["x-request-id"]
points = signals.points(METRIC_HTTP_SERVER_DURATION)
(span,) = signals.finished_spans()
assert span.name == "GET /api/v1/items/{item_id}"
assert span.kind == SpanKind.SERVER
assert span.parent is None
assert dict(span.attributes) == {
"http.request.method": "GET",
"http.route": "/api/v1/items/{item_id}",
"http.response.status_code": 200,
"messenger.request_id": request_id,
}
(point,) = points
assert dict(point.attributes) == {
"http.request.method": "GET",
"http.route": "/api/v1/items/{item_id}",
"http.response.status_code": 200,
}
assert point.count == 1
# записи запроса, включая итоговую, — с trace_id и span_id этого span
trace_id = format(span.context.trace_id, "032x")
span_id = format(span.context.span_id, "016x")
records = [r for r in _records(stdout) if r.get("request_id")]
assert {r["event"] for r in records} == {
"Inside handler",
"HTTP request completed",
}
for record in records:
assert record["trace_id"] == trace_id
assert record["span_id"] == span_id
async def test_unknown_method_and_path(signals: Signals) -> None:
uuid_path = "/nope/6f1c2d3e-0000-4000-8000-000000000000"
await _request("BREW", uuid_path)
points = signals.points(METRIC_HTTP_SERVER_DURATION)
spans = signals.finished_spans()
(span,) = spans
assert span.name == "_OTHER"
assert "http.route" not in span.attributes
assert span.attributes["http.request.method"] == "_OTHER"
(point,) = points
assert "http.route" not in point.attributes
values = " ".join(str(value) for value in point.attributes.values())
assert "BREW" not in values
assert "6f1c2d3e" not in values
async def test_health_has_no_span_and_metric(signals: Signals) -> None:
response = await _request("GET", "/health/live")
assert response.status_code == 200
assert signals.points(METRIC_HTTP_SERVER_DURATION) == []
assert signals.finished_spans() == []
async def test_5xx_marks_error_without_text(signals: Signals) -> None:
response = await _request("GET", "/api/v1/items/x/boom")
assert response.status_code == 500
(span,) = signals.finished_spans()
assert span.status.status_code == StatusCode.ERROR
assert span.attributes["error.type"] == "RuntimeError"
assert span.attributes["http.response.status_code"] == 500
assert span.events == ()
dumped = json.dumps(
{
"attributes": dict(span.attributes),
"status": span.status.description,
},
)
assert SECRET_TEXT not in dumped
async def test_incoming_traceparent_is_not_continued(
signals: Signals,
) -> None:
foreign = "0af7651916cd43dd8448eb211c80319c"
await _request(
"GET",
"/api/v1/items/abc",
headers={"traceparent": f"00-{foreign}-b7ad6b7169203331-01"},
)
(span,) = signals.finished_spans()
assert span.parent is None
assert format(span.context.trace_id, "032x") != foreign
# --- журнал ---------------------------------------------------------------
def test_log_record_export(signals: Signals, stdout: io.StringIO) -> None:
log = get_logger().bind(module="probe")
with core.tracer().start_as_current_span("parent") as parent:
log.info(
"Exported record",
count=3,
ratio=0.5,
flag=True,
nothing=None,
variants=[{"name": "thumbnail", "width": 10}],
)
log.warning("Outside span")
exported = signals.finished_logs()
inside, outside = exported
assert inside.body == "Exported record"
assert inside.severity_number == SeverityNumber.INFO
assert inside.severity_text == "info"
assert inside.trace_id == parent.get_span_context().trace_id
assert inside.span_id == parent.get_span_context().span_id
attributes = dict(inside.attributes)
assert attributes["count"] == 3
assert attributes["ratio"] == 0.5
assert attributes["flag"] is True
assert "nothing" not in attributes
assert json.loads(attributes["variants"]) == [
{"name": "thumbnail", "width": 10},
]
assert attributes["module"] == "probe"
assert attributes["instance_id"] == "test-instance"
assert "event" not in attributes
assert "trace_id" not in attributes
assert outside.severity_number == SeverityNumber.WARN
assert not outside.trace_id
first, second = _records(stdout)
assert first["trace_id"] == format(
parent.get_span_context().trace_id,
"032x",
)
assert len(first["span_id"]) == 16
assert "trace_id" not in second
def test_uvicorn_traceback_is_exported(signals: Signals) -> None:
try:
raise ValueError("boom")
except ValueError as e:
logging.getLogger("uvicorn.error").error(
"Exception in ASGI application",
exc_info=e,
)
(record,) = signals.finished_logs()
assert record.body == "Exception in ASGI application"
assert record.severity_number == SeverityNumber.ERROR
assert "Traceback" in record.attributes["exception"]
assert record.attributes["logger"] == "uvicorn.error"
def test_sdk_records_not_exported_and_thinned(
signals: Signals,
stdout: io.StringIO,
) -> None:
sdk = logging.getLogger("opentelemetry.probe.thinned")
for _ in range(5):
sdk.warning("Failed to export batch")
assert signals.finished_logs() == []
records = [r for r in _records(stdout) if r.get("logger") == sdk.name]
assert len(records) == 1
# --- правило корня и клиенты ----------------------------------------------
def test_child_span_needs_recording_parent(signals: Signals) -> None:
with telemetry.child_span("SELECT") as orphan:
assert orphan is None
with (
core.tracer().start_as_current_span("parent") as parent,
telemetry.child_span("SELECT") as child,
):
assert child is not None
spans = {span.name: span for span in signals.finished_spans()}
assert set(spans) == {"parent", "SELECT"}
assert spans["SELECT"].parent.span_id == (
parent.get_span_context().span_id
)
def test_child_span_error_has_class_only(signals: Signals) -> None:
with (
core.tracer().start_as_current_span("parent"),
pytest.raises(RuntimeError),
telemetry.child_span("GET"),
):
raise RuntimeError(SECRET_TEXT)
spans = {span.name: span for span in signals.finished_spans()}
child = spans["GET"]
assert child.status.status_code == StatusCode.ERROR
assert child.attributes["error.type"] == "RuntimeError"
assert child.events == ()
class _Redis:
def __init__(self, values: dict[str, str], fail: bool = False) -> None:
self.values = values
self.fail = fail
async def get(self, key: str) -> str | None:
if self.fail:
raise RedisError("down")
return self.values.get(key)
async def set(self, key: str, value: str, *, ex: int) -> None:
if self.fail:
raise RedisError("down")
self.values[key] = value
async def test_cache_spans(signals: Signals) -> None:
cache = Cache(
_Redis({"k": "v"}),
namespace="db",
timeout_seconds=1,
retry_after_seconds=60,
)
await cache.get("k") # без родителя — span нет
with core.tracer().start_as_current_span("parent"):
await cache.get("k")
await cache.get("absent-chat-key")
await cache.set("k2", "v2", ttl_seconds=10)
cache._client.fail = True
await cache.get("k")
await cache.get("k") # окно пропуска
spans = [s for s in signals.finished_spans() if s.name != "parent"]
results = [(s.name, s.attributes["messenger.cache.result"]) for s in spans]
assert results == [
("GET", "hit"),
("GET", "miss"),
("SET", "stored"),
("GET", "error"),
("GET", "skipped"),
]
assert spans[3].status.status_code == StatusCode.ERROR
for span in spans:
assert span.attributes["db.system.name"] == "valkey"
assert "absent-chat-key" not in json.dumps(dict(span.attributes))
class _S3Client:
async def put_object(self, **params: Any) -> dict[str, Any]:
return {}
async def head_object(self, **params: Any) -> dict[str, Any]:
raise ClientError(
{"Error": {"Code": "404"}, "ResponseMetadata": {}},
"HeadObject",
)
async def test_s3_spans_without_key(
signals: Signals,
monkeypatch: pytest.MonkeyPatch,
) -> None:
storage = ObjectStorage(settings.s3, namespace="db")
async def client() -> _S3Client:
return _S3Client()
monkeypatch.setattr(storage, "_get_client", client)
key = "db/attachments/6f1c2d3e-0000-4000-8000-000000000000/original"
with core.tracer().start_as_current_span("parent"):
await storage.put(key, io.BytesIO(b"abc"), size=3, content_type="x")
assert await storage.head(key) is None
spans = {s.name: s for s in signals.finished_spans()}
put = spans["S3.PutObject"]
head = spans["S3.HeadObject"]
assert put.attributes["rpc.method"] == "PutObject"
assert put.attributes["messenger.object.size"] == 3
assert put.attributes["messenger.s3.result"] == "ok"
assert head.attributes["messenger.s3.result"] == "missing"
assert head.status.status_code == StatusCode.UNSET
for span in (put, head):
assert "6f1c2d3e" not in json.dumps(dict(span.attributes))
# --- закрытие -------------------------------------------------------------
class _BlockingExporter(SpanExporter):
def export(self, spans: Any) -> SpanExportResult:
time.sleep(5)
return SpanExportResult.SUCCESS
def shutdown(self) -> None:
time.sleep(5)
async def test_shutdown_is_bounded() -> None:
telemetry.start_process_telemetry(
SERVICE,
exporters=telemetry.Exporters(
spans=_BlockingExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
),
)
try:
with core.tracer().start_as_current_span("queued"):
pass
started = time.monotonic()
await telemetry.shutdown(timeout=0.3)
assert time.monotonic() - started < 1
assert not telemetry.is_enabled()
finally:
telemetry.remove_log_hooks()
def test_exporters_ignore_proxy_environment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Данные идут только в Collector из `[telemetry]`: прокси окружения
(`HTTP(S)_PROXY`) экспортёры не берут (verify S5). Отправка подменена на
уровне `requests.Session.send` — сети нет, видно, с какими прокси она бы
ушла."""
for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
monkeypatch.setenv(name, PROXY)
monkeypatch.setenv(name.lower(), PROXY)
for name in ("NO_PROXY", "no_proxy"):
monkeypatch.delenv(name, raising=False)
sent: list[tuple[str, dict[str, str]]] = []
def send(
self: requests.Session, # noqa: ARG001
request: requests.PreparedRequest,
**kwargs: Any,
) -> requests.Response:
sent.append((str(request.url), dict(kwargs.get("proxies") or {})))
response = requests.Response()
response.status_code = 200
response._content = b""
response.request = request
return response
monkeypatch.setattr(requests.Session, "send", send)
config = TelemetrySettings(otlp_endpoint=COLLECTOR)
reader = sdk.otlp_metric_reader(config)
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(
SimpleSpanProcessor(
sdk.otlp_span_exporter(config, meter_provider=None),
),
)
logger_provider = LoggerProvider()
logger_provider.add_log_record_processor(
SimpleLogRecordProcessor(
sdk.otlp_log_exporter(config, meter_provider=None),
),
)
meter_provider = MeterProvider(metric_readers=[reader])
try:
tracer_provider.get_tracer("t").start_span("probe").end()
logger_provider.get_logger("t").emit(body="probe")
meter_provider.get_meter("t").create_counter("probe").add(1)
reader.force_flush()
finally:
tracer_provider.shutdown()
logger_provider.shutdown()
meter_provider.shutdown()
assert {url for url, _ in sent} == {
f"{COLLECTOR}/v1/traces",
f"{COLLECTOR}/v1/logs",
f"{COLLECTOR}/v1/metrics",
}
assert all(proxies == {} for _, proxies in sent), sent
# --- исход доставки worker ------------------------------------------------
@dataclass
class _Delivery:
body: bytes = b"{}"
redelivered: bool = False
settled: list[str] = field(default_factory=list)
trace_context: dataclasses.TraceContextDTO | None = None
async def ack(self) -> None:
self.settled.append("ack")
async def drop(self) -> None:
self.settled.append("drop")
async def requeue(self) -> None:
self.settled.append("requeue")
class _Service:
def __init__(
self,
outcome: JobDeliveryOutcome | None,
result: dataclasses.JobDeliveryResultDTO | None = None,
) -> None:
self.outcome = outcome
self.result = result
async def handle(self, *, body: bytes, redelivered: bool) -> Any:
if self.result is not None:
return self.result
if self.outcome is None:
raise RuntimeError(SECRET_TEXT)
return dataclasses.JobDeliveryResultDTO(outcome=self.outcome)
@pytest.mark.parametrize(
("outcome", "settled"),
[
(JobDeliveryOutcome.READY, "ack"),
(JobDeliveryOutcome.FAILED, "ack"),
(JobDeliveryOutcome.RETRY, "ack"),
(JobDeliveryOutcome.SKIPPED, "ack"),
(JobDeliveryOutcome.REQUEUED, "requeue"),
(JobDeliveryOutcome.DROPPED, "drop"),
(None, "requeue"),
],
ids=lambda value: getattr(value, "value", value or "error"),
)
async def test_delivery_outcome_span_and_histogram(
signals: Signals,
outcome: JobDeliveryOutcome | None,
settled: str,
) -> None:
delivery = _Delivery()
await worker_main._handle(
lambda: _Service(outcome),
delivery,
asyncio.Semaphore(1),
asyncio.Event(),
)
expected = "error" if outcome is None else outcome.value
assert delivery.settled == [settled]
(point,) = signals.points(METRIC_JOB_DURATION)
assert dict(point.attributes) == {
"messenger.job.type": ATTACHMENT_JOB_TYPE,
"messenger.job.outcome": expected,
}
assert point.count == 1
(span,) = signals.finished_spans()
assert span.kind == SpanKind.CONSUMER
assert span.name.startswith("process ")
assert span.name.endswith(".attachment.process.v1")
assert span.parent is None
assert span.attributes["messenger.job.outcome"] == expected
assert span.attributes["messaging.system"] == "rabbitmq"
assert span.attributes["messaging.operation.type"] == "process"
if outcome is None:
assert span.status.status_code == StatusCode.ERROR
assert span.attributes["error.type"] == "RuntimeError"
assert SECRET_TEXT not in json.dumps(dict(span.attributes))
else:
assert span.status.status_code == StatusCode.UNSET
async def test_delivery_without_telemetry_has_no_signals() -> None:
delivery = _Delivery()
await worker_main._handle(
lambda: _Service(JobDeliveryOutcome.READY),
delivery,
asyncio.Semaphore(1),
asyncio.Event(),
)
assert delivery.settled == ["ack"]
assert not trace.get_current_span().is_recording()
# --- сквозная трасса (design change job-trace-propagation) ----------------
QUEUE = "testdb.attachment.process.v1"
SAVED_TRACE = "0af7651916cd43dd8448eb211c80319c"
SAVED_SPAN = "b7ad6b7169203331"
SAVED = dataclasses.TraceContextDTO(
traceparent=f"00-{SAVED_TRACE}-{SAVED_SPAN}-01",
tracestate="vendor=value",
)
UNSAMPLED = dataclasses.TraceContextDTO(
traceparent=f"00-{SAVED_TRACE}-{SAVED_SPAN}-00",
)
def _start(**kwargs: Any) -> Signals:
collected = Signals(
spans=InMemorySpanExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
)
assert telemetry.start_process_telemetry(
kwargs.pop("service", SERVICE),
exporters=telemetry.Exporters(
spans=collected.spans,
metrics=collected.metrics,
logs=collected.logs,
),
**kwargs,
)
return collected
def _publication(
trace_context: dataclasses.TraceContextDTO | None = SAVED,
*,
attempt: int = 1,
replayed: bool = False,
retry_reason: str | None = None,
) -> dataclasses.JobPublicationDTO:
return dataclasses.JobPublicationDTO(
job_id=uuid4(),
body=b"{}",
attempt=attempt,
replayed=replayed,
retry_reason=retry_reason,
trace_context=trace_context,
)
def _hex(value: int, width: int) -> str:
return format(value, f"0{width}x")
async def test_upload_context_is_server_span() -> None:
"""Контекст загрузки — контекст server span запроса; вне span и без
телеметрии — `None` (решение 4)."""
seen: list[dataclasses.TraceContextDTO | None] = []
app = FastAPI()
@app.post("/api/v1/upload")
async def upload() -> dict[str, str]:
seen.append(telemetry.current_trace_context())
return {}
app.add_middleware(RequestContextMiddleware)
async def call() -> None:
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(
transport=transport,
base_url="http://test",
) as client:
await client.post("/api/v1/upload")
await call()
assert seen == [None]
signals = _start()
try:
await call()
assert telemetry.current_trace_context() is None
(span,) = signals.finished_spans()
finally:
signals.finish()
telemetry.remove_log_hooks()
context = seen[-1]
assert context is not None
# флаги W3C — от SDK: выбрано и, у SDK 1.45, «random trace id»
flags = int(span.context.trace_flags)
assert flags & 0x01
assert context.traceparent == (
f"00-{_hex(span.context.trace_id, 32)}-"
f"{_hex(span.context.span_id, 16)}-{_hex(flags, 2)}"
)
assert context.tracestate is None
def test_producer_span_under_saved_context(signals: Signals) -> None:
publication = _publication(attempt=2, retry_reason="storage_unavailable")
with telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=publication,
) as headers:
pass
(span,) = signals.finished_spans()
assert span.kind == SpanKind.PRODUCER
assert span.name == f"publish {QUEUE}"
assert _hex(span.context.trace_id, 32) == SAVED_TRACE
assert span.parent is not None
assert _hex(span.parent.span_id, 16) == SAVED_SPAN
assert span.parent.is_remote
assert dict(span.attributes) == {
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"messaging.operation.type": "send",
"messaging.operation.name": "publish",
"messenger.job.type": ATTACHMENT_JOB_TYPE,
"messenger.job.attempt": 2,
"messenger.job.replayed": False,
"messenger.job.retry_reason": "storage_unavailable",
}
# заголовки — контекст span producer, tracestate прежний
assert headers == {
"traceparent": (
f"00-{SAVED_TRACE}-{_hex(span.context.span_id, 16)}-01"
),
"tracestate": "vendor=value",
}
assert str(publication.job_id) not in json.dumps(dict(span.attributes))
@pytest.mark.parametrize("error", [RuntimeError, asyncio.CancelledError])
def test_producer_span_error(signals: Signals, error: type) -> None:
with (
pytest.raises(error),
telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=_publication(replayed=True),
),
):
raise error(SECRET_TEXT)
(span,) = signals.finished_spans()
assert span.status.status_code == StatusCode.ERROR
assert span.attributes["error.type"] == error.__name__
assert span.attributes["messenger.job.replayed"] is True
assert SECRET_TEXT not in json.dumps(dict(span.attributes))
@pytest.mark.parametrize(
"saved",
[
None,
dataclasses.TraceContextDTO(traceparent="00-zz-bad-01"),
dataclasses.TraceContextDTO(
traceparent=f"00-{'0' * 32}-{'0' * 16}-01",
),
],
ids=["none", "garbage", "invalid"],
)
def test_producer_without_context_has_no_span(
signals: Signals,
saved: dataclasses.TraceContextDTO | None,
) -> None:
"""Проходы relay трасс не начинают: без контекста — ни span, ни
заголовков."""
with telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=_publication(saved),
) as headers:
assert headers == {}
assert signals.finished_spans() == []
def test_producer_without_telemetry_passes_context() -> None:
"""Телеметрия relay выключена — сохранённый контекст как есть."""
with telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=_publication(),
) as headers:
assert headers == {
"traceparent": SAVED.traceparent,
"tracestate": "vendor=value",
}
@dataclass
class _Exchange:
messages: list[Any] = field(default_factory=list)
async def publish(
self,
message: Any,
*,
routing_key: str,
mandatory: bool,
) -> None:
self.messages.append((message, routing_key, mandatory))
@dataclass
class _Channel:
default_exchange: _Exchange = field(default_factory=_Exchange)
async def test_publisher_puts_context_in_headers(signals: Signals) -> None:
"""`JobPublisher._publish_one`: span на каждое сообщение, его контекст —
заголовки; без контекста — сообщение без заголовков, тело прежнее."""
publisher = JobPublisher(settings.broker, namespace="testdb")
channel = _Channel()
traced = _publication()
plain = _publication(None)
await publisher._publish_one(channel, traced)
await publisher._publish_one(channel, plain)
(span,) = signals.finished_spans()
(first, key, mandatory), (second, _, _) = channel.default_exchange.messages
assert key == QUEUE
assert mandatory is True
assert first.body == traced.body
assert first.headers["traceparent"] == (
f"00-{SAVED_TRACE}-{_hex(span.context.span_id, 16)}-01"
)
assert first.headers["tracestate"] == "vendor=value"
assert first.message_id == str(traced.job_id)
assert not second.headers
assert second.body == plain.body
def test_consumer_span_child_with_link(signals: Signals) -> None:
with (
telemetry.consumer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
parent=SAVED,
redelivered=True,
),
telemetry.child_span("SELECT"),
):
pass
spans = {span.name: span for span in signals.finished_spans()}
consumer = spans[f"process {QUEUE}"]
assert _hex(consumer.context.trace_id, 32) == SAVED_TRACE
assert _hex(consumer.parent.span_id, 16) == SAVED_SPAN
(link,) = consumer.links
assert _hex(link.context.span_id, 16) == SAVED_SPAN
assert _hex(link.context.trace_id, 32) == SAVED_TRACE
assert consumer.attributes["messaging.operation.name"] == "process"
assert consumer.attributes["messenger.job.redelivered"] is True
assert spans["SELECT"].parent.span_id == consumer.context.span_id
@pytest.mark.parametrize(
"parent",
[None, dataclasses.TraceContextDTO(traceparent="not-w3c")],
ids=["none", "garbage"],
)
def test_consumer_span_root_without_context(
signals: Signals,
parent: dataclasses.TraceContextDTO | None,
) -> None:
with telemetry.consumer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
parent=parent,
):
pass
(span,) = signals.finished_spans()
assert span.parent is None
assert span.links == ()
def test_unsampled_context_is_not_recorded(signals: Signals) -> None:
"""Флаг «не выбрано» соблюдается: ни producer, ни consumer, ни
дочерних span; исход и метрика доставки те же (решение 5)."""
publication = _publication(UNSAMPLED)
with telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=publication,
) as headers:
pass
assert headers["traceparent"].startswith(f"00-{SAVED_TRACE}-")
assert headers["traceparent"].endswith("-00")
parent = core.to_trace_context(headers)
with telemetry.consumer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
parent=parent,
) as observation:
with telemetry.child_span("SELECT") as child:
assert child is None
observation.finish(
result=dataclasses.JobDeliveryResultDTO(
outcome=JobDeliveryOutcome.READY,
attempt=1,
time_to_ready_seconds=1.5,
),
duration_seconds=0.2,
)
points = signals.points(METRIC_JOB_DURATION)
assert signals.finished_spans() == []
(point,) = points
assert point.attributes["messenger.job.outcome"] == "ready"
async def test_delivery_result_attributes_and_time_to_ready(
signals: Signals,
) -> None:
result = dataclasses.JobDeliveryResultDTO(
outcome=JobDeliveryOutcome.READY,
attempt=2,
time_to_ready_seconds=12.5,
)
delivery = _Delivery(trace_context=SAVED)
await worker_main._handle(
lambda: _Service(None, result),
delivery,
asyncio.Semaphore(1),
asyncio.Event(),
)
(point,) = signals.points("messenger.job.time_to_ready")
assert dict(point.attributes) == {
"messenger.job.type": ATTACHMENT_JOB_TYPE,
}
assert point.count == 1
assert point.sum == 12.5
assert point.explicit_bounds[0] == 0.25
assert point.explicit_bounds[-1] == 600
(span,) = signals.finished_spans()
assert _hex(span.context.trace_id, 32) == SAVED_TRACE
assert span.attributes["messenger.job.attempt"] == 2
assert "messenger.job.reason" not in span.attributes
async def test_failed_result_names_reason_without_time(
signals: Signals,
) -> None:
result = dataclasses.JobDeliveryResultDTO(
outcome=JobDeliveryOutcome.FAILED,
attempt=1,
reason=AttachmentJobReason.UNDECODABLE,
error_code=AttachmentErrorCode.INVALID_IMAGE,
)
await worker_main._handle(
lambda: _Service(None, result),
_Delivery(),
asyncio.Semaphore(1),
asyncio.Event(),
)
assert signals.points("messenger.job.time_to_ready") == []
(span,) = signals.finished_spans()
assert span.attributes["messenger.job.reason"] == "undecodable"
assert span.attributes["messenger.job.error_code"] == "invalid_image"
# --- метрики уровня 4 -------------------------------------------------------
@dataclass
class _Pool:
used: int = 2
idle: int = 1
waiting: int = 3
timed_out: int = 4
def checkedout(self) -> int:
return self.used
def checkedin(self) -> int:
return self.idle
def size(self) -> int:
return 10
def pending(self) -> int:
return self.waiting
def timeouts(self) -> int:
return self.timed_out
@dataclass
class _SyncEngine:
pool: Any
url: Any
@dataclass
class _Engine:
sync_engine: _SyncEngine
def test_pool_metrics(signals: Signals) -> None:
from sqlalchemy.engine import make_url
engine = _Engine(
_SyncEngine(
pool=_Pool(),
url=make_url("postgresql+asyncpg://user@db:5432/app"),
),
)
metrics.observe_pool(engine, max_overflow=2)
name = {"db.client.connection.pool.name": "db:5432/app"}
counts = {
point.attributes["db.client.connection.state"]: point.value
for point in signals.points("db.client.connection.count")
}
assert counts == {"used": 2, "idle": 1}
(limit,) = signals.points("db.client.connection.max")
assert limit.value == 12
assert dict(limit.attributes) == name
(pending,) = signals.points("db.client.connection.pending_requests")
assert pending.value == 3
(timeouts,) = signals.points("db.client.connection.timeouts")
assert timeouts.value == 4
# dispose(): новый пул читается на следующем сборе
engine.sync_engine.pool = _Pool(used=0, idle=0, waiting=0, timed_out=0)
(pending,) = signals.points("db.client.connection.pending_requests")
assert pending.value == 0
async def test_worker_saturation(signals: Signals) -> None:
"""Конкуренция 2: две доставки заняли места, третья ждёт; после всех —
ноль занятых и ожидающих."""
load = telemetry.WorkerLoad()
telemetry.observe_worker(capacity=2, load=load)
release = asyncio.Event()
class _Blocking:
async def handle(self, *, body: bytes, redelivered: bool) -> Any:
await release.wait()
return dataclasses.JobDeliveryResultDTO(
outcome=JobDeliveryOutcome.READY,
)
slots = asyncio.Semaphore(2)
stop = asyncio.Event()
tasks = [
asyncio.create_task(
worker_main._handle(
_Blocking,
_Delivery(),
slots,
stop,
load=load,
),
)
for _ in range(3)
]
for _ in range(50):
await asyncio.sleep(0)
values = {
name: [point.value for point in signals.points(name)]
for name in (
"messenger.worker.busy",
"messenger.worker.capacity",
"messenger.worker.waiting",
)
}
assert values == {
"messenger.worker.busy": [2],
"messenger.worker.capacity": [2],
"messenger.worker.waiting": [1],
}
release.set()
await asyncio.gather(*tasks)
assert (load.busy, load.waiting) == (0, 0)
(busy,) = signals.points("messenger.worker.busy")
assert busy.value == 0
def _snapshot(taken_ago: float) -> dataclasses.QueueSnapshotDTO:
from datetime import timedelta
from src.utils.utc import utcnow
taken_at = utcnow() - timedelta(seconds=taken_ago)
return dataclasses.QueueSnapshotDTO(
taken_at=taken_at,
scheduled=1,
outbox=2,
published=3,
processing=4,
oldest_created_at=taken_at - timedelta(seconds=42),
)
def test_queue_gauges_fresh_and_stale(signals: Signals) -> None:
latest: list[dataclasses.QueueSnapshotDTO | None] = [None]
telemetry.observe_queue(queue=QUEUE, latest=lambda: latest[0])
assert signals.points("messenger.queue.depth") == []
latest[0] = _snapshot(31)
# снимок старше 30 с не отдаётся
assert signals.points("messenger.queue.depth") == []
assert signals.points("messenger.queue.oldest_age") == []
latest[0] = _snapshot(1)
depth = {
point.attributes["messenger.queue.stage"]: point.value
for point in signals.points("messenger.queue.depth")
}
assert depth == {
"scheduled": 1,
"outbox": 2,
"published": 3,
"processing": 4,
}
(age,) = signals.points("messenger.queue.oldest_age")
assert age.value == 42
assert dict(age.attributes) == {"messaging.destination.name": QUEUE}
def test_queue_age_without_unfinished_is_zero() -> None:
snapshot = dataclasses.QueueSnapshotDTO(
taken_at=_snapshot(0).taken_at,
scheduled=0,
outbox=0,
published=0,
processing=0,
)
depth, (age,) = metrics.queue_observations(snapshot, queue=QUEUE)
assert [item.value for item in depth] == [0, 0, 0, 0]
assert age.value == 0
def _cgroup(root: Any, *, memory_max: str = "268435456") -> None:
(root / "cpu.stat").write_text(
"usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n",
)
(root / "memory.current").write_text("1048576\n")
(root / "cpu.max").write_text("25000 100000\n")
(root / "memory.max").write_text(f"{memory_max}\n")
def test_container_metrics_from_cgroup(tmp_path: Any) -> None:
_cgroup(tmp_path)
signals = _start(cgroup_root=tmp_path)
try:
(cpu,) = signals.points("container.cpu.time")
(memory,) = signals.points("container.memory.usage")
(cpu_limit,) = signals.points("messenger.container.cpu.limit")
(memory_limit,) = signals.points("messenger.container.memory.limit")
finally:
signals.finish()
telemetry.remove_log_hooks()
assert cpu.value == 2.5
assert memory.value == 1048576
assert cpu_limit.value == 0.25
assert memory_limit.value == 268435456
def test_container_without_limit(tmp_path: Any) -> None:
from src.infra.telemetry import container
_cgroup(tmp_path, memory_max="max")
(tmp_path / "cpu.max").write_text("max 100000\n")
assert container.memory_limit(tmp_path) is None
assert container.cpu_limit(tmp_path) is None
assert container.available(tmp_path)
def test_container_unavailable_one_record(
tmp_path: Any,
stdout: io.StringIO,
) -> None:
signals = _start(cgroup_root=tmp_path / "missing")
try:
assert signals.points("container.cpu.time") == []
finally:
signals.finish()
telemetry.remove_log_hooks()
records = [
record
for record in _records(stdout)
if record["event"] == "Container metrics are unavailable"
]
assert len(records) == 1
assert records[0]["level"] == "warning"
# запись до хуков журнала: в хранилище логов не уходит
assert signals.finished_logs() == []
def test_relay_resource() -> None:
signals = _start(service="messenger-relay")
try:
with telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=_publication(),
):
pass
(span,) = signals.finished_spans()
finally:
signals.finish()
telemetry.remove_log_hooks()
assert span.resource.attributes["service.name"] == "messenger-relay"
assert span.resource.attributes["service.instance.id"] == "test-instance"
@pytest.mark.parametrize(
("headers", "expected"),
[
(None, None),
({}, None),
({"traceparent": SAVED.traceparent}, (SAVED.traceparent, None)),
(
{
"traceparent": SAVED.traceparent.encode(),
"tracestate": b"vendor=value",
},
(SAVED.traceparent, "vendor=value"),
),
({"traceparent": "01-bad"}, None),
({"traceparent": SAVED.traceparent.upper()}, None),
({"traceparent": "\xff".encode("latin-1")}, None),
({"traceparent": 42}, None),
({"traceparent": SAVED.traceparent, "tracestate": "x" * 513}, None),
# пустой tracestate по W3C равен отсутствующему (verify S4)
(
{"traceparent": SAVED.traceparent, "tracestate": ""},
(SAVED.traceparent, None),
),
(
{"traceparent": SAVED.traceparent.encode(), "tracestate": b""},
(SAVED.traceparent, None),
),
],
ids=[
"no-headers",
"empty",
"str",
"bytes",
"garbage",
"upper",
"not-ascii",
"number",
"long-tracestate",
"empty-tracestate",
"empty-tracestate-bytes",
],
)
def test_delivery_headers(
headers: Any,
expected: tuple[str, str | None] | None,
) -> None:
from src.infra.broker.client import trace_context_from_headers
context = trace_context_from_headers(headers)
if expected is None:
assert context is None
else:
assert context == dataclasses.TraceContextDTO(*expected)
def _loaded_modules(imports: str) -> set[str]:
"""Модули SDK, экспортёра, protobuf и `requests` после импорта в
отдельном интерпретаторе с выключенной телеметрией."""
import os
import subprocess
import sys
from pathlib import Path
code = (
"import json, sys\n"
f"import {imports}\n"
"prefixes = ('opentelemetry.sdk', 'opentelemetry.exporter',\n"
" 'google.protobuf', 'requests')\n"
"print(json.dumps(sorted(m for m in sys.modules\n"
" if m.startswith(prefixes))))\n"
)
root = Path(__file__).resolve().parents[2]
env = {**os.environ, "TELEMETRY__ENABLED": "false"}
result = subprocess.run( # noqa: S603
[sys.executable, "-c", code],
cwd=root,
env=env,
capture_output=True,
text=True,
check=True,
timeout=60,
)
return set(json.loads(result.stdout.strip().splitlines()[-1]))
def test_sdk_modules_not_loaded_without_telemetry() -> None:
"""Выключенная телеметрия — ни провайдеров SDK, ни экспортёра, ни
protobuf, ни HTTP-клиента экспортёра в памяти точек входа (решение 7).
Исключение — модули метрик SDK, которые redis-py 8.1 сам загружает при
импорте (`redis.observability.providers`: проба «установлен ли SDK»):
точки входа не грузят ничего сверх `import redis.asyncio` (design,
«Уточнения apply»)."""
loaded = _loaded_modules(
"src.main, src.worker.main, src.relay.main, src.cleaner.main",
)
forbidden = (
"opentelemetry.exporter",
"opentelemetry.sdk.trace",
"opentelemetry.sdk._logs",
"google.protobuf",
"requests",
)
assert not {name for name in loaded if name.startswith(forbidden)}
assert loaded <= _loaded_modules("redis.asyncio")
# --- внутренние метрики SDK (design change slo-alerts, решения 8–9) -------
INTERNAL_METRICS_ENV = "OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED"
SPAN_PROCESSED = "otel.sdk.processor.span.processed"
LOG_PROCESSED = "otel.sdk.processor.log.processed"
SPAN_EXPORTED = "otel.sdk.exporter.span.exported"
LOG_EXPORTED = "otel.sdk.exporter.log.exported"
SDK_HISTOGRAMS = (
"otel.sdk.exporter.operation.duration",
"otel.sdk.metric_reader.collection.duration",
)
class _GateSpanExporter(SpanExporter):
"""Отправка ждёт, пока тест не откроет ворота: очередь процессора
заполняется, как при медленном Collector."""
def __init__(self) -> None:
self.gate = threading.Event()
def export(self, spans: Any) -> SpanExportResult:
self.gate.wait(5)
return SpanExportResult.SUCCESS
def shutdown(self) -> None:
self.gate.set()
class _GateLogExporter(LogRecordExporter):
def __init__(self) -> None:
self.gate = threading.Event()
def export(self, batch: Any) -> LogRecordExportResult:
self.gate.wait(5)
return LogRecordExportResult.SUCCESS
def shutdown(self) -> None:
self.gate.set()
def force_flush(self, timeout_millis: int = 30000) -> bool: # noqa: ARG002
return True
def _metric_names(reader: InMemoryMetricReader) -> set[str]:
data = reader.get_metrics_data()
if data is None:
return set()
return {
metric.name
for resource in data.resource_metrics
for scope in resource.scope_metrics
for metric in scope.metrics
}
def _sum_where(signals: Signals, name: str, **attributes: Any) -> int:
return sum(
point.value
for point in signals.points(name)
if all(
point.attributes.get(key) == value
for key, value in attributes.items()
)
)
@pytest.fixture
def small_queue(monkeypatch: pytest.MonkeyPatch) -> None:
"""Очередь процессоров 2, пакет 1, пауза 10 мс: третий span ждёт места."""
monkeypatch.setattr(settings.telemetry, "max_queue_size", 2)
monkeypatch.setattr(settings.telemetry, "max_export_batch_size", 1)
monkeypatch.setattr(settings.telemetry, "schedule_delay_seconds", 0.01)
@pytest.mark.usefixtures("small_queue")
def test_sdk_counts_queue_full() -> None:
"""Полная очередь процессора — счёт `error.type` = `queue_full` у span и
записей журнала, ёмкость очереди — как `[telemetry] max_queue_size`."""
spans, logs = _GateSpanExporter(), _GateLogExporter()
collected = Signals(
spans=InMemorySpanExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
)
assert telemetry.start_process_telemetry(
SERVICE,
exporters=telemetry.Exporters(
spans=spans,
metrics=collected.metrics,
logs=logs,
),
)
try:
for number in range(10):
with core.tracer().start_as_current_span(f"span-{number}"):
pass
otel_logger = core.otel_logger()
for number in range(10):
otel_logger.emit(body=f"record-{number}")
dropped_spans = _sum_where(
collected,
SPAN_PROCESSED,
**{"error.type": "queue_full"},
)
dropped_logs = _sum_where(
collected,
LOG_PROCESSED,
**{"error.type": "queue_full"},
)
capacity = {
point.value
for name in (
"otel.sdk.processor.span.queue.capacity",
"otel.sdk.processor.log.queue.capacity",
)
for point in collected.points(name)
}
finally:
spans.gate.set()
logs.gate.set()
collected.finish()
telemetry.remove_log_hooks()
# в полёте — не больше одного пакета, в очереди — два: остальные отброшены
assert dropped_spans >= 10 - 2 - 1
assert dropped_logs >= 10 - 2 - 1
assert capacity == {2}
def test_sdk_counts_failed_export(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Экспортёр OTLP span и журнала, отправка которого падает
`ConnectionError` (подставная `requests.Session.send`, сети нет), —
`otel.sdk.exporter.*.exported` с `error.type`; гистограмм SDK нет."""
def send(
self: requests.Session, # noqa: ARG001
request: requests.PreparedRequest, # noqa: ARG001
**kwargs: Any, # noqa: ARG001
) -> requests.Response:
raise requests.exceptions.ConnectionError("collector is down")
monkeypatch.setattr(requests.Session, "send", send)
monkeypatch.setattr(settings.telemetry, "otlp_endpoint", COLLECTOR)
monkeypatch.setattr(settings.telemetry, "export_timeout_seconds", 0.2)
monkeypatch.setattr(settings.telemetry, "schedule_delay_seconds", 0.01)
collected = Signals(
spans=InMemorySpanExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
)
assert telemetry.start_process_telemetry(
SERVICE,
exporters=telemetry.Exporters(
spans=None,
metrics=collected.metrics,
logs=None,
),
)
try:
with core.tracer().start_as_current_span("probe"):
pass
core.otel_logger().emit(body="probe")
deadline = time.monotonic() + 5
failed: dict[str, int] = {}
while time.monotonic() < deadline:
failed = {
name: _sum_where(
collected,
name,
**{"error.type": "ConnectionError"},
)
for name in (SPAN_EXPORTED, LOG_EXPORTED)
}
if all(failed.values()):
break
time.sleep(0.05)
names = _metric_names(collected.metrics)
exported = collected.points(SPAN_EXPORTED)
finally:
collected.finish()
telemetry.remove_log_hooks()
assert failed == {SPAN_EXPORTED: 1, LOG_EXPORTED: 1}
assert not names & set(SDK_HISTOGRAMS)
# метки — компонент и адрес Collector, без id
(point,) = exported
assert set(point.attributes) == {
"otel.component.type",
"otel.component.name",
"server.address",
"server.port",
"error.type",
}
assert point.attributes["server.address"] == "otel-collector"
def test_sdk_internal_metrics_environment(
monkeypatch: pytest.MonkeyPatch,
stdout: io.StringIO,
) -> None:
"""Внешняя переменная внутренних метрик удаляется с `warning` и ни на что
не влияет: метрики SDK есть, после `start()` переменной в окружении нет."""
import os
monkeypatch.setenv(INTERNAL_METRICS_ENV, "false")
collected = Signals(
spans=InMemorySpanExporter(),
metrics=InMemoryMetricReader(),
logs=InMemoryLogRecordExporter(),
)
try:
assert telemetry.start_process_telemetry(
SERVICE,
exporters=telemetry.Exporters(
spans=collected.spans,
metrics=collected.metrics,
logs=collected.logs,
),
)
assert INTERNAL_METRICS_ENV not in os.environ
with core.tracer().start_as_current_span("probe"):
pass
names = _metric_names(collected.metrics)
finally:
collected.finish()
telemetry.remove_log_hooks()
assert "otel.sdk.processor.span.queue.capacity" in names
assert "otel.sdk.processor.log.queue.capacity" in names
assert not names & set(SDK_HISTOGRAMS)
(warning,) = [
record
for record in _records(stdout)
if record["event"] == "Ignored OpenTelemetry environment variables"
]
assert warning["names"] == [INTERNAL_METRICS_ENV]
def test_sdk_internal_metrics_off_without_telemetry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Выключенная телеметрия — ничего: ни провайдеров, ни переменной."""
import os
monkeypatch.setattr(settings.telemetry, "enabled", False)
monkeypatch.delenv(INTERNAL_METRICS_ENV, raising=False)
assert telemetry.start_process_telemetry(SERVICE) is False
assert not telemetry.is_enabled()
assert INTERNAL_METRICS_ENV not in os.environ
# --- выбор трасс в голове и наборы границ ---------------------------------
# (design change track1-telemetry-cost, решения 3–4)
SPAN_STARTED = "otel.sdk.span.started"
SPAN_LIVE = "otel.sdk.span.live"
SAMPLING_RESULT = "otel.span.sampling_result"
PARENT_ORIGIN = "otel.span.parent.origin"
# доля 0,5: корень выбран, если младшие 64 бита trace_id меньше 2**63
TRACE_KEPT = (1 << 63) - 1
TRACE_DROPPED = 1 << 63
def _sampled_app() -> FastAPI:
"""Ручка с дочерним span и записью журнала."""
app = FastAPI()
@app.get("/api/v1/probe")
async def probe() -> dict[str, str]:
with telemetry.child_span("SELECT") as child:
get_logger().info("Inside probe", child=child is not None)
return {}
app.add_middleware(RequestContextMiddleware)
return app
async def _probe(times: int = 1) -> list[str]:
transport = httpx.ASGITransport(app=_sampled_app())
async with httpx.AsyncClient(
transport=transport,
base_url="http://test",
) as client:
return [
(await client.get("/api/v1/probe")).headers["x-request-id"]
for _ in range(times)
]
def _started(signals: Signals) -> dict[tuple[str, str], int]:
return {
(
point.attributes[PARENT_ORIGIN],
point.attributes[SAMPLING_RESULT],
): point.value
for point in signals.points(SPAN_STARTED)
}
def _ratio(monkeypatch: pytest.MonkeyPatch, ratio: float) -> None:
monkeypatch.setattr(settings.telemetry, "trace_sample_ratio", ratio)
async def test_ratio_one_traces_every_request(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_ratio(monkeypatch, 1)
signals = _start()
try:
request_ids = await _probe(5)
started = _started(signals)
spans = signals.finished_spans()
finally:
signals.finish()
telemetry.remove_log_hooks()
servers = [span for span in spans if span.kind == SpanKind.SERVER]
assert sorted(
span.attributes["messenger.request_id"] for span in servers
) == sorted(request_ids)
assert len(spans) == 10
# отброшенных выборкой нет; живые span не отдаются
assert started == {
("none", "RECORD_AND_SAMPLE"): 5,
("local", "RECORD_AND_SAMPLE"): 5,
}
assert signals.points(SPAN_LIVE) == []
async def test_ratio_zero_keeps_metrics_and_logs(
monkeypatch: pytest.MonkeyPatch,
stdout: io.StringIO,
) -> None:
"""Невыбранный запрос: ни server span, ни дочерних; histogram и запись
журнала — те же, у записи в хранилище логов `trace_id` с флагом «не
выбрано», в stdout полей трассы нет."""
_ratio(monkeypatch, 0)
signals = _start()
try:
await _probe(3)
points = signals.points(METRIC_HTTP_SERVER_DURATION)
started = _started(signals)
spans = signals.finished_spans()
exported = [
record
for record in signals.finished_logs()
if record.body == "Inside probe"
]
finally:
signals.finish()
telemetry.remove_log_hooks()
assert spans == []
(point,) = points
assert point.count == 3
assert started == {("none", "DROP"): 3}
assert len(exported) == 3
for record in exported:
assert record.trace_id != 0
assert not int(record.trace_flags) & 0x01
assert record.attributes["child"] is False
records = [r for r in _records(stdout) if r.get("event") == "Inside probe"]
assert len(records) == 3
for record in records:
assert "trace_id" not in record
assert "span_id" not in record
async def test_ratio_half_decides_by_trace_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Решение — по младшим 64 битам `trace_id` (`TraceIdRatioBased`)."""
from opentelemetry.sdk.trace import RandomIdGenerator
ids = iter([TRACE_KEPT, TRACE_DROPPED])
monkeypatch.setattr(
RandomIdGenerator,
"generate_trace_id",
lambda self: next(ids), # noqa: ARG005
)
_ratio(monkeypatch, 0.5)
signals = _start()
try:
kept, dropped = await _probe(2)
started = _started(signals)
spans = signals.finished_spans()
finally:
signals.finish()
telemetry.remove_log_hooks()
servers = [span for span in spans if span.kind == SpanKind.SERVER]
(server,) = servers
assert server.context.trace_id == TRACE_KEPT
assert server.attributes["messenger.request_id"] == kept
assert dropped != kept
assert started == {
("none", "RECORD_AND_SAMPLE"): 1,
("none", "DROP"): 1,
("local", "RECORD_AND_SAMPLE"): 1,
}
def test_remote_parent_flag_wins_over_ratio(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Span с удалённым родителем следует его флагу: выбранный пишется и
при доле 0, невыбранный не пишется и при доле 1 (producer и consumer)."""
def chain(parent: dataclasses.TraceContextDTO) -> None:
with telemetry.producer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
publication=_publication(parent),
):
pass
with telemetry.consumer_span(
queue=QUEUE,
job_type=ATTACHMENT_JOB_TYPE,
parent=parent,
):
pass
_ratio(monkeypatch, 0)
signals = _start()
try:
chain(SAVED)
zero = signals.finished_spans()
finally:
signals.finish()
telemetry.remove_log_hooks()
assert {span.kind for span in zero} == {
SpanKind.PRODUCER,
SpanKind.CONSUMER,
}
assert {_hex(span.context.trace_id, 32) for span in zero} == {SAVED_TRACE}
_ratio(monkeypatch, 1)
signals = _start()
try:
chain(UNSAMPLED)
started = _started(signals)
one = signals.finished_spans()
finally:
signals.finish()
telemetry.remove_log_hooks()
assert one == []
assert started == {("remote", "DROP"): 2}
async def test_coarse_buckets_in_export(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
settings.telemetry,
"metric_buckets",
MetricBucketsEnum.COARSE,
)
signals = _start()
try:
await _probe()
(point,) = signals.points(METRIC_HTTP_SERVER_DURATION)
histograms = core.job_histograms()
assert histograms is not None
histograms[0].record(0.3, {})
histograms[1].record(12, {})
(job,) = signals.points(METRIC_JOB_DURATION)
(ready,) = signals.points(METRIC_JOB_TIME_TO_READY)
finally:
signals.finish()
telemetry.remove_log_hooks()
assert tuple(point.explicit_bounds) == HTTP_DURATION_BUCKETS_COARSE
assert tuple(job.explicit_bounds) == JOB_DURATION_BUCKETS_COARSE
assert tuple(ready.explicit_bounds) == TIME_TO_READY_BUCKETS_COARSE
assert 0.25 in point.explicit_bounds
assert 30 in ready.explicit_bounds
async def test_default_buckets_unchanged() -> None:
signals = _start()
try:
await _probe()
(point,) = signals.points(METRIC_HTTP_SERVER_DURATION)
finally:
signals.finish()
telemetry.remove_log_hooks()
assert tuple(point.explicit_bounds) == HTTP_DURATION_BUCKETS
def test_sampling_without_telemetry_has_nothing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings.telemetry, "enabled", False)
_ratio(monkeypatch, 0.1)
monkeypatch.setattr(
settings.telemetry,
"metric_buckets",
MetricBucketsEnum.COARSE,
)
assert telemetry.start_process_telemetry(SERVICE) is False
assert core.tracer() is None
assert core.meter() is None
def test_startup_fields(monkeypatch: pytest.MonkeyPatch) -> None:
"""Запись о старте называет долю, набор и интервал при включённой
телеметрии и `None` при выключенной (спек `service-configuration`)."""
_ratio(monkeypatch, 0.1)
monkeypatch.setattr(
settings.telemetry,
"metric_buckets",
MetricBucketsEnum.COARSE,
)
monkeypatch.setattr(settings.telemetry, "otlp_endpoint", COLLECTOR)
assert telemetry.startup_fields(True) == {
"telemetry_enabled": True,
"telemetry_endpoint": COLLECTOR,
"telemetry_environment": settings.telemetry.environment,
"telemetry_trace_sample_ratio": 0.1,
"telemetry_metric_buckets": "coarse",
"telemetry_metric_export_interval_seconds": 15,
}
disabled = telemetry.startup_fields(False)
assert disabled["telemetry_enabled"] is False
assert set(disabled) == set(telemetry.startup_fields(True))
assert all(
value is None
for key, value in disabled.items()
if key != "telemetry_enabled"
)
@pytest.mark.parametrize(
"module",
["src/main.py", "src/worker/main.py", "src/relay/main.py"],
)
def test_startup_records_use_fields(module: str) -> None:
from src.config.settings import BASE_DIR
source = BASE_DIR.joinpath(module).read_text(encoding="utf-8")
assert "**telemetry.startup_fields(telemetry_enabled)" in source
def test_sampler_choice() -> None:
"""Доля 1 — тот же `ParentBased(ALWAYS_ON)`, что раньше; иначе —
`ParentBased(TraceIdRatioBased)` (`parentbased_traceidratio`)."""
from opentelemetry.sdk.trace.sampling import ALWAYS_ON, TraceIdRatioBased
always = sdk.sampler(1)
assert always._root is ALWAYS_ON
ratio = sdk.sampler(0.1)
assert isinstance(ratio._root, TraceIdRatioBased)
assert ratio._root.rate == 0.1