- [telemetry] trace_sample_ratio (ParentBased(TraceIdRatioBased), 1 keeps ParentBased(ALWAYS_ON)) and metric_buckets (default|coarse with SLO bounds), export interval <= 30 s; otel.sdk.span.started, span.live dropped; startup records name ratio, buckets and interval - bench: telemetry services on mtusi-bench, TELEMETRY=on switch for uploads and queue, make bench_telemetry[_quick] with off/full/full-aa/head/tail/ metrics/balanced, rare events, lookup from Tempo/Loki/Prometheus, cost sampler, verdicts H1-H8 and B1-B5; collector layers and lean Prometheus in bench/otel with preflight checks - REPORT: telemetry cost hypotheses and criteria before the label Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2027 lines
66 KiB
Python
2027 lines
66 KiB
Python
"""Телеметрия на in-memory экспортёрах SDK, без сети (design change
|
||
`otel-telemetry`, решение 21): выключенная телеметрия, `OTEL_*`, ресурс,
|
||
server span и метрика HTTP, запись журнала, правило корня, span Valkey и S3,
|
||
закрытие со сроком, исход доставки worker → span и histogram."""
|
||
|
||
import asyncio
|
||
import io
|
||
import json
|
||
import logging
|
||
import threading
|
||
import time
|
||
from collections.abc import Iterator
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
from uuid import uuid4
|
||
|
||
import httpx
|
||
import pytest
|
||
import requests
|
||
from botocore.exceptions import ClientError
|
||
from fastapi import APIRouter, FastAPI
|
||
from opentelemetry import trace
|
||
from opentelemetry._logs import SeverityNumber
|
||
from opentelemetry.sdk._logs import LoggerProvider
|
||
from opentelemetry.sdk._logs.export import (
|
||
InMemoryLogRecordExporter,
|
||
LogRecordExporter,
|
||
LogRecordExportResult,
|
||
SimpleLogRecordProcessor,
|
||
)
|
||
from opentelemetry.sdk.metrics import MeterProvider
|
||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
|
||
from opentelemetry.sdk.trace.export import (
|
||
SimpleSpanProcessor,
|
||
SpanExporter,
|
||
SpanExportResult,
|
||
)
|
||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
||
InMemorySpanExporter,
|
||
)
|
||
from opentelemetry.trace import SpanKind, StatusCode
|
||
from redis.exceptions import RedisError
|
||
|
||
from src.api.middlewares import RequestContextMiddleware
|
||
from src.config import settings
|
||
from src.config.settings import Telemetry as TelemetrySettings
|
||
from src.infra import logger as app_logger
|
||
from src.infra import telemetry
|
||
from src.infra.broker import JobPublisher
|
||
from src.infra.cache.client import Cache
|
||
from src.infra.logger import get_logger
|
||
from src.infra.s3.client import ObjectStorage
|
||
from src.infra.telemetry import core, metrics, sdk
|
||
from src.schemas import dataclasses
|
||
from src.schemas.enums import (
|
||
AttachmentErrorCode,
|
||
AttachmentJobReason,
|
||
JobDeliveryOutcome,
|
||
MetricBucketsEnum,
|
||
)
|
||
from src.utils.constants import (
|
||
ATTACHMENT_JOB_TYPE,
|
||
HTTP_DURATION_BUCKETS,
|
||
HTTP_DURATION_BUCKETS_COARSE,
|
||
JOB_DURATION_BUCKETS_COARSE,
|
||
METRIC_HTTP_SERVER_DURATION,
|
||
METRIC_JOB_DURATION,
|
||
METRIC_JOB_TIME_TO_READY,
|
||
SERVICE_VERSION,
|
||
TIME_TO_READY_BUCKETS_COARSE,
|
||
)
|
||
from src.worker import main as worker_main
|
||
|
||
pytestmark = pytest.mark.anyio
|
||
|
||
SERVICE = "messenger-api"
|
||
SECRET_TEXT = "secret-text-in-exception"
|
||
COLLECTOR = "http://otel-collector:4318"
|
||
PROXY = "http://proxy.invalid:3128"
|
||
|
||
|
||
@dataclass
|
||
class Signals:
|
||
spans: InMemorySpanExporter
|
||
metrics: InMemoryMetricReader
|
||
logs: InMemoryLogRecordExporter
|
||
closed: bool = False
|
||
|
||
def finish(self) -> None:
|
||
"""Закрыть провайдеры: батчи отправлены в in-memory экспортёры."""
|
||
if self.closed:
|
||
return
|
||
self.closed = True
|
||
for thread in telemetry.begin_shutdown():
|
||
thread.join(5)
|
||
|
||
def finished_spans(self) -> list[ReadableSpan]:
|
||
self.finish()
|
||
return list(self.spans.get_finished_spans())
|
||
|
||
def finished_logs(self) -> list[Any]:
|
||
self.finish()
|
||
return [item.log_record for item in self.logs.get_finished_logs()]
|
||
|
||
def points(self, name: str) -> list[Any]:
|
||
data = self.metrics.get_metrics_data()
|
||
if data is None:
|
||
return []
|
||
return [
|
||
point
|
||
for resource in data.resource_metrics
|
||
for scope in resource.scope_metrics
|
||
for metric in scope.metrics
|
||
if metric.name == name
|
||
for point in metric.data.data_points
|
||
]
|
||
|
||
|
||
@pytest.fixture
|
||
def stdout() -> Iterator[io.StringIO]:
|
||
"""Вывод handler журнала (он пишет в sys.stdout, взятый при настройке)."""
|
||
get_logger()
|
||
handler = logging.getLogger("src").handlers[0]
|
||
buffer = io.StringIO()
|
||
previous = handler.setStream(buffer)
|
||
try:
|
||
yield buffer
|
||
finally:
|
||
handler.setStream(previous)
|
||
|
||
|
||
@pytest.fixture
|
||
def signals(monkeypatch: pytest.MonkeyPatch) -> Iterator[Signals]:
|
||
monkeypatch.setattr(settings.telemetry, "environment", "lab")
|
||
collected = Signals(
|
||
spans=InMemorySpanExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
)
|
||
started = telemetry.start_process_telemetry(
|
||
SERVICE,
|
||
exporters=telemetry.Exporters(
|
||
spans=collected.spans,
|
||
metrics=collected.metrics,
|
||
logs=collected.logs,
|
||
),
|
||
)
|
||
assert started
|
||
try:
|
||
yield collected
|
||
finally:
|
||
collected.finish()
|
||
telemetry.remove_log_hooks()
|
||
|
||
|
||
def _records(buffer: io.StringIO) -> list[dict[str, Any]]:
|
||
return [json.loads(line) for line in buffer.getvalue().splitlines()]
|
||
|
||
|
||
# --- выключенная телеметрия ---------------------------------------------
|
||
|
||
|
||
def test_disabled_has_no_providers_and_hooks(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
monkeypatch.setattr(settings.telemetry, "enabled", False)
|
||
monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
|
||
assert telemetry.start_process_telemetry(SERVICE) is False
|
||
assert not telemetry.is_enabled()
|
||
assert app_logger._export_hook is None
|
||
assert app_logger._trace_fields_hook is None
|
||
# выключенная телеметрия окружение не трогает
|
||
assert "OTEL_SDK_DISABLED" in __import__("os").environ
|
||
with telemetry.child_span("GET") as span:
|
||
assert span is None
|
||
get_logger().info("Plain record", answer=42)
|
||
(record,) = _records(stdout)
|
||
assert "trace_id" not in record
|
||
assert "span_id" not in record
|
||
assert record["event"] == "Plain record"
|
||
assert record["answer"] == 42
|
||
|
||
|
||
def test_disabled_record_fields_unchanged(stdout: io.StringIO) -> None:
|
||
"""Поля записи без телеметрии — прежние: полей трассы нет."""
|
||
get_logger().bind(module="probe").info("Plain record", value=1)
|
||
(record,) = _records(stdout)
|
||
assert list(record)[:2] == ["event", "instance_id"]
|
||
assert set(record) == {
|
||
"event",
|
||
"instance_id",
|
||
"module",
|
||
"value",
|
||
"level",
|
||
"timestamp",
|
||
"filename",
|
||
"func_name",
|
||
"lineno",
|
||
}
|
||
|
||
|
||
# --- запуск, OTEL_*, ресурс ----------------------------------------------
|
||
|
||
|
||
def test_otel_environment_removed_and_named(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
|
||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://elsewhere:9")
|
||
collected = Signals(
|
||
spans=InMemorySpanExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
)
|
||
try:
|
||
telemetry.start_process_telemetry(
|
||
SERVICE,
|
||
exporters=telemetry.Exporters(
|
||
spans=collected.spans,
|
||
metrics=collected.metrics,
|
||
logs=collected.logs,
|
||
),
|
||
)
|
||
import os
|
||
|
||
assert "OTEL_SDK_DISABLED" not in os.environ
|
||
assert "OTEL_EXPORTER_OTLP_ENDPOINT" not in os.environ
|
||
# SDK не выключен переменной: span пишется
|
||
with core.tracer().start_as_current_span("probe"):
|
||
pass
|
||
assert [span.name for span in collected.finished_spans()] == ["probe"]
|
||
finally:
|
||
collected.finish()
|
||
telemetry.remove_log_hooks()
|
||
output = stdout.getvalue()
|
||
assert "elsewhere" not in output
|
||
(warning,) = [
|
||
record
|
||
for record in _records(stdout)
|
||
if record["event"] == "Ignored OpenTelemetry environment variables"
|
||
]
|
||
assert warning["level"] == "warning"
|
||
assert warning["names"] == [
|
||
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||
"OTEL_SDK_DISABLED",
|
||
]
|
||
|
||
|
||
def test_resource_has_four_attributes(signals: Signals) -> None:
|
||
with core.tracer().start_as_current_span("probe"):
|
||
pass
|
||
(span,) = signals.finished_spans()
|
||
assert dict(span.resource.attributes) == {
|
||
"service.name": SERVICE,
|
||
"service.version": SERVICE_VERSION,
|
||
"service.instance.id": "test-instance",
|
||
"deployment.environment.name": "lab",
|
||
}
|
||
|
||
|
||
# --- server span и метрика HTTP -------------------------------------------
|
||
|
||
|
||
def _app() -> FastAPI:
|
||
app = FastAPI()
|
||
v1 = APIRouter(prefix="/api/v1")
|
||
items = APIRouter(prefix="/items")
|
||
|
||
@items.get("/{item_id}")
|
||
async def get_item(item_id: str) -> dict[str, str]:
|
||
get_logger().info("Inside handler")
|
||
return {"id": item_id}
|
||
|
||
@items.get("/{item_id}/boom")
|
||
async def boom(item_id: str) -> None:
|
||
raise RuntimeError(SECRET_TEXT)
|
||
|
||
@app.get("/health/live")
|
||
async def live() -> dict[str, str]:
|
||
return {"status": "ok"}
|
||
|
||
v1.include_router(items)
|
||
app.include_router(v1)
|
||
app.add_middleware(RequestContextMiddleware)
|
||
return app
|
||
|
||
|
||
async def _request(
|
||
method: str,
|
||
path: str,
|
||
headers: dict[str, str] | None = None,
|
||
) -> httpx.Response:
|
||
transport = httpx.ASGITransport(app=_app(), raise_app_exceptions=False)
|
||
async with httpx.AsyncClient(
|
||
transport=transport,
|
||
base_url="http://test",
|
||
) as client:
|
||
return await client.request(method, path, headers=headers)
|
||
|
||
|
||
async def test_server_span_and_metric(
|
||
signals: Signals,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
response = await _request("GET", "/api/v1/items/abc")
|
||
assert response.status_code == 200
|
||
request_id = response.headers["x-request-id"]
|
||
points = signals.points(METRIC_HTTP_SERVER_DURATION)
|
||
(span,) = signals.finished_spans()
|
||
assert span.name == "GET /api/v1/items/{item_id}"
|
||
assert span.kind == SpanKind.SERVER
|
||
assert span.parent is None
|
||
assert dict(span.attributes) == {
|
||
"http.request.method": "GET",
|
||
"http.route": "/api/v1/items/{item_id}",
|
||
"http.response.status_code": 200,
|
||
"messenger.request_id": request_id,
|
||
}
|
||
(point,) = points
|
||
assert dict(point.attributes) == {
|
||
"http.request.method": "GET",
|
||
"http.route": "/api/v1/items/{item_id}",
|
||
"http.response.status_code": 200,
|
||
}
|
||
assert point.count == 1
|
||
# записи запроса, включая итоговую, — с trace_id и span_id этого span
|
||
trace_id = format(span.context.trace_id, "032x")
|
||
span_id = format(span.context.span_id, "016x")
|
||
records = [r for r in _records(stdout) if r.get("request_id")]
|
||
assert {r["event"] for r in records} == {
|
||
"Inside handler",
|
||
"HTTP request completed",
|
||
}
|
||
for record in records:
|
||
assert record["trace_id"] == trace_id
|
||
assert record["span_id"] == span_id
|
||
|
||
|
||
async def test_unknown_method_and_path(signals: Signals) -> None:
|
||
uuid_path = "/nope/6f1c2d3e-0000-4000-8000-000000000000"
|
||
await _request("BREW", uuid_path)
|
||
points = signals.points(METRIC_HTTP_SERVER_DURATION)
|
||
spans = signals.finished_spans()
|
||
(span,) = spans
|
||
assert span.name == "_OTHER"
|
||
assert "http.route" not in span.attributes
|
||
assert span.attributes["http.request.method"] == "_OTHER"
|
||
(point,) = points
|
||
assert "http.route" not in point.attributes
|
||
values = " ".join(str(value) for value in point.attributes.values())
|
||
assert "BREW" not in values
|
||
assert "6f1c2d3e" not in values
|
||
|
||
|
||
async def test_health_has_no_span_and_metric(signals: Signals) -> None:
|
||
response = await _request("GET", "/health/live")
|
||
assert response.status_code == 200
|
||
assert signals.points(METRIC_HTTP_SERVER_DURATION) == []
|
||
assert signals.finished_spans() == []
|
||
|
||
|
||
async def test_5xx_marks_error_without_text(signals: Signals) -> None:
|
||
response = await _request("GET", "/api/v1/items/x/boom")
|
||
assert response.status_code == 500
|
||
(span,) = signals.finished_spans()
|
||
assert span.status.status_code == StatusCode.ERROR
|
||
assert span.attributes["error.type"] == "RuntimeError"
|
||
assert span.attributes["http.response.status_code"] == 500
|
||
assert span.events == ()
|
||
dumped = json.dumps(
|
||
{
|
||
"attributes": dict(span.attributes),
|
||
"status": span.status.description,
|
||
},
|
||
)
|
||
assert SECRET_TEXT not in dumped
|
||
|
||
|
||
async def test_incoming_traceparent_is_not_continued(
|
||
signals: Signals,
|
||
) -> None:
|
||
foreign = "0af7651916cd43dd8448eb211c80319c"
|
||
await _request(
|
||
"GET",
|
||
"/api/v1/items/abc",
|
||
headers={"traceparent": f"00-{foreign}-b7ad6b7169203331-01"},
|
||
)
|
||
(span,) = signals.finished_spans()
|
||
assert span.parent is None
|
||
assert format(span.context.trace_id, "032x") != foreign
|
||
|
||
|
||
# --- журнал ---------------------------------------------------------------
|
||
|
||
|
||
def test_log_record_export(signals: Signals, stdout: io.StringIO) -> None:
|
||
log = get_logger().bind(module="probe")
|
||
with core.tracer().start_as_current_span("parent") as parent:
|
||
log.info(
|
||
"Exported record",
|
||
count=3,
|
||
ratio=0.5,
|
||
flag=True,
|
||
nothing=None,
|
||
variants=[{"name": "thumbnail", "width": 10}],
|
||
)
|
||
log.warning("Outside span")
|
||
exported = signals.finished_logs()
|
||
inside, outside = exported
|
||
assert inside.body == "Exported record"
|
||
assert inside.severity_number == SeverityNumber.INFO
|
||
assert inside.severity_text == "info"
|
||
assert inside.trace_id == parent.get_span_context().trace_id
|
||
assert inside.span_id == parent.get_span_context().span_id
|
||
attributes = dict(inside.attributes)
|
||
assert attributes["count"] == 3
|
||
assert attributes["ratio"] == 0.5
|
||
assert attributes["flag"] is True
|
||
assert "nothing" not in attributes
|
||
assert json.loads(attributes["variants"]) == [
|
||
{"name": "thumbnail", "width": 10},
|
||
]
|
||
assert attributes["module"] == "probe"
|
||
assert attributes["instance_id"] == "test-instance"
|
||
assert "event" not in attributes
|
||
assert "trace_id" not in attributes
|
||
assert outside.severity_number == SeverityNumber.WARN
|
||
assert not outside.trace_id
|
||
first, second = _records(stdout)
|
||
assert first["trace_id"] == format(
|
||
parent.get_span_context().trace_id,
|
||
"032x",
|
||
)
|
||
assert len(first["span_id"]) == 16
|
||
assert "trace_id" not in second
|
||
|
||
|
||
def test_uvicorn_traceback_is_exported(signals: Signals) -> None:
|
||
try:
|
||
raise ValueError("boom")
|
||
except ValueError as e:
|
||
logging.getLogger("uvicorn.error").error(
|
||
"Exception in ASGI application",
|
||
exc_info=e,
|
||
)
|
||
(record,) = signals.finished_logs()
|
||
assert record.body == "Exception in ASGI application"
|
||
assert record.severity_number == SeverityNumber.ERROR
|
||
assert "Traceback" in record.attributes["exception"]
|
||
assert record.attributes["logger"] == "uvicorn.error"
|
||
|
||
|
||
def test_sdk_records_not_exported_and_thinned(
|
||
signals: Signals,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
sdk = logging.getLogger("opentelemetry.probe.thinned")
|
||
for _ in range(5):
|
||
sdk.warning("Failed to export batch")
|
||
assert signals.finished_logs() == []
|
||
records = [r for r in _records(stdout) if r.get("logger") == sdk.name]
|
||
assert len(records) == 1
|
||
|
||
|
||
# --- правило корня и клиенты ----------------------------------------------
|
||
|
||
|
||
def test_child_span_needs_recording_parent(signals: Signals) -> None:
|
||
with telemetry.child_span("SELECT") as orphan:
|
||
assert orphan is None
|
||
with (
|
||
core.tracer().start_as_current_span("parent") as parent,
|
||
telemetry.child_span("SELECT") as child,
|
||
):
|
||
assert child is not None
|
||
spans = {span.name: span for span in signals.finished_spans()}
|
||
assert set(spans) == {"parent", "SELECT"}
|
||
assert spans["SELECT"].parent.span_id == (
|
||
parent.get_span_context().span_id
|
||
)
|
||
|
||
|
||
def test_child_span_error_has_class_only(signals: Signals) -> None:
|
||
with (
|
||
core.tracer().start_as_current_span("parent"),
|
||
pytest.raises(RuntimeError),
|
||
telemetry.child_span("GET"),
|
||
):
|
||
raise RuntimeError(SECRET_TEXT)
|
||
spans = {span.name: span for span in signals.finished_spans()}
|
||
child = spans["GET"]
|
||
assert child.status.status_code == StatusCode.ERROR
|
||
assert child.attributes["error.type"] == "RuntimeError"
|
||
assert child.events == ()
|
||
|
||
|
||
class _Redis:
|
||
def __init__(self, values: dict[str, str], fail: bool = False) -> None:
|
||
self.values = values
|
||
self.fail = fail
|
||
|
||
async def get(self, key: str) -> str | None:
|
||
if self.fail:
|
||
raise RedisError("down")
|
||
return self.values.get(key)
|
||
|
||
async def set(self, key: str, value: str, *, ex: int) -> None:
|
||
if self.fail:
|
||
raise RedisError("down")
|
||
self.values[key] = value
|
||
|
||
|
||
async def test_cache_spans(signals: Signals) -> None:
|
||
cache = Cache(
|
||
_Redis({"k": "v"}),
|
||
namespace="db",
|
||
timeout_seconds=1,
|
||
retry_after_seconds=60,
|
||
)
|
||
await cache.get("k") # без родителя — span нет
|
||
with core.tracer().start_as_current_span("parent"):
|
||
await cache.get("k")
|
||
await cache.get("absent-chat-key")
|
||
await cache.set("k2", "v2", ttl_seconds=10)
|
||
cache._client.fail = True
|
||
await cache.get("k")
|
||
await cache.get("k") # окно пропуска
|
||
spans = [s for s in signals.finished_spans() if s.name != "parent"]
|
||
results = [(s.name, s.attributes["messenger.cache.result"]) for s in spans]
|
||
assert results == [
|
||
("GET", "hit"),
|
||
("GET", "miss"),
|
||
("SET", "stored"),
|
||
("GET", "error"),
|
||
("GET", "skipped"),
|
||
]
|
||
assert spans[3].status.status_code == StatusCode.ERROR
|
||
for span in spans:
|
||
assert span.attributes["db.system.name"] == "valkey"
|
||
assert "absent-chat-key" not in json.dumps(dict(span.attributes))
|
||
|
||
|
||
class _S3Client:
|
||
async def put_object(self, **params: Any) -> dict[str, Any]:
|
||
return {}
|
||
|
||
async def head_object(self, **params: Any) -> dict[str, Any]:
|
||
raise ClientError(
|
||
{"Error": {"Code": "404"}, "ResponseMetadata": {}},
|
||
"HeadObject",
|
||
)
|
||
|
||
|
||
async def test_s3_spans_without_key(
|
||
signals: Signals,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
storage = ObjectStorage(settings.s3, namespace="db")
|
||
|
||
async def client() -> _S3Client:
|
||
return _S3Client()
|
||
|
||
monkeypatch.setattr(storage, "_get_client", client)
|
||
key = "db/attachments/6f1c2d3e-0000-4000-8000-000000000000/original"
|
||
with core.tracer().start_as_current_span("parent"):
|
||
await storage.put(key, io.BytesIO(b"abc"), size=3, content_type="x")
|
||
assert await storage.head(key) is None
|
||
spans = {s.name: s for s in signals.finished_spans()}
|
||
put = spans["S3.PutObject"]
|
||
head = spans["S3.HeadObject"]
|
||
assert put.attributes["rpc.method"] == "PutObject"
|
||
assert put.attributes["messenger.object.size"] == 3
|
||
assert put.attributes["messenger.s3.result"] == "ok"
|
||
assert head.attributes["messenger.s3.result"] == "missing"
|
||
assert head.status.status_code == StatusCode.UNSET
|
||
for span in (put, head):
|
||
assert "6f1c2d3e" not in json.dumps(dict(span.attributes))
|
||
|
||
|
||
# --- закрытие -------------------------------------------------------------
|
||
|
||
|
||
class _BlockingExporter(SpanExporter):
|
||
def export(self, spans: Any) -> SpanExportResult:
|
||
time.sleep(5)
|
||
return SpanExportResult.SUCCESS
|
||
|
||
def shutdown(self) -> None:
|
||
time.sleep(5)
|
||
|
||
|
||
async def test_shutdown_is_bounded() -> None:
|
||
telemetry.start_process_telemetry(
|
||
SERVICE,
|
||
exporters=telemetry.Exporters(
|
||
spans=_BlockingExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
),
|
||
)
|
||
try:
|
||
with core.tracer().start_as_current_span("queued"):
|
||
pass
|
||
started = time.monotonic()
|
||
await telemetry.shutdown(timeout=0.3)
|
||
assert time.monotonic() - started < 1
|
||
assert not telemetry.is_enabled()
|
||
finally:
|
||
telemetry.remove_log_hooks()
|
||
|
||
|
||
def test_exporters_ignore_proxy_environment(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Данные идут только в Collector из `[telemetry]`: прокси окружения
|
||
(`HTTP(S)_PROXY`) экспортёры не берут (verify S5). Отправка подменена на
|
||
уровне `requests.Session.send` — сети нет, видно, с какими прокси она бы
|
||
ушла."""
|
||
for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
|
||
monkeypatch.setenv(name, PROXY)
|
||
monkeypatch.setenv(name.lower(), PROXY)
|
||
for name in ("NO_PROXY", "no_proxy"):
|
||
monkeypatch.delenv(name, raising=False)
|
||
sent: list[tuple[str, dict[str, str]]] = []
|
||
|
||
def send(
|
||
self: requests.Session, # noqa: ARG001
|
||
request: requests.PreparedRequest,
|
||
**kwargs: Any,
|
||
) -> requests.Response:
|
||
sent.append((str(request.url), dict(kwargs.get("proxies") or {})))
|
||
response = requests.Response()
|
||
response.status_code = 200
|
||
response._content = b""
|
||
response.request = request
|
||
return response
|
||
|
||
monkeypatch.setattr(requests.Session, "send", send)
|
||
config = TelemetrySettings(otlp_endpoint=COLLECTOR)
|
||
reader = sdk.otlp_metric_reader(config)
|
||
tracer_provider = TracerProvider()
|
||
tracer_provider.add_span_processor(
|
||
SimpleSpanProcessor(
|
||
sdk.otlp_span_exporter(config, meter_provider=None),
|
||
),
|
||
)
|
||
logger_provider = LoggerProvider()
|
||
logger_provider.add_log_record_processor(
|
||
SimpleLogRecordProcessor(
|
||
sdk.otlp_log_exporter(config, meter_provider=None),
|
||
),
|
||
)
|
||
meter_provider = MeterProvider(metric_readers=[reader])
|
||
try:
|
||
tracer_provider.get_tracer("t").start_span("probe").end()
|
||
logger_provider.get_logger("t").emit(body="probe")
|
||
meter_provider.get_meter("t").create_counter("probe").add(1)
|
||
reader.force_flush()
|
||
finally:
|
||
tracer_provider.shutdown()
|
||
logger_provider.shutdown()
|
||
meter_provider.shutdown()
|
||
assert {url for url, _ in sent} == {
|
||
f"{COLLECTOR}/v1/traces",
|
||
f"{COLLECTOR}/v1/logs",
|
||
f"{COLLECTOR}/v1/metrics",
|
||
}
|
||
assert all(proxies == {} for _, proxies in sent), sent
|
||
|
||
|
||
# --- исход доставки worker ------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class _Delivery:
|
||
body: bytes = b"{}"
|
||
redelivered: bool = False
|
||
settled: list[str] = field(default_factory=list)
|
||
trace_context: dataclasses.TraceContextDTO | None = None
|
||
|
||
async def ack(self) -> None:
|
||
self.settled.append("ack")
|
||
|
||
async def drop(self) -> None:
|
||
self.settled.append("drop")
|
||
|
||
async def requeue(self) -> None:
|
||
self.settled.append("requeue")
|
||
|
||
|
||
class _Service:
|
||
def __init__(
|
||
self,
|
||
outcome: JobDeliveryOutcome | None,
|
||
result: dataclasses.JobDeliveryResultDTO | None = None,
|
||
) -> None:
|
||
self.outcome = outcome
|
||
self.result = result
|
||
|
||
async def handle(self, *, body: bytes, redelivered: bool) -> Any:
|
||
if self.result is not None:
|
||
return self.result
|
||
if self.outcome is None:
|
||
raise RuntimeError(SECRET_TEXT)
|
||
return dataclasses.JobDeliveryResultDTO(outcome=self.outcome)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("outcome", "settled"),
|
||
[
|
||
(JobDeliveryOutcome.READY, "ack"),
|
||
(JobDeliveryOutcome.FAILED, "ack"),
|
||
(JobDeliveryOutcome.RETRY, "ack"),
|
||
(JobDeliveryOutcome.SKIPPED, "ack"),
|
||
(JobDeliveryOutcome.REQUEUED, "requeue"),
|
||
(JobDeliveryOutcome.DROPPED, "drop"),
|
||
(None, "requeue"),
|
||
],
|
||
ids=lambda value: getattr(value, "value", value or "error"),
|
||
)
|
||
async def test_delivery_outcome_span_and_histogram(
|
||
signals: Signals,
|
||
outcome: JobDeliveryOutcome | None,
|
||
settled: str,
|
||
) -> None:
|
||
delivery = _Delivery()
|
||
await worker_main._handle(
|
||
lambda: _Service(outcome),
|
||
delivery,
|
||
asyncio.Semaphore(1),
|
||
asyncio.Event(),
|
||
)
|
||
expected = "error" if outcome is None else outcome.value
|
||
assert delivery.settled == [settled]
|
||
(point,) = signals.points(METRIC_JOB_DURATION)
|
||
assert dict(point.attributes) == {
|
||
"messenger.job.type": ATTACHMENT_JOB_TYPE,
|
||
"messenger.job.outcome": expected,
|
||
}
|
||
assert point.count == 1
|
||
(span,) = signals.finished_spans()
|
||
assert span.kind == SpanKind.CONSUMER
|
||
assert span.name.startswith("process ")
|
||
assert span.name.endswith(".attachment.process.v1")
|
||
assert span.parent is None
|
||
assert span.attributes["messenger.job.outcome"] == expected
|
||
assert span.attributes["messaging.system"] == "rabbitmq"
|
||
assert span.attributes["messaging.operation.type"] == "process"
|
||
if outcome is None:
|
||
assert span.status.status_code == StatusCode.ERROR
|
||
assert span.attributes["error.type"] == "RuntimeError"
|
||
assert SECRET_TEXT not in json.dumps(dict(span.attributes))
|
||
else:
|
||
assert span.status.status_code == StatusCode.UNSET
|
||
|
||
|
||
async def test_delivery_without_telemetry_has_no_signals() -> None:
|
||
delivery = _Delivery()
|
||
await worker_main._handle(
|
||
lambda: _Service(JobDeliveryOutcome.READY),
|
||
delivery,
|
||
asyncio.Semaphore(1),
|
||
asyncio.Event(),
|
||
)
|
||
assert delivery.settled == ["ack"]
|
||
assert not trace.get_current_span().is_recording()
|
||
|
||
|
||
# --- сквозная трасса (design change job-trace-propagation) ----------------
|
||
|
||
QUEUE = "testdb.attachment.process.v1"
|
||
SAVED_TRACE = "0af7651916cd43dd8448eb211c80319c"
|
||
SAVED_SPAN = "b7ad6b7169203331"
|
||
SAVED = dataclasses.TraceContextDTO(
|
||
traceparent=f"00-{SAVED_TRACE}-{SAVED_SPAN}-01",
|
||
tracestate="vendor=value",
|
||
)
|
||
UNSAMPLED = dataclasses.TraceContextDTO(
|
||
traceparent=f"00-{SAVED_TRACE}-{SAVED_SPAN}-00",
|
||
)
|
||
|
||
|
||
def _start(**kwargs: Any) -> Signals:
|
||
collected = Signals(
|
||
spans=InMemorySpanExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
)
|
||
assert telemetry.start_process_telemetry(
|
||
kwargs.pop("service", SERVICE),
|
||
exporters=telemetry.Exporters(
|
||
spans=collected.spans,
|
||
metrics=collected.metrics,
|
||
logs=collected.logs,
|
||
),
|
||
**kwargs,
|
||
)
|
||
return collected
|
||
|
||
|
||
def _publication(
|
||
trace_context: dataclasses.TraceContextDTO | None = SAVED,
|
||
*,
|
||
attempt: int = 1,
|
||
replayed: bool = False,
|
||
retry_reason: str | None = None,
|
||
) -> dataclasses.JobPublicationDTO:
|
||
return dataclasses.JobPublicationDTO(
|
||
job_id=uuid4(),
|
||
body=b"{}",
|
||
attempt=attempt,
|
||
replayed=replayed,
|
||
retry_reason=retry_reason,
|
||
trace_context=trace_context,
|
||
)
|
||
|
||
|
||
def _hex(value: int, width: int) -> str:
|
||
return format(value, f"0{width}x")
|
||
|
||
|
||
async def test_upload_context_is_server_span() -> None:
|
||
"""Контекст загрузки — контекст server span запроса; вне span и без
|
||
телеметрии — `None` (решение 4)."""
|
||
seen: list[dataclasses.TraceContextDTO | None] = []
|
||
app = FastAPI()
|
||
|
||
@app.post("/api/v1/upload")
|
||
async def upload() -> dict[str, str]:
|
||
seen.append(telemetry.current_trace_context())
|
||
return {}
|
||
|
||
app.add_middleware(RequestContextMiddleware)
|
||
|
||
async def call() -> None:
|
||
transport = httpx.ASGITransport(app=app)
|
||
async with httpx.AsyncClient(
|
||
transport=transport,
|
||
base_url="http://test",
|
||
) as client:
|
||
await client.post("/api/v1/upload")
|
||
|
||
await call()
|
||
assert seen == [None]
|
||
signals = _start()
|
||
try:
|
||
await call()
|
||
assert telemetry.current_trace_context() is None
|
||
(span,) = signals.finished_spans()
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
context = seen[-1]
|
||
assert context is not None
|
||
# флаги W3C — от SDK: выбрано и, у SDK 1.45, «random trace id»
|
||
flags = int(span.context.trace_flags)
|
||
assert flags & 0x01
|
||
assert context.traceparent == (
|
||
f"00-{_hex(span.context.trace_id, 32)}-"
|
||
f"{_hex(span.context.span_id, 16)}-{_hex(flags, 2)}"
|
||
)
|
||
assert context.tracestate is None
|
||
|
||
|
||
def test_producer_span_under_saved_context(signals: Signals) -> None:
|
||
publication = _publication(attempt=2, retry_reason="storage_unavailable")
|
||
with telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=publication,
|
||
) as headers:
|
||
pass
|
||
(span,) = signals.finished_spans()
|
||
assert span.kind == SpanKind.PRODUCER
|
||
assert span.name == f"publish {QUEUE}"
|
||
assert _hex(span.context.trace_id, 32) == SAVED_TRACE
|
||
assert span.parent is not None
|
||
assert _hex(span.parent.span_id, 16) == SAVED_SPAN
|
||
assert span.parent.is_remote
|
||
assert dict(span.attributes) == {
|
||
"messaging.system": "rabbitmq",
|
||
"messaging.destination.name": QUEUE,
|
||
"messaging.operation.type": "send",
|
||
"messaging.operation.name": "publish",
|
||
"messenger.job.type": ATTACHMENT_JOB_TYPE,
|
||
"messenger.job.attempt": 2,
|
||
"messenger.job.replayed": False,
|
||
"messenger.job.retry_reason": "storage_unavailable",
|
||
}
|
||
# заголовки — контекст span producer, tracestate прежний
|
||
assert headers == {
|
||
"traceparent": (
|
||
f"00-{SAVED_TRACE}-{_hex(span.context.span_id, 16)}-01"
|
||
),
|
||
"tracestate": "vendor=value",
|
||
}
|
||
assert str(publication.job_id) not in json.dumps(dict(span.attributes))
|
||
|
||
|
||
@pytest.mark.parametrize("error", [RuntimeError, asyncio.CancelledError])
|
||
def test_producer_span_error(signals: Signals, error: type) -> None:
|
||
with (
|
||
pytest.raises(error),
|
||
telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=_publication(replayed=True),
|
||
),
|
||
):
|
||
raise error(SECRET_TEXT)
|
||
(span,) = signals.finished_spans()
|
||
assert span.status.status_code == StatusCode.ERROR
|
||
assert span.attributes["error.type"] == error.__name__
|
||
assert span.attributes["messenger.job.replayed"] is True
|
||
assert SECRET_TEXT not in json.dumps(dict(span.attributes))
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"saved",
|
||
[
|
||
None,
|
||
dataclasses.TraceContextDTO(traceparent="00-zz-bad-01"),
|
||
dataclasses.TraceContextDTO(
|
||
traceparent=f"00-{'0' * 32}-{'0' * 16}-01",
|
||
),
|
||
],
|
||
ids=["none", "garbage", "invalid"],
|
||
)
|
||
def test_producer_without_context_has_no_span(
|
||
signals: Signals,
|
||
saved: dataclasses.TraceContextDTO | None,
|
||
) -> None:
|
||
"""Проходы relay трасс не начинают: без контекста — ни span, ни
|
||
заголовков."""
|
||
with telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=_publication(saved),
|
||
) as headers:
|
||
assert headers == {}
|
||
assert signals.finished_spans() == []
|
||
|
||
|
||
def test_producer_without_telemetry_passes_context() -> None:
|
||
"""Телеметрия relay выключена — сохранённый контекст как есть."""
|
||
with telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=_publication(),
|
||
) as headers:
|
||
assert headers == {
|
||
"traceparent": SAVED.traceparent,
|
||
"tracestate": "vendor=value",
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class _Exchange:
|
||
messages: list[Any] = field(default_factory=list)
|
||
|
||
async def publish(
|
||
self,
|
||
message: Any,
|
||
*,
|
||
routing_key: str,
|
||
mandatory: bool,
|
||
) -> None:
|
||
self.messages.append((message, routing_key, mandatory))
|
||
|
||
|
||
@dataclass
|
||
class _Channel:
|
||
default_exchange: _Exchange = field(default_factory=_Exchange)
|
||
|
||
|
||
async def test_publisher_puts_context_in_headers(signals: Signals) -> None:
|
||
"""`JobPublisher._publish_one`: span на каждое сообщение, его контекст —
|
||
заголовки; без контекста — сообщение без заголовков, тело прежнее."""
|
||
publisher = JobPublisher(settings.broker, namespace="testdb")
|
||
channel = _Channel()
|
||
traced = _publication()
|
||
plain = _publication(None)
|
||
await publisher._publish_one(channel, traced)
|
||
await publisher._publish_one(channel, plain)
|
||
(span,) = signals.finished_spans()
|
||
(first, key, mandatory), (second, _, _) = channel.default_exchange.messages
|
||
assert key == QUEUE
|
||
assert mandatory is True
|
||
assert first.body == traced.body
|
||
assert first.headers["traceparent"] == (
|
||
f"00-{SAVED_TRACE}-{_hex(span.context.span_id, 16)}-01"
|
||
)
|
||
assert first.headers["tracestate"] == "vendor=value"
|
||
assert first.message_id == str(traced.job_id)
|
||
assert not second.headers
|
||
assert second.body == plain.body
|
||
|
||
|
||
def test_consumer_span_child_with_link(signals: Signals) -> None:
|
||
with (
|
||
telemetry.consumer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
parent=SAVED,
|
||
redelivered=True,
|
||
),
|
||
telemetry.child_span("SELECT"),
|
||
):
|
||
pass
|
||
spans = {span.name: span for span in signals.finished_spans()}
|
||
consumer = spans[f"process {QUEUE}"]
|
||
assert _hex(consumer.context.trace_id, 32) == SAVED_TRACE
|
||
assert _hex(consumer.parent.span_id, 16) == SAVED_SPAN
|
||
(link,) = consumer.links
|
||
assert _hex(link.context.span_id, 16) == SAVED_SPAN
|
||
assert _hex(link.context.trace_id, 32) == SAVED_TRACE
|
||
assert consumer.attributes["messaging.operation.name"] == "process"
|
||
assert consumer.attributes["messenger.job.redelivered"] is True
|
||
assert spans["SELECT"].parent.span_id == consumer.context.span_id
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"parent",
|
||
[None, dataclasses.TraceContextDTO(traceparent="not-w3c")],
|
||
ids=["none", "garbage"],
|
||
)
|
||
def test_consumer_span_root_without_context(
|
||
signals: Signals,
|
||
parent: dataclasses.TraceContextDTO | None,
|
||
) -> None:
|
||
with telemetry.consumer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
parent=parent,
|
||
):
|
||
pass
|
||
(span,) = signals.finished_spans()
|
||
assert span.parent is None
|
||
assert span.links == ()
|
||
|
||
|
||
def test_unsampled_context_is_not_recorded(signals: Signals) -> None:
|
||
"""Флаг «не выбрано» соблюдается: ни producer, ни consumer, ни
|
||
дочерних span; исход и метрика доставки те же (решение 5)."""
|
||
publication = _publication(UNSAMPLED)
|
||
with telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=publication,
|
||
) as headers:
|
||
pass
|
||
assert headers["traceparent"].startswith(f"00-{SAVED_TRACE}-")
|
||
assert headers["traceparent"].endswith("-00")
|
||
parent = core.to_trace_context(headers)
|
||
with telemetry.consumer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
parent=parent,
|
||
) as observation:
|
||
with telemetry.child_span("SELECT") as child:
|
||
assert child is None
|
||
observation.finish(
|
||
result=dataclasses.JobDeliveryResultDTO(
|
||
outcome=JobDeliveryOutcome.READY,
|
||
attempt=1,
|
||
time_to_ready_seconds=1.5,
|
||
),
|
||
duration_seconds=0.2,
|
||
)
|
||
points = signals.points(METRIC_JOB_DURATION)
|
||
assert signals.finished_spans() == []
|
||
(point,) = points
|
||
assert point.attributes["messenger.job.outcome"] == "ready"
|
||
|
||
|
||
async def test_delivery_result_attributes_and_time_to_ready(
|
||
signals: Signals,
|
||
) -> None:
|
||
result = dataclasses.JobDeliveryResultDTO(
|
||
outcome=JobDeliveryOutcome.READY,
|
||
attempt=2,
|
||
time_to_ready_seconds=12.5,
|
||
)
|
||
delivery = _Delivery(trace_context=SAVED)
|
||
await worker_main._handle(
|
||
lambda: _Service(None, result),
|
||
delivery,
|
||
asyncio.Semaphore(1),
|
||
asyncio.Event(),
|
||
)
|
||
(point,) = signals.points("messenger.job.time_to_ready")
|
||
assert dict(point.attributes) == {
|
||
"messenger.job.type": ATTACHMENT_JOB_TYPE,
|
||
}
|
||
assert point.count == 1
|
||
assert point.sum == 12.5
|
||
assert point.explicit_bounds[0] == 0.25
|
||
assert point.explicit_bounds[-1] == 600
|
||
(span,) = signals.finished_spans()
|
||
assert _hex(span.context.trace_id, 32) == SAVED_TRACE
|
||
assert span.attributes["messenger.job.attempt"] == 2
|
||
assert "messenger.job.reason" not in span.attributes
|
||
|
||
|
||
async def test_failed_result_names_reason_without_time(
|
||
signals: Signals,
|
||
) -> None:
|
||
result = dataclasses.JobDeliveryResultDTO(
|
||
outcome=JobDeliveryOutcome.FAILED,
|
||
attempt=1,
|
||
reason=AttachmentJobReason.UNDECODABLE,
|
||
error_code=AttachmentErrorCode.INVALID_IMAGE,
|
||
)
|
||
await worker_main._handle(
|
||
lambda: _Service(None, result),
|
||
_Delivery(),
|
||
asyncio.Semaphore(1),
|
||
asyncio.Event(),
|
||
)
|
||
assert signals.points("messenger.job.time_to_ready") == []
|
||
(span,) = signals.finished_spans()
|
||
assert span.attributes["messenger.job.reason"] == "undecodable"
|
||
assert span.attributes["messenger.job.error_code"] == "invalid_image"
|
||
|
||
|
||
# --- метрики уровня 4 -------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class _Pool:
|
||
used: int = 2
|
||
idle: int = 1
|
||
waiting: int = 3
|
||
timed_out: int = 4
|
||
|
||
def checkedout(self) -> int:
|
||
return self.used
|
||
|
||
def checkedin(self) -> int:
|
||
return self.idle
|
||
|
||
def size(self) -> int:
|
||
return 10
|
||
|
||
def pending(self) -> int:
|
||
return self.waiting
|
||
|
||
def timeouts(self) -> int:
|
||
return self.timed_out
|
||
|
||
|
||
@dataclass
|
||
class _SyncEngine:
|
||
pool: Any
|
||
url: Any
|
||
|
||
|
||
@dataclass
|
||
class _Engine:
|
||
sync_engine: _SyncEngine
|
||
|
||
|
||
def test_pool_metrics(signals: Signals) -> None:
|
||
from sqlalchemy.engine import make_url
|
||
|
||
engine = _Engine(
|
||
_SyncEngine(
|
||
pool=_Pool(),
|
||
url=make_url("postgresql+asyncpg://user@db:5432/app"),
|
||
),
|
||
)
|
||
metrics.observe_pool(engine, max_overflow=2)
|
||
name = {"db.client.connection.pool.name": "db:5432/app"}
|
||
counts = {
|
||
point.attributes["db.client.connection.state"]: point.value
|
||
for point in signals.points("db.client.connection.count")
|
||
}
|
||
assert counts == {"used": 2, "idle": 1}
|
||
(limit,) = signals.points("db.client.connection.max")
|
||
assert limit.value == 12
|
||
assert dict(limit.attributes) == name
|
||
(pending,) = signals.points("db.client.connection.pending_requests")
|
||
assert pending.value == 3
|
||
(timeouts,) = signals.points("db.client.connection.timeouts")
|
||
assert timeouts.value == 4
|
||
# dispose(): новый пул читается на следующем сборе
|
||
engine.sync_engine.pool = _Pool(used=0, idle=0, waiting=0, timed_out=0)
|
||
(pending,) = signals.points("db.client.connection.pending_requests")
|
||
assert pending.value == 0
|
||
|
||
|
||
async def test_worker_saturation(signals: Signals) -> None:
|
||
"""Конкуренция 2: две доставки заняли места, третья ждёт; после всех —
|
||
ноль занятых и ожидающих."""
|
||
load = telemetry.WorkerLoad()
|
||
telemetry.observe_worker(capacity=2, load=load)
|
||
release = asyncio.Event()
|
||
|
||
class _Blocking:
|
||
async def handle(self, *, body: bytes, redelivered: bool) -> Any:
|
||
await release.wait()
|
||
return dataclasses.JobDeliveryResultDTO(
|
||
outcome=JobDeliveryOutcome.READY,
|
||
)
|
||
|
||
slots = asyncio.Semaphore(2)
|
||
stop = asyncio.Event()
|
||
tasks = [
|
||
asyncio.create_task(
|
||
worker_main._handle(
|
||
_Blocking,
|
||
_Delivery(),
|
||
slots,
|
||
stop,
|
||
load=load,
|
||
),
|
||
)
|
||
for _ in range(3)
|
||
]
|
||
for _ in range(50):
|
||
await asyncio.sleep(0)
|
||
values = {
|
||
name: [point.value for point in signals.points(name)]
|
||
for name in (
|
||
"messenger.worker.busy",
|
||
"messenger.worker.capacity",
|
||
"messenger.worker.waiting",
|
||
)
|
||
}
|
||
assert values == {
|
||
"messenger.worker.busy": [2],
|
||
"messenger.worker.capacity": [2],
|
||
"messenger.worker.waiting": [1],
|
||
}
|
||
release.set()
|
||
await asyncio.gather(*tasks)
|
||
assert (load.busy, load.waiting) == (0, 0)
|
||
(busy,) = signals.points("messenger.worker.busy")
|
||
assert busy.value == 0
|
||
|
||
|
||
def _snapshot(taken_ago: float) -> dataclasses.QueueSnapshotDTO:
|
||
from datetime import timedelta
|
||
|
||
from src.utils.utc import utcnow
|
||
|
||
taken_at = utcnow() - timedelta(seconds=taken_ago)
|
||
return dataclasses.QueueSnapshotDTO(
|
||
taken_at=taken_at,
|
||
scheduled=1,
|
||
outbox=2,
|
||
published=3,
|
||
processing=4,
|
||
oldest_created_at=taken_at - timedelta(seconds=42),
|
||
)
|
||
|
||
|
||
def test_queue_gauges_fresh_and_stale(signals: Signals) -> None:
|
||
latest: list[dataclasses.QueueSnapshotDTO | None] = [None]
|
||
telemetry.observe_queue(queue=QUEUE, latest=lambda: latest[0])
|
||
assert signals.points("messenger.queue.depth") == []
|
||
latest[0] = _snapshot(31)
|
||
# снимок старше 30 с не отдаётся
|
||
assert signals.points("messenger.queue.depth") == []
|
||
assert signals.points("messenger.queue.oldest_age") == []
|
||
latest[0] = _snapshot(1)
|
||
depth = {
|
||
point.attributes["messenger.queue.stage"]: point.value
|
||
for point in signals.points("messenger.queue.depth")
|
||
}
|
||
assert depth == {
|
||
"scheduled": 1,
|
||
"outbox": 2,
|
||
"published": 3,
|
||
"processing": 4,
|
||
}
|
||
(age,) = signals.points("messenger.queue.oldest_age")
|
||
assert age.value == 42
|
||
assert dict(age.attributes) == {"messaging.destination.name": QUEUE}
|
||
|
||
|
||
def test_queue_age_without_unfinished_is_zero() -> None:
|
||
snapshot = dataclasses.QueueSnapshotDTO(
|
||
taken_at=_snapshot(0).taken_at,
|
||
scheduled=0,
|
||
outbox=0,
|
||
published=0,
|
||
processing=0,
|
||
)
|
||
depth, (age,) = metrics.queue_observations(snapshot, queue=QUEUE)
|
||
assert [item.value for item in depth] == [0, 0, 0, 0]
|
||
assert age.value == 0
|
||
|
||
|
||
def _cgroup(root: Any, *, memory_max: str = "268435456") -> None:
|
||
(root / "cpu.stat").write_text(
|
||
"usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n",
|
||
)
|
||
(root / "memory.current").write_text("1048576\n")
|
||
(root / "cpu.max").write_text("25000 100000\n")
|
||
(root / "memory.max").write_text(f"{memory_max}\n")
|
||
|
||
|
||
def test_container_metrics_from_cgroup(tmp_path: Any) -> None:
|
||
_cgroup(tmp_path)
|
||
signals = _start(cgroup_root=tmp_path)
|
||
try:
|
||
(cpu,) = signals.points("container.cpu.time")
|
||
(memory,) = signals.points("container.memory.usage")
|
||
(cpu_limit,) = signals.points("messenger.container.cpu.limit")
|
||
(memory_limit,) = signals.points("messenger.container.memory.limit")
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert cpu.value == 2.5
|
||
assert memory.value == 1048576
|
||
assert cpu_limit.value == 0.25
|
||
assert memory_limit.value == 268435456
|
||
|
||
|
||
def test_container_without_limit(tmp_path: Any) -> None:
|
||
from src.infra.telemetry import container
|
||
|
||
_cgroup(tmp_path, memory_max="max")
|
||
(tmp_path / "cpu.max").write_text("max 100000\n")
|
||
assert container.memory_limit(tmp_path) is None
|
||
assert container.cpu_limit(tmp_path) is None
|
||
assert container.available(tmp_path)
|
||
|
||
|
||
def test_container_unavailable_one_record(
|
||
tmp_path: Any,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
signals = _start(cgroup_root=tmp_path / "missing")
|
||
try:
|
||
assert signals.points("container.cpu.time") == []
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
records = [
|
||
record
|
||
for record in _records(stdout)
|
||
if record["event"] == "Container metrics are unavailable"
|
||
]
|
||
assert len(records) == 1
|
||
assert records[0]["level"] == "warning"
|
||
# запись до хуков журнала: в хранилище логов не уходит
|
||
assert signals.finished_logs() == []
|
||
|
||
|
||
def test_relay_resource() -> None:
|
||
signals = _start(service="messenger-relay")
|
||
try:
|
||
with telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=_publication(),
|
||
):
|
||
pass
|
||
(span,) = signals.finished_spans()
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert span.resource.attributes["service.name"] == "messenger-relay"
|
||
assert span.resource.attributes["service.instance.id"] == "test-instance"
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("headers", "expected"),
|
||
[
|
||
(None, None),
|
||
({}, None),
|
||
({"traceparent": SAVED.traceparent}, (SAVED.traceparent, None)),
|
||
(
|
||
{
|
||
"traceparent": SAVED.traceparent.encode(),
|
||
"tracestate": b"vendor=value",
|
||
},
|
||
(SAVED.traceparent, "vendor=value"),
|
||
),
|
||
({"traceparent": "01-bad"}, None),
|
||
({"traceparent": SAVED.traceparent.upper()}, None),
|
||
({"traceparent": "\xff".encode("latin-1")}, None),
|
||
({"traceparent": 42}, None),
|
||
({"traceparent": SAVED.traceparent, "tracestate": "x" * 513}, None),
|
||
# пустой tracestate по W3C равен отсутствующему (verify S4)
|
||
(
|
||
{"traceparent": SAVED.traceparent, "tracestate": ""},
|
||
(SAVED.traceparent, None),
|
||
),
|
||
(
|
||
{"traceparent": SAVED.traceparent.encode(), "tracestate": b""},
|
||
(SAVED.traceparent, None),
|
||
),
|
||
],
|
||
ids=[
|
||
"no-headers",
|
||
"empty",
|
||
"str",
|
||
"bytes",
|
||
"garbage",
|
||
"upper",
|
||
"not-ascii",
|
||
"number",
|
||
"long-tracestate",
|
||
"empty-tracestate",
|
||
"empty-tracestate-bytes",
|
||
],
|
||
)
|
||
def test_delivery_headers(
|
||
headers: Any,
|
||
expected: tuple[str, str | None] | None,
|
||
) -> None:
|
||
from src.infra.broker.client import trace_context_from_headers
|
||
|
||
context = trace_context_from_headers(headers)
|
||
if expected is None:
|
||
assert context is None
|
||
else:
|
||
assert context == dataclasses.TraceContextDTO(*expected)
|
||
|
||
|
||
def _loaded_modules(imports: str) -> set[str]:
|
||
"""Модули SDK, экспортёра, protobuf и `requests` после импорта в
|
||
отдельном интерпретаторе с выключенной телеметрией."""
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
code = (
|
||
"import json, sys\n"
|
||
f"import {imports}\n"
|
||
"prefixes = ('opentelemetry.sdk', 'opentelemetry.exporter',\n"
|
||
" 'google.protobuf', 'requests')\n"
|
||
"print(json.dumps(sorted(m for m in sys.modules\n"
|
||
" if m.startswith(prefixes))))\n"
|
||
)
|
||
root = Path(__file__).resolve().parents[2]
|
||
env = {**os.environ, "TELEMETRY__ENABLED": "false"}
|
||
result = subprocess.run( # noqa: S603
|
||
[sys.executable, "-c", code],
|
||
cwd=root,
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
check=True,
|
||
timeout=60,
|
||
)
|
||
return set(json.loads(result.stdout.strip().splitlines()[-1]))
|
||
|
||
|
||
def test_sdk_modules_not_loaded_without_telemetry() -> None:
|
||
"""Выключенная телеметрия — ни провайдеров SDK, ни экспортёра, ни
|
||
protobuf, ни HTTP-клиента экспортёра в памяти точек входа (решение 7).
|
||
|
||
Исключение — модули метрик SDK, которые redis-py 8.1 сам загружает при
|
||
импорте (`redis.observability.providers`: проба «установлен ли SDK»):
|
||
точки входа не грузят ничего сверх `import redis.asyncio` (design,
|
||
«Уточнения apply»)."""
|
||
loaded = _loaded_modules(
|
||
"src.main, src.worker.main, src.relay.main, src.cleaner.main",
|
||
)
|
||
forbidden = (
|
||
"opentelemetry.exporter",
|
||
"opentelemetry.sdk.trace",
|
||
"opentelemetry.sdk._logs",
|
||
"google.protobuf",
|
||
"requests",
|
||
)
|
||
assert not {name for name in loaded if name.startswith(forbidden)}
|
||
assert loaded <= _loaded_modules("redis.asyncio")
|
||
|
||
|
||
# --- внутренние метрики SDK (design change slo-alerts, решения 8–9) -------
|
||
|
||
INTERNAL_METRICS_ENV = "OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED"
|
||
SPAN_PROCESSED = "otel.sdk.processor.span.processed"
|
||
LOG_PROCESSED = "otel.sdk.processor.log.processed"
|
||
SPAN_EXPORTED = "otel.sdk.exporter.span.exported"
|
||
LOG_EXPORTED = "otel.sdk.exporter.log.exported"
|
||
SDK_HISTOGRAMS = (
|
||
"otel.sdk.exporter.operation.duration",
|
||
"otel.sdk.metric_reader.collection.duration",
|
||
)
|
||
|
||
|
||
class _GateSpanExporter(SpanExporter):
|
||
"""Отправка ждёт, пока тест не откроет ворота: очередь процессора
|
||
заполняется, как при медленном Collector."""
|
||
|
||
def __init__(self) -> None:
|
||
self.gate = threading.Event()
|
||
|
||
def export(self, spans: Any) -> SpanExportResult:
|
||
self.gate.wait(5)
|
||
return SpanExportResult.SUCCESS
|
||
|
||
def shutdown(self) -> None:
|
||
self.gate.set()
|
||
|
||
|
||
class _GateLogExporter(LogRecordExporter):
|
||
def __init__(self) -> None:
|
||
self.gate = threading.Event()
|
||
|
||
def export(self, batch: Any) -> LogRecordExportResult:
|
||
self.gate.wait(5)
|
||
return LogRecordExportResult.SUCCESS
|
||
|
||
def shutdown(self) -> None:
|
||
self.gate.set()
|
||
|
||
def force_flush(self, timeout_millis: int = 30000) -> bool: # noqa: ARG002
|
||
return True
|
||
|
||
|
||
def _metric_names(reader: InMemoryMetricReader) -> set[str]:
|
||
data = reader.get_metrics_data()
|
||
if data is None:
|
||
return set()
|
||
return {
|
||
metric.name
|
||
for resource in data.resource_metrics
|
||
for scope in resource.scope_metrics
|
||
for metric in scope.metrics
|
||
}
|
||
|
||
|
||
def _sum_where(signals: Signals, name: str, **attributes: Any) -> int:
|
||
return sum(
|
||
point.value
|
||
for point in signals.points(name)
|
||
if all(
|
||
point.attributes.get(key) == value
|
||
for key, value in attributes.items()
|
||
)
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def small_queue(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Очередь процессоров 2, пакет 1, пауза 10 мс: третий span ждёт места."""
|
||
monkeypatch.setattr(settings.telemetry, "max_queue_size", 2)
|
||
monkeypatch.setattr(settings.telemetry, "max_export_batch_size", 1)
|
||
monkeypatch.setattr(settings.telemetry, "schedule_delay_seconds", 0.01)
|
||
|
||
|
||
@pytest.mark.usefixtures("small_queue")
|
||
def test_sdk_counts_queue_full() -> None:
|
||
"""Полная очередь процессора — счёт `error.type` = `queue_full` у span и
|
||
записей журнала, ёмкость очереди — как `[telemetry] max_queue_size`."""
|
||
spans, logs = _GateSpanExporter(), _GateLogExporter()
|
||
collected = Signals(
|
||
spans=InMemorySpanExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
)
|
||
assert telemetry.start_process_telemetry(
|
||
SERVICE,
|
||
exporters=telemetry.Exporters(
|
||
spans=spans,
|
||
metrics=collected.metrics,
|
||
logs=logs,
|
||
),
|
||
)
|
||
try:
|
||
for number in range(10):
|
||
with core.tracer().start_as_current_span(f"span-{number}"):
|
||
pass
|
||
otel_logger = core.otel_logger()
|
||
for number in range(10):
|
||
otel_logger.emit(body=f"record-{number}")
|
||
dropped_spans = _sum_where(
|
||
collected,
|
||
SPAN_PROCESSED,
|
||
**{"error.type": "queue_full"},
|
||
)
|
||
dropped_logs = _sum_where(
|
||
collected,
|
||
LOG_PROCESSED,
|
||
**{"error.type": "queue_full"},
|
||
)
|
||
capacity = {
|
||
point.value
|
||
for name in (
|
||
"otel.sdk.processor.span.queue.capacity",
|
||
"otel.sdk.processor.log.queue.capacity",
|
||
)
|
||
for point in collected.points(name)
|
||
}
|
||
finally:
|
||
spans.gate.set()
|
||
logs.gate.set()
|
||
collected.finish()
|
||
telemetry.remove_log_hooks()
|
||
# в полёте — не больше одного пакета, в очереди — два: остальные отброшены
|
||
assert dropped_spans >= 10 - 2 - 1
|
||
assert dropped_logs >= 10 - 2 - 1
|
||
assert capacity == {2}
|
||
|
||
|
||
def test_sdk_counts_failed_export(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Экспортёр OTLP span и журнала, отправка которого падает
|
||
`ConnectionError` (подставная `requests.Session.send`, сети нет), —
|
||
`otel.sdk.exporter.*.exported` с `error.type`; гистограмм SDK нет."""
|
||
|
||
def send(
|
||
self: requests.Session, # noqa: ARG001
|
||
request: requests.PreparedRequest, # noqa: ARG001
|
||
**kwargs: Any, # noqa: ARG001
|
||
) -> requests.Response:
|
||
raise requests.exceptions.ConnectionError("collector is down")
|
||
|
||
monkeypatch.setattr(requests.Session, "send", send)
|
||
monkeypatch.setattr(settings.telemetry, "otlp_endpoint", COLLECTOR)
|
||
monkeypatch.setattr(settings.telemetry, "export_timeout_seconds", 0.2)
|
||
monkeypatch.setattr(settings.telemetry, "schedule_delay_seconds", 0.01)
|
||
collected = Signals(
|
||
spans=InMemorySpanExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
)
|
||
assert telemetry.start_process_telemetry(
|
||
SERVICE,
|
||
exporters=telemetry.Exporters(
|
||
spans=None,
|
||
metrics=collected.metrics,
|
||
logs=None,
|
||
),
|
||
)
|
||
try:
|
||
with core.tracer().start_as_current_span("probe"):
|
||
pass
|
||
core.otel_logger().emit(body="probe")
|
||
deadline = time.monotonic() + 5
|
||
failed: dict[str, int] = {}
|
||
while time.monotonic() < deadline:
|
||
failed = {
|
||
name: _sum_where(
|
||
collected,
|
||
name,
|
||
**{"error.type": "ConnectionError"},
|
||
)
|
||
for name in (SPAN_EXPORTED, LOG_EXPORTED)
|
||
}
|
||
if all(failed.values()):
|
||
break
|
||
time.sleep(0.05)
|
||
names = _metric_names(collected.metrics)
|
||
exported = collected.points(SPAN_EXPORTED)
|
||
finally:
|
||
collected.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert failed == {SPAN_EXPORTED: 1, LOG_EXPORTED: 1}
|
||
assert not names & set(SDK_HISTOGRAMS)
|
||
# метки — компонент и адрес Collector, без id
|
||
(point,) = exported
|
||
assert set(point.attributes) == {
|
||
"otel.component.type",
|
||
"otel.component.name",
|
||
"server.address",
|
||
"server.port",
|
||
"error.type",
|
||
}
|
||
assert point.attributes["server.address"] == "otel-collector"
|
||
|
||
|
||
def test_sdk_internal_metrics_environment(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
"""Внешняя переменная внутренних метрик удаляется с `warning` и ни на что
|
||
не влияет: метрики SDK есть, после `start()` переменной в окружении нет."""
|
||
import os
|
||
|
||
monkeypatch.setenv(INTERNAL_METRICS_ENV, "false")
|
||
collected = Signals(
|
||
spans=InMemorySpanExporter(),
|
||
metrics=InMemoryMetricReader(),
|
||
logs=InMemoryLogRecordExporter(),
|
||
)
|
||
try:
|
||
assert telemetry.start_process_telemetry(
|
||
SERVICE,
|
||
exporters=telemetry.Exporters(
|
||
spans=collected.spans,
|
||
metrics=collected.metrics,
|
||
logs=collected.logs,
|
||
),
|
||
)
|
||
assert INTERNAL_METRICS_ENV not in os.environ
|
||
with core.tracer().start_as_current_span("probe"):
|
||
pass
|
||
names = _metric_names(collected.metrics)
|
||
finally:
|
||
collected.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert "otel.sdk.processor.span.queue.capacity" in names
|
||
assert "otel.sdk.processor.log.queue.capacity" in names
|
||
assert not names & set(SDK_HISTOGRAMS)
|
||
(warning,) = [
|
||
record
|
||
for record in _records(stdout)
|
||
if record["event"] == "Ignored OpenTelemetry environment variables"
|
||
]
|
||
assert warning["names"] == [INTERNAL_METRICS_ENV]
|
||
|
||
|
||
def test_sdk_internal_metrics_off_without_telemetry(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Выключенная телеметрия — ничего: ни провайдеров, ни переменной."""
|
||
import os
|
||
|
||
monkeypatch.setattr(settings.telemetry, "enabled", False)
|
||
monkeypatch.delenv(INTERNAL_METRICS_ENV, raising=False)
|
||
assert telemetry.start_process_telemetry(SERVICE) is False
|
||
assert not telemetry.is_enabled()
|
||
assert INTERNAL_METRICS_ENV not in os.environ
|
||
|
||
|
||
# --- выбор трасс в голове и наборы границ ---------------------------------
|
||
# (design change track1-telemetry-cost, решения 3–4)
|
||
|
||
SPAN_STARTED = "otel.sdk.span.started"
|
||
SPAN_LIVE = "otel.sdk.span.live"
|
||
SAMPLING_RESULT = "otel.span.sampling_result"
|
||
PARENT_ORIGIN = "otel.span.parent.origin"
|
||
# доля 0,5: корень выбран, если младшие 64 бита trace_id меньше 2**63
|
||
TRACE_KEPT = (1 << 63) - 1
|
||
TRACE_DROPPED = 1 << 63
|
||
|
||
|
||
def _sampled_app() -> FastAPI:
|
||
"""Ручка с дочерним span и записью журнала."""
|
||
app = FastAPI()
|
||
|
||
@app.get("/api/v1/probe")
|
||
async def probe() -> dict[str, str]:
|
||
with telemetry.child_span("SELECT") as child:
|
||
get_logger().info("Inside probe", child=child is not None)
|
||
return {}
|
||
|
||
app.add_middleware(RequestContextMiddleware)
|
||
return app
|
||
|
||
|
||
async def _probe(times: int = 1) -> list[str]:
|
||
transport = httpx.ASGITransport(app=_sampled_app())
|
||
async with httpx.AsyncClient(
|
||
transport=transport,
|
||
base_url="http://test",
|
||
) as client:
|
||
return [
|
||
(await client.get("/api/v1/probe")).headers["x-request-id"]
|
||
for _ in range(times)
|
||
]
|
||
|
||
|
||
def _started(signals: Signals) -> dict[tuple[str, str], int]:
|
||
return {
|
||
(
|
||
point.attributes[PARENT_ORIGIN],
|
||
point.attributes[SAMPLING_RESULT],
|
||
): point.value
|
||
for point in signals.points(SPAN_STARTED)
|
||
}
|
||
|
||
|
||
def _ratio(monkeypatch: pytest.MonkeyPatch, ratio: float) -> None:
|
||
monkeypatch.setattr(settings.telemetry, "trace_sample_ratio", ratio)
|
||
|
||
|
||
async def test_ratio_one_traces_every_request(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
_ratio(monkeypatch, 1)
|
||
signals = _start()
|
||
try:
|
||
request_ids = await _probe(5)
|
||
started = _started(signals)
|
||
spans = signals.finished_spans()
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
servers = [span for span in spans if span.kind == SpanKind.SERVER]
|
||
assert sorted(
|
||
span.attributes["messenger.request_id"] for span in servers
|
||
) == sorted(request_ids)
|
||
assert len(spans) == 10
|
||
# отброшенных выборкой нет; живые span не отдаются
|
||
assert started == {
|
||
("none", "RECORD_AND_SAMPLE"): 5,
|
||
("local", "RECORD_AND_SAMPLE"): 5,
|
||
}
|
||
assert signals.points(SPAN_LIVE) == []
|
||
|
||
|
||
async def test_ratio_zero_keeps_metrics_and_logs(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
stdout: io.StringIO,
|
||
) -> None:
|
||
"""Невыбранный запрос: ни server span, ни дочерних; histogram и запись
|
||
журнала — те же, у записи в хранилище логов `trace_id` с флагом «не
|
||
выбрано», в stdout полей трассы нет."""
|
||
_ratio(monkeypatch, 0)
|
||
signals = _start()
|
||
try:
|
||
await _probe(3)
|
||
points = signals.points(METRIC_HTTP_SERVER_DURATION)
|
||
started = _started(signals)
|
||
spans = signals.finished_spans()
|
||
exported = [
|
||
record
|
||
for record in signals.finished_logs()
|
||
if record.body == "Inside probe"
|
||
]
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert spans == []
|
||
(point,) = points
|
||
assert point.count == 3
|
||
assert started == {("none", "DROP"): 3}
|
||
assert len(exported) == 3
|
||
for record in exported:
|
||
assert record.trace_id != 0
|
||
assert not int(record.trace_flags) & 0x01
|
||
assert record.attributes["child"] is False
|
||
records = [r for r in _records(stdout) if r.get("event") == "Inside probe"]
|
||
assert len(records) == 3
|
||
for record in records:
|
||
assert "trace_id" not in record
|
||
assert "span_id" not in record
|
||
|
||
|
||
async def test_ratio_half_decides_by_trace_id(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Решение — по младшим 64 битам `trace_id` (`TraceIdRatioBased`)."""
|
||
from opentelemetry.sdk.trace import RandomIdGenerator
|
||
|
||
ids = iter([TRACE_KEPT, TRACE_DROPPED])
|
||
monkeypatch.setattr(
|
||
RandomIdGenerator,
|
||
"generate_trace_id",
|
||
lambda self: next(ids), # noqa: ARG005
|
||
)
|
||
_ratio(monkeypatch, 0.5)
|
||
signals = _start()
|
||
try:
|
||
kept, dropped = await _probe(2)
|
||
started = _started(signals)
|
||
spans = signals.finished_spans()
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
servers = [span for span in spans if span.kind == SpanKind.SERVER]
|
||
(server,) = servers
|
||
assert server.context.trace_id == TRACE_KEPT
|
||
assert server.attributes["messenger.request_id"] == kept
|
||
assert dropped != kept
|
||
assert started == {
|
||
("none", "RECORD_AND_SAMPLE"): 1,
|
||
("none", "DROP"): 1,
|
||
("local", "RECORD_AND_SAMPLE"): 1,
|
||
}
|
||
|
||
|
||
def test_remote_parent_flag_wins_over_ratio(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Span с удалённым родителем следует его флагу: выбранный пишется и
|
||
при доле 0, невыбранный не пишется и при доле 1 (producer и consumer)."""
|
||
|
||
def chain(parent: dataclasses.TraceContextDTO) -> None:
|
||
with telemetry.producer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
publication=_publication(parent),
|
||
):
|
||
pass
|
||
with telemetry.consumer_span(
|
||
queue=QUEUE,
|
||
job_type=ATTACHMENT_JOB_TYPE,
|
||
parent=parent,
|
||
):
|
||
pass
|
||
|
||
_ratio(monkeypatch, 0)
|
||
signals = _start()
|
||
try:
|
||
chain(SAVED)
|
||
zero = signals.finished_spans()
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert {span.kind for span in zero} == {
|
||
SpanKind.PRODUCER,
|
||
SpanKind.CONSUMER,
|
||
}
|
||
assert {_hex(span.context.trace_id, 32) for span in zero} == {SAVED_TRACE}
|
||
|
||
_ratio(monkeypatch, 1)
|
||
signals = _start()
|
||
try:
|
||
chain(UNSAMPLED)
|
||
started = _started(signals)
|
||
one = signals.finished_spans()
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert one == []
|
||
assert started == {("remote", "DROP"): 2}
|
||
|
||
|
||
async def test_coarse_buckets_in_export(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
monkeypatch.setattr(
|
||
settings.telemetry,
|
||
"metric_buckets",
|
||
MetricBucketsEnum.COARSE,
|
||
)
|
||
signals = _start()
|
||
try:
|
||
await _probe()
|
||
(point,) = signals.points(METRIC_HTTP_SERVER_DURATION)
|
||
histograms = core.job_histograms()
|
||
assert histograms is not None
|
||
histograms[0].record(0.3, {})
|
||
histograms[1].record(12, {})
|
||
(job,) = signals.points(METRIC_JOB_DURATION)
|
||
(ready,) = signals.points(METRIC_JOB_TIME_TO_READY)
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert tuple(point.explicit_bounds) == HTTP_DURATION_BUCKETS_COARSE
|
||
assert tuple(job.explicit_bounds) == JOB_DURATION_BUCKETS_COARSE
|
||
assert tuple(ready.explicit_bounds) == TIME_TO_READY_BUCKETS_COARSE
|
||
assert 0.25 in point.explicit_bounds
|
||
assert 30 in ready.explicit_bounds
|
||
|
||
|
||
async def test_default_buckets_unchanged() -> None:
|
||
signals = _start()
|
||
try:
|
||
await _probe()
|
||
(point,) = signals.points(METRIC_HTTP_SERVER_DURATION)
|
||
finally:
|
||
signals.finish()
|
||
telemetry.remove_log_hooks()
|
||
assert tuple(point.explicit_bounds) == HTTP_DURATION_BUCKETS
|
||
|
||
|
||
def test_sampling_without_telemetry_has_nothing(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
monkeypatch.setattr(settings.telemetry, "enabled", False)
|
||
_ratio(monkeypatch, 0.1)
|
||
monkeypatch.setattr(
|
||
settings.telemetry,
|
||
"metric_buckets",
|
||
MetricBucketsEnum.COARSE,
|
||
)
|
||
assert telemetry.start_process_telemetry(SERVICE) is False
|
||
assert core.tracer() is None
|
||
assert core.meter() is None
|
||
|
||
|
||
def test_startup_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Запись о старте называет долю, набор и интервал при включённой
|
||
телеметрии и `None` при выключенной (спек `service-configuration`)."""
|
||
_ratio(monkeypatch, 0.1)
|
||
monkeypatch.setattr(
|
||
settings.telemetry,
|
||
"metric_buckets",
|
||
MetricBucketsEnum.COARSE,
|
||
)
|
||
monkeypatch.setattr(settings.telemetry, "otlp_endpoint", COLLECTOR)
|
||
assert telemetry.startup_fields(True) == {
|
||
"telemetry_enabled": True,
|
||
"telemetry_endpoint": COLLECTOR,
|
||
"telemetry_environment": settings.telemetry.environment,
|
||
"telemetry_trace_sample_ratio": 0.1,
|
||
"telemetry_metric_buckets": "coarse",
|
||
"telemetry_metric_export_interval_seconds": 15,
|
||
}
|
||
disabled = telemetry.startup_fields(False)
|
||
assert disabled["telemetry_enabled"] is False
|
||
assert set(disabled) == set(telemetry.startup_fields(True))
|
||
assert all(
|
||
value is None
|
||
for key, value in disabled.items()
|
||
if key != "telemetry_enabled"
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"module",
|
||
["src/main.py", "src/worker/main.py", "src/relay/main.py"],
|
||
)
|
||
def test_startup_records_use_fields(module: str) -> None:
|
||
from src.config.settings import BASE_DIR
|
||
|
||
source = BASE_DIR.joinpath(module).read_text(encoding="utf-8")
|
||
assert "**telemetry.startup_fields(telemetry_enabled)" in source
|
||
|
||
|
||
def test_sampler_choice() -> None:
|
||
"""Доля 1 — тот же `ParentBased(ALWAYS_ON)`, что раньше; иначе —
|
||
`ParentBased(TraceIdRatioBased)` (`parentbased_traceidratio`)."""
|
||
from opentelemetry.sdk.trace.sampling import ALWAYS_ON, TraceIdRatioBased
|
||
|
||
always = sdk.sampler(1)
|
||
assert always._root is ALWAYS_ON
|
||
ratio = sdk.sampler(0.1)
|
||
assert isinstance(ratio._root, TraceIdRatioBased)
|
||
assert ratio._root.rate == 0.1
|