- [telemetry] trace_sample_ratio (ParentBased(TraceIdRatioBased), 1 keeps ParentBased(ALWAYS_ON)) and metric_buckets (default|coarse with SLO bounds), export interval <= 30 s; otel.sdk.span.started, span.live dropped; startup records name ratio, buckets and interval - bench: telemetry services on mtusi-bench, TELEMETRY=on switch for uploads and queue, make bench_telemetry[_quick] with off/full/full-aa/head/tail/ metrics/balanced, rare events, lookup from Tempo/Loki/Prometheus, cost sampler, verdicts H1-H8 and B1-B5; collector layers and lean Prometheus in bench/otel with preflight checks - REPORT: telemetry cost hypotheses and criteria before the label Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
218 lines
8.0 KiB
Python
218 lines
8.0 KiB
Python
"""Значения `[uploads]` (спек `service-configuration`, «Неверный параметр
|
||
загрузок») и сочетание срока остановки API с телеметрией (спек
|
||
`graceful-shutdown`, «Остановка при недоступном Collector»). Секция или
|
||
`Settings` только из аргументов: настоящий читает `.env` и `config.toml`.
|
||
"""
|
||
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from pydantic import ValidationError
|
||
from pydantic_settings import (
|
||
BaseSettings,
|
||
PydanticBaseSettingsSource,
|
||
SettingsConfigDict,
|
||
TomlConfigSettingsSource,
|
||
)
|
||
|
||
from src.config.settings import BASE_DIR, Settings, Telemetry, Uploads, Worker
|
||
from src.schemas.enums import MetricBucketsEnum
|
||
from src.utils.constants import (
|
||
API_STOP_BUDGET_SECONDS,
|
||
CONTAINER_STOP_TIMEOUT_SECONDS,
|
||
HTTP_DURATION_BUCKETS,
|
||
HTTP_DURATION_BUCKETS_COARSE,
|
||
JOB_DURATION_BUCKETS,
|
||
JOB_DURATION_BUCKETS_COARSE,
|
||
TELEMETRY_EXPORTER_SERIES_EXPIRATION_SECONDS,
|
||
TELEMETRY_METRIC_EXPORT_INTERVAL_MAX_SECONDS,
|
||
TELEMETRY_SHUTDOWN_TIMEOUT_SECONDS,
|
||
TIME_TO_READY_BUCKETS,
|
||
TIME_TO_READY_BUCKETS_COARSE,
|
||
)
|
||
from src.worker import main as worker_main
|
||
|
||
EXAMPLE_TOML_PATH = BASE_DIR.joinpath("config_example.toml")
|
||
|
||
|
||
class _ArgumentsOnly(Settings):
|
||
"""`Settings` без окружения, `.env` и `config.toml`."""
|
||
|
||
model_config = SettingsConfigDict(
|
||
**{**Settings.model_config, "env_file": None, "toml_file": None},
|
||
)
|
||
|
||
@classmethod
|
||
def settings_customise_sources(
|
||
cls,
|
||
settings_cls: type[BaseSettings], # noqa: ARG003
|
||
init_settings: PydanticBaseSettingsSource,
|
||
*args: PydanticBaseSettingsSource, # noqa: ARG003
|
||
**kwargs: Any, # noqa: ARG003
|
||
) -> tuple[PydanticBaseSettingsSource, ...]:
|
||
return (init_settings,)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("field", "value"),
|
||
[
|
||
("max_concurrent", 0),
|
||
("max_waiting", -1),
|
||
("max_unfinished_jobs", -1),
|
||
("body_timeout_seconds", "abc"),
|
||
],
|
||
)
|
||
def test_wrong_value_names_field(field: str, value: object) -> None:
|
||
with pytest.raises(ValidationError) as caught:
|
||
Uploads(**{field: value})
|
||
assert [error["loc"] for error in caught.value.errors()] == [(field,)]
|
||
|
||
|
||
def test_zero_turns_backlog_limit_off() -> None:
|
||
assert Uploads(max_unfinished_jobs=0).max_unfinished_jobs == 0
|
||
assert Uploads().max_unfinished_jobs == 250
|
||
|
||
|
||
def test_api_stop_with_telemetry_over_budget_names_field() -> None:
|
||
"""9 с дренажа + 1 с закрытия SDK — SIGKILL docker (verify W3)."""
|
||
with pytest.raises(ValidationError) as caught:
|
||
_ArgumentsOnly(
|
||
app={"graceful_shutdown_seconds": 9},
|
||
telemetry={"enabled": True},
|
||
)
|
||
errors = caught.value.errors(include_input=False)
|
||
assert [error["loc"] for error in errors] == [("telemetry",)]
|
||
assert "app.graceful_shutdown_seconds" in errors[0]["msg"]
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("graceful", "enabled"),
|
||
[(9, False), (8, True), (1, True)],
|
||
)
|
||
def test_api_stop_within_budget(graceful: int, enabled: bool) -> None:
|
||
loaded = _ArgumentsOnly(
|
||
app={"graceful_shutdown_seconds": graceful},
|
||
telemetry={"enabled": enabled},
|
||
)
|
||
assert loaded.app.graceful_shutdown_seconds == graceful
|
||
|
||
|
||
def test_budgets_fit_container_stop() -> None:
|
||
"""API: дренаж до 8 с + закрытие SDK; worker: ожидание до 8 с, отмена,
|
||
потребитель и одно окно `gather` с закрытием SDK — по 0,5 с."""
|
||
assert API_STOP_BUDGET_SECONDS < CONTAINER_STOP_TIMEOUT_SECONDS
|
||
assert 8 + TELEMETRY_SHUTDOWN_TIMEOUT_SECONDS <= API_STOP_BUDGET_SECONDS
|
||
with pytest.raises(ValidationError):
|
||
Worker(graceful_shutdown_seconds=8.01)
|
||
worker_worst = 8 + 3 * worker_main._CLOSE_TIMEOUT_SECONDS
|
||
assert worker_worst < CONTAINER_STOP_TIMEOUT_SECONDS
|
||
|
||
|
||
# выбор трасс в голове и набор границ (design change `track1-telemetry-cost`,
|
||
# решения 3–4; спек `service-configuration`, «Параметры телеметрии»)
|
||
|
||
|
||
class _EnvironmentOnly(_ArgumentsOnly):
|
||
"""`Settings` из окружения процесса, без `.env` и `config.toml`."""
|
||
|
||
@classmethod
|
||
def settings_customise_sources(
|
||
cls,
|
||
settings_cls: type[BaseSettings], # noqa: ARG003
|
||
init_settings: PydanticBaseSettingsSource,
|
||
env_settings: PydanticBaseSettingsSource,
|
||
*args: PydanticBaseSettingsSource, # noqa: ARG003
|
||
**kwargs: Any, # noqa: ARG003
|
||
) -> tuple[PydanticBaseSettingsSource, ...]:
|
||
return (init_settings, env_settings)
|
||
|
||
|
||
class _ExampleToml(_ArgumentsOnly):
|
||
"""`Settings` только из `config_example.toml`, как из `config.toml`."""
|
||
|
||
model_config = SettingsConfigDict(
|
||
**{**_ArgumentsOnly.model_config, "toml_file": EXAMPLE_TOML_PATH},
|
||
)
|
||
|
||
@classmethod
|
||
def settings_customise_sources(
|
||
cls,
|
||
settings_cls: type[BaseSettings],
|
||
init_settings: PydanticBaseSettingsSource,
|
||
*args: PydanticBaseSettingsSource, # noqa: ARG003
|
||
**kwargs: Any, # noqa: ARG003
|
||
) -> tuple[PydanticBaseSettingsSource, ...]:
|
||
return (init_settings, TomlConfigSettingsSource(settings_cls))
|
||
|
||
|
||
def test_telemetry_defaults_keep_previous_behaviour() -> None:
|
||
loaded = Telemetry()
|
||
assert loaded.trace_sample_ratio == 1
|
||
assert loaded.metric_buckets is MetricBucketsEnum.DEFAULT
|
||
assert loaded.metric_export_interval_seconds == 15
|
||
|
||
|
||
@pytest.mark.parametrize("ratio", [0, 0.1, 0.5, 1])
|
||
def test_trace_sample_ratio_accepts_range(ratio: float) -> None:
|
||
assert Telemetry(trace_sample_ratio=ratio).trace_sample_ratio == ratio
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("field", "value"),
|
||
[
|
||
("trace_sample_ratio", 1.5),
|
||
("trace_sample_ratio", -0.1),
|
||
("trace_sample_ratio", float("nan")),
|
||
("trace_sample_ratio", "nan"),
|
||
("metric_buckets", "fine"),
|
||
("metric_export_interval_seconds", 60),
|
||
("metric_export_interval_seconds", -1),
|
||
],
|
||
)
|
||
def test_wrong_telemetry_value_names_field(field: str, value: object) -> None:
|
||
with pytest.raises(ValidationError) as caught:
|
||
Telemetry(**{field: value})
|
||
assert [error["loc"] for error in caught.value.errors()] == [(field,)]
|
||
|
||
|
||
def test_export_interval_up_to_half_of_series_expiration() -> None:
|
||
limit = TELEMETRY_METRIC_EXPORT_INTERVAL_MAX_SECONDS
|
||
assert limit * 2 == TELEMETRY_EXPORTER_SERIES_EXPIRATION_SECONDS
|
||
assert Telemetry(metric_export_interval_seconds=limit)
|
||
with pytest.raises(ValidationError):
|
||
Telemetry(metric_export_interval_seconds=limit + 0.1)
|
||
|
||
|
||
def test_coarse_buckets_keep_slo_bounds() -> None:
|
||
assert 0.25 in HTTP_DURATION_BUCKETS_COARSE
|
||
assert 30 in TIME_TO_READY_BUCKETS_COARSE
|
||
assert HTTP_DURATION_BUCKETS_COARSE[-1] == HTTP_DURATION_BUCKETS[-1]
|
||
assert JOB_DURATION_BUCKETS_COARSE[-1] == JOB_DURATION_BUCKETS[-1]
|
||
assert TIME_TO_READY_BUCKETS_COARSE[-1] == TIME_TO_READY_BUCKETS[-1]
|
||
|
||
|
||
def test_telemetry_from_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setenv("TELEMETRY__TRACE_SAMPLE_RATIO", "0.1")
|
||
monkeypatch.setenv("TELEMETRY__METRIC_BUCKETS", "coarse")
|
||
loaded = _EnvironmentOnly()
|
||
assert loaded.telemetry.trace_sample_ratio == 0.1
|
||
assert loaded.telemetry.metric_buckets is MetricBucketsEnum.COARSE
|
||
|
||
|
||
def test_wrong_ratio_from_environment_names_field(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
monkeypatch.setenv("TELEMETRY__TRACE_SAMPLE_RATIO", "1.5")
|
||
with pytest.raises(ValidationError) as caught:
|
||
_EnvironmentOnly()
|
||
assert [error["loc"] for error in caught.value.errors()] == [
|
||
("telemetry", "trace_sample_ratio"),
|
||
]
|
||
|
||
|
||
def test_example_toml_loads_as_config_toml() -> None:
|
||
"""`config_example.toml` — рабочий `config.toml` (задача 2.4)."""
|
||
loaded = _ExampleToml()
|
||
assert loaded.telemetry.trace_sample_ratio == 1
|
||
assert loaded.telemetry.metric_buckets is MetricBucketsEnum.DEFAULT
|