Исправила замечание 2 (согласование UID датасорсов): явно задала UID prometheus/loki/tempo в datasources.yaml и согласовала их во всех ссылках (панели, alert rules, derived fields), плюс тест проверяет существование UID и health=ok у rules

This commit is contained in:
vasyukova
2026-10-30 23:53:56 +03:00
parent d169076e30
commit 55057eab6e
3 changed files with 72 additions and 17 deletions
@@ -2,7 +2,7 @@
"uid": "messenger-lab6",
"title": "Messenger — lab6 observability",
"schemaVersion": 39,
"version": 1,
"version": 2,
"editable": true,
"timezone": "browser",
"time": {"from": "now-30m", "to": "now"},
@@ -23,8 +23,9 @@
"type": "timeseries",
"title": "API RPS (total)",
"gridPos": {"x": 0, "y": 0, "w": 8, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps"}
{"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps", "refId": "A"}
]
},
{
@@ -32,8 +33,9 @@
"type": "timeseries",
"title": "API error ratio (5xx / total)",
"gridPos": {"x": 8, "y": 0, "w": 8, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio"}
{"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio", "refId": "A"}
]
},
{
@@ -41,10 +43,11 @@
"type": "timeseries",
"title": "HTTP latency p50/p95/p99 (s)",
"gridPos": {"x": 16, "y": 0, "w": 8, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99"}
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
]
},
{
@@ -52,8 +55,9 @@
"type": "timeseries",
"title": "Jobs processed by outcome",
"gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}"}
{"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A"}
]
},
{
@@ -61,8 +65,9 @@
"type": "timeseries",
"title": "Job duration p95 (s) by outcome",
"gridPos": {"x": 12, "y": 8, "w": 12, "h": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"}
{"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}", "refId": "A"}
]
},
{
@@ -72,7 +77,7 @@
"gridPos": {"x": 0, "y": 16, "w": 24, "h": 10},
"datasource": {"type": "loki", "uid": "loki"},
"targets": [
{"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}"}
{"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}", "refId": "A"}
]
}
]
@@ -2,6 +2,7 @@ apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
@@ -9,6 +10,7 @@ datasources:
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
+56 -8
View File
@@ -344,6 +344,12 @@ def test_logs_in_loki(c):
print(" Loki OK: api+worker logs with non-zero trace_id found")
def _grafana_ds_uids(auth) -> dict[str, dict]:
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
check(code == 200, "grafana datasources API %s" % code)
return {d.get("uid"): d for d in json.loads(body)}
def test_grafana_provisioning(c):
deadline = time.monotonic() + 60
auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD)
@@ -355,24 +361,40 @@ def test_grafana_provisioning(c):
else:
raise Failure("Grafana API не поднялась за 60 с")
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
check(code == 200, "grafana datasources API %s" % code)
datasources = {d["type"]: d for d in json.loads(body)}
for t in ("prometheus", "loki", "tempo"):
check(t in datasources, "Grafana missing datasource: %s (have %s)"
% (t, list(datasources)))
print(" datasources:", sorted(datasources))
by_uid = _grafana_ds_uids(auth)
# Требуемые UID явно заданы в provisioning и совпадают со ссылками в
# dashboard/rules/derived fields.
for uid, type_ in (("prometheus", "prometheus"),
("loki", "loki"),
("tempo", "tempo")):
check(uid in by_uid, "Grafana missing datasource uid=%r (have %s)"
% (uid, sorted(by_uid)))
check(by_uid[uid].get("type") == type_,
"datasource uid=%r has wrong type: %r"
% (uid, by_uid[uid].get("type")))
print(" datasources by uid:", sorted(by_uid))
code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6",
basic_auth=auth)
check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code)
dash = json.loads(body)
panels = dash.get("dashboard", {}).get("panels", [])
dashboard = dash.get("dashboard", {})
panels = dashboard.get("panels", [])
titles = {p.get("title") for p in panels}
check(any("p95" in (t or "") for t in titles),
"dashboard missing p95 panel")
check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles),
"dashboard missing jobs panel")
# Проверяем, что панель логов ссылается на существующий uid Loki,
# а не на «магическое» имя.
for p in panels:
ds = p.get("datasource") or {}
uid = ds.get("uid")
if uid is not None:
check(uid in by_uid,
"panel %r references unknown datasource uid %r"
% (p.get("title"), uid))
print(" dashboard panels:", sorted(t for t in titles if t))
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
@@ -380,6 +402,32 @@ def test_grafana_provisioning(c):
check(code == 200, "grafana alert-rules API %s" % code)
rules = json.loads(body)
check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules))
for r in rules:
# datasourceUid в data должен существовать.
for item in r.get("data", []):
uid = item.get("datasourceUid")
if uid and uid != "__expr__":
check(uid in by_uid,
"rule %r references unknown datasource uid %r"
% (r.get("title"), uid))
# Все правила должны быть health=ok. Если datasources не согласованы,
# Grafana вернёт health=error "data source not found", как было
# в замечании проверяющего.
deadline = time.monotonic() + 180
while True:
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
basic_auth=auth)
check(code == 200, "grafana alert-rules API %s" % code)
rules = json.loads(body)
bad = [r.get("title") for r in rules
if r.get("health") not in (None, "ok")]
if not bad:
break
if time.monotonic() > deadline:
raise Failure("Grafana alert rules with non-ok health: %s" % bad)
time.sleep(3)
print(" alert rules:", [r.get("title") for r in rules])