Исправила замечание 2 (согласование UID датасорсов): явно задала UID prometheus/loki/tempo в datasources.yaml и согласовала их во всех ссылках (панели, alert rules, derived fields), плюс тест проверяет существование UID и health=ok у rules
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger — lab6 observability",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"editable": true,
|
||||
"timezone": "browser",
|
||||
"time": {"from": "now-30m", "to": "now"},
|
||||
@@ -23,8 +23,9 @@
|
||||
"type": "timeseries",
|
||||
"title": "API RPS (total)",
|
||||
"gridPos": {"x": 0, "y": 0, "w": 8, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps"}
|
||||
{"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -32,8 +33,9 @@
|
||||
"type": "timeseries",
|
||||
"title": "API error ratio (5xx / total)",
|
||||
"gridPos": {"x": 8, "y": 0, "w": 8, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio"}
|
||||
{"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -41,10 +43,11 @@
|
||||
"type": "timeseries",
|
||||
"title": "HTTP latency p50/p95/p99 (s)",
|
||||
"gridPos": {"x": 16, "y": 0, "w": 8, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99"}
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -52,8 +55,9 @@
|
||||
"type": "timeseries",
|
||||
"title": "Jobs processed by outcome",
|
||||
"gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}"}
|
||||
{"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -61,8 +65,9 @@
|
||||
"type": "timeseries",
|
||||
"title": "Job duration p95 (s) by outcome",
|
||||
"gridPos": {"x": 12, "y": 8, "w": 12, "h": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"}
|
||||
{"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -72,7 +77,7 @@
|
||||
"gridPos": {"x": 0, "y": 16, "w": 24, "h": 10},
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"targets": [
|
||||
{"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}"}
|
||||
{"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}", "refId": "A"}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -2,6 +2,7 @@ apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
@@ -9,6 +10,7 @@ datasources:
|
||||
editable: false
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
|
||||
@@ -344,6 +344,12 @@ def test_logs_in_loki(c):
|
||||
print(" Loki OK: api+worker logs with non-zero trace_id found")
|
||||
|
||||
|
||||
def _grafana_ds_uids(auth) -> dict[str, dict]:
|
||||
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
|
||||
check(code == 200, "grafana datasources API %s" % code)
|
||||
return {d.get("uid"): d for d in json.loads(body)}
|
||||
|
||||
|
||||
def test_grafana_provisioning(c):
|
||||
deadline = time.monotonic() + 60
|
||||
auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD)
|
||||
@@ -355,24 +361,40 @@ def test_grafana_provisioning(c):
|
||||
else:
|
||||
raise Failure("Grafana API не поднялась за 60 с")
|
||||
|
||||
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
|
||||
check(code == 200, "grafana datasources API %s" % code)
|
||||
datasources = {d["type"]: d for d in json.loads(body)}
|
||||
for t in ("prometheus", "loki", "tempo"):
|
||||
check(t in datasources, "Grafana missing datasource: %s (have %s)"
|
||||
% (t, list(datasources)))
|
||||
print(" datasources:", sorted(datasources))
|
||||
by_uid = _grafana_ds_uids(auth)
|
||||
# Требуемые UID явно заданы в provisioning и совпадают со ссылками в
|
||||
# dashboard/rules/derived fields.
|
||||
for uid, type_ in (("prometheus", "prometheus"),
|
||||
("loki", "loki"),
|
||||
("tempo", "tempo")):
|
||||
check(uid in by_uid, "Grafana missing datasource uid=%r (have %s)"
|
||||
% (uid, sorted(by_uid)))
|
||||
check(by_uid[uid].get("type") == type_,
|
||||
"datasource uid=%r has wrong type: %r"
|
||||
% (uid, by_uid[uid].get("type")))
|
||||
print(" datasources by uid:", sorted(by_uid))
|
||||
|
||||
code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6",
|
||||
basic_auth=auth)
|
||||
check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code)
|
||||
dash = json.loads(body)
|
||||
panels = dash.get("dashboard", {}).get("panels", [])
|
||||
dashboard = dash.get("dashboard", {})
|
||||
panels = dashboard.get("panels", [])
|
||||
titles = {p.get("title") for p in panels}
|
||||
check(any("p95" in (t or "") for t in titles),
|
||||
"dashboard missing p95 panel")
|
||||
check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles),
|
||||
"dashboard missing jobs panel")
|
||||
|
||||
# Проверяем, что панель логов ссылается на существующий uid Loki,
|
||||
# а не на «магическое» имя.
|
||||
for p in panels:
|
||||
ds = p.get("datasource") or {}
|
||||
uid = ds.get("uid")
|
||||
if uid is not None:
|
||||
check(uid in by_uid,
|
||||
"panel %r references unknown datasource uid %r"
|
||||
% (p.get("title"), uid))
|
||||
print(" dashboard panels:", sorted(t for t in titles if t))
|
||||
|
||||
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
|
||||
@@ -380,6 +402,32 @@ def test_grafana_provisioning(c):
|
||||
check(code == 200, "grafana alert-rules API %s" % code)
|
||||
rules = json.loads(body)
|
||||
check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules))
|
||||
|
||||
for r in rules:
|
||||
# datasourceUid в data должен существовать.
|
||||
for item in r.get("data", []):
|
||||
uid = item.get("datasourceUid")
|
||||
if uid and uid != "__expr__":
|
||||
check(uid in by_uid,
|
||||
"rule %r references unknown datasource uid %r"
|
||||
% (r.get("title"), uid))
|
||||
|
||||
# Все правила должны быть health=ok. Если datasources не согласованы,
|
||||
# Grafana вернёт health=error "data source not found", как было
|
||||
# в замечании проверяющего.
|
||||
deadline = time.monotonic() + 180
|
||||
while True:
|
||||
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
|
||||
basic_auth=auth)
|
||||
check(code == 200, "grafana alert-rules API %s" % code)
|
||||
rules = json.loads(body)
|
||||
bad = [r.get("title") for r in rules
|
||||
if r.get("health") not in (None, "ok")]
|
||||
if not bad:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise Failure("Grafana alert rules with non-ok health: %s" % bad)
|
||||
time.sleep(3)
|
||||
print(" alert rules:", [r.get("title") for r in rules])
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user