diff --git a/infra/grafana/provisioning/dashboards/json/messenger.json b/infra/grafana/provisioning/dashboards/json/messenger.json index 76f5acb..bf20e02 100644 --- a/infra/grafana/provisioning/dashboards/json/messenger.json +++ b/infra/grafana/provisioning/dashboards/json/messenger.json @@ -2,7 +2,7 @@ "uid": "messenger-lab6", "title": "Messenger — lab6 observability", "schemaVersion": 39, - "version": 1, + "version": 2, "editable": true, "timezone": "browser", "time": {"from": "now-30m", "to": "now"}, @@ -23,8 +23,9 @@ "type": "timeseries", "title": "API RPS (total)", "gridPos": {"x": 0, "y": 0, "w": 8, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ - {"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps"} + {"expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "rps", "refId": "A"} ] }, { @@ -32,8 +33,9 @@ "type": "timeseries", "title": "API error ratio (5xx / total)", "gridPos": {"x": 8, "y": 0, "w": 8, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ - {"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio"} + {"expr": "sum(rate(http_requests_total{http_status_class=\"5xx\"}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1e-9)", "legendFormat": "ratio", "refId": "A"} ] }, { @@ -41,10 +43,11 @@ "type": "timeseries", "title": "HTTP latency p50/p95/p99 (s)", "gridPos": {"x": 16, "y": 0, "w": 8, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ - {"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50"}, - {"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95"}, - {"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99"} + {"expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"} ] }, { @@ -52,8 +55,9 @@ "type": "timeseries", "title": "Jobs processed by outcome", "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ - {"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}"} + {"expr": "sum by (outcome) (rate(jobs_processed_total[1m]))", "legendFormat": "{{outcome}}", "refId": "A"} ] }, { @@ -61,8 +65,9 @@ "type": "timeseries", "title": "Job duration p95 (s) by outcome", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"} + {"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(job_processing_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}", "refId": "A"} ] }, { @@ -72,7 +77,7 @@ "gridPos": {"x": 0, "y": 16, "w": 24, "h": 10}, "datasource": {"type": "loki", "uid": "loki"}, "targets": [ - {"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}"} + {"expr": "{service_name=~\"messenger-api|messenger-worker|messenger-outbox\"}", "refId": "A"} ] } ] diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml index 4a94656..e3a970b 100644 --- a/infra/grafana/provisioning/datasources/datasources.yaml +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Prometheus + uid: prometheus type: prometheus access: proxy url: http://prometheus:9090 @@ -9,6 +10,7 @@ datasources: editable: false - name: Loki + uid: loki type: loki access: proxy url: http://loki:3100 diff --git a/student-tests/lab6_test_level4.py b/student-tests/lab6_test_level4.py index c505091..9177afb 100644 --- a/student-tests/lab6_test_level4.py +++ b/student-tests/lab6_test_level4.py @@ -344,6 +344,12 @@ def test_logs_in_loki(c): print(" Loki OK: api+worker logs with non-zero trace_id found") +def _grafana_ds_uids(auth) -> dict[str, dict]: + code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth) + check(code == 200, "grafana datasources API %s" % code) + return {d.get("uid"): d for d in json.loads(body)} + + def test_grafana_provisioning(c): deadline = time.monotonic() + 60 auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD) @@ -355,24 +361,40 @@ def test_grafana_provisioning(c): else: raise Failure("Grafana API не поднялась за 60 с") - code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth) - check(code == 200, "grafana datasources API %s" % code) - datasources = {d["type"]: d for d in json.loads(body)} - for t in ("prometheus", "loki", "tempo"): - check(t in datasources, "Grafana missing datasource: %s (have %s)" - % (t, list(datasources))) - print(" datasources:", sorted(datasources)) + by_uid = _grafana_ds_uids(auth) + # Требуемые UID явно заданы в provisioning и совпадают со ссылками в + # dashboard/rules/derived fields. + for uid, type_ in (("prometheus", "prometheus"), + ("loki", "loki"), + ("tempo", "tempo")): + check(uid in by_uid, "Grafana missing datasource uid=%r (have %s)" + % (uid, sorted(by_uid))) + check(by_uid[uid].get("type") == type_, + "datasource uid=%r has wrong type: %r" + % (uid, by_uid[uid].get("type"))) + print(" datasources by uid:", sorted(by_uid)) code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6", basic_auth=auth) check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code) dash = json.loads(body) - panels = dash.get("dashboard", {}).get("panels", []) + dashboard = dash.get("dashboard", {}) + panels = dashboard.get("panels", []) titles = {p.get("title") for p in panels} check(any("p95" in (t or "") for t in titles), "dashboard missing p95 panel") check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles), "dashboard missing jobs panel") + + # Проверяем, что панель логов ссылается на существующий uid Loki, + # а не на «магическое» имя. + for p in panels: + ds = p.get("datasource") or {} + uid = ds.get("uid") + if uid is not None: + check(uid in by_uid, + "panel %r references unknown datasource uid %r" + % (p.get("title"), uid)) print(" dashboard panels:", sorted(t for t in titles if t)) code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules", @@ -380,6 +402,32 @@ def test_grafana_provisioning(c): check(code == 200, "grafana alert-rules API %s" % code) rules = json.loads(body) check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules)) + + for r in rules: + # datasourceUid в data должен существовать. + for item in r.get("data", []): + uid = item.get("datasourceUid") + if uid and uid != "__expr__": + check(uid in by_uid, + "rule %r references unknown datasource uid %r" + % (r.get("title"), uid)) + + # Все правила должны быть health=ok. Если datasources не согласованы, + # Grafana вернёт health=error "data source not found", как было + # в замечании проверяющего. + deadline = time.monotonic() + 180 + while True: + code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules", + basic_auth=auth) + check(code == 200, "grafana alert-rules API %s" % code) + rules = json.loads(body) + bad = [r.get("title") for r in rules + if r.get("health") not in (None, "ok")] + if not bad: + break + if time.monotonic() > deadline: + raise Failure("Grafana alert rules with non-ok health: %s" % bad) + time.sleep(3) print(" alert rules:", [r.get("title") for r in rules])