forked from mtusi2026/mtusi2026lab7
111 lines
6.4 KiB
Python
111 lines
6.4 KiB
Python
"""Lab6/lab7: конфигурации наблюдаемости из Git проверены «родными» инструментами тех же версий + provisioning Grafana.
|
|
|
|
python student-tests/check_configs.py
|
|
|
|
В Docker-стенде (по умолчанию) инструменты запускаются в образах из compose.yaml:
|
|
otelcol-contrib validate, promtool check config/rules, promtool test rules (unit-тесты алертов),
|
|
loki -verify-config, tempo -config.verify; затем через HTTP API Grafana: источники данных созданы
|
|
provisioning-ом и отвечают (health), dashboard импортирован, его панели ссылаются на эти источники.
|
|
OBS_BIN_DIR=<каталог с бинарниками> — проверить конфиги локальными бинарниками без Docker (Grafana пропускается,
|
|
если GRAFANA_URL недоступен).
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from common import ROOT, check
|
|
|
|
BIN = os.environ.get("OBS_BIN_DIR")
|
|
GRAFANA_URL = os.environ.get("GRAFANA_URL", "https://127.0.0.1:3000").rstrip("/") # lab7: HTTPS, CA стенда
|
|
|
|
|
|
def run(title, docker_args, local_args):
|
|
if BIN:
|
|
command = [str(Path(BIN) / local_args[0]), *local_args[1:]]
|
|
else:
|
|
command = ["docker", "compose", *docker_args]
|
|
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
|
|
output = (result.stdout + result.stderr).strip()
|
|
tail = "\n".join(output.splitlines()[-6:])
|
|
print(f"--- {title}: exit {result.returncode}\n{tail}")
|
|
check(result.returncode == 0, f"{title} failed")
|
|
|
|
|
|
def local_copy(path, replacements):
|
|
text = (ROOT / path).read_text(encoding="utf-8")
|
|
for old, new in replacements:
|
|
text = text.replace(old, new)
|
|
handle = tempfile.NamedTemporaryFile("w", suffix=Path(path).suffix, delete=False, encoding="utf-8")
|
|
handle.write(text)
|
|
handle.close()
|
|
return handle.name
|
|
|
|
|
|
def grafana(path):
|
|
password = (ROOT / "secrets/grafana_admin_password").read_text(encoding="utf-8").strip()
|
|
request = urllib.request.Request(GRAFANA_URL + path, headers={
|
|
"Authorization": "Basic " + base64.b64encode(f"admin:{password}".encode()).decode(), "Accept": "application/json"})
|
|
with urllib.request.urlopen(request, timeout=10) as reply:
|
|
return json.loads(reply.read().decode("utf-8"))
|
|
|
|
|
|
def main():
|
|
tmp = tempfile.mkdtemp()
|
|
run("otelcol-contrib validate (infra/otel/collector.yaml)",
|
|
["run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
|
|
["otelcol-contrib", "validate", f"--config={ROOT / 'infra/otel/collector.yaml'}"])
|
|
run("promtool check config (infra/prometheus/prometheus.yml)",
|
|
["exec", "-T", "prometheus", "promtool", "check", "config", "/etc/prometheus/prometheus.yml"],
|
|
["promtool", "check", "config", local_copy("infra/prometheus/prometheus.yml", [
|
|
("/etc/prometheus/rules/", str(ROOT / "infra/prometheus/rules") + "/")])])
|
|
run("promtool check rules (recording + alerts)",
|
|
["exec", "-T", "prometheus", "sh", "-c", "promtool check rules /etc/prometheus/rules/*.yml"],
|
|
["promtool", "check", "rules", *map(str, sorted((ROOT / "infra/prometheus/rules").glob("*.yml")))])
|
|
run("promtool check web-config (lab7: TLS, client certs, allowed SANs)",
|
|
["exec", "-T", "prometheus", "promtool", "check", "web-config", "/etc/prometheus/web.yml"],
|
|
["promtool", "check", "web-config", str(ROOT / "infra/prometheus/web.yml")])
|
|
run("promtool test rules (unit tests of alert thresholds/windows/for)",
|
|
["exec", "-T", "prometheus", "promtool", "test", "rules", "/etc/prometheus/tests/alerts_test.yml"],
|
|
["promtool", "test", "rules", str(ROOT / "infra/prometheus/tests/alerts_test.yml")])
|
|
run("loki -verify-config (infra/loki/loki.yaml)",
|
|
["run", "--rm", "--no-deps", "loki", "-config.file=/etc/loki/loki.yaml", "-verify-config"],
|
|
["loki", f"-config.file={local_copy('infra/loki/loki.yaml', [('/loki', tmp + '/loki')])}", "-verify-config"])
|
|
run("tempo -config.verify (infra/tempo/tempo.yaml)",
|
|
["run", "--rm", "--no-deps", "tempo", "-config.file=/etc/tempo/tempo.yaml", "-config.verify=true"],
|
|
[os.environ.get("TEMPO_BIN", "tempo"),
|
|
f"-config.file={local_copy('infra/tempo/tempo.yaml', [('/var/tempo', tmp + '/tempo')])}", "-config.verify=true"])
|
|
|
|
try:
|
|
health = grafana("/api/health")
|
|
except (OSError, urllib.error.URLError) as exc:
|
|
check(BIN, f"Grafana is not reachable at {GRAFANA_URL}: {exc}")
|
|
print(f"--- Grafana: not reachable at {GRAFANA_URL}, skipped (local binaries mode)")
|
|
print("PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid")
|
|
return
|
|
print(f"--- Grafana {health.get('version')}: database {health.get('database')}")
|
|
sources = {d["uid"]: d for d in grafana("/api/datasources")}
|
|
for uid, kind in (("prometheus", "prometheus"), ("loki", "loki"), ("tempo", "tempo")):
|
|
check(uid in sources and sources[uid]["type"] == kind, f"datasource {uid} is not provisioned")
|
|
state = grafana(f"/api/datasources/uid/{uid}/health")
|
|
print(f"datasource {uid} ({kind}): {state.get('status')} {state.get('message', '')[:80]}")
|
|
check(state.get("status") == "OK", f"datasource {uid} is not healthy: {state}")
|
|
dashboard = grafana("/api/dashboards/uid/messenger-lab6")
|
|
panels = [p for p in dashboard["dashboard"]["panels"] if p["type"] != "row"]
|
|
used = {t.get("datasource", {}).get("uid") for p in panels for t in p.get("targets", [])}
|
|
print(f"dashboard '{dashboard['dashboard']['title']}' in folder '{dashboard['meta'].get('folderTitle')}': "
|
|
f"{len(panels)} panels, provisioned={dashboard['meta'].get('provisioned')}, datasources {sorted(used)}")
|
|
check(dashboard["meta"].get("provisioned"), "dashboard must come from provisioning, not the UI")
|
|
check(used <= set(sources), "panels reference a datasource that is not provisioned")
|
|
print("PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana "
|
|
"provisioned 3 healthy datasources and the dashboard without manual steps")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|