Files
mtusi2026lab2/scripts/obs_check.py
T

498 lines
27 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import base64
import json
import os
import re
import secrets
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PROMQL_WORDS = {
"sum", "min", "max", "avg", "count", "by", "without", "on", "ignoring", "group_left", "group_right", "and", "or",
"unless", "offset", "bool", "rate", "irate", "increase", "delta", "deriv", "histogram_quantile", "label_replace",
"label_join", "absent", "absent_over_time", "vector", "scalar", "clamp_min", "clamp_max", "topk", "bottomk",
"max_over_time", "min_over_time", "avg_over_time", "sum_over_time", "count_over_time", "quantile_over_time",
"time", "timestamp", "sort", "sort_desc", "round", "abs", "le", "inf", "nan", "predict_linear", "changes", "resets",
}
class Failure(Exception):
pass
def now():
return datetime.now(timezone.utc).isoformat(timespec="seconds")
def http(method, url, body=None, headers=None, timeout=10, expect=None):
data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
head = {"Accept": "application/json"}
if body is not None and not isinstance(body, bytes):
head["Content-Type"] = "application/json"
head.update(headers or {})
request = urllib.request.Request(url, data=data, headers=head, method=method)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
status, raw, got = response.status, response.read(), dict(response.headers)
except urllib.error.HTTPError as error:
status, raw, got = error.code, error.read(), dict(error.headers)
if expect is not None and status not in expect:
raise Failure(f"{method} {url.split('?')[0]} -> {status}: {raw[:200]!r}")
try:
parsed = json.loads(raw) if raw else None
except ValueError:
parsed = None
return status, parsed, got, raw
def get_json(url, headers=None, params=None):
if params:
url += ("&" if "?" in url else "?") + urllib.parse.urlencode(params)
return http("GET", url, headers=headers, expect=(200,))[1]
def wait_for(what, check, timeout, interval=2.0):
deadline = time.monotonic() + timeout
while True:
try:
result = check()
if result:
return result
except (Failure, OSError):
pass
if time.monotonic() > deadline:
raise Failure(f"timeout waiting for {what}")
time.sleep(interval)
class Backends:
def __init__(self, args):
self.prometheus = args.prometheus.rstrip("/")
self.loki = args.loki.rstrip("/")
self.tempo = args.tempo.rstrip("/")
self.grafana = args.grafana.rstrip("/")
self.grafana_password_file = args.grafana_password_file
def grafana_headers(self):
password = Path(self.grafana_password_file).read_text().strip()
token = base64.b64encode(f"admin:{password}".encode()).decode()
return {"Authorization": "Basic " + token}
def promql(self, query, at=None, base=None):
params = {"query": query}
if at is not None:
params["time"] = f"{at:.3f}"
result = get_json((base or self.prometheus) + "/api/v1/query", params=params)
if result.get("status") != "success":
raise Failure(f"PromQL failed: {query}")
return result["data"]["result"]
def logs(self, query, start, end, limit=100, base=None):
params = {"query": query, "start": str(int(start * 1e9)), "end": str(int(end * 1e9)), "limit": str(limit),
"direction": "backward"}
result = get_json((base or self.loki) + "/loki/api/v1/query_range", params=params)
return [(stream["stream"], line) for stream in result["data"]["result"] for _, line in stream["values"]]
def traceql(self, query, start, end, limit=20, base=None):
params = {"q": query, "start": str(int(start)), "end": str(int(end) + 1), "limit": str(limit)}
return get_json((base or self.tempo) + "/api/search", params=params).get("traces") or []
def trace(self, trace_id, base=None):
status, body, _, raw = http("GET", (base or self.tempo) + f"/api/v2/traces/{trace_id}", expect=(200, 404))
return (body, raw.decode("utf-8", "replace")) if status == 200 else (None, "")
def otlp_attrs(values):
return [{"key": k, "value": {"intValue": str(v)} if isinstance(v, int) else {"stringValue": str(v)}}
for k, v in values.items()]
def otlp_resource(service, instance):
return {"attributes": otlp_attrs({"service.name": service, "service.version": "probe",
"service.instance.id": instance, "deployment.environment.name": "lab"})}
def tempo_spans(body):
spans = []
for batch in (body or {}).get("trace", {}).get("resourceSpans", []):
service = next((a["value"].get("stringValue") for a in batch["resource"].get("attributes", [])
if a["key"] == "service.name"), "?")
for scope in batch.get("scopeSpans", []):
for span in scope.get("spans", []):
values = {a["key"]: next(iter(a["value"].values()), None) for a in span.get("attributes", [])}
duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6
spans.append({"service": service, "name": span["name"], "kind": span.get("kind"),
"status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"),
"duration_ms": round(duration, 3), "links": len(span.get("links", [])),
"attributes": values})
return spans
def probe(args, b):
leak = "probe-secret-" + secrets.token_hex(8)
rid = uuid.uuid4().hex
trace_id, server, producer, consumer = secrets.token_hex(16), secrets.token_hex(8), secrets.token_hex(8), secrets.token_hex(8)
t0 = time.time()
ns = time.time_ns()
api, worker = args.service_name, args.service_name + "-worker"
scope = {"name": "obs-check", "version": "1"}
def span(span_id, parent, name, kind, start_ms, end_ms, values, links=()):
return {"traceId": trace_id, "spanId": span_id, "parentSpanId": parent, "name": name, "kind": kind,
"startTimeUnixNano": str(ns + start_ms * 1_000_000), "endTimeUnixNano": str(ns + end_ms * 1_000_000),
"attributes": otlp_attrs(values), "status": {"code": 1},
"links": [{"traceId": trace_id, "spanId": s} for s in links]}
traces = {"resourceSpans": [
{"resource": otlp_resource(api, api + "-1"), "scopeSpans": [{"scope": scope, "spans": [
span(server, "", "POST /api/v1/probe", 2, 0, 40, {
"http.request.method": "POST", "http.route": "/api/v1/probe", "http.response.status_code": 202,
"messenger.request_id": rid, "http.request.header.authorization": "Bearer " + leak,
"url.query": "filename=" + leak, "user.password": leak, "messenger.session.token": leak}),
span(producer, server, "probe.jobs publish", 4, 30, 35, {"messaging.operation.type": "send"})]}]},
{"resource": otlp_resource(worker, worker + "-1"), "scopeSpans": [{"scope": scope, "spans": [
span(consumer, producer, "probe.jobs process", 5, 50, 90, {"messaging.operation.type": "process"}, [producer])]}]},
]}
logs = {"resourceLogs": [{"resource": otlp_resource(api, api + "-1"), "scopeLogs": [{"scope": scope, "logRecords": [{
"timeUnixNano": str(ns), "observedTimeUnixNano": str(ns), "severityNumber": 9, "severityText": "INFO",
"body": {"stringValue": "probe.request.completed"}, "traceId": trace_id, "spanId": server,
"attributes": otlp_attrs({"event": "probe.request.completed", "request_id": rid, "authorization": "Bearer " + leak})}]}]}]}
point = {"attributes": otlp_attrs({"probe.id": rid}), "startTimeUnixNano": str(ns - 10_000_000_000), "timeUnixNano": str(ns)}
metrics = {"resourceMetrics": [{"resource": otlp_resource(api, api + "-1"), "scopeMetrics": [{"scope": scope, "metrics": [
{"name": "obs.probe.heartbeat", "unit": "{probe}", "sum": {"aggregationTemporality": 2, "isMonotonic": True,
"dataPoints": [dict(point, asInt="1")]}},
{"name": "obs.probe.latency", "unit": "s", "histogram": {"aggregationTemporality": 2, "dataPoints": [dict(
point, count="1", sum=0.04, bucketCounts=["0", "1", "0"], explicitBounds=[0.01, 0.1])]}},
]}]}]}
otlp = args.otlp.rstrip("/")
for path, payload in (("/v1/traces", traces), ("/v1/logs", logs), ("/v1/metrics", metrics)):
http("POST", otlp + path, payload, expect=(200,))
report = {"probe_id": rid, "trace_id": trace_id, "service": api, "sent_at": now()}
def tempo_ok(base=None):
body, raw = b.trace(trace_id, base)
spans = tempo_spans(body)
return {"spans": len(spans), "services": sorted({s["service"] for s in spans}), "raw": raw} if len(spans) == 3 else None
tempo = wait_for("trace in Tempo", tempo_ok, args.timeout)
found = wait_for("TraceQL search", lambda: b.traceql(f'{{ span.messenger.request_id = "{rid}" }}', t0 - 60, time.time() + 60), args.timeout)
loki_query = f'{{service_name="{api}"}} | request_id="{rid}"'
lines = wait_for("log in Loki", lambda: b.logs(loki_query, t0 - 60, time.time() + 60), args.timeout)
series = wait_for("metrics in Prometheus", lambda: (lambda r: r if len(r) == 2 else None)(b.promql(
f'{{__name__=~"obs_probe_heartbeat_total|obs_probe_latency_seconds_count", probe_id="{rid}"}}')), args.timeout)
labels = lines[0][0]
raw_logs = json.dumps(lines)
report.update({
"tempo": {"spans": tempo["spans"], "services": tempo["services"], "traceql_hits": len(found)},
"loki": {"lines": len(lines), "trace_id_matches": labels.get("trace_id") == trace_id,
"indexed_labels": sorted(k for k in labels if k in {"service_name", "deployment_environment_name"})},
"prometheus": {"series": sorted(s["metric"].get("__name__", "") for s in series),
"job": series[0]["metric"].get("job")},
"redaction": {"secret_in_tempo": leak in tempo["raw"], "secret_in_loki": leak in raw_logs,
"masked_in_tempo": "****" in tempo["raw"]},
})
if not args.no_grafana:
g = b.grafana + "/api/datasources/proxy/uid/"
headers = b.grafana_headers()
report["grafana"] = {
"prometheus": len(get_json(g + "prometheus/api/v1/query", headers, {"query": f'obs_probe_heartbeat_total{{probe_id="{rid}"}}'})["data"]["result"]),
"loki": len(get_json(g + "loki/loki/api/v1/query_range", headers, {"query": loki_query, "start": str(int((t0 - 60) * 1e9)), "end": str(int((time.time() + 60) * 1e9))})["data"]["result"]),
"tempo": http("GET", g + f"tempo/api/v2/traces/{trace_id}", headers=headers)[0],
}
print(json.dumps(report, ensure_ascii=False, indent=2))
ok = (report["tempo"]["spans"] == 3 and report["loki"]["trace_id_matches"] and len(series) == 2
and not report["redaction"]["secret_in_tempo"] and not report["redaction"]["secret_in_loki"]
and ("grafana" not in report or (report["grafana"]["prometheus"] and report["grafana"]["loki"] and report["grafana"]["tempo"] == 200)))
if not ok:
raise Failure("probe verification failed")
class Api:
def __init__(self, base):
self.base = base.rstrip("/")
def call(self, method, path, body=None, headers=None, expect=(200,)):
status, parsed, got, _ = http(method, self.base + path, body, headers, expect=expect)
return parsed, got.get("X-Request-Id") or got.get("x-request-id")
def register(self):
username = "u_obs_" + secrets.token_hex(6)
password = "T3st-" + secrets.token_hex(16)
user, rid = self.call("POST", "/api/v1/users", {"username": username, "display_name": "obs check", "password": password}, expect=(201,))
return user, username, password, rid
def login(self, username, password):
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
session, _ = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, expect=(201,))
return {"Authorization": "Bearer " + session["token"]}, session["token"]
def chat(self, auth, user):
chat, _ = self.call("POST", "/api/v1/chats", {"title": "obs", "member_ids": [user["id"]]}, auth, expect=(201,))
return chat
def logout(self, auth):
self.call("DELETE", "/api/v1/auth/sessions/current", headers=auth, expect=(204,))
def request_check(args, b):
api = Api(args.base_url)
t0 = time.time()
_, _, _, rid = api.register()
query = f'{{service_name="{args.api_service}"}} | {args.log_request_id_key}="{rid}"'
lines = wait_for("API log by request_id", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
labels, line = lines[0]
trace_id = labels.get("trace_id")
if not trace_id:
raise Failure("log record has no trace_id")
spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout)
if not any(s["kind"] == "SPAN_KIND_SERVER" and s["service"] == args.api_service for s in spans):
raise Failure("no real API server span")
if not any(s["kind"] == "SPAN_KIND_CLIENT" and s["attributes"].get("db.system") == "postgresql" for s in spans):
raise Failure("no PostgreSQL client span inside API trace")
print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id,
"span_id": labels.get("span_id"),
"service_name": labels.get("service_name"),
"severity": labels.get("severity_text") or labels.get("detected_level")},
"trace": [{k: s[k] for k in ("service", "name", "kind", "status", "duration_ms")} for s in spans]},
ensure_ascii=False, indent=2))
def job_check(args, b):
api = Api(args.base_url)
t0 = time.time()
user, username, password, _ = api.register()
auth, _ = api.login(username, password)
chat = api.chat(auth, user)
image = Path(args.image).read_bytes()
headers = dict(auth, **{"Content-Type": "image/png"})
attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=obs.png", image, headers, expect=(202,))
ready = wait_for("attachment ready", lambda: (lambda a: a if a["status"] in ("ready", "failed") else None)(
api.call("GET", f"/api/v1/attachments/{attachment['id']}", headers=auth)[0]), 60, 0.5)
key = args.attachment_attr
traces = wait_for("traces for attachment", lambda: b.traceql(f'{{ span.{key} = "{attachment["id"]}" }}', t0 - 30, time.time() + 30), args.timeout)
chain = []
for item in traces:
for span in tempo_spans(b.trace(item["traceID"])[0]):
chain.append(dict({k: span[k] for k in ("service", "name", "kind", "status", "duration_ms", "links")},
trace_id=item["traceID"], attempt=span["attributes"].get(args.attempt_attr)))
query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"'
worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
api.logout(auth)
if ready["status"] != "ready" or not any(s["kind"] == "SPAN_KIND_CONSUMER" and s["service"] == args.worker_service for s in chain):
raise Failure("job is not ready or no real consumer span")
if not any(labels.get("trace_id") and labels.get("span_id") for labels, _ in worker_logs):
raise Failure("worker log is not correlated")
print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"],
"traces": sorted({c["trace_id"] for c in chain}), "spans": chain,
"worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")}
for labels, line in worker_logs[:10]]}, ensure_ascii=False, indent=2))
def leaks_check(args, b):
api = Api(args.base_url)
t0 = time.time()
user, username, password, _ = api.register()
auth, token = api.login(username, password)
chat = api.chat(auth, user)
text = "obs-leak-text-" + secrets.token_hex(8)
filename = "obs-leak-file-" + secrets.token_hex(6) + ".png"
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": text}, dict(auth, **{"Idempotency-Key": uuid.uuid4().hex}), expect=(201,))
if Path(args.image).is_file():
api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=" + urllib.parse.quote(filename),
Path(args.image).read_bytes(), dict(auth, **{"Content-Type": "image/png"}), expect=(202,))
api.logout(auth)
time.sleep(args.flush)
needles = {"password": password, "token": token, "message_text": text, "filename": filename}
end = time.time() + 5
hits = {}
for name, needle in needles.items():
lines = b.logs(f'{{service_name=~"{args.service_regex}"}} |= `{needle}`', t0 - 30, end, limit=50)
hits[name] = {"loki": len(lines)}
traces = b.traceql(f'{{ resource.service.name =~ "{args.service_regex}" }}', t0 - 30, end, limit=500)
raws = [b.trace(t["traceID"])[1] for t in traces]
for name, needle in needles.items():
hits[name]["tempo"] = sum(needle in raw for raw in raws)
report = {"checked_at": now(), "window_seconds": round(end - t0), "traces_scanned": len(raws), "hits": hits}
print(json.dumps(report, indent=2))
if any(v["loki"] or v["tempo"] for v in hits.values()):
raise Failure("secret or message body found in telemetry")
def uploads_burst(args, b):
api = Api(args.base_url)
user, username, password, _ = api.register()
auth, _ = api.login(username, password)
chat = api.chat(auth, user)
image = Path(args.image).read_bytes()
accepted = []
for index in range(args.count):
attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=burst-{index}.png", image,
dict(auth, **{"Content-Type": "image/png"}), expect=(202,))
accepted.append({"attachment_id": attachment["id"], "request_id": rid, "status": attachment["status"]})
time.sleep(args.interval)
api.logout(auth)
print(json.dumps({"ts": now(), "uploaded": accepted}, ensure_ascii=False))
def query(args, b):
results = {expr: [{"labels": {k: v for k, v in r["metric"].items() if k != "__name__"}, "value": r["value"][1]}
for r in b.promql(expr)] for expr in args.expr}
print(json.dumps({"ts": now(), "results": results}, ensure_ascii=False))
def expand(expr):
expr = re.sub(r"\$\{?__(rate_interval|interval|range)\}?", "1m", expr)
return re.sub(r"\$\{?(\w+)(?::\w+)?\}?", ".*", expr)
def metric_names(expr):
stripped = re.sub(r'"[^"]*"|`[^`]*`|\{[^}]*\}|\b(by|without|on|ignoring|group_left|group_right)\s*\([^)]*\)', " ", expr)
stripped = re.sub(r"\[[^\]]*\]", " ", stripped)
return {t for t in re.findall(r"[a-zA-Z_:][a-zA-Z0-9_:]*", stripped) if ("_" in t or ":" in t) and t not in PROMQL_WORDS}
def walk_panels(dashboard):
for panel in dashboard.get("panels", []):
yield panel
yield from panel.get("panels", [])
def dashboards_check(args, b):
known = set(get_json(b.prometheus + "/api/v1/label/__name__/values")["data"])
problems, report = [], []
for path in sorted(Path(args.dir).glob("*.json")):
dashboard = json.loads(path.read_text())
for panel in walk_panels(dashboard):
for target in panel.get("targets", []):
uid = (target.get("datasource") or panel.get("datasource") or {}).get("uid")
entry = {"dashboard": path.name, "panel": panel.get("title"), "datasource": uid}
try:
if uid == "prometheus":
expr = expand(target["expr"])
result = b.promql(expr)
missing = sorted(metric_names(expr) - known)
entry.update(series=len(result), missing_metrics=missing)
if missing and args.strict:
problems.append(f"{path.name}: {panel.get('title')}: missing {missing}")
elif uid == "loki":
entry["lines"] = len(b.logs(expand(target["expr"]), time.time() - 3600, time.time(), limit=5))
elif uid == "tempo":
entry["traces"] = len(b.traceql(expand(target["query"]), time.time() - 3600, time.time(), limit=5))
except Failure as error:
problems.append(f"{path.name}: {panel.get('title')}: {error}")
report.append(entry)
print(json.dumps({"checked_at": now(), "targets": report, "problems": problems}, ensure_ascii=False, indent=2))
if problems:
raise Failure(f"{len(problems)} dashboard queries failed")
def grafana_check(args, b):
headers = b.grafana_headers()
health = get_json(b.grafana + "/api/health")
sources = {}
for uid in ("prometheus", "loki", "tempo"):
status, body, _, _ = http("GET", b.grafana + f"/api/datasources/uid/{uid}/health", headers=headers)
sources[uid] = {"http": status, "status": (body or {}).get("status"), "message": (body or {}).get("message")}
found = get_json(b.grafana + "/api/search", headers, {"type": "dash-db"})
boards = {}
for item in found:
status, body, _, _ = http("GET", b.grafana + f"/api/dashboards/uid/{item['uid']}", headers=headers)
meta = (body or {}).get("meta", {})
boards[item["uid"]] = {"title": item["title"], "http": status, "provisioned": meta.get("provisioned"),
"folder": meta.get("folderTitle"), "panels": len(list(walk_panels((body or {}).get("dashboard", {}))))}
expected = {json.loads(p.read_text())["uid"] for p in Path(args.dir).glob("*.json")}
print(json.dumps({"checked_at": now(), "grafana": health.get("version"), "datasources": sources, "dashboards": boards,
"missing_dashboards": sorted(expected - set(boards))}, ensure_ascii=False, indent=2))
if expected - set(boards) or any(s["http"] != 200 or s["status"] != "OK" for s in sources.values()):
raise Failure("grafana provisioning incomplete")
def alerts_watch(args, b):
want_state, _, wanted = args.until.partition(":") if args.until else ("", "", "")
names = set(filter(None, wanted.split("|")))
only = {n for value in args.only for n in value.split("|")}
deadline = time.monotonic() + args.timeout
last = None
output = open(args.output, "a") if args.output else None
while True:
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
state = sorted((a["labels"]["alertname"], a["state"], a.get("activeAt", "")) for a in alerts
if not only or a["labels"]["alertname"] in only)
if state != last:
line = json.dumps({"ts": now(), "alerts": [{"alertname": n, "state": s, "active_at": at} for n, s, at in state]}, ensure_ascii=False)
print(line, flush=True)
if output:
output.write(line + "\n")
output.flush()
last = state
current = {n: s for n, s, _ in state}
firing = {n for n in names if current.get(n) == "firing"}
if names and ((want_state == "firing" and firing) or (want_state == "resolved" and not firing)):
return
if time.monotonic() > deadline:
raise Failure(f"timeout waiting for {args.until}")
time.sleep(args.interval)
def main():
parser = argparse.ArgumentParser(description="Telemetry checks through public HTTP APIs of the messenger and of the observability stack")
parser.add_argument("--prometheus", default=os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090"))
parser.add_argument("--loki", default=os.environ.get("LOKI_URL", "http://127.0.0.1:3100"))
parser.add_argument("--tempo", default=os.environ.get("TEMPO_URL", "http://127.0.0.1:3200"))
parser.add_argument("--grafana", default=os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000"))
parser.add_argument("--grafana-password-file", default=os.environ.get("GRAFANA_PASSWORD_FILE", str(ROOT / "secrets/grafana_admin_password")))
parser.add_argument("--timeout", type=float, default=60)
sub = parser.add_subparsers(dest="command", required=True)
p = sub.add_parser("probe", help="send a synthetic OTLP trace, log and metric and find them in every backend and in Grafana")
p.add_argument("--otlp", default=os.environ.get("OTLP_URL", "http://127.0.0.1:4318"))
p.add_argument("--service-name", default="obs-probe")
p.add_argument("--no-grafana", action="store_true")
for name, fn in (("request", request_check), ("job", job_check), ("leaks", leaks_check), ("uploads", uploads_burst)):
q = sub.add_parser(name)
q.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
q.add_argument("--api-service", default="messenger-api")
q.add_argument("--worker-service", default="messenger-worker")
q.add_argument("--service-regex", default="messenger-.*")
q.add_argument("--log-request-id-key", default="request_id")
q.add_argument("--log-attachment-key", default="attachment_id")
q.add_argument("--attachment-attr", default="messenger.attachment_id")
q.add_argument("--attempt-attr", default="messenger.job.attempt")
q.add_argument("--image", default=str(ROOT / "tests/fixtures/sample.png"))
q.add_argument("--flush", type=float, default=15)
q.add_argument("--count", type=int, default=10)
q.add_argument("--interval", type=float, default=1)
q.set_defaults(fn=fn)
q = sub.add_parser("query", help="print instant PromQL results with a timestamp")
q.add_argument("expr", nargs="+")
q.set_defaults(fn=query)
for name, fn in (("dashboards", dashboards_check), ("grafana", grafana_check)):
q = sub.add_parser(name)
q.add_argument("--dir", default=str(ROOT / "infra/grafana/dashboards"))
q.add_argument("--strict", action="store_true", help="fail when a referenced metric does not exist yet")
q.set_defaults(fn=fn)
w = sub.add_parser("alerts", help="record alert state changes until firing:<a|b> or resolved:<a|b>")
w.add_argument("--until", default="")
w.add_argument("--only", nargs="*", default=[])
w.add_argument("--interval", type=float, default=5)
w.add_argument("--output", default="")
p.set_defaults(fn=probe)
w.set_defaults(fn=alerts_watch)
args = parser.parse_args()
try:
args.fn(args, Backends(args))
except (Failure, OSError, KeyError, ValueError) as error:
print(f"FAIL: {error}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()