#!/usr/bin/env python3 import argparse import base64 import json import os import re import secrets import sys import time import urllib.error import urllib.parse import urllib.request import uuid from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] PROMQL_WORDS = { "sum", "min", "max", "avg", "count", "by", "without", "on", "ignoring", "group_left", "group_right", "and", "or", "unless", "offset", "bool", "rate", "irate", "increase", "delta", "deriv", "histogram_quantile", "label_replace", "label_join", "absent", "absent_over_time", "vector", "scalar", "clamp_min", "clamp_max", "topk", "bottomk", "max_over_time", "min_over_time", "avg_over_time", "sum_over_time", "count_over_time", "quantile_over_time", "time", "timestamp", "sort", "sort_desc", "round", "abs", "le", "inf", "nan", "predict_linear", "changes", "resets", } class Failure(Exception): pass def now(): return datetime.now(timezone.utc).isoformat(timespec="seconds") def http(method, url, body=None, headers=None, timeout=10, expect=None): data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode() head = {"Accept": "application/json"} if body is not None and not isinstance(body, bytes): head["Content-Type"] = "application/json" head.update(headers or {}) request = urllib.request.Request(url, data=data, headers=head, method=method) try: with urllib.request.urlopen(request, timeout=timeout) as response: status, raw, got = response.status, response.read(), dict(response.headers) except urllib.error.HTTPError as error: status, raw, got = error.code, error.read(), dict(error.headers) if expect is not None and status not in expect: raise Failure(f"{method} {url.split('?')[0]} -> {status}: {raw[:200]!r}") try: parsed = json.loads(raw) if raw else None except ValueError: parsed = None return status, parsed, got, raw def get_json(url, headers=None, params=None): if params: url += ("&" if "?" in url else "?") + urllib.parse.urlencode(params) return http("GET", url, headers=headers, expect=(200,))[1] def wait_for(what, check, timeout, interval=2.0): deadline = time.monotonic() + timeout while True: try: result = check() if result: return result except (Failure, OSError): pass if time.monotonic() > deadline: raise Failure(f"timeout waiting for {what}") time.sleep(interval) class Backends: def __init__(self, args): self.prometheus = args.prometheus.rstrip("/") self.loki = args.loki.rstrip("/") self.tempo = args.tempo.rstrip("/") self.grafana = args.grafana.rstrip("/") self.grafana_password_file = args.grafana_password_file def grafana_headers(self): password = Path(self.grafana_password_file).read_text().strip() token = base64.b64encode(f"admin:{password}".encode()).decode() return {"Authorization": "Basic " + token} def promql(self, query, at=None, base=None): params = {"query": query} if at is not None: params["time"] = f"{at:.3f}" result = get_json((base or self.prometheus) + "/api/v1/query", params=params) if result.get("status") != "success": raise Failure(f"PromQL failed: {query}") return result["data"]["result"] def logs(self, query, start, end, limit=100, base=None): params = {"query": query, "start": str(int(start * 1e9)), "end": str(int(end * 1e9)), "limit": str(limit), "direction": "backward"} result = get_json((base or self.loki) + "/loki/api/v1/query_range", params=params) return [(stream["stream"], line) for stream in result["data"]["result"] for _, line in stream["values"]] def traceql(self, query, start, end, limit=20, base=None): params = {"q": query, "start": str(int(start)), "end": str(int(end) + 1), "limit": str(limit)} return get_json((base or self.tempo) + "/api/search", params=params).get("traces") or [] def trace(self, trace_id, base=None): status, body, _, raw = http("GET", (base or self.tempo) + f"/api/v2/traces/{trace_id}", expect=(200, 404)) return (body, raw.decode("utf-8", "replace")) if status == 200 else (None, "") def otlp_attrs(values): return [{"key": k, "value": {"intValue": str(v)} if isinstance(v, int) else {"stringValue": str(v)}} for k, v in values.items()] def otlp_resource(service, instance): return {"attributes": otlp_attrs({"service.name": service, "service.version": "probe", "service.instance.id": instance, "deployment.environment.name": "lab"})} def tempo_spans(body): spans = [] for batch in (body or {}).get("trace", {}).get("resourceSpans", []): service = next((a["value"].get("stringValue") for a in batch["resource"].get("attributes", []) if a["key"] == "service.name"), "?") for scope in batch.get("scopeSpans", []): for span in scope.get("spans", []): values = {a["key"]: next(iter(a["value"].values()), None) for a in span.get("attributes", [])} duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6 spans.append({"service": service, "name": span["name"], "kind": span.get("kind"), "status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"), "span_id": span.get("spanId"), "parent_span_id": span.get("parentSpanId"), "duration_ms": round(duration, 3), "links": len(span.get("links", [])), "attributes": values}) return spans def probe(args, b): leak = "probe-secret-" + secrets.token_hex(8) rid = uuid.uuid4().hex trace_id, server, producer, consumer = secrets.token_hex(16), secrets.token_hex(8), secrets.token_hex(8), secrets.token_hex(8) t0 = time.time() ns = time.time_ns() api, worker = args.service_name, args.service_name + "-worker" scope = {"name": "obs-check", "version": "1"} def span(span_id, parent, name, kind, start_ms, end_ms, values, links=()): return {"traceId": trace_id, "spanId": span_id, "parentSpanId": parent, "name": name, "kind": kind, "startTimeUnixNano": str(ns + start_ms * 1_000_000), "endTimeUnixNano": str(ns + end_ms * 1_000_000), "attributes": otlp_attrs(values), "status": {"code": 1}, "links": [{"traceId": trace_id, "spanId": s} for s in links]} traces = {"resourceSpans": [ {"resource": otlp_resource(api, api + "-1"), "scopeSpans": [{"scope": scope, "spans": [ span(server, "", "POST /api/v1/probe", 2, 0, 40, { "http.request.method": "POST", "http.route": "/api/v1/probe", "http.response.status_code": 202, "messenger.request_id": rid, "http.request.header.authorization": "Bearer " + leak, "url.query": "filename=" + leak, "user.password": leak, "messenger.session.token": leak}), span(producer, server, "probe.jobs publish", 4, 30, 35, {"messaging.operation.type": "send"})]}]}, {"resource": otlp_resource(worker, worker + "-1"), "scopeSpans": [{"scope": scope, "spans": [ span(consumer, producer, "probe.jobs process", 5, 50, 90, {"messaging.operation.type": "process"}, [producer])]}]}, ]} logs = {"resourceLogs": [{"resource": otlp_resource(api, api + "-1"), "scopeLogs": [{"scope": scope, "logRecords": [{ "timeUnixNano": str(ns), "observedTimeUnixNano": str(ns), "severityNumber": 9, "severityText": "INFO", "body": {"stringValue": "probe.request.completed"}, "traceId": trace_id, "spanId": server, "attributes": otlp_attrs({"event": "probe.request.completed", "request_id": rid, "authorization": "Bearer " + leak})}]}]}]} point = {"attributes": otlp_attrs({"probe.id": rid}), "startTimeUnixNano": str(ns - 10_000_000_000), "timeUnixNano": str(ns)} metrics = {"resourceMetrics": [{"resource": otlp_resource(api, api + "-1"), "scopeMetrics": [{"scope": scope, "metrics": [ {"name": "obs.probe.heartbeat", "unit": "{probe}", "sum": {"aggregationTemporality": 2, "isMonotonic": True, "dataPoints": [dict(point, asInt="1")]}}, {"name": "obs.probe.latency", "unit": "s", "histogram": {"aggregationTemporality": 2, "dataPoints": [dict( point, count="1", sum=0.04, bucketCounts=["0", "1", "0"], explicitBounds=[0.01, 0.1])]}}, ]}]}]} otlp = args.otlp.rstrip("/") for path, payload in (("/v1/traces", traces), ("/v1/logs", logs), ("/v1/metrics", metrics)): http("POST", otlp + path, payload, expect=(200,)) report = {"probe_id": rid, "trace_id": trace_id, "service": api, "sent_at": now()} def tempo_ok(base=None): body, raw = b.trace(trace_id, base) spans = tempo_spans(body) return {"spans": len(spans), "services": sorted({s["service"] for s in spans}), "raw": raw} if len(spans) == 3 else None tempo = wait_for("trace in Tempo", tempo_ok, args.timeout) found = wait_for("TraceQL search", lambda: b.traceql(f'{{ span.messenger.request_id = "{rid}" }}', t0 - 60, time.time() + 60), args.timeout) loki_query = f'{{service_name="{api}"}} | request_id="{rid}"' lines = wait_for("log in Loki", lambda: b.logs(loki_query, t0 - 60, time.time() + 60), args.timeout) series = wait_for("metrics in Prometheus", lambda: (lambda r: r if len(r) == 2 else None)(b.promql( f'{{__name__=~"obs_probe_heartbeat_total|obs_probe_latency_seconds_count", probe_id="{rid}"}}')), args.timeout) labels = lines[0][0] raw_logs = json.dumps(lines) report.update({ "tempo": {"spans": tempo["spans"], "services": tempo["services"], "traceql_hits": len(found)}, "loki": {"lines": len(lines), "trace_id_matches": labels.get("trace_id") == trace_id, "indexed_labels": sorted(k for k in labels if k in {"service_name", "deployment_environment_name"})}, "prometheus": {"series": sorted(s["metric"].get("__name__", "") for s in series), "job": series[0]["metric"].get("job")}, "redaction": {"secret_in_tempo": leak in tempo["raw"], "secret_in_loki": leak in raw_logs, "masked_in_tempo": "****" in tempo["raw"]}, }) if not args.no_grafana: g = b.grafana + "/api/datasources/proxy/uid/" headers = b.grafana_headers() report["grafana"] = { "prometheus": len(get_json(g + "prometheus/api/v1/query", headers, {"query": f'obs_probe_heartbeat_total{{probe_id="{rid}"}}'})["data"]["result"]), "loki": len(get_json(g + "loki/loki/api/v1/query_range", headers, {"query": loki_query, "start": str(int((t0 - 60) * 1e9)), "end": str(int((time.time() + 60) * 1e9))})["data"]["result"]), "tempo": http("GET", g + f"tempo/api/v2/traces/{trace_id}", headers=headers)[0], } print(json.dumps(report, ensure_ascii=False, indent=2)) ok = (report["tempo"]["spans"] == 3 and report["loki"]["trace_id_matches"] and len(series) == 2 and not report["redaction"]["secret_in_tempo"] and not report["redaction"]["secret_in_loki"] and ("grafana" not in report or (report["grafana"]["prometheus"] and report["grafana"]["loki"] and report["grafana"]["tempo"] == 200))) if not ok: raise Failure("probe verification failed") class Api: def __init__(self, base): self.base = base.rstrip("/") def call(self, method, path, body=None, headers=None, expect=(200,)): status, parsed, got, _ = http(method, self.base + path, body, headers, expect=expect) return parsed, got.get("X-Request-Id") or got.get("x-request-id") def register(self): username = "u_obs_" + secrets.token_hex(6) password = "T3st-" + secrets.token_hex(16) user, rid = self.call("POST", "/api/v1/users", {"username": username, "display_name": "obs check", "password": password}, expect=(201,)) return user, username, password, rid def login(self, username, password): basic = base64.b64encode(f"{username}:{password}".encode()).decode() session, _ = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, expect=(201,)) return {"Authorization": "Bearer " + session["token"]}, session["token"] def chat(self, auth, user): chat, _ = self.call("POST", "/api/v1/chats", {"title": "obs", "member_ids": [user["id"]]}, auth, expect=(201,)) return chat def logout(self, auth): self.call("DELETE", "/api/v1/auth/sessions/current", headers=auth, expect=(204,)) def request_check(args, b): api = Api(args.base_url) t0 = time.time() _, _, _, rid = api.register() query = f'{{service_name="{args.api_service}"}} | {args.log_request_id_key}="{rid}"' lines = wait_for("API log by request_id", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout) labels, line = lines[0] trace_id = labels.get("trace_id") if not trace_id: raise Failure("log record has no trace_id") spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout) if not any(s["kind"] == "SPAN_KIND_SERVER" and s["service"] == args.api_service for s in spans): raise Failure("no real API server span") if not any(s["kind"] == "SPAN_KIND_CLIENT" and s["attributes"].get("db.system") == "postgresql" for s in spans): raise Failure("no PostgreSQL client span inside API trace") print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id, "span_id": labels.get("span_id"), "service_name": labels.get("service_name"), "severity": labels.get("severity_text") or labels.get("detected_level")}, "trace": [{k: s[k] for k in ("service", "name", "kind", "status", "duration_ms")} for s in spans]}, ensure_ascii=False, indent=2)) def job_check(args, b): api = Api(args.base_url) t0 = time.time() user, username, password, _ = api.register() auth, _ = api.login(username, password) chat = api.chat(auth, user) image = Path(args.image).read_bytes() headers = dict(auth, **{"Content-Type": "image/png"}) attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=obs.png", image, headers, expect=(202,)) ready = wait_for("attachment ready", lambda: (lambda a: a if a["status"] in ("ready", "failed") else None)( api.call("GET", f"/api/v1/attachments/{attachment['id']}", headers=auth)[0]), 60, 0.5) key = args.attachment_attr traces = wait_for("traces for attachment", lambda: b.traceql(f'{{ span.{key} = "{attachment["id"]}" }}', t0 - 30, time.time() + 30), args.timeout) chain = [] for item in traces: for span in tempo_spans(b.trace(item["traceID"])[0]): chain.append(dict({k: span[k] for k in ("service", "name", "kind", "status", "duration_ms", "links")}, trace_id=item["traceID"], attempt=span["attributes"].get(args.attempt_attr))) query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"' worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout) api.logout(auth) if ready["status"] != "ready" or not any(s["kind"] == "SPAN_KIND_CONSUMER" and s["service"] == args.worker_service for s in chain): raise Failure("job is not ready or no real consumer span") if not any(labels.get("trace_id") and labels.get("span_id") for labels, _ in worker_logs): raise Failure("worker log is not correlated") print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"], "traces": sorted({c["trace_id"] for c in chain}), "spans": chain, "worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")} for labels, line in worker_logs[:10]]}, ensure_ascii=False, indent=2)) def leaks_check(args, b): api = Api(args.base_url) t0 = time.time() user, username, password, _ = api.register() auth, token = api.login(username, password) chat = api.chat(auth, user) text = "obs-leak-text-" + secrets.token_hex(8) filename = "obs-leak-file-" + secrets.token_hex(6) + ".png" api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": text}, dict(auth, **{"Idempotency-Key": uuid.uuid4().hex}), expect=(201,)) if Path(args.image).is_file(): api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=" + urllib.parse.quote(filename), Path(args.image).read_bytes(), dict(auth, **{"Content-Type": "image/png"}), expect=(202,)) api.logout(auth) time.sleep(args.flush) needles = {"password": password, "token": token, "message_text": text, "filename": filename} end = time.time() + 5 hits = {} for name, needle in needles.items(): lines = b.logs(f'{{service_name=~"{args.service_regex}"}} |= `{needle}`', t0 - 30, end, limit=50) hits[name] = {"loki": len(lines)} traces = b.traceql(f'{{ resource.service.name =~ "{args.service_regex}" }}', t0 - 30, end, limit=500) raws = [b.trace(t["traceID"])[1] for t in traces] for name, needle in needles.items(): hits[name]["tempo"] = sum(needle in raw for raw in raws) report = {"checked_at": now(), "window_seconds": round(end - t0), "traces_scanned": len(raws), "hits": hits} print(json.dumps(report, indent=2)) if any(v["loki"] or v["tempo"] for v in hits.values()): raise Failure("secret or message body found in telemetry") def uploads_burst(args, b): api = Api(args.base_url) user, username, password, _ = api.register() auth, _ = api.login(username, password) chat = api.chat(auth, user) image = Path(args.image).read_bytes() accepted = [] for index in range(args.count): attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=burst-{index}.png", image, dict(auth, **{"Content-Type": "image/png"}), expect=(202,)) accepted.append({"attachment_id": attachment["id"], "request_id": rid, "status": attachment["status"]}) time.sleep(args.interval) api.logout(auth) print(json.dumps({"ts": now(), "uploaded": accepted}, ensure_ascii=False)) def query(args, b): results = {expr: [{"labels": {k: v for k, v in r["metric"].items() if k != "__name__"}, "value": r["value"][1]} for r in b.promql(expr)] for expr in args.expr} print(json.dumps({"ts": now(), "results": results}, ensure_ascii=False)) def expand(expr): expr = re.sub(r"\$\{?__(rate_interval|interval|range)\}?", "1m", expr) return re.sub(r"\$\{?(\w+)(?::\w+)?\}?", ".*", expr) def metric_names(expr): stripped = re.sub(r'"[^"]*"|`[^`]*`|\{[^}]*\}|\b(by|without|on|ignoring|group_left|group_right)\s*\([^)]*\)', " ", expr) stripped = re.sub(r"\[[^\]]*\]", " ", stripped) return {t for t in re.findall(r"[a-zA-Z_:][a-zA-Z0-9_:]*", stripped) if ("_" in t or ":" in t) and t not in PROMQL_WORDS} def walk_panels(dashboard): for panel in dashboard.get("panels", []): yield panel yield from panel.get("panels", []) def dashboards_check(args, b): known = set(get_json(b.prometheus + "/api/v1/label/__name__/values")["data"]) problems, report = [], [] for path in sorted(Path(args.dir).glob("*.json")): dashboard = json.loads(path.read_text()) for panel in walk_panels(dashboard): for target in panel.get("targets", []): uid = (target.get("datasource") or panel.get("datasource") or {}).get("uid") entry = {"dashboard": path.name, "panel": panel.get("title"), "datasource": uid} try: if uid == "prometheus": expr = expand(target["expr"]) result = b.promql(expr) missing = sorted(metric_names(expr) - known) entry.update(series=len(result), missing_metrics=missing) if missing and args.strict: problems.append(f"{path.name}: {panel.get('title')}: missing {missing}") elif uid == "loki": entry["lines"] = len(b.logs(expand(target["expr"]), time.time() - 3600, time.time(), limit=5)) elif uid == "tempo": entry["traces"] = len(b.traceql(expand(target["query"]), time.time() - 3600, time.time(), limit=5)) except Failure as error: problems.append(f"{path.name}: {panel.get('title')}: {error}") report.append(entry) print(json.dumps({"checked_at": now(), "targets": report, "problems": problems}, ensure_ascii=False, indent=2)) if problems: raise Failure(f"{len(problems)} dashboard queries failed") def grafana_check(args, b): headers = b.grafana_headers() health = get_json(b.grafana + "/api/health") sources = {} for uid in ("prometheus", "loki", "tempo"): status, body, _, _ = http("GET", b.grafana + f"/api/datasources/uid/{uid}/health", headers=headers) sources[uid] = {"http": status, "status": (body or {}).get("status"), "message": (body or {}).get("message")} found = get_json(b.grafana + "/api/search", headers, {"type": "dash-db"}) boards = {} for item in found: status, body, _, _ = http("GET", b.grafana + f"/api/dashboards/uid/{item['uid']}", headers=headers) meta = (body or {}).get("meta", {}) boards[item["uid"]] = {"title": item["title"], "http": status, "provisioned": meta.get("provisioned"), "folder": meta.get("folderTitle"), "panels": len(list(walk_panels((body or {}).get("dashboard", {}))))} expected = {json.loads(p.read_text())["uid"] for p in Path(args.dir).glob("*.json")} print(json.dumps({"checked_at": now(), "grafana": health.get("version"), "datasources": sources, "dashboards": boards, "missing_dashboards": sorted(expected - set(boards))}, ensure_ascii=False, indent=2)) if expected - set(boards) or any(s["http"] != 200 or s["status"] != "OK" for s in sources.values()): raise Failure("grafana provisioning incomplete") def alerts_watch(args, b): want_state, _, wanted = args.until.partition(":") if args.until else ("", "", "") names = set(filter(None, wanted.split("|"))) only = {n for value in args.only for n in value.split("|")} deadline = time.monotonic() + args.timeout last = None output = open(args.output, "a") if args.output else None while True: alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"] state = sorted((a["labels"]["alertname"], a["state"], a.get("activeAt", "")) for a in alerts if not only or a["labels"]["alertname"] in only) if state != last: line = json.dumps({"ts": now(), "alerts": [{"alertname": n, "state": s, "active_at": at} for n, s, at in state]}, ensure_ascii=False) print(line, flush=True) if output: output.write(line + "\n") output.flush() last = state current = {n: s for n, s, _ in state} firing = {n for n in names if current.get(n) == "firing"} if names and ((want_state == "firing" and firing) or (want_state == "resolved" and not firing)): return if time.monotonic() > deadline: raise Failure(f"timeout waiting for {args.until}") time.sleep(args.interval) def main(): parser = argparse.ArgumentParser(description="Telemetry checks through public HTTP APIs of the messenger and of the observability stack") parser.add_argument("--prometheus", default=os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")) parser.add_argument("--loki", default=os.environ.get("LOKI_URL", "http://127.0.0.1:3100")) parser.add_argument("--tempo", default=os.environ.get("TEMPO_URL", "http://127.0.0.1:3200")) parser.add_argument("--grafana", default=os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000")) parser.add_argument("--grafana-password-file", default=os.environ.get("GRAFANA_PASSWORD_FILE", str(ROOT / "secrets/grafana_admin_password"))) parser.add_argument("--timeout", type=float, default=60) sub = parser.add_subparsers(dest="command", required=True) p = sub.add_parser("probe", help="send a synthetic OTLP trace, log and metric and find them in every backend and in Grafana") p.add_argument("--otlp", default=os.environ.get("OTLP_URL", "http://127.0.0.1:4318")) p.add_argument("--service-name", default="obs-probe") p.add_argument("--no-grafana", action="store_true") for name, fn in (("request", request_check), ("job", job_check), ("leaks", leaks_check), ("uploads", uploads_burst)): q = sub.add_parser(name) q.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) q.add_argument("--api-service", default="messenger-api") q.add_argument("--worker-service", default="messenger-worker") q.add_argument("--service-regex", default="messenger-.*") q.add_argument("--log-request-id-key", default="request_id") q.add_argument("--log-attachment-key", default="attachment_id") q.add_argument("--attachment-attr", default="messenger.attachment_id") q.add_argument("--attempt-attr", default="messenger.job.attempt") q.add_argument("--image", default=str(ROOT / "tests/fixtures/sample.png")) q.add_argument("--flush", type=float, default=15) q.add_argument("--count", type=int, default=10) q.add_argument("--interval", type=float, default=1) q.set_defaults(fn=fn) q = sub.add_parser("query", help="print instant PromQL results with a timestamp") q.add_argument("expr", nargs="+") q.set_defaults(fn=query) for name, fn in (("dashboards", dashboards_check), ("grafana", grafana_check)): q = sub.add_parser(name) q.add_argument("--dir", default=str(ROOT / "infra/grafana/dashboards")) q.add_argument("--strict", action="store_true", help="fail when a referenced metric does not exist yet") q.set_defaults(fn=fn) w = sub.add_parser("alerts", help="record alert state changes until firing: or resolved:") w.add_argument("--until", default="") w.add_argument("--only", nargs="*", default=[]) w.add_argument("--interval", type=float, default=5) w.add_argument("--output", default="") p.set_defaults(fn=probe) w.set_defaults(fn=alerts_watch) args = parser.parse_args() try: args.fn(args, Backends(args)) except (Failure, OSError, KeyError, ValueError) as error: print(f"FAIL: {error}", file=sys.stderr) sys.exit(1) if __name__ == "__main__": main()