Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files. Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
490 lines
26 KiB
Python
Executable File
490 lines
26 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROMQL_WORDS = {
|
|
"sum", "min", "max", "avg", "count", "by", "without", "on", "ignoring", "group_left", "group_right", "and", "or",
|
|
"unless", "offset", "bool", "rate", "irate", "increase", "delta", "deriv", "histogram_quantile", "label_replace",
|
|
"label_join", "absent", "absent_over_time", "vector", "scalar", "clamp_min", "clamp_max", "topk", "bottomk",
|
|
"max_over_time", "min_over_time", "avg_over_time", "sum_over_time", "count_over_time", "quantile_over_time",
|
|
"time", "timestamp", "sort", "sort_desc", "round", "abs", "le", "inf", "nan", "predict_linear", "changes", "resets",
|
|
}
|
|
|
|
|
|
class Failure(Exception):
|
|
pass
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
def http(method, url, body=None, headers=None, timeout=10, expect=None):
|
|
data = body if isinstance(body, (bytes, type(None))) else json.dumps(body).encode()
|
|
head = {"Accept": "application/json"}
|
|
if body is not None and not isinstance(body, bytes):
|
|
head["Content-Type"] = "application/json"
|
|
head.update(headers or {})
|
|
request = urllib.request.Request(url, data=data, headers=head, method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
status, raw, got = response.status, response.read(), dict(response.headers)
|
|
except urllib.error.HTTPError as error:
|
|
status, raw, got = error.code, error.read(), dict(error.headers)
|
|
if expect is not None and status not in expect:
|
|
raise Failure(f"{method} {url.split('?')[0]} -> {status}: {raw[:200]!r}")
|
|
try:
|
|
parsed = json.loads(raw) if raw else None
|
|
except ValueError:
|
|
parsed = None
|
|
return status, parsed, got, raw
|
|
|
|
|
|
def get_json(url, headers=None, params=None):
|
|
if params:
|
|
url += ("&" if "?" in url else "?") + urllib.parse.urlencode(params)
|
|
return http("GET", url, headers=headers, expect=(200,))[1]
|
|
|
|
|
|
def wait_for(what, check, timeout, interval=2.0):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
try:
|
|
result = check()
|
|
if result:
|
|
return result
|
|
except Failure:
|
|
pass
|
|
if time.monotonic() > deadline:
|
|
raise Failure(f"timeout waiting for {what}")
|
|
time.sleep(interval)
|
|
|
|
|
|
class Backends:
|
|
def __init__(self, args):
|
|
self.prometheus = args.prometheus.rstrip("/")
|
|
self.loki = args.loki.rstrip("/")
|
|
self.tempo = args.tempo.rstrip("/")
|
|
self.grafana = args.grafana.rstrip("/")
|
|
self.grafana_password_file = args.grafana_password_file
|
|
|
|
def grafana_headers(self):
|
|
password = Path(self.grafana_password_file).read_text().strip()
|
|
token = base64.b64encode(f"admin:{password}".encode()).decode()
|
|
return {"Authorization": "Basic " + token}
|
|
|
|
def promql(self, query, at=None, base=None):
|
|
params = {"query": query}
|
|
if at is not None:
|
|
params["time"] = f"{at:.3f}"
|
|
result = get_json((base or self.prometheus) + "/api/v1/query", params=params)
|
|
if result.get("status") != "success":
|
|
raise Failure(f"PromQL failed: {query}")
|
|
return result["data"]["result"]
|
|
|
|
def logs(self, query, start, end, limit=100, base=None):
|
|
params = {"query": query, "start": str(int(start * 1e9)), "end": str(int(end * 1e9)), "limit": str(limit),
|
|
"direction": "backward"}
|
|
result = get_json((base or self.loki) + "/loki/api/v1/query_range", params=params)
|
|
return [(stream["stream"], line) for stream in result["data"]["result"] for _, line in stream["values"]]
|
|
|
|
def traceql(self, query, start, end, limit=20, base=None):
|
|
params = {"q": query, "start": str(int(start)), "end": str(int(end) + 1), "limit": str(limit)}
|
|
return get_json((base or self.tempo) + "/api/search", params=params).get("traces") or []
|
|
|
|
def trace(self, trace_id, base=None):
|
|
status, body, _, raw = http("GET", (base or self.tempo) + f"/api/v2/traces/{trace_id}", expect=(200, 404))
|
|
return (body, raw.decode("utf-8", "replace")) if status == 200 else (None, "")
|
|
|
|
|
|
def otlp_attrs(values):
|
|
return [{"key": k, "value": {"intValue": str(v)} if isinstance(v, int) else {"stringValue": str(v)}}
|
|
for k, v in values.items()]
|
|
|
|
|
|
def otlp_resource(service, instance):
|
|
return {"attributes": otlp_attrs({"service.name": service, "service.version": "probe",
|
|
"service.instance.id": instance, "deployment.environment.name": "lab"})}
|
|
|
|
|
|
def tempo_spans(body):
|
|
spans = []
|
|
for batch in (body or {}).get("trace", {}).get("resourceSpans", []):
|
|
service = next((a["value"].get("stringValue") for a in batch["resource"].get("attributes", [])
|
|
if a["key"] == "service.name"), "?")
|
|
for scope in batch.get("scopeSpans", []):
|
|
for span in scope.get("spans", []):
|
|
values = {a["key"]: next(iter(a["value"].values()), None) for a in span.get("attributes", [])}
|
|
duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6
|
|
spans.append({"service": service, "name": span["name"], "kind": span.get("kind"),
|
|
"status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"),
|
|
"duration_ms": round(duration, 3), "links": len(span.get("links", [])),
|
|
"attributes": values})
|
|
return spans
|
|
|
|
|
|
def probe(args, b):
|
|
leak = "probe-secret-" + secrets.token_hex(8)
|
|
rid = uuid.uuid4().hex
|
|
trace_id, server, producer, consumer = secrets.token_hex(16), secrets.token_hex(8), secrets.token_hex(8), secrets.token_hex(8)
|
|
t0 = time.time()
|
|
ns = time.time_ns()
|
|
api, worker = args.service_name, args.service_name + "-worker"
|
|
scope = {"name": "obs-check", "version": "1"}
|
|
|
|
def span(span_id, parent, name, kind, start_ms, end_ms, values, links=()):
|
|
return {"traceId": trace_id, "spanId": span_id, "parentSpanId": parent, "name": name, "kind": kind,
|
|
"startTimeUnixNano": str(ns + start_ms * 1_000_000), "endTimeUnixNano": str(ns + end_ms * 1_000_000),
|
|
"attributes": otlp_attrs(values), "status": {"code": 1},
|
|
"links": [{"traceId": trace_id, "spanId": s} for s in links]}
|
|
|
|
traces = {"resourceSpans": [
|
|
{"resource": otlp_resource(api, api + "-1"), "scopeSpans": [{"scope": scope, "spans": [
|
|
span(server, "", "POST /api/v1/probe", 2, 0, 40, {
|
|
"http.request.method": "POST", "http.route": "/api/v1/probe", "http.response.status_code": 202,
|
|
"messenger.request_id": rid, "http.request.header.authorization": "Bearer " + leak,
|
|
"url.query": "filename=" + leak, "user.password": leak, "messenger.session.token": leak}),
|
|
span(producer, server, "probe.jobs publish", 4, 30, 35, {"messaging.operation.type": "send"})]}]},
|
|
{"resource": otlp_resource(worker, worker + "-1"), "scopeSpans": [{"scope": scope, "spans": [
|
|
span(consumer, producer, "probe.jobs process", 5, 50, 90, {"messaging.operation.type": "process"}, [producer])]}]},
|
|
]}
|
|
logs = {"resourceLogs": [{"resource": otlp_resource(api, api + "-1"), "scopeLogs": [{"scope": scope, "logRecords": [{
|
|
"timeUnixNano": str(ns), "observedTimeUnixNano": str(ns), "severityNumber": 9, "severityText": "INFO",
|
|
"body": {"stringValue": "probe.request.completed"}, "traceId": trace_id, "spanId": server,
|
|
"attributes": otlp_attrs({"event": "probe.request.completed", "request_id": rid, "authorization": "Bearer " + leak})}]}]}]}
|
|
point = {"attributes": otlp_attrs({"probe.id": rid}), "startTimeUnixNano": str(ns - 10_000_000_000), "timeUnixNano": str(ns)}
|
|
metrics = {"resourceMetrics": [{"resource": otlp_resource(api, api + "-1"), "scopeMetrics": [{"scope": scope, "metrics": [
|
|
{"name": "obs.probe.heartbeat", "unit": "{probe}", "sum": {"aggregationTemporality": 2, "isMonotonic": True,
|
|
"dataPoints": [dict(point, asInt="1")]}},
|
|
{"name": "obs.probe.latency", "unit": "s", "histogram": {"aggregationTemporality": 2, "dataPoints": [dict(
|
|
point, count="1", sum=0.04, bucketCounts=["0", "1", "0"], explicitBounds=[0.01, 0.1])]}},
|
|
]}]}]}
|
|
otlp = args.otlp.rstrip("/")
|
|
for path, payload in (("/v1/traces", traces), ("/v1/logs", logs), ("/v1/metrics", metrics)):
|
|
http("POST", otlp + path, payload, expect=(200,))
|
|
report = {"probe_id": rid, "trace_id": trace_id, "service": api, "sent_at": now()}
|
|
|
|
def tempo_ok(base=None):
|
|
body, raw = b.trace(trace_id, base)
|
|
spans = tempo_spans(body)
|
|
return {"spans": len(spans), "services": sorted({s["service"] for s in spans}), "raw": raw} if len(spans) == 3 else None
|
|
|
|
tempo = wait_for("trace in Tempo", tempo_ok, args.timeout)
|
|
found = wait_for("TraceQL search", lambda: b.traceql(f'{{ span.messenger.request_id = "{rid}" }}', t0 - 60, time.time() + 60), args.timeout)
|
|
loki_query = f'{{service_name="{api}"}} | request_id="{rid}"'
|
|
lines = wait_for("log in Loki", lambda: b.logs(loki_query, t0 - 60, time.time() + 60), args.timeout)
|
|
series = wait_for("metrics in Prometheus", lambda: (lambda r: r if len(r) == 2 else None)(b.promql(
|
|
f'{{__name__=~"obs_probe_heartbeat_total|obs_probe_latency_seconds_count", probe_id="{rid}"}}')), args.timeout)
|
|
labels = lines[0][0]
|
|
raw_logs = json.dumps(lines)
|
|
report.update({
|
|
"tempo": {"spans": tempo["spans"], "services": tempo["services"], "traceql_hits": len(found)},
|
|
"loki": {"lines": len(lines), "trace_id_matches": labels.get("trace_id") == trace_id,
|
|
"indexed_labels": sorted(k for k in labels if k in {"service_name", "deployment_environment_name"})},
|
|
"prometheus": {"series": sorted(s["metric"].get("__name__", "") for s in series),
|
|
"job": series[0]["metric"].get("job")},
|
|
"redaction": {"secret_in_tempo": leak in tempo["raw"], "secret_in_loki": leak in raw_logs,
|
|
"masked_in_tempo": "****" in tempo["raw"]},
|
|
})
|
|
if not args.no_grafana:
|
|
g = b.grafana + "/api/datasources/proxy/uid/"
|
|
headers = b.grafana_headers()
|
|
report["grafana"] = {
|
|
"prometheus": len(get_json(g + "prometheus/api/v1/query", headers, {"query": f'obs_probe_heartbeat_total{{probe_id="{rid}"}}'})["data"]["result"]),
|
|
"loki": len(get_json(g + "loki/loki/api/v1/query_range", headers, {"query": loki_query, "start": str(int((t0 - 60) * 1e9)), "end": str(int((time.time() + 60) * 1e9))})["data"]["result"]),
|
|
"tempo": http("GET", g + f"tempo/api/v2/traces/{trace_id}", headers=headers)[0],
|
|
}
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
ok = (report["tempo"]["spans"] == 3 and report["loki"]["trace_id_matches"] and len(series) == 2
|
|
and not report["redaction"]["secret_in_tempo"] and not report["redaction"]["secret_in_loki"]
|
|
and ("grafana" not in report or (report["grafana"]["prometheus"] and report["grafana"]["loki"] and report["grafana"]["tempo"] == 200)))
|
|
if not ok:
|
|
raise Failure("probe verification failed")
|
|
|
|
|
|
class Api:
|
|
def __init__(self, base):
|
|
self.base = base.rstrip("/")
|
|
|
|
def call(self, method, path, body=None, headers=None, expect=(200,)):
|
|
status, parsed, got, _ = http(method, self.base + path, body, headers, expect=expect)
|
|
return parsed, got.get("X-Request-Id") or got.get("x-request-id")
|
|
|
|
def register(self):
|
|
username = "u_obs_" + secrets.token_hex(6)
|
|
password = "T3st-" + secrets.token_hex(16)
|
|
user, rid = self.call("POST", "/api/v1/users", {"username": username, "display_name": "obs check", "password": password}, expect=(201,))
|
|
return user, username, password, rid
|
|
|
|
def login(self, username, password):
|
|
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
session, _ = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, expect=(201,))
|
|
return {"Authorization": "Bearer " + session["token"]}, session["token"]
|
|
|
|
def chat(self, auth, user):
|
|
chat, _ = self.call("POST", "/api/v1/chats", {"title": "obs", "member_ids": [user["id"]]}, auth, expect=(201,))
|
|
return chat
|
|
|
|
def logout(self, auth):
|
|
self.call("DELETE", "/api/v1/auth/sessions/current", headers=auth, expect=(204,))
|
|
|
|
|
|
def request_check(args, b):
|
|
api = Api(args.base_url)
|
|
t0 = time.time()
|
|
_, _, _, rid = api.register()
|
|
query = f'{{service_name="{args.api_service}"}} | {args.log_request_id_key}="{rid}"'
|
|
lines = wait_for("API log by request_id", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
|
|
labels, line = lines[0]
|
|
trace_id = labels.get("trace_id")
|
|
if not trace_id:
|
|
raise Failure("log record has no trace_id")
|
|
spans = wait_for("API trace in Tempo", lambda: tempo_spans(b.trace(trace_id)[0]), args.timeout)
|
|
print(json.dumps({"checked_at": now(), "request_id": rid, "log": {"line": line[:200], "trace_id": trace_id,
|
|
"span_id": labels.get("span_id"),
|
|
"service_name": labels.get("service_name"),
|
|
"severity": labels.get("severity_text") or labels.get("detected_level")},
|
|
"trace": [{k: s[k] for k in ("service", "name", "kind", "status", "duration_ms")} for s in spans]},
|
|
ensure_ascii=False, indent=2))
|
|
|
|
|
|
def job_check(args, b):
|
|
api = Api(args.base_url)
|
|
t0 = time.time()
|
|
user, username, password, _ = api.register()
|
|
auth, _ = api.login(username, password)
|
|
chat = api.chat(auth, user)
|
|
image = Path(args.image).read_bytes()
|
|
headers = dict(auth, **{"Content-Type": "image/png"})
|
|
attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=obs.png", image, headers, expect=(202,))
|
|
ready = wait_for("attachment ready", lambda: (lambda a: a if a["status"] in ("ready", "failed") else None)(
|
|
api.call("GET", f"/api/v1/attachments/{attachment['id']}", headers=auth)[0]), 60, 0.5)
|
|
key = args.attachment_attr
|
|
traces = wait_for("traces for attachment", lambda: b.traceql(f'{{ span.{key} = "{attachment["id"]}" }}', t0 - 30, time.time() + 30), args.timeout)
|
|
chain = []
|
|
for item in traces:
|
|
for span in tempo_spans(b.trace(item["traceID"])[0]):
|
|
chain.append(dict({k: span[k] for k in ("service", "name", "kind", "status", "duration_ms", "links")},
|
|
trace_id=item["traceID"], attempt=span["attributes"].get(args.attempt_attr)))
|
|
query = f'{{service_name="{args.worker_service}"}} | {args.log_attachment_key}="{attachment["id"]}"'
|
|
worker_logs = wait_for("worker logs", lambda: b.logs(query, t0 - 30, time.time() + 30), args.timeout)
|
|
api.logout(auth)
|
|
print(json.dumps({"checked_at": now(), "upload_request_id": rid, "attachment_status": ready["status"],
|
|
"traces": sorted({c["trace_id"] for c in chain}), "spans": chain,
|
|
"worker_logs": [{"line": line[:200], "trace_id": labels.get("trace_id"), "span_id": labels.get("span_id")}
|
|
for labels, line in worker_logs[:10]]}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
def leaks_check(args, b):
|
|
api = Api(args.base_url)
|
|
t0 = time.time()
|
|
user, username, password, _ = api.register()
|
|
auth, token = api.login(username, password)
|
|
chat = api.chat(auth, user)
|
|
text = "obs-leak-text-" + secrets.token_hex(8)
|
|
filename = "obs-leak-file-" + secrets.token_hex(6) + ".png"
|
|
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": text}, dict(auth, **{"Idempotency-Key": uuid.uuid4().hex}), expect=(201,))
|
|
if Path(args.image).is_file():
|
|
api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=" + urllib.parse.quote(filename),
|
|
Path(args.image).read_bytes(), dict(auth, **{"Content-Type": "image/png"}), expect=(202,))
|
|
api.logout(auth)
|
|
time.sleep(args.flush)
|
|
needles = {"password": password, "token": token, "message_text": text, "filename": filename}
|
|
end = time.time() + 5
|
|
hits = {}
|
|
for name, needle in needles.items():
|
|
lines = b.logs(f'{{service_name=~"{args.service_regex}"}} |= `{needle}`', t0 - 30, end, limit=50)
|
|
hits[name] = {"loki": len(lines)}
|
|
traces = b.traceql(f'{{ resource.service.name =~ "{args.service_regex}" }}', t0 - 30, end, limit=500)
|
|
raws = [b.trace(t["traceID"])[1] for t in traces]
|
|
for name, needle in needles.items():
|
|
hits[name]["tempo"] = sum(needle in raw for raw in raws)
|
|
report = {"checked_at": now(), "window_seconds": round(end - t0), "traces_scanned": len(raws), "hits": hits}
|
|
print(json.dumps(report, indent=2))
|
|
if any(v["loki"] or v["tempo"] for v in hits.values()):
|
|
raise Failure("secret or message body found in telemetry")
|
|
|
|
|
|
def uploads_burst(args, b):
|
|
api = Api(args.base_url)
|
|
user, username, password, _ = api.register()
|
|
auth, _ = api.login(username, password)
|
|
chat = api.chat(auth, user)
|
|
image = Path(args.image).read_bytes()
|
|
accepted = []
|
|
for index in range(args.count):
|
|
attachment, rid = api.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=burst-{index}.png", image,
|
|
dict(auth, **{"Content-Type": "image/png"}), expect=(202,))
|
|
accepted.append({"attachment_id": attachment["id"], "request_id": rid, "status": attachment["status"]})
|
|
time.sleep(args.interval)
|
|
api.logout(auth)
|
|
print(json.dumps({"ts": now(), "uploaded": accepted}, ensure_ascii=False))
|
|
|
|
|
|
def query(args, b):
|
|
results = {expr: [{"labels": {k: v for k, v in r["metric"].items() if k != "__name__"}, "value": r["value"][1]}
|
|
for r in b.promql(expr)] for expr in args.expr}
|
|
print(json.dumps({"ts": now(), "results": results}, ensure_ascii=False))
|
|
|
|
|
|
def expand(expr):
|
|
expr = re.sub(r"\$\{?__(rate_interval|interval|range)\}?", "1m", expr)
|
|
return re.sub(r"\$\{?(\w+)(?::\w+)?\}?", ".*", expr)
|
|
|
|
|
|
def metric_names(expr):
|
|
stripped = re.sub(r'"[^"]*"|`[^`]*`|\{[^}]*\}|\b(by|without|on|ignoring|group_left|group_right)\s*\([^)]*\)', " ", expr)
|
|
stripped = re.sub(r"\[[^\]]*\]", " ", stripped)
|
|
return {t for t in re.findall(r"[a-zA-Z_:][a-zA-Z0-9_:]*", stripped) if ("_" in t or ":" in t) and t not in PROMQL_WORDS}
|
|
|
|
|
|
def walk_panels(dashboard):
|
|
for panel in dashboard.get("panels", []):
|
|
yield panel
|
|
yield from panel.get("panels", [])
|
|
|
|
|
|
def dashboards_check(args, b):
|
|
known = set(get_json(b.prometheus + "/api/v1/label/__name__/values")["data"])
|
|
problems, report = [], []
|
|
for path in sorted(Path(args.dir).glob("*.json")):
|
|
dashboard = json.loads(path.read_text())
|
|
for panel in walk_panels(dashboard):
|
|
for target in panel.get("targets", []):
|
|
uid = (target.get("datasource") or panel.get("datasource") or {}).get("uid")
|
|
entry = {"dashboard": path.name, "panel": panel.get("title"), "datasource": uid}
|
|
try:
|
|
if uid == "prometheus":
|
|
expr = expand(target["expr"])
|
|
result = b.promql(expr)
|
|
missing = sorted(metric_names(expr) - known)
|
|
entry.update(series=len(result), missing_metrics=missing)
|
|
if missing and args.strict:
|
|
problems.append(f"{path.name}: {panel.get('title')}: missing {missing}")
|
|
elif uid == "loki":
|
|
entry["lines"] = len(b.logs(expand(target["expr"]), time.time() - 3600, time.time(), limit=5))
|
|
elif uid == "tempo":
|
|
entry["traces"] = len(b.traceql(expand(target["query"]), time.time() - 3600, time.time(), limit=5))
|
|
except Failure as error:
|
|
problems.append(f"{path.name}: {panel.get('title')}: {error}")
|
|
report.append(entry)
|
|
print(json.dumps({"checked_at": now(), "targets": report, "problems": problems}, ensure_ascii=False, indent=2))
|
|
if problems:
|
|
raise Failure(f"{len(problems)} dashboard queries failed")
|
|
|
|
|
|
def grafana_check(args, b):
|
|
headers = b.grafana_headers()
|
|
health = get_json(b.grafana + "/api/health")
|
|
sources = {}
|
|
for uid in ("prometheus", "loki", "tempo"):
|
|
status, body, _, _ = http("GET", b.grafana + f"/api/datasources/uid/{uid}/health", headers=headers)
|
|
sources[uid] = {"http": status, "status": (body or {}).get("status"), "message": (body or {}).get("message")}
|
|
found = get_json(b.grafana + "/api/search", headers, {"type": "dash-db"})
|
|
boards = {}
|
|
for item in found:
|
|
status, body, _, _ = http("GET", b.grafana + f"/api/dashboards/uid/{item['uid']}", headers=headers)
|
|
meta = (body or {}).get("meta", {})
|
|
boards[item["uid"]] = {"title": item["title"], "http": status, "provisioned": meta.get("provisioned"),
|
|
"folder": meta.get("folderTitle"), "panels": len(list(walk_panels((body or {}).get("dashboard", {}))))}
|
|
expected = {json.loads(p.read_text())["uid"] for p in Path(args.dir).glob("*.json")}
|
|
print(json.dumps({"checked_at": now(), "grafana": health.get("version"), "datasources": sources, "dashboards": boards,
|
|
"missing_dashboards": sorted(expected - set(boards))}, ensure_ascii=False, indent=2))
|
|
if expected - set(boards) or any(s["http"] != 200 or s["status"] != "OK" for s in sources.values()):
|
|
raise Failure("grafana provisioning incomplete")
|
|
|
|
|
|
def alerts_watch(args, b):
|
|
want_state, _, wanted = args.until.partition(":") if args.until else ("", "", "")
|
|
names = set(filter(None, wanted.split("|")))
|
|
only = {n for value in args.only for n in value.split("|")}
|
|
deadline = time.monotonic() + args.timeout
|
|
last = None
|
|
output = open(args.output, "a") if args.output else None
|
|
while True:
|
|
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
|
|
state = sorted((a["labels"]["alertname"], a["state"], a.get("activeAt", "")) for a in alerts
|
|
if not only or a["labels"]["alertname"] in only)
|
|
if state != last:
|
|
line = json.dumps({"ts": now(), "alerts": [{"alertname": n, "state": s, "active_at": at} for n, s, at in state]}, ensure_ascii=False)
|
|
print(line, flush=True)
|
|
if output:
|
|
output.write(line + "\n")
|
|
output.flush()
|
|
last = state
|
|
current = {n: s for n, s, _ in state}
|
|
firing = {n for n in names if current.get(n) == "firing"}
|
|
if names and ((want_state == "firing" and firing) or (want_state == "resolved" and not firing)):
|
|
return
|
|
if time.monotonic() > deadline:
|
|
raise Failure(f"timeout waiting for {args.until}")
|
|
time.sleep(args.interval)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Telemetry checks through public HTTP APIs of the messenger and of the observability stack")
|
|
parser.add_argument("--prometheus", default=os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090"))
|
|
parser.add_argument("--loki", default=os.environ.get("LOKI_URL", "http://127.0.0.1:3100"))
|
|
parser.add_argument("--tempo", default=os.environ.get("TEMPO_URL", "http://127.0.0.1:3200"))
|
|
parser.add_argument("--grafana", default=os.environ.get("GRAFANA_URL", "http://127.0.0.1:3000"))
|
|
parser.add_argument("--grafana-password-file", default=os.environ.get("GRAFANA_PASSWORD_FILE", str(ROOT / "secrets/grafana_admin_password")))
|
|
parser.add_argument("--timeout", type=float, default=60)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
p = sub.add_parser("probe", help="send a synthetic OTLP trace, log and metric and find them in every backend and in Grafana")
|
|
p.add_argument("--otlp", default=os.environ.get("OTLP_URL", "http://127.0.0.1:4318"))
|
|
p.add_argument("--service-name", default="obs-probe")
|
|
p.add_argument("--no-grafana", action="store_true")
|
|
for name, fn in (("request", request_check), ("job", job_check), ("leaks", leaks_check), ("uploads", uploads_burst)):
|
|
q = sub.add_parser(name)
|
|
q.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
q.add_argument("--api-service", default="messenger-api")
|
|
q.add_argument("--worker-service", default="messenger-worker")
|
|
q.add_argument("--service-regex", default="messenger-.*")
|
|
q.add_argument("--log-request-id-key", default="request_id")
|
|
q.add_argument("--log-attachment-key", default="attachment_id")
|
|
q.add_argument("--attachment-attr", default="messenger.attachment_id")
|
|
q.add_argument("--attempt-attr", default="messenger.job.attempt")
|
|
q.add_argument("--image", default=str(ROOT / "tests/fixtures/sample.png"))
|
|
q.add_argument("--flush", type=float, default=15)
|
|
q.add_argument("--count", type=int, default=10)
|
|
q.add_argument("--interval", type=float, default=1)
|
|
q.set_defaults(fn=fn)
|
|
q = sub.add_parser("query", help="print instant PromQL results with a timestamp")
|
|
q.add_argument("expr", nargs="+")
|
|
q.set_defaults(fn=query)
|
|
for name, fn in (("dashboards", dashboards_check), ("grafana", grafana_check)):
|
|
q = sub.add_parser(name)
|
|
q.add_argument("--dir", default=str(ROOT / "infra/grafana/dashboards"))
|
|
q.add_argument("--strict", action="store_true", help="fail when a referenced metric does not exist yet")
|
|
q.set_defaults(fn=fn)
|
|
w = sub.add_parser("alerts", help="record alert state changes until firing:<a|b> or resolved:<a|b>")
|
|
w.add_argument("--until", default="")
|
|
w.add_argument("--only", nargs="*", default=[])
|
|
w.add_argument("--interval", type=float, default=5)
|
|
w.add_argument("--output", default="")
|
|
p.set_defaults(fn=probe)
|
|
w.set_defaults(fn=alerts_watch)
|
|
args = parser.parse_args()
|
|
try:
|
|
args.fn(args, Backends(args))
|
|
except (Failure, OSError, KeyError, ValueError) as error:
|
|
print(f"FAIL: {error}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|