99 lines
3.5 KiB
Python
99 lines
3.5 KiB
Python
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
|
|
from obs_check import Api, Backends, get_json, http, now, tempo_spans, wait_for
|
|
|
|
|
|
def backends():
|
|
return Backends(
|
|
argparse.Namespace(
|
|
prometheus=os.environ["PROMETHEUS_URL"],
|
|
loki=os.environ["LOKI_URL"],
|
|
tempo=os.environ["TEMPO_URL"],
|
|
grafana=os.environ["GRAFANA_URL"],
|
|
grafana_password_file=os.environ.get(
|
|
"GRAFANA_PASSWORD_FILE", "/".join(("secrets", "grafana_admin_password"))
|
|
),
|
|
)
|
|
)
|
|
|
|
|
|
def compose(*args):
|
|
return subprocess.run(
|
|
["docker", "compose", *args], check=True, capture_output=True, text=True
|
|
).stdout.strip()
|
|
|
|
|
|
def history(b):
|
|
started = time.time()
|
|
_, _, _, rid = Api(os.environ["BASE_URL"]).register()
|
|
query = '{service_name="messenger-api"} | request_id="' + rid + '"'
|
|
logs = wait_for("log", lambda: b.logs(query, started - 1, time.time() + 1), 60)
|
|
trace_id = logs[0][0]["trace_id"]
|
|
spans = wait_for("trace", lambda: tempo_spans(b.trace(trace_id)[0]), 60)
|
|
expr = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})'
|
|
before = wait_for("metrics", lambda: b.promql(expr), 60)
|
|
at = before[0]["value"][0]
|
|
compose("restart", "prometheus", "loki", "tempo", "grafana")
|
|
wait_for("Grafana", lambda: get_json(b.grafana + "/api/health"), 90)
|
|
after_logs = wait_for("retained log", lambda: b.logs(query, started - 1, time.time() + 1), 90)
|
|
after_spans = wait_for("retained trace", lambda: tempo_spans(b.trace(trace_id)[0]), 90)
|
|
after = wait_for("retained metric", lambda: b.promql(expr, at=at), 90)
|
|
if before[0]["value"][1] != after[0]["value"][1] or len(after_spans) < len(spans):
|
|
raise RuntimeError("telemetry history changed after restart")
|
|
return {
|
|
"request_id": rid,
|
|
"trace_id": trace_id,
|
|
"spans_before": len(spans),
|
|
"spans_after": len(after_spans),
|
|
"logs_after": len(after_logs),
|
|
"metric_before": before,
|
|
"metric_after": after,
|
|
}
|
|
|
|
|
|
def collector():
|
|
compose("stop", "otel-collector")
|
|
try:
|
|
status = http("GET", os.environ["BASE_URL"] + "/health/ready")[0]
|
|
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=False)
|
|
if status != 200 or result.returncode:
|
|
raise RuntimeError("application failed without collector")
|
|
replica = compose("ps", "-q", "api").splitlines()[0]
|
|
started = time.monotonic()
|
|
subprocess.run(["docker", "stop", "-t", "8", replica], check=True, capture_output=True)
|
|
elapsed = time.monotonic() - started
|
|
code = subprocess.check_output(
|
|
["docker", "inspect", "-f", "{{.State.ExitCode}}", replica], text=True
|
|
).strip()
|
|
if code != "0" or elapsed > 8:
|
|
raise RuntimeError("unbounded API shutdown")
|
|
return {
|
|
"ready": status,
|
|
"smoke": result.stdout.strip(),
|
|
"api_exit_code": code,
|
|
"api_shutdown_seconds": round(elapsed, 3),
|
|
}
|
|
finally:
|
|
compose("up", "-d", "--no-deps", "--wait", "api", "otel-collector")
|
|
subprocess.run(["scripts/lb-wait.sh"], check=True)
|
|
|
|
|
|
def main():
|
|
b = backends()
|
|
report = {
|
|
"utc": now(),
|
|
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
|
}
|
|
report["history"] = history(b)
|
|
report["collector_down"] = collector()
|
|
report["result"] = "PASS"
|
|
print(json.dumps(report, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|