Files
mtusi2026lab6/scripts/obs_recovery.py
T

99 lines
3.5 KiB
Python

import argparse
import json
import os
import subprocess
import time
from obs_check import Api, Backends, get_json, http, now, tempo_spans, wait_for
def backends():
return Backends(
argparse.Namespace(
prometheus=os.environ["PROMETHEUS_URL"],
loki=os.environ["LOKI_URL"],
tempo=os.environ["TEMPO_URL"],
grafana=os.environ["GRAFANA_URL"],
grafana_password_file=os.environ.get(
"GRAFANA_PASSWORD_FILE", "/".join(("secrets", "grafana_admin_password"))
),
)
)
def compose(*args):
return subprocess.run(
["docker", "compose", *args], check=True, capture_output=True, text=True
).stdout.strip()
def history(b):
started = time.time()
_, _, _, rid = Api(os.environ["BASE_URL"]).register()
query = '{service_name="messenger-api"} | request_id="' + rid + '"'
logs = wait_for("log", lambda: b.logs(query, started - 1, time.time() + 1), 60)
trace_id = logs[0][0]["trace_id"]
spans = wait_for("trace", lambda: tempo_spans(b.trace(trace_id)[0]), 60)
expr = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})'
before = wait_for("metrics", lambda: b.promql(expr), 60)
at = before[0]["value"][0]
compose("restart", "prometheus", "loki", "tempo", "grafana")
wait_for("Grafana", lambda: get_json(b.grafana + "/api/health"), 90)
after_logs = wait_for("retained log", lambda: b.logs(query, started - 1, time.time() + 1), 90)
after_spans = wait_for("retained trace", lambda: tempo_spans(b.trace(trace_id)[0]), 90)
after = wait_for("retained metric", lambda: b.promql(expr, at=at), 90)
if before[0]["value"][1] != after[0]["value"][1] or len(after_spans) < len(spans):
raise RuntimeError("telemetry history changed after restart")
return {
"request_id": rid,
"trace_id": trace_id,
"spans_before": len(spans),
"spans_after": len(after_spans),
"logs_after": len(after_logs),
"metric_before": before,
"metric_after": after,
}
def collector():
compose("stop", "otel-collector")
try:
status = http("GET", os.environ["BASE_URL"] + "/health/ready")[0]
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=False)
if status != 200 or result.returncode:
raise RuntimeError("application failed without collector")
replica = compose("ps", "-q", "api").splitlines()[0]
started = time.monotonic()
subprocess.run(["docker", "stop", "-t", "8", replica], check=True, capture_output=True)
elapsed = time.monotonic() - started
code = subprocess.check_output(
["docker", "inspect", "-f", "{{.State.ExitCode}}", replica], text=True
).strip()
if code != "0" or elapsed > 8:
raise RuntimeError("unbounded API shutdown")
return {
"ready": status,
"smoke": result.stdout.strip(),
"api_exit_code": code,
"api_shutdown_seconds": round(elapsed, 3),
}
finally:
compose("up", "-d", "--no-deps", "--wait", "api", "otel-collector")
subprocess.run(["scripts/lb-wait.sh"], check=True)
def main():
b = backends()
report = {
"utc": now(),
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
}
report["history"] = history(b)
report["collector_down"] = collector()
report["result"] = "PASS"
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()