forked from mtusi2026/mtusi2026lab6
110 lines
4.1 KiB
Python
110 lines
4.1 KiB
Python
"""Re-check worker-stop alerts and a Tempo outage after the rule fix."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402
|
|
|
|
OUT = ROOT / "evidence" / "incidents.json"
|
|
|
|
|
|
def prom(expr):
|
|
url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
|
|
with urllib.request.urlopen(url, timeout=10) as response:
|
|
return json.load(response)["data"]["result"]
|
|
|
|
|
|
def alerts():
|
|
return {
|
|
item["metric"].get("alertname"): item["metric"].get("alertstate")
|
|
for item in prom("ALERTS")
|
|
}
|
|
|
|
|
|
def reload_rules():
|
|
request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST")
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
return response.status
|
|
|
|
|
|
def main():
|
|
print("reload", reload_rules(), flush=True)
|
|
client = Client("http://127.0.0.1:8080", timeout=8)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
report = {"alerts_before": alerts()}
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
|
client.attachment(alice, chat)
|
|
started = time.monotonic()
|
|
seen = {}
|
|
samples = []
|
|
while time.monotonic() - started < 130:
|
|
state = alerts()
|
|
age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})')
|
|
consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})')
|
|
sample = {
|
|
"t": round(time.monotonic() - started, 1),
|
|
"alerts": state,
|
|
"age": age[0]["value"][1] if age else None,
|
|
"api_consumers": consumers[0]["value"][1] if consumers else None,
|
|
}
|
|
samples.append(sample)
|
|
for name in ("QueueLagHigh", "WorkerConsumersZero"):
|
|
if name not in seen and state.get(name) == "firing":
|
|
seen[name] = sample["t"]
|
|
if len(seen) == 2:
|
|
break
|
|
time.sleep(5)
|
|
report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]}
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
|
cleared = {}
|
|
restore_started = time.monotonic()
|
|
while time.monotonic() - restore_started < 90:
|
|
state = alerts()
|
|
for name in ("QueueLagHigh", "WorkerConsumersZero"):
|
|
if name not in cleared and name not in state:
|
|
cleared[name] = round(time.monotonic() - restore_started, 1)
|
|
if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared:
|
|
break
|
|
time.sleep(5)
|
|
report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()}
|
|
|
|
before_failed = prom("otelcol_exporter_send_failed_spans")
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
|
counts = {}
|
|
deadline = time.monotonic() + 40
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
|
except Exception:
|
|
status = "error"
|
|
counts[str(status)] = counts.get(str(status), 0) + 1
|
|
time.sleep(0.5)
|
|
time.sleep(5)
|
|
memory = subprocess.check_output(
|
|
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
|
text=True,
|
|
).strip()
|
|
queue = prom("otelcol_exporter_queue_size")
|
|
failed = prom("otelcol_exporter_send_failed_spans")
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
|
report["tempo_down_40s"] = {
|
|
"api_results": counts,
|
|
"failed_before": [item["value"][1] for item in before_failed],
|
|
"failed_after": [item["value"][1] for item in failed],
|
|
"queue_size": [item["value"][1] for item in queue],
|
|
"collector_memory": memory,
|
|
}
|
|
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print("wrote", OUT, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|