Files
mtusi2026lab6/scripts/incident_followup.py
T

110 lines
4.1 KiB
Python

"""Re-check worker-stop alerts and a Tempo outage after the rule fix."""
import json
import subprocess
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
OUT = ROOT / "evidence" / "incidents.json"
def prom(expr):
url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)["data"]["result"]
def alerts():
return {
item["metric"].get("alertname"): item["metric"].get("alertstate")
for item in prom("ALERTS")
}
def reload_rules():
request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST")
with urllib.request.urlopen(request, timeout=10) as response:
return response.status
def main():
print("reload", reload_rules(), flush=True)
client = Client("http://127.0.0.1:8080", timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
report = {"alerts_before": alerts()}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
client.attachment(alice, chat)
started = time.monotonic()
seen = {}
samples = []
while time.monotonic() - started < 130:
state = alerts()
age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})')
consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})')
sample = {
"t": round(time.monotonic() - started, 1),
"alerts": state,
"age": age[0]["value"][1] if age else None,
"api_consumers": consumers[0]["value"][1] if consumers else None,
}
samples.append(sample)
for name in ("QueueLagHigh", "WorkerConsumersZero"):
if name not in seen and state.get(name) == "firing":
seen[name] = sample["t"]
if len(seen) == 2:
break
time.sleep(5)
report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
cleared = {}
restore_started = time.monotonic()
while time.monotonic() - restore_started < 90:
state = alerts()
for name in ("QueueLagHigh", "WorkerConsumersZero"):
if name not in cleared and name not in state:
cleared[name] = round(time.monotonic() - restore_started, 1)
if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared:
break
time.sleep(5)
report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()}
before_failed = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
counts = {}
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
time.sleep(0.5)
time.sleep(5)
memory = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
text=True,
).strip()
queue = prom("otelcol_exporter_queue_size")
failed = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
report["tempo_down_40s"] = {
"api_results": counts,
"failed_before": [item["value"][1] for item in before_failed],
"failed_after": [item["value"][1] for item in failed],
"queue_size": [item["value"][1] for item in queue],
"collector_memory": memory,
}
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print("wrote", OUT, flush=True)
if __name__ == "__main__":
main()