"""Re-check worker-stop alerts and a Tempo outage after the rule fix.""" import json import subprocess import sys import time import urllib.parse import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 OUT = ROOT / "evidence" / "incidents.json" def prom(expr): url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr) with urllib.request.urlopen(url, timeout=10) as response: return json.load(response)["data"]["result"] def alerts(): return { item["metric"].get("alertname"): item["metric"].get("alertstate") for item in prom("ALERTS") } def reload_rules(): request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST") with urllib.request.urlopen(request, timeout=10) as response: return response.status def main(): print("reload", reload_rules(), flush=True) client = Client("http://127.0.0.1:8080", timeout=8) alice, _bob, _eve, chat = client.bootstrap() report = {"alerts_before": alerts()} subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT) client.attachment(alice, chat) started = time.monotonic() seen = {} samples = [] while time.monotonic() - started < 130: state = alerts() age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})') consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})') sample = { "t": round(time.monotonic() - started, 1), "alerts": state, "age": age[0]["value"][1] if age else None, "api_consumers": consumers[0]["value"][1] if consumers else None, } samples.append(sample) for name in ("QueueLagHigh", "WorkerConsumersZero"): if name not in seen and state.get(name) == "firing": seen[name] = sample["t"] if len(seen) == 2: break time.sleep(5) report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]} subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT) cleared = {} restore_started = time.monotonic() while time.monotonic() - restore_started < 90: state = alerts() for name in ("QueueLagHigh", "WorkerConsumersZero"): if name not in cleared and name not in state: cleared[name] = round(time.monotonic() - restore_started, 1) if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared: break time.sleep(5) report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()} before_failed = prom("otelcol_exporter_send_failed_spans") subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT) counts = {} deadline = time.monotonic() + 40 while time.monotonic() < deadline: try: status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"] except Exception: status = "error" counts[str(status)] = counts.get(str(status), 0) + 1 time.sleep(0.5) time.sleep(5) memory = subprocess.check_output( ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"], text=True, ).strip() queue = prom("otelcol_exporter_queue_size") failed = prom("otelcol_exporter_send_failed_spans") subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT) report["tempo_down_40s"] = { "api_results": counts, "failed_before": [item["value"][1] for item in before_failed], "failed_after": [item["value"][1] for item in failed], "queue_size": [item["value"][1] for item in queue], "collector_memory": memory, } OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") print("wrote", OUT, flush=True) if __name__ == "__main__": main()