forked from mtusi2026/mtusi2026lab6
203 lines
11 KiB
Python
Executable File
203 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
|
|
|
|
def load(path):
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def check(condition, message, errors):
|
|
if not condition:
|
|
errors.append(message)
|
|
|
|
|
|
def query_scalar(snapshot, name, source, errors):
|
|
query = snapshot.get("queries", {}).get(name, {})
|
|
check("error" not in query, source + " query failed: " + name, errors)
|
|
series = query.get("series", [])
|
|
check(len(series) == 1, source + " query must return one series: " + name, errors)
|
|
if len(series) != 1:
|
|
return None
|
|
try:
|
|
value = float(series[0]["value"])
|
|
except (KeyError, TypeError, ValueError):
|
|
check(False, source + " query returned a non-numeric value: " + name, errors)
|
|
return None
|
|
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
|
|
return value if math.isfinite(value) else None
|
|
|
|
|
|
def query_exporters(snapshot, name, source, errors):
|
|
expected = {"otlp_grpc/tempo", "otlp_http/loki"}
|
|
query = snapshot.get("queries", {}).get(name, {})
|
|
check("error" not in query, source + " query failed: " + name, errors)
|
|
values = {}
|
|
for series in query.get("series", []):
|
|
exporter = series.get("labels", {}).get("exporter")
|
|
check(exporter in expected, source + " query returned unexpected exporter: " + name, errors)
|
|
check(exporter not in values, source + " query returned duplicate exporter: " + name, errors)
|
|
try:
|
|
value = float(series["value"])
|
|
except (KeyError, TypeError, ValueError):
|
|
check(False, source + " query returned a non-numeric value: " + name, errors)
|
|
continue
|
|
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
|
|
if exporter in expected and exporter not in values and math.isfinite(value):
|
|
values[exporter] = value
|
|
check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors)
|
|
return values if set(values) == expected else None
|
|
|
|
|
|
def validate_telemetry_outage(directory, errors):
|
|
phase_names = ("baseline", "degraded", "recovered")
|
|
exporter_metric_names = (
|
|
"collector_queue_size",
|
|
"collector_queue_capacity",
|
|
"collector_export_failures_total",
|
|
"collector_dropped_records_total",
|
|
)
|
|
scalar_metric_names = (
|
|
"collector_memory_bytes",
|
|
"collector_memory_peak_bytes",
|
|
)
|
|
phases = {}
|
|
for phase in phase_names:
|
|
path = directory / (phase + ".json")
|
|
check(path.exists(), str(path) + " is missing", errors)
|
|
if not path.exists():
|
|
return
|
|
snapshot = load(path)
|
|
phases[phase] = {
|
|
**{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names},
|
|
**{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names},
|
|
}
|
|
if any(value is None for values in phases.values() for value in values.values()):
|
|
return
|
|
|
|
for phase, values in phases.items():
|
|
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
|
|
capacity = values["collector_queue_capacity"][exporter]
|
|
size = values["collector_queue_size"][exporter]
|
|
check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors)
|
|
check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors)
|
|
check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors)
|
|
check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors)
|
|
|
|
baseline = phases["baseline"]
|
|
degraded = phases["degraded"]
|
|
recovered = phases["recovered"]
|
|
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
|
|
for name in ("collector_export_failures_total", "collector_dropped_records_total"):
|
|
check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors)
|
|
check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors)
|
|
outage_observed = (
|
|
degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter]
|
|
or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter]
|
|
or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter]
|
|
)
|
|
check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors)
|
|
drained_limit = max(
|
|
baseline["collector_queue_size"][exporter],
|
|
recovered["collector_queue_capacity"][exporter] * 0.05,
|
|
)
|
|
check(
|
|
recovered["collector_queue_size"][exporter] <= drained_limit,
|
|
"collector queue did not drain after recovery: " + exporter,
|
|
errors,
|
|
)
|
|
memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024)
|
|
check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors)
|
|
check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence")
|
|
parser.add_argument("directory")
|
|
parser.add_argument("--require-incidents", action="store_true")
|
|
parser.add_argument("--require-alerts", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
root = Path(args.directory)
|
|
errors = []
|
|
baseline = root / "baseline-traffic.json"
|
|
check(baseline.exists(), "baseline-traffic.json is missing", errors)
|
|
if baseline.exists():
|
|
report = load(baseline)
|
|
check(report.get("requests", 0) > 0, "baseline sent no requests", errors)
|
|
check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors)
|
|
|
|
correlation = root / "baseline-correlation.json"
|
|
check(correlation.exists(), "baseline-correlation.json is missing", errors)
|
|
if correlation.exists():
|
|
check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors)
|
|
|
|
observability = root / "baseline-observability.json"
|
|
check(observability.exists(), "baseline-observability.json is missing", errors)
|
|
if observability.exists():
|
|
firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"]
|
|
check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors)
|
|
|
|
summaries = list(root.rglob("summary.json"))
|
|
if args.require_incidents:
|
|
names = {load(path).get("incident") for path in summaries}
|
|
for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"):
|
|
check(required in names, "incident evidence is missing: " + required, errors)
|
|
|
|
for summary_path in summaries:
|
|
summary = load(summary_path)
|
|
incident = summary.get("incident")
|
|
traffic_path = summary_path.parent / "traffic.json"
|
|
check(traffic_path.exists(), str(traffic_path) + " is missing", errors)
|
|
if not traffic_path.exists():
|
|
continue
|
|
traffic = load(traffic_path)
|
|
check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors)
|
|
if incident == "worker_unavailable":
|
|
terminal = traffic.get("attachment_terminal_states") or {}
|
|
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
|
|
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors)
|
|
check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors)
|
|
check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors)
|
|
elif incident == "postgres_unavailable":
|
|
statuses = traffic.get("status_counts", {})
|
|
failures = sum(count for status, count in statuses.items() if not status.startswith("2"))
|
|
check(failures > 0, incident + " produced no observable failed requests", errors)
|
|
correlation_path = summary_path.parent / "database-correlation.json"
|
|
check(correlation_path.exists(), str(correlation_path) + " is missing", errors)
|
|
if correlation_path.exists():
|
|
check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors)
|
|
elif incident == "telemetry_backends_unavailable":
|
|
terminal = traffic.get("attachment_terminal_states") or {}
|
|
check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors)
|
|
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
|
|
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors)
|
|
check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors)
|
|
check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors)
|
|
validate_telemetry_outage(summary_path.parent, errors)
|
|
|
|
if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"):
|
|
for name in ("alert-firing.json", "alert-resolved.json"):
|
|
path = summary_path.parent / name
|
|
check(path.exists(), str(path) + " is missing", errors)
|
|
if path.exists():
|
|
check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors)
|
|
if incident == "worker_unavailable" and summary.get("profile") == "long":
|
|
for name in ("burn-fast-firing.json", "burn-slow-firing.json"):
|
|
path = summary_path.parent / name
|
|
check(path.exists(), str(path) + " is missing", errors)
|
|
if path.exists():
|
|
check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors)
|
|
|
|
result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)}
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
if errors:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|