forked from mtusi2026/mtusi2026lab6
fix: validate telemetry outage bounds
This commit is contained in:
@@ -58,6 +58,13 @@ service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
|
||||
@@ -1010,7 +1010,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)",
|
||||
"expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001)",
|
||||
"legendFormat": "ready in 30s",
|
||||
"refId": "A"
|
||||
}
|
||||
@@ -1045,17 +1045,17 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "5m burn",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "1h burn",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
|
||||
"expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01",
|
||||
"legendFormat": "6h burn",
|
||||
"refId": "C"
|
||||
}
|
||||
@@ -1410,12 +1410,12 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))",
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m]))",
|
||||
"legendFormat": "{{exporter}} spans failed/s",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))",
|
||||
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m]))",
|
||||
"legendFormat": "{{exporter}} logs failed/s",
|
||||
"refId": "B"
|
||||
},
|
||||
|
||||
@@ -18,7 +18,12 @@ QUERIES = {
|
||||
"worker_capacity": 'sum(messenger_attachment_worker_capacity)',
|
||||
"pending_outbox": 'sum(messenger_outbox_pending)',
|
||||
"worker_up": 'min(up{job="attachment-worker"})',
|
||||
"collector_export_failures_5m": 'sum(increase(otelcol_exporter_send_failed_spans_total[5m])) + sum(increase(otelcol_exporter_send_failed_log_records_total[5m]))',
|
||||
"collector_queue_size": 'max by (exporter) (otelcol_exporter_queue_size)',
|
||||
"collector_queue_capacity": 'max by (exporter) (otelcol_exporter_queue_capacity)',
|
||||
"collector_export_failures_total": 'sum by (exporter) (otelcol_exporter_send_failed_spans) or sum by (exporter) (otelcol_exporter_send_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))',
|
||||
"collector_dropped_records_total": 'sum by (exporter) (otelcol_exporter_enqueue_failed_spans) or sum by (exporter) (otelcol_exporter_enqueue_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))',
|
||||
"collector_memory_bytes": 'max(otelcol_process_memory_rss) or max(process_resident_memory_bytes{job="otel-collector"})',
|
||||
"collector_memory_peak_bytes": 'max(max_over_time(otelcol_process_memory_rss[2m])) or max(max_over_time(process_resident_memory_bytes{job="otel-collector"}[2m]))',
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -14,6 +15,105 @@ def check(condition, message, errors):
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def query_scalar(snapshot, name, source, errors):
|
||||
query = snapshot.get("queries", {}).get(name, {})
|
||||
check("error" not in query, source + " query failed: " + name, errors)
|
||||
series = query.get("series", [])
|
||||
check(len(series) == 1, source + " query must return one series: " + name, errors)
|
||||
if len(series) != 1:
|
||||
return None
|
||||
try:
|
||||
value = float(series[0]["value"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
check(False, source + " query returned a non-numeric value: " + name, errors)
|
||||
return None
|
||||
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
|
||||
return value if math.isfinite(value) else None
|
||||
|
||||
|
||||
def query_exporters(snapshot, name, source, errors):
|
||||
expected = {"otlp_grpc/tempo", "otlp_http/loki"}
|
||||
query = snapshot.get("queries", {}).get(name, {})
|
||||
check("error" not in query, source + " query failed: " + name, errors)
|
||||
values = {}
|
||||
for series in query.get("series", []):
|
||||
exporter = series.get("labels", {}).get("exporter")
|
||||
check(exporter in expected, source + " query returned unexpected exporter: " + name, errors)
|
||||
check(exporter not in values, source + " query returned duplicate exporter: " + name, errors)
|
||||
try:
|
||||
value = float(series["value"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
check(False, source + " query returned a non-numeric value: " + name, errors)
|
||||
continue
|
||||
check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors)
|
||||
if exporter in expected and exporter not in values and math.isfinite(value):
|
||||
values[exporter] = value
|
||||
check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors)
|
||||
return values if set(values) == expected else None
|
||||
|
||||
|
||||
def validate_telemetry_outage(directory, errors):
|
||||
phase_names = ("baseline", "degraded", "recovered")
|
||||
exporter_metric_names = (
|
||||
"collector_queue_size",
|
||||
"collector_queue_capacity",
|
||||
"collector_export_failures_total",
|
||||
"collector_dropped_records_total",
|
||||
)
|
||||
scalar_metric_names = (
|
||||
"collector_memory_bytes",
|
||||
"collector_memory_peak_bytes",
|
||||
)
|
||||
phases = {}
|
||||
for phase in phase_names:
|
||||
path = directory / (phase + ".json")
|
||||
check(path.exists(), str(path) + " is missing", errors)
|
||||
if not path.exists():
|
||||
return
|
||||
snapshot = load(path)
|
||||
phases[phase] = {
|
||||
**{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names},
|
||||
**{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names},
|
||||
}
|
||||
if any(value is None for values in phases.values() for value in values.values()):
|
||||
return
|
||||
|
||||
for phase, values in phases.items():
|
||||
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
|
||||
capacity = values["collector_queue_capacity"][exporter]
|
||||
size = values["collector_queue_size"][exporter]
|
||||
check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors)
|
||||
check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors)
|
||||
check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors)
|
||||
check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors)
|
||||
|
||||
baseline = phases["baseline"]
|
||||
degraded = phases["degraded"]
|
||||
recovered = phases["recovered"]
|
||||
for exporter in ("otlp_grpc/tempo", "otlp_http/loki"):
|
||||
for name in ("collector_export_failures_total", "collector_dropped_records_total"):
|
||||
check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors)
|
||||
check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors)
|
||||
outage_observed = (
|
||||
degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter]
|
||||
or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter]
|
||||
or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter]
|
||||
)
|
||||
check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors)
|
||||
drained_limit = max(
|
||||
baseline["collector_queue_size"][exporter],
|
||||
recovered["collector_queue_capacity"][exporter] * 0.05,
|
||||
)
|
||||
check(
|
||||
recovered["collector_queue_size"][exporter] <= drained_limit,
|
||||
"collector queue did not drain after recovery: " + exporter,
|
||||
errors,
|
||||
)
|
||||
memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024)
|
||||
check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors)
|
||||
check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence")
|
||||
parser.add_argument("directory")
|
||||
@@ -77,6 +177,7 @@ def main():
|
||||
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors)
|
||||
check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors)
|
||||
check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors)
|
||||
validate_telemetry_outage(summary_path.parent, errors)
|
||||
|
||||
if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"):
|
||||
for name in ("alert-firing.json", "alert-resolved.json"):
|
||||
|
||||
Reference in New Issue
Block a user