From c9155474491d0307227085ccf062092c44ef68fb Mon Sep 17 00:00:00 2001 From: dizi Date: Wed, 4 Nov 2026 20:56:35 +0300 Subject: [PATCH] fix: validate telemetry outage bounds --- infra/otel/collector.yaml | 7 ++ .../grafana/dashboards/messenger-lab6.json | 12 +-- scripts/lab6-observe.py | 7 +- scripts/validate-lab6-evidence.py | 101 ++++++++++++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index e912195..b22fa0e 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -58,6 +58,13 @@ service: telemetry: logs: level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 pipelines: metrics: receivers: [otlp] diff --git a/monitoring/grafana/dashboards/messenger-lab6.json b/monitoring/grafana/dashboards/messenger-lab6.json index f20cabe..d3b39d9 100644 --- a/monitoring/grafana/dashboards/messenger-lab6.json +++ b/monitoring/grafana/dashboards/messenger-lab6.json @@ -1010,7 +1010,7 @@ }, "targets": [ { - "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001)", + "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001)", "legendFormat": "ready in 30s", "refId": "A" } @@ -1045,17 +1045,17 @@ }, "targets": [ { - "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", "legendFormat": "5m burn", "refId": "A" }, { - "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", "legendFormat": "1h burn", "refId": "B" }, { - "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_worker_jobs_total{result=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", "legendFormat": "6h burn", "refId": "C" } @@ -1410,12 +1410,12 @@ }, "targets": [ { - "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m]))", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m]))", "legendFormat": "{{exporter}} spans failed/s", "refId": "A" }, { - "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m]))", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m]))", "legendFormat": "{{exporter}} logs failed/s", "refId": "B" }, diff --git a/scripts/lab6-observe.py b/scripts/lab6-observe.py index 4614176..6ff2489 100644 --- a/scripts/lab6-observe.py +++ b/scripts/lab6-observe.py @@ -18,7 +18,12 @@ QUERIES = { "worker_capacity": 'sum(messenger_attachment_worker_capacity)', "pending_outbox": 'sum(messenger_outbox_pending)', "worker_up": 'min(up{job="attachment-worker"})', - "collector_export_failures_5m": 'sum(increase(otelcol_exporter_send_failed_spans_total[5m])) + sum(increase(otelcol_exporter_send_failed_log_records_total[5m]))', + "collector_queue_size": 'max by (exporter) (otelcol_exporter_queue_size)', + "collector_queue_capacity": 'max by (exporter) (otelcol_exporter_queue_capacity)', + "collector_export_failures_total": 'sum by (exporter) (otelcol_exporter_send_failed_spans) or sum by (exporter) (otelcol_exporter_send_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))', + "collector_dropped_records_total": 'sum by (exporter) (otelcol_exporter_enqueue_failed_spans) or sum by (exporter) (otelcol_exporter_enqueue_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))', + "collector_memory_bytes": 'max(otelcol_process_memory_rss) or max(process_resident_memory_bytes{job="otel-collector"})', + "collector_memory_peak_bytes": 'max(max_over_time(otelcol_process_memory_rss[2m])) or max(max_over_time(process_resident_memory_bytes{job="otel-collector"}[2m]))', } diff --git a/scripts/validate-lab6-evidence.py b/scripts/validate-lab6-evidence.py index 733ad02..643a6fb 100755 --- a/scripts/validate-lab6-evidence.py +++ b/scripts/validate-lab6-evidence.py @@ -2,6 +2,7 @@ import argparse import json +import math from pathlib import Path @@ -14,6 +15,105 @@ def check(condition, message, errors): errors.append(message) +def query_scalar(snapshot, name, source, errors): + query = snapshot.get("queries", {}).get(name, {}) + check("error" not in query, source + " query failed: " + name, errors) + series = query.get("series", []) + check(len(series) == 1, source + " query must return one series: " + name, errors) + if len(series) != 1: + return None + try: + value = float(series[0]["value"]) + except (KeyError, TypeError, ValueError): + check(False, source + " query returned a non-numeric value: " + name, errors) + return None + check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors) + return value if math.isfinite(value) else None + + +def query_exporters(snapshot, name, source, errors): + expected = {"otlp_grpc/tempo", "otlp_http/loki"} + query = snapshot.get("queries", {}).get(name, {}) + check("error" not in query, source + " query failed: " + name, errors) + values = {} + for series in query.get("series", []): + exporter = series.get("labels", {}).get("exporter") + check(exporter in expected, source + " query returned unexpected exporter: " + name, errors) + check(exporter not in values, source + " query returned duplicate exporter: " + name, errors) + try: + value = float(series["value"]) + except (KeyError, TypeError, ValueError): + check(False, source + " query returned a non-numeric value: " + name, errors) + continue + check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors) + if exporter in expected and exporter not in values and math.isfinite(value): + values[exporter] = value + check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors) + return values if set(values) == expected else None + + +def validate_telemetry_outage(directory, errors): + phase_names = ("baseline", "degraded", "recovered") + exporter_metric_names = ( + "collector_queue_size", + "collector_queue_capacity", + "collector_export_failures_total", + "collector_dropped_records_total", + ) + scalar_metric_names = ( + "collector_memory_bytes", + "collector_memory_peak_bytes", + ) + phases = {} + for phase in phase_names: + path = directory / (phase + ".json") + check(path.exists(), str(path) + " is missing", errors) + if not path.exists(): + return + snapshot = load(path) + phases[phase] = { + **{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names}, + **{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names}, + } + if any(value is None for values in phases.values() for value in values.values()): + return + + for phase, values in phases.items(): + for exporter in ("otlp_grpc/tempo", "otlp_http/loki"): + capacity = values["collector_queue_capacity"][exporter] + size = values["collector_queue_size"][exporter] + check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors) + check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors) + check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors) + check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors) + + baseline = phases["baseline"] + degraded = phases["degraded"] + recovered = phases["recovered"] + for exporter in ("otlp_grpc/tempo", "otlp_http/loki"): + for name in ("collector_export_failures_total", "collector_dropped_records_total"): + check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors) + check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors) + outage_observed = ( + degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter] + or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter] + or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter] + ) + check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors) + drained_limit = max( + baseline["collector_queue_size"][exporter], + recovered["collector_queue_capacity"][exporter] * 0.05, + ) + check( + recovered["collector_queue_size"][exporter] <= drained_limit, + "collector queue did not drain after recovery: " + exporter, + errors, + ) + memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024) + check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors) + check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors) + + def main(): parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence") parser.add_argument("directory") @@ -77,6 +177,7 @@ def main(): check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors) check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors) check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors) + validate_telemetry_outage(summary_path.parent, errors) if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"): for name in ("alert-firing.json", "alert-resolved.json"):