Files
mtusi2026lab7/scripts/validate-lab6-evidence.py
T
2026-11-01 17:59:39 +03:00

102 lines
5.4 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
def load(path):
return json.loads(path.read_text(encoding="utf-8"))
def check(condition, message, errors):
if not condition:
errors.append(message)
def main():
parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence")
parser.add_argument("directory")
parser.add_argument("--require-incidents", action="store_true")
parser.add_argument("--require-alerts", action="store_true")
args = parser.parse_args()
root = Path(args.directory)
errors = []
baseline = root / "baseline-traffic.json"
check(baseline.exists(), "baseline-traffic.json is missing", errors)
if baseline.exists():
report = load(baseline)
check(report.get("requests", 0) > 0, "baseline sent no requests", errors)
check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors)
correlation = root / "baseline-correlation.json"
check(correlation.exists(), "baseline-correlation.json is missing", errors)
if correlation.exists():
check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors)
observability = root / "baseline-observability.json"
check(observability.exists(), "baseline-observability.json is missing", errors)
if observability.exists():
firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"]
check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors)
summaries = list(root.rglob("summary.json"))
if args.require_incidents:
names = {load(path).get("incident") for path in summaries}
for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"):
check(required in names, "incident evidence is missing: " + required, errors)
for summary_path in summaries:
summary = load(summary_path)
incident = summary.get("incident")
traffic_path = summary_path.parent / "traffic.json"
check(traffic_path.exists(), str(traffic_path) + " is missing", errors)
if not traffic_path.exists():
continue
traffic = load(traffic_path)
check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors)
if incident == "worker_unavailable":
terminal = traffic.get("attachment_terminal_states") or {}
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors)
check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors)
check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors)
elif incident == "postgres_unavailable":
statuses = traffic.get("status_counts", {})
failures = sum(count for status, count in statuses.items() if not status.startswith("2"))
check(failures > 0, incident + " produced no observable failed requests", errors)
correlation_path = summary_path.parent / "database-correlation.json"
check(correlation_path.exists(), str(correlation_path) + " is missing", errors)
if correlation_path.exists():
check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors)
elif incident == "telemetry_backends_unavailable":
terminal = traffic.get("attachment_terminal_states") or {}
check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors)
check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors)
check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors)
check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors)
check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors)
if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"):
for name in ("alert-firing.json", "alert-resolved.json"):
path = summary_path.parent / name
check(path.exists(), str(path) + " is missing", errors)
if path.exists():
check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors)
if incident == "worker_unavailable" and summary.get("profile") == "long":
for name in ("burn-fast-firing.json", "burn-slow-firing.json"):
path = summary_path.parent / name
check(path.exists(), str(path) + " is missing", errors)
if path.exists():
check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors)
result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)}
print(json.dumps(result, ensure_ascii=False, indent=2))
if errors:
raise SystemExit(1)
if __name__ == "__main__":
main()