import csv import json import math import statistics from common import EXPERIMENT, ROOT, ExperimentError, digest, load, save def percentile(values, fraction): if not values: return None return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] def json_lines(path): if not path.exists(): return [] return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] def case_metrics(directory, protocol): case = load(directory / "case.json") outcome = ( load(directory / "outcome.json") if (directory / "outcome.json").exists() else {"status": "interrupted"} ) requests, verification = ( json_lines(directory / "requests.jsonl"), json_lines(directory / "verification.jsonl"), ) expected = protocol["clients"] * protocol["waves"] result = { **case, "status": outcome["status"], "expected_requests": expected, "requests_recorded": len(requests), "ready": sum(r.get("ready", False) for r in requests), "errors": sum( not r.get("ready", False) and r.get("error") != "interrupted" for r in requests ), "cancelled": sum(r.get("error") == "interrupted" for r in requests), "verified": sum(v.get("verified", False) for v in verification), "system_failures": [], "instrumentation_issues": [], "containers": [], } system, issues = result["system_failures"], result["instrumentation_issues"] if result["errors"]: system.append("upload_errors") if ( len({r["attachment"]["id"] for r in requests if r.get("ready")}) != result["ready"] ): system.append("duplicate_attachment_ids") if any( not v.get("verified", False) and v.get("error") != "interrupted" for v in verification ): system.append("original_verification_failed") warmups = json_lines(directory / "warmup.jsonl") if any( not r.get("ready", False) and r.get("error") != "interrupted" for r in warmups ): system.append("warmup_upload_errors") if len(requests) != expected or result["verified"] != result["ready"]: issues.append("missing_request_or_verification_records") if outcome["status"] != "completed": issues.append("case_" + outcome["status"]) if outcome.get("sampler_errors"): issues.append("sampler_error") before = ( load(directory / "containers-before.json") if (directory / "containers-before.json").exists() else [] ) after = ( load(directory / "containers-after.json") if (directory / "containers-after.json").exists() else [] ) for c in after: previous = next((p for p in before if p["id"] == c["id"]), None) if ( c["oom_killed"] or c["status"] != "running" or ( previous and ( c["restarts"] != previous["restarts"] or c["started_at"] != previous["started_at"] ) ) ): system.append("container_oom_restart_or_exit:" + c["service"]) if not (directory / "measurement.json").exists(): issues.append("missing_measurement") return result m = load(directory / "measurement.json") seconds = (m["finished_ns"] - m["started_ns"]) / 1e9 result.update( window_seconds=seconds, generator_cpu_cores=m["generator_cpu_seconds"] / seconds, ready_rps=result["ready"] / seconds, ) durations = [ (r["finished_ns"] - r["started_ns"]) / 1e9 for r in requests if r.get("ready") ] result["latency_seconds"] = { "p50": percentile(durations, 0.50), "p95": percentile(durations, 0.95), "p99": percentile(durations, 0.99), } spreads = [] for wave in range(protocol["waves"]): starts = [r["started_ns"] for r in requests if r["wave"] == wave] if len(starts) == protocol["clients"]: spreads.append((max(starts) - min(starts)) / 1e9) result["max_start_spread_seconds"] = max(spreads, default=None) criteria = protocol["criteria"] if len(spreads) != protocol["waves"] or any( s > criteria["start_spread_seconds_max"] for s in spreads ): issues.append("burst_start_spread") if result["generator_cpu_cores"] >= criteria["generator_cpu_cores_max"]: issues.append("possible_generator_cpu_bottleneck") instances = {r["attachment"]["instance"] for r in requests if r.get("ready")} if len(instances) != protocol["resources"]["api_replicas"]: issues.append("not_all_api_replicas_exercised") warm_instances = {r["attachment"]["instance"] for r in warmups if r.get("ready")} if len(warm_instances) != protocol["resources"]["api_replicas"]: issues.append("not_all_api_replicas_warmed") samples = json_lines(directory / "samples.jsonl") api_traffic, background = 0, 0.0 for c in before: cid = c["id"] if cid not in m["before"] or cid not in m["after"]: issues.append("missing_counter_snapshot:" + c["service"]) continue start, end = m["before"][cid], m["after"][cid] points = [ s for s in samples if s["container"] == cid and m["started_ns"] <= s["monotonic_ns"] <= m["finished_ns"] ] times = ( [m["started_ns"]] + [s["monotonic_ns"] for s in points] + [m["finished_ns"]] ) gaps = [(b - a) / 1e9 for a, b in zip(times, times[1:])] coverage = len(points) * protocol["sample_interval_seconds"] / seconds rx, tx = ( end["rx_bytes"] - start["rx_bytes"], end["tx_bytes"] - start["tx_bytes"], ) if rx < 0 or tx < 0: issues.append("network_counter_reset:" + c["service"]) idle_start, idle_end = m["idle_before"][cid], m["idle_after"][cid] idle_seconds = ( idle_end["read_started_ns"] - idle_start["read_started_ns"] ) / 1e9 idle_rate = ( (idle_end["rx_bytes"] + idle_end["tx_bytes"]) - (idle_start["rx_bytes"] + idle_start["tx_bytes"]) ) / idle_seconds cpu_usec = end["cpu.stat"]["usage_usec"] - start["cpu.stat"]["usage_usec"] peak = max( [s["memory_current"] for s in points] + [start["memory.current"], end["memory.current"]] ) events = { key: end["memory.events"].get(key, 0) - start["memory.events"].get(key, 0) for key in ("oom", "oom_kill", "max") } if end["memory.events"].get("oom", 0) or end["memory.events"].get( "oom_kill", 0 ): system.append("oom_event:" + c["service"]) if ( coverage < criteria["sampler_coverage_fraction_min"] or max(gaps) > criteria["sampler_gap_seconds_max"] ): issues.append("sampler_coverage:" + c["service"]) item = { "service": c["service"], "hostname": c["hostname"], "rx_bytes": rx, "tx_bytes": tx, "cpu_seconds": cpu_usec / 1e6, "cpu_fraction_of_limit": cpu_usec / 1e6 / seconds / (c["nano_cpus"] / 1e9), "throttled_seconds": ( end["cpu.stat"].get("throttled_usec", 0) - start["cpu.stat"].get("throttled_usec", 0) ) / 1e6, "throttled_periods": end["cpu.stat"].get("nr_throttled", 0) - start["cpu.stat"].get("nr_throttled", 0), "baseline_memory": start["memory.current"], "sampled_peak_window": peak, "peak_lifetime_before": start["memory.peak"], "peak_lifetime_after": end["memory.peak"], "memory_limit": end["memory.max"], "memory_events_delta": events, "idle_bytes_per_second": idle_rate, "sample_count": len(points), "sample_coverage": coverage, "sample_gap_max_seconds": max(gaps), } result["containers"].append(item) if c["service"] == "api": api_traffic += rx + tx background += idle_rate * seconds if end["memory.peak"] >= criteria["api_peak_strictly_below_bytes"]: system.append("api_lifetime_peak_reached_limit") if sum(c["service"] == "api" for c in result["containers"]) != 2: issues.append("missing_api_counters") result["api_rx_tx_bytes"] = api_traffic result["api_bytes_per_ready"] = ( api_traffic / result["ready"] if result["ready"] else None ) api_metrics = [c for c in result["containers"] if c["service"] == "api"] result["api_sampled_peak_window_max"] = max( (c["sampled_peak_window"] for c in api_metrics), default=None ) result["api_peak_lifetime_max"] = max( (c["peak_lifetime_after"] for c in api_metrics), default=None ) result["api_cpu_seconds_total"] = sum(c["cpu_seconds"] for c in api_metrics) result["s3_rx_tx_bytes"] = sum( c["rx_bytes"] + c["tx_bytes"] for c in result["containers"] if c["service"] == "s3" ) result["estimated_background_fraction"] = background / ( expected * case["size_bytes"] ) if ( result["estimated_background_fraction"] > criteria["idle_background_payload_fraction_max"] ): issues.append("excessive_background_traffic") result["system_failures"] = sorted(set(system)) result["instrumentation_issues"] = sorted(set(issues)) return result def decision(protocol, cases, plan): pairs, issues, failures, profiles = [], [], [], [] if len(cases) != len(plan): issues.append("not_all_planned_cases_finished") for c in cases: failures.extend(c["system_failures"]) issues.extend(c["instrumentation_issues"]) for profile in protocol["profiles"]: valid_ratios = [] for repetition in range(1, protocol["repetitions"] + 1): pair = [ c for c in cases if c["profile"] == profile["name"] and c["repetition"] == repetition ] modes = {c["mode"]: c for c in pair} if set(modes) != {"proxy", "direct"} or not all( m.get("api_bytes_per_ready") for m in modes.values() ): issues.append("incomplete_pair:" + profile["name"]) continue ratio = ( modes["direct"]["api_bytes_per_ready"] / modes["proxy"]["api_bytes_per_ready"] ) valid = all(not m["instrumentation_issues"] for m in modes.values()) if valid: valid_ratios.append(ratio) pairs.append( { "profile": profile["name"], "repetition": repetition, "direct_proxy_ratio": ratio, "instrumentation_valid": valid, } ) profiles.append( { "profile": profile["name"], "valid_pairs": len(valid_ratios), "median_traffic_ratio": statistics.median(valid_ratios) if valid_ratios else None, "min_traffic_ratio": min(valid_ratios, default=None), "max_traffic_ratio": max(valid_ratios, default=None), } ) if any( r > protocol["criteria"]["each_pair_traffic_ratio_max"] for r in valid_ratios ) or ( len(valid_ratios) == protocol["repetitions"] and statistics.median(valid_ratios) > protocol["criteria"]["median_direct_proxy_traffic_ratio_max"] ): failures.append("traffic_threshold:" + profile["name"]) if protocol["purpose"] == "pilot": status = "pilot_not_for_hypothesis" elif failures: # OOM, errors and incorrect bytes are contrary outcomes, never exclusions. status = "rejected" elif issues: status = "inconclusive" else: status = "confirmed_in_tested_conditions" return { "criterion_status": status, "system_or_threshold_failures": sorted(set(failures)), "instrumentation_issues": sorted(set(issues)), "pairs": pairs, "profiles": profiles, } def summarize(campaign): manifest = load(campaign / "source-manifest.json") analysis_sources = [EXPERIMENT / "src/analysis.py", EXPERIMENT / "src/common.py"] if any( manifest.get(str(path.relative_to(ROOT))) != digest(path) for path in analysis_sources ): raise ExperimentError( "analysis code differs from frozen snapshot; use the archived experiment source" ) protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json") cases = [] for c in plan: directory = campaign / "cases" / c["directory"] if (directory / "case.json").exists(): cases.append(case_metrics(directory, protocol)) result = { "protocol_version": protocol["version"], "purpose": protocol["purpose"], "decision": decision(protocol, cases, plan), "cases": cases, } if (campaign / "execution-failure.json").exists(): result["decision"]["instrumentation_issues"].append( "campaign_execution_failure" ) if result["decision"]["criterion_status"] == "confirmed_in_tested_conditions": result["decision"]["criterion_status"] = "inconclusive" save(campaign / "metrics.json", result) save( campaign / "analysis-provenance.json", { "source_sha256": { str(p.relative_to(ROOT)): digest(p) for p in analysis_sources }, "seal_sha256": digest(campaign / "seal.json"), }, ) with (campaign / "metrics.csv").open("w", newline="") as output: columns = [ "profile", "mode", "repetition", "ready", "errors", "verified", "api_bytes_per_ready", "api_sampled_peak_window_max", "api_peak_lifetime_max", "api_cpu_seconds_total", "s3_rx_tx_bytes", "window_seconds", "ready_rps", "generator_cpu_cores", "p50_seconds", "p95_seconds", "p99_seconds", ] writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore") writer.writeheader() writer.writerows( { **case, **{ name + "_seconds": value for name, value in case.get("latency_seconds", {}).items() }, } for case in cases ) return result