Files
mtusi2026lab4-fork/experiments/src/analysis.py
T

393 lines
15 KiB
Python

import csv
import json
import math
import statistics
from common import EXPERIMENT, ROOT, ExperimentError, digest, load, save
def percentile(values, fraction):
if not values:
return None
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
def json_lines(path):
if not path.exists():
return []
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
def case_metrics(directory, protocol):
case = load(directory / "case.json")
outcome = (
load(directory / "outcome.json")
if (directory / "outcome.json").exists()
else {"status": "interrupted"}
)
requests, verification = (
json_lines(directory / "requests.jsonl"),
json_lines(directory / "verification.jsonl"),
)
expected = protocol["clients"] * protocol["waves"]
result = {
**case,
"status": outcome["status"],
"expected_requests": expected,
"requests_recorded": len(requests),
"ready": sum(r.get("ready", False) for r in requests),
"errors": sum(
not r.get("ready", False) and r.get("error") != "interrupted"
for r in requests
),
"cancelled": sum(r.get("error") == "interrupted" for r in requests),
"verified": sum(v.get("verified", False) for v in verification),
"system_failures": [],
"instrumentation_issues": [],
"containers": [],
}
system, issues = result["system_failures"], result["instrumentation_issues"]
if result["errors"]:
system.append("upload_errors")
if (
len({r["attachment"]["id"] for r in requests if r.get("ready")})
!= result["ready"]
):
system.append("duplicate_attachment_ids")
if any(
not v.get("verified", False) and v.get("error") != "interrupted"
for v in verification
):
system.append("original_verification_failed")
warmups = json_lines(directory / "warmup.jsonl")
if any(
not r.get("ready", False) and r.get("error") != "interrupted" for r in warmups
):
system.append("warmup_upload_errors")
if len(requests) != expected or result["verified"] != result["ready"]:
issues.append("missing_request_or_verification_records")
if outcome["status"] != "completed":
issues.append("case_" + outcome["status"])
if outcome.get("sampler_errors"):
issues.append("sampler_error")
before = (
load(directory / "containers-before.json")
if (directory / "containers-before.json").exists()
else []
)
after = (
load(directory / "containers-after.json")
if (directory / "containers-after.json").exists()
else []
)
for c in after:
previous = next((p for p in before if p["id"] == c["id"]), None)
if (
c["oom_killed"]
or c["status"] != "running"
or (
previous
and (
c["restarts"] != previous["restarts"]
or c["started_at"] != previous["started_at"]
)
)
):
system.append("container_oom_restart_or_exit:" + c["service"])
if not (directory / "measurement.json").exists():
issues.append("missing_measurement")
return result
m = load(directory / "measurement.json")
seconds = (m["finished_ns"] - m["started_ns"]) / 1e9
result.update(
window_seconds=seconds,
generator_cpu_cores=m["generator_cpu_seconds"] / seconds,
ready_rps=result["ready"] / seconds,
)
durations = [
(r["finished_ns"] - r["started_ns"]) / 1e9 for r in requests if r.get("ready")
]
result["latency_seconds"] = {
"p50": percentile(durations, 0.50),
"p95": percentile(durations, 0.95),
"p99": percentile(durations, 0.99),
}
spreads = []
for wave in range(protocol["waves"]):
starts = [r["started_ns"] for r in requests if r["wave"] == wave]
if len(starts) == protocol["clients"]:
spreads.append((max(starts) - min(starts)) / 1e9)
result["max_start_spread_seconds"] = max(spreads, default=None)
criteria = protocol["criteria"]
if len(spreads) != protocol["waves"] or any(
s > criteria["start_spread_seconds_max"] for s in spreads
):
issues.append("burst_start_spread")
if result["generator_cpu_cores"] >= criteria["generator_cpu_cores_max"]:
issues.append("possible_generator_cpu_bottleneck")
instances = {r["attachment"]["instance"] for r in requests if r.get("ready")}
if len(instances) != protocol["resources"]["api_replicas"]:
issues.append("not_all_api_replicas_exercised")
warm_instances = {r["attachment"]["instance"] for r in warmups if r.get("ready")}
if len(warm_instances) != protocol["resources"]["api_replicas"]:
issues.append("not_all_api_replicas_warmed")
samples = json_lines(directory / "samples.jsonl")
api_traffic, background = 0, 0.0
for c in before:
cid = c["id"]
if cid not in m["before"] or cid not in m["after"]:
issues.append("missing_counter_snapshot:" + c["service"])
continue
start, end = m["before"][cid], m["after"][cid]
points = [
s
for s in samples
if s["container"] == cid
and m["started_ns"] <= s["monotonic_ns"] <= m["finished_ns"]
]
times = (
[m["started_ns"]] + [s["monotonic_ns"] for s in points] + [m["finished_ns"]]
)
gaps = [(b - a) / 1e9 for a, b in zip(times, times[1:])]
coverage = len(points) * protocol["sample_interval_seconds"] / seconds
rx, tx = (
end["rx_bytes"] - start["rx_bytes"],
end["tx_bytes"] - start["tx_bytes"],
)
if rx < 0 or tx < 0:
issues.append("network_counter_reset:" + c["service"])
idle_start, idle_end = m["idle_before"][cid], m["idle_after"][cid]
idle_seconds = (
idle_end["read_started_ns"] - idle_start["read_started_ns"]
) / 1e9
idle_rate = (
(idle_end["rx_bytes"] + idle_end["tx_bytes"])
- (idle_start["rx_bytes"] + idle_start["tx_bytes"])
) / idle_seconds
cpu_usec = end["cpu.stat"]["usage_usec"] - start["cpu.stat"]["usage_usec"]
peak = max(
[s["memory_current"] for s in points]
+ [start["memory.current"], end["memory.current"]]
)
events = {
key: end["memory.events"].get(key, 0) - start["memory.events"].get(key, 0)
for key in ("oom", "oom_kill", "max")
}
if end["memory.events"].get("oom", 0) or end["memory.events"].get(
"oom_kill", 0
):
system.append("oom_event:" + c["service"])
if (
coverage < criteria["sampler_coverage_fraction_min"]
or max(gaps) > criteria["sampler_gap_seconds_max"]
):
issues.append("sampler_coverage:" + c["service"])
item = {
"service": c["service"],
"hostname": c["hostname"],
"rx_bytes": rx,
"tx_bytes": tx,
"cpu_seconds": cpu_usec / 1e6,
"cpu_fraction_of_limit": cpu_usec / 1e6 / seconds / (c["nano_cpus"] / 1e9),
"throttled_seconds": (
end["cpu.stat"].get("throttled_usec", 0)
- start["cpu.stat"].get("throttled_usec", 0)
)
/ 1e6,
"throttled_periods": end["cpu.stat"].get("nr_throttled", 0)
- start["cpu.stat"].get("nr_throttled", 0),
"baseline_memory": start["memory.current"],
"sampled_peak_window": peak,
"peak_lifetime_before": start["memory.peak"],
"peak_lifetime_after": end["memory.peak"],
"memory_limit": end["memory.max"],
"memory_events_delta": events,
"idle_bytes_per_second": idle_rate,
"sample_count": len(points),
"sample_coverage": coverage,
"sample_gap_max_seconds": max(gaps),
}
result["containers"].append(item)
if c["service"] == "api":
api_traffic += rx + tx
background += idle_rate * seconds
if end["memory.peak"] >= criteria["api_peak_strictly_below_bytes"]:
system.append("api_lifetime_peak_reached_limit")
if sum(c["service"] == "api" for c in result["containers"]) != 2:
issues.append("missing_api_counters")
result["api_rx_tx_bytes"] = api_traffic
result["api_bytes_per_ready"] = (
api_traffic / result["ready"] if result["ready"] else None
)
api_metrics = [c for c in result["containers"] if c["service"] == "api"]
result["api_sampled_peak_window_max"] = max(
(c["sampled_peak_window"] for c in api_metrics), default=None
)
result["api_peak_lifetime_max"] = max(
(c["peak_lifetime_after"] for c in api_metrics), default=None
)
result["api_cpu_seconds_total"] = sum(c["cpu_seconds"] for c in api_metrics)
result["s3_rx_tx_bytes"] = sum(
c["rx_bytes"] + c["tx_bytes"]
for c in result["containers"]
if c["service"] == "s3"
)
result["estimated_background_fraction"] = background / (
expected * case["size_bytes"]
)
if (
result["estimated_background_fraction"]
> criteria["idle_background_payload_fraction_max"]
):
issues.append("excessive_background_traffic")
result["system_failures"] = sorted(set(system))
result["instrumentation_issues"] = sorted(set(issues))
return result
def decision(protocol, cases, plan):
pairs, issues, failures, profiles = [], [], [], []
if len(cases) != len(plan):
issues.append("not_all_planned_cases_finished")
for c in cases:
failures.extend(c["system_failures"])
issues.extend(c["instrumentation_issues"])
for profile in protocol["profiles"]:
valid_ratios = []
for repetition in range(1, protocol["repetitions"] + 1):
pair = [
c
for c in cases
if c["profile"] == profile["name"] and c["repetition"] == repetition
]
modes = {c["mode"]: c for c in pair}
if set(modes) != {"proxy", "direct"} or not all(
m.get("api_bytes_per_ready") for m in modes.values()
):
issues.append("incomplete_pair:" + profile["name"])
continue
ratio = (
modes["direct"]["api_bytes_per_ready"]
/ modes["proxy"]["api_bytes_per_ready"]
)
valid = all(not m["instrumentation_issues"] for m in modes.values())
if valid:
valid_ratios.append(ratio)
pairs.append(
{
"profile": profile["name"],
"repetition": repetition,
"direct_proxy_ratio": ratio,
"instrumentation_valid": valid,
}
)
profiles.append(
{
"profile": profile["name"],
"valid_pairs": len(valid_ratios),
"median_traffic_ratio": statistics.median(valid_ratios)
if valid_ratios
else None,
"min_traffic_ratio": min(valid_ratios, default=None),
"max_traffic_ratio": max(valid_ratios, default=None),
}
)
if any(
r > protocol["criteria"]["each_pair_traffic_ratio_max"]
for r in valid_ratios
) or (
len(valid_ratios) == protocol["repetitions"]
and statistics.median(valid_ratios)
> protocol["criteria"]["median_direct_proxy_traffic_ratio_max"]
):
failures.append("traffic_threshold:" + profile["name"])
if protocol["purpose"] == "pilot":
status = "pilot_not_for_hypothesis"
elif failures:
# OOM, errors and incorrect bytes are contrary outcomes, never exclusions.
status = "rejected"
elif issues:
status = "inconclusive"
else:
status = "confirmed_in_tested_conditions"
return {
"criterion_status": status,
"system_or_threshold_failures": sorted(set(failures)),
"instrumentation_issues": sorted(set(issues)),
"pairs": pairs,
"profiles": profiles,
}
def summarize(campaign):
manifest = load(campaign / "source-manifest.json")
analysis_sources = [EXPERIMENT / "src/analysis.py", EXPERIMENT / "src/common.py"]
if any(
manifest.get(str(path.relative_to(ROOT))) != digest(path)
for path in analysis_sources
):
raise ExperimentError(
"analysis code differs from frozen snapshot; use the archived experiment source"
)
protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json")
cases = []
for c in plan:
directory = campaign / "cases" / c["directory"]
if (directory / "case.json").exists():
cases.append(case_metrics(directory, protocol))
result = {
"protocol_version": protocol["version"],
"purpose": protocol["purpose"],
"decision": decision(protocol, cases, plan),
"cases": cases,
}
if (campaign / "execution-failure.json").exists():
result["decision"]["instrumentation_issues"].append(
"campaign_execution_failure"
)
if result["decision"]["criterion_status"] == "confirmed_in_tested_conditions":
result["decision"]["criterion_status"] = "inconclusive"
save(campaign / "metrics.json", result)
save(
campaign / "analysis-provenance.json",
{
"source_sha256": {
str(p.relative_to(ROOT)): digest(p) for p in analysis_sources
},
"seal_sha256": digest(campaign / "seal.json"),
},
)
with (campaign / "metrics.csv").open("w", newline="") as output:
columns = [
"profile",
"mode",
"repetition",
"ready",
"errors",
"verified",
"api_bytes_per_ready",
"api_sampled_peak_window_max",
"api_peak_lifetime_max",
"api_cpu_seconds_total",
"s3_rx_tx_bytes",
"window_seconds",
"ready_rps",
"generator_cpu_cores",
"p50_seconds",
"p95_seconds",
"p99_seconds",
]
writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore")
writer.writeheader()
writer.writerows(
{
**case,
**{
name + "_seconds": value
for name, value in case.get("latency_seconds", {}).items()
},
}
for case in cases
)
return result