forked from mtusi2026/mtusi2026lab4
393 lines
15 KiB
Python
393 lines
15 KiB
Python
import csv
|
|
import json
|
|
import math
|
|
import statistics
|
|
|
|
from common import EXPERIMENT, ROOT, ExperimentError, digest, load, save
|
|
|
|
|
|
def percentile(values, fraction):
|
|
if not values:
|
|
return None
|
|
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
|
|
|
|
|
|
def json_lines(path):
|
|
if not path.exists():
|
|
return []
|
|
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
|
|
|
|
|
def case_metrics(directory, protocol):
|
|
case = load(directory / "case.json")
|
|
outcome = (
|
|
load(directory / "outcome.json")
|
|
if (directory / "outcome.json").exists()
|
|
else {"status": "interrupted"}
|
|
)
|
|
requests, verification = (
|
|
json_lines(directory / "requests.jsonl"),
|
|
json_lines(directory / "verification.jsonl"),
|
|
)
|
|
expected = protocol["clients"] * protocol["waves"]
|
|
result = {
|
|
**case,
|
|
"status": outcome["status"],
|
|
"expected_requests": expected,
|
|
"requests_recorded": len(requests),
|
|
"ready": sum(r.get("ready", False) for r in requests),
|
|
"errors": sum(
|
|
not r.get("ready", False) and r.get("error") != "interrupted"
|
|
for r in requests
|
|
),
|
|
"cancelled": sum(r.get("error") == "interrupted" for r in requests),
|
|
"verified": sum(v.get("verified", False) for v in verification),
|
|
"system_failures": [],
|
|
"instrumentation_issues": [],
|
|
"containers": [],
|
|
}
|
|
system, issues = result["system_failures"], result["instrumentation_issues"]
|
|
if result["errors"]:
|
|
system.append("upload_errors")
|
|
if (
|
|
len({r["attachment"]["id"] for r in requests if r.get("ready")})
|
|
!= result["ready"]
|
|
):
|
|
system.append("duplicate_attachment_ids")
|
|
if any(
|
|
not v.get("verified", False) and v.get("error") != "interrupted"
|
|
for v in verification
|
|
):
|
|
system.append("original_verification_failed")
|
|
warmups = json_lines(directory / "warmup.jsonl")
|
|
if any(
|
|
not r.get("ready", False) and r.get("error") != "interrupted" for r in warmups
|
|
):
|
|
system.append("warmup_upload_errors")
|
|
if len(requests) != expected or result["verified"] != result["ready"]:
|
|
issues.append("missing_request_or_verification_records")
|
|
if outcome["status"] != "completed":
|
|
issues.append("case_" + outcome["status"])
|
|
if outcome.get("sampler_errors"):
|
|
issues.append("sampler_error")
|
|
before = (
|
|
load(directory / "containers-before.json")
|
|
if (directory / "containers-before.json").exists()
|
|
else []
|
|
)
|
|
after = (
|
|
load(directory / "containers-after.json")
|
|
if (directory / "containers-after.json").exists()
|
|
else []
|
|
)
|
|
for c in after:
|
|
previous = next((p for p in before if p["id"] == c["id"]), None)
|
|
if (
|
|
c["oom_killed"]
|
|
or c["status"] != "running"
|
|
or (
|
|
previous
|
|
and (
|
|
c["restarts"] != previous["restarts"]
|
|
or c["started_at"] != previous["started_at"]
|
|
)
|
|
)
|
|
):
|
|
system.append("container_oom_restart_or_exit:" + c["service"])
|
|
if not (directory / "measurement.json").exists():
|
|
issues.append("missing_measurement")
|
|
return result
|
|
m = load(directory / "measurement.json")
|
|
seconds = (m["finished_ns"] - m["started_ns"]) / 1e9
|
|
result.update(
|
|
window_seconds=seconds,
|
|
generator_cpu_cores=m["generator_cpu_seconds"] / seconds,
|
|
ready_rps=result["ready"] / seconds,
|
|
)
|
|
durations = [
|
|
(r["finished_ns"] - r["started_ns"]) / 1e9 for r in requests if r.get("ready")
|
|
]
|
|
result["latency_seconds"] = {
|
|
"p50": percentile(durations, 0.50),
|
|
"p95": percentile(durations, 0.95),
|
|
"p99": percentile(durations, 0.99),
|
|
}
|
|
spreads = []
|
|
for wave in range(protocol["waves"]):
|
|
starts = [r["started_ns"] for r in requests if r["wave"] == wave]
|
|
if len(starts) == protocol["clients"]:
|
|
spreads.append((max(starts) - min(starts)) / 1e9)
|
|
result["max_start_spread_seconds"] = max(spreads, default=None)
|
|
criteria = protocol["criteria"]
|
|
if len(spreads) != protocol["waves"] or any(
|
|
s > criteria["start_spread_seconds_max"] for s in spreads
|
|
):
|
|
issues.append("burst_start_spread")
|
|
if result["generator_cpu_cores"] >= criteria["generator_cpu_cores_max"]:
|
|
issues.append("possible_generator_cpu_bottleneck")
|
|
instances = {r["attachment"]["instance"] for r in requests if r.get("ready")}
|
|
if len(instances) != protocol["resources"]["api_replicas"]:
|
|
issues.append("not_all_api_replicas_exercised")
|
|
warm_instances = {r["attachment"]["instance"] for r in warmups if r.get("ready")}
|
|
if len(warm_instances) != protocol["resources"]["api_replicas"]:
|
|
issues.append("not_all_api_replicas_warmed")
|
|
samples = json_lines(directory / "samples.jsonl")
|
|
api_traffic, background = 0, 0.0
|
|
for c in before:
|
|
cid = c["id"]
|
|
if cid not in m["before"] or cid not in m["after"]:
|
|
issues.append("missing_counter_snapshot:" + c["service"])
|
|
continue
|
|
start, end = m["before"][cid], m["after"][cid]
|
|
points = [
|
|
s
|
|
for s in samples
|
|
if s["container"] == cid
|
|
and m["started_ns"] <= s["monotonic_ns"] <= m["finished_ns"]
|
|
]
|
|
times = (
|
|
[m["started_ns"]] + [s["monotonic_ns"] for s in points] + [m["finished_ns"]]
|
|
)
|
|
gaps = [(b - a) / 1e9 for a, b in zip(times, times[1:])]
|
|
coverage = len(points) * protocol["sample_interval_seconds"] / seconds
|
|
rx, tx = (
|
|
end["rx_bytes"] - start["rx_bytes"],
|
|
end["tx_bytes"] - start["tx_bytes"],
|
|
)
|
|
if rx < 0 or tx < 0:
|
|
issues.append("network_counter_reset:" + c["service"])
|
|
idle_start, idle_end = m["idle_before"][cid], m["idle_after"][cid]
|
|
idle_seconds = (
|
|
idle_end["read_started_ns"] - idle_start["read_started_ns"]
|
|
) / 1e9
|
|
idle_rate = (
|
|
(idle_end["rx_bytes"] + idle_end["tx_bytes"])
|
|
- (idle_start["rx_bytes"] + idle_start["tx_bytes"])
|
|
) / idle_seconds
|
|
cpu_usec = end["cpu.stat"]["usage_usec"] - start["cpu.stat"]["usage_usec"]
|
|
peak = max(
|
|
[s["memory_current"] for s in points]
|
|
+ [start["memory.current"], end["memory.current"]]
|
|
)
|
|
events = {
|
|
key: end["memory.events"].get(key, 0) - start["memory.events"].get(key, 0)
|
|
for key in ("oom", "oom_kill", "max")
|
|
}
|
|
if end["memory.events"].get("oom", 0) or end["memory.events"].get(
|
|
"oom_kill", 0
|
|
):
|
|
system.append("oom_event:" + c["service"])
|
|
if (
|
|
coverage < criteria["sampler_coverage_fraction_min"]
|
|
or max(gaps) > criteria["sampler_gap_seconds_max"]
|
|
):
|
|
issues.append("sampler_coverage:" + c["service"])
|
|
item = {
|
|
"service": c["service"],
|
|
"hostname": c["hostname"],
|
|
"rx_bytes": rx,
|
|
"tx_bytes": tx,
|
|
"cpu_seconds": cpu_usec / 1e6,
|
|
"cpu_fraction_of_limit": cpu_usec / 1e6 / seconds / (c["nano_cpus"] / 1e9),
|
|
"throttled_seconds": (
|
|
end["cpu.stat"].get("throttled_usec", 0)
|
|
- start["cpu.stat"].get("throttled_usec", 0)
|
|
)
|
|
/ 1e6,
|
|
"throttled_periods": end["cpu.stat"].get("nr_throttled", 0)
|
|
- start["cpu.stat"].get("nr_throttled", 0),
|
|
"baseline_memory": start["memory.current"],
|
|
"sampled_peak_window": peak,
|
|
"peak_lifetime_before": start["memory.peak"],
|
|
"peak_lifetime_after": end["memory.peak"],
|
|
"memory_limit": end["memory.max"],
|
|
"memory_events_delta": events,
|
|
"idle_bytes_per_second": idle_rate,
|
|
"sample_count": len(points),
|
|
"sample_coverage": coverage,
|
|
"sample_gap_max_seconds": max(gaps),
|
|
}
|
|
result["containers"].append(item)
|
|
if c["service"] == "api":
|
|
api_traffic += rx + tx
|
|
background += idle_rate * seconds
|
|
if end["memory.peak"] >= criteria["api_peak_strictly_below_bytes"]:
|
|
system.append("api_lifetime_peak_reached_limit")
|
|
if sum(c["service"] == "api" for c in result["containers"]) != 2:
|
|
issues.append("missing_api_counters")
|
|
result["api_rx_tx_bytes"] = api_traffic
|
|
result["api_bytes_per_ready"] = (
|
|
api_traffic / result["ready"] if result["ready"] else None
|
|
)
|
|
api_metrics = [c for c in result["containers"] if c["service"] == "api"]
|
|
result["api_sampled_peak_window_max"] = max(
|
|
(c["sampled_peak_window"] for c in api_metrics), default=None
|
|
)
|
|
result["api_peak_lifetime_max"] = max(
|
|
(c["peak_lifetime_after"] for c in api_metrics), default=None
|
|
)
|
|
result["api_cpu_seconds_total"] = sum(c["cpu_seconds"] for c in api_metrics)
|
|
result["s3_rx_tx_bytes"] = sum(
|
|
c["rx_bytes"] + c["tx_bytes"]
|
|
for c in result["containers"]
|
|
if c["service"] == "s3"
|
|
)
|
|
result["estimated_background_fraction"] = background / (
|
|
expected * case["size_bytes"]
|
|
)
|
|
if (
|
|
result["estimated_background_fraction"]
|
|
> criteria["idle_background_payload_fraction_max"]
|
|
):
|
|
issues.append("excessive_background_traffic")
|
|
result["system_failures"] = sorted(set(system))
|
|
result["instrumentation_issues"] = sorted(set(issues))
|
|
return result
|
|
|
|
|
|
def decision(protocol, cases, plan):
|
|
pairs, issues, failures, profiles = [], [], [], []
|
|
if len(cases) != len(plan):
|
|
issues.append("not_all_planned_cases_finished")
|
|
for c in cases:
|
|
failures.extend(c["system_failures"])
|
|
issues.extend(c["instrumentation_issues"])
|
|
for profile in protocol["profiles"]:
|
|
valid_ratios = []
|
|
for repetition in range(1, protocol["repetitions"] + 1):
|
|
pair = [
|
|
c
|
|
for c in cases
|
|
if c["profile"] == profile["name"] and c["repetition"] == repetition
|
|
]
|
|
modes = {c["mode"]: c for c in pair}
|
|
if set(modes) != {"proxy", "direct"} or not all(
|
|
m.get("api_bytes_per_ready") for m in modes.values()
|
|
):
|
|
issues.append("incomplete_pair:" + profile["name"])
|
|
continue
|
|
ratio = (
|
|
modes["direct"]["api_bytes_per_ready"]
|
|
/ modes["proxy"]["api_bytes_per_ready"]
|
|
)
|
|
valid = all(not m["instrumentation_issues"] for m in modes.values())
|
|
if valid:
|
|
valid_ratios.append(ratio)
|
|
pairs.append(
|
|
{
|
|
"profile": profile["name"],
|
|
"repetition": repetition,
|
|
"direct_proxy_ratio": ratio,
|
|
"instrumentation_valid": valid,
|
|
}
|
|
)
|
|
profiles.append(
|
|
{
|
|
"profile": profile["name"],
|
|
"valid_pairs": len(valid_ratios),
|
|
"median_traffic_ratio": statistics.median(valid_ratios)
|
|
if valid_ratios
|
|
else None,
|
|
"min_traffic_ratio": min(valid_ratios, default=None),
|
|
"max_traffic_ratio": max(valid_ratios, default=None),
|
|
}
|
|
)
|
|
if any(
|
|
r > protocol["criteria"]["each_pair_traffic_ratio_max"]
|
|
for r in valid_ratios
|
|
) or (
|
|
len(valid_ratios) == protocol["repetitions"]
|
|
and statistics.median(valid_ratios)
|
|
> protocol["criteria"]["median_direct_proxy_traffic_ratio_max"]
|
|
):
|
|
failures.append("traffic_threshold:" + profile["name"])
|
|
if protocol["purpose"] == "pilot":
|
|
status = "pilot_not_for_hypothesis"
|
|
elif failures:
|
|
# OOM, errors and incorrect bytes are contrary outcomes, never exclusions.
|
|
status = "rejected"
|
|
elif issues:
|
|
status = "inconclusive"
|
|
else:
|
|
status = "confirmed_in_tested_conditions"
|
|
return {
|
|
"criterion_status": status,
|
|
"system_or_threshold_failures": sorted(set(failures)),
|
|
"instrumentation_issues": sorted(set(issues)),
|
|
"pairs": pairs,
|
|
"profiles": profiles,
|
|
}
|
|
|
|
|
|
def summarize(campaign):
|
|
manifest = load(campaign / "source-manifest.json")
|
|
analysis_sources = [EXPERIMENT / "src/analysis.py", EXPERIMENT / "src/common.py"]
|
|
if any(
|
|
manifest.get(str(path.relative_to(ROOT))) != digest(path)
|
|
for path in analysis_sources
|
|
):
|
|
raise ExperimentError(
|
|
"analysis code differs from frozen snapshot; use the archived experiment source"
|
|
)
|
|
protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json")
|
|
cases = []
|
|
for c in plan:
|
|
directory = campaign / "cases" / c["directory"]
|
|
if (directory / "case.json").exists():
|
|
cases.append(case_metrics(directory, protocol))
|
|
result = {
|
|
"protocol_version": protocol["version"],
|
|
"purpose": protocol["purpose"],
|
|
"decision": decision(protocol, cases, plan),
|
|
"cases": cases,
|
|
}
|
|
if (campaign / "execution-failure.json").exists():
|
|
result["decision"]["instrumentation_issues"].append(
|
|
"campaign_execution_failure"
|
|
)
|
|
if result["decision"]["criterion_status"] == "confirmed_in_tested_conditions":
|
|
result["decision"]["criterion_status"] = "inconclusive"
|
|
save(campaign / "metrics.json", result)
|
|
save(
|
|
campaign / "analysis-provenance.json",
|
|
{
|
|
"source_sha256": {
|
|
str(p.relative_to(ROOT)): digest(p) for p in analysis_sources
|
|
},
|
|
"seal_sha256": digest(campaign / "seal.json"),
|
|
},
|
|
)
|
|
with (campaign / "metrics.csv").open("w", newline="") as output:
|
|
columns = [
|
|
"profile",
|
|
"mode",
|
|
"repetition",
|
|
"ready",
|
|
"errors",
|
|
"verified",
|
|
"api_bytes_per_ready",
|
|
"api_sampled_peak_window_max",
|
|
"api_peak_lifetime_max",
|
|
"api_cpu_seconds_total",
|
|
"s3_rx_tx_bytes",
|
|
"window_seconds",
|
|
"ready_rps",
|
|
"generator_cpu_cores",
|
|
"p50_seconds",
|
|
"p95_seconds",
|
|
"p99_seconds",
|
|
]
|
|
writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore")
|
|
writer.writeheader()
|
|
writer.writerows(
|
|
{
|
|
**case,
|
|
**{
|
|
name + "_seconds": value
|
|
for name, value in case.get("latency_seconds", {}).items()
|
|
},
|
|
}
|
|
for case in cases
|
|
)
|
|
return result
|