feat: plan the experiment
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot saved summaries without changing raw evidence or the H1 decision."""
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
from common import campaign_path
|
||||
|
||||
|
||||
def sha256(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def create_plots(campaign):
|
||||
inputs = [campaign / "metrics.csv", campaign / "metrics.json"]
|
||||
frame = pd.read_csv(inputs[0])
|
||||
summary = json.loads(inputs[1].read_text())
|
||||
protocol = json.loads((campaign / "protocol.json").read_text())
|
||||
profiles = [p["name"] for p in protocol["profiles"]]
|
||||
status = summary["decision"]["criterion_status"]
|
||||
output = campaign / "plots" / datetime.datetime.now(
|
||||
datetime.timezone.utc
|
||||
).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
output.mkdir(parents=True)
|
||||
|
||||
def finish(fig, ax, title, ylabel, name):
|
||||
ax.set_title(title + "\n" + status)
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.set_xticks(range(len(profiles)), profiles, rotation=20, ha="right")
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
ax.legend(loc="best", fontsize=8)
|
||||
fig.tight_layout()
|
||||
fig.savefig(output / (name + ".png"), dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
def points(column, scale, title, ylabel, name, limit=None):
|
||||
fig, ax = plt.subplots(figsize=(10, 5))
|
||||
for mode, offset, color in [("proxy", -0.12, "#2563eb"),
|
||||
("direct", 0.12, "#d97706")]:
|
||||
subset = frame[frame["mode"] == mode]
|
||||
x, y = [], []
|
||||
for index, profile in enumerate(profiles):
|
||||
values = pd.to_numeric(
|
||||
subset.loc[subset["profile"] == profile, column], errors="coerce"
|
||||
).dropna().tolist()
|
||||
for j, value in enumerate(values):
|
||||
x.append(index + offset + (j - (len(values) - 1) / 2) * 0.035)
|
||||
y.append(value / scale)
|
||||
ax.scatter(x, y, label=mode, color=color, alpha=0.85)
|
||||
if limit is not None:
|
||||
ax.axhline(limit, color="red", linestyle="--", label="API limit")
|
||||
finish(fig, ax, title + " (each point = one run)", ylabel, name)
|
||||
|
||||
points("api_bytes_per_ready", 1024 ** 2, "API RX+TX per ready attachment",
|
||||
"MiB / ready", "api-traffic")
|
||||
points("api_sampled_peak_window_max", 1024 ** 2,
|
||||
"API sampled memory peak: maximum of two replicas", "MiB",
|
||||
"api-memory-window", limit=512)
|
||||
points("api_peak_lifetime_max", 1024 ** 2,
|
||||
"API lifetime memory peak (includes startup and warmup)", "MiB",
|
||||
"api-memory-lifetime", limit=512)
|
||||
points("p95_seconds", 1, "Successful upload flow p95", "seconds", "latency-p95")
|
||||
points("errors", 1, "Upload errors (missing runs are absent)", "count", "errors")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 5))
|
||||
pairs = summary["decision"].get("pairs", [])
|
||||
for valid, marker, label in [(True, "o", "instrumentation valid"),
|
||||
(False, "x", "instrumentation issues")]:
|
||||
selected = [p for p in pairs if p["instrumentation_valid"] == valid]
|
||||
ax.scatter([profiles.index(p["profile"]) +
|
||||
(p["repetition"] - 2) * 0.06 for p in selected],
|
||||
[p["direct_proxy_ratio"] for p in selected],
|
||||
marker=marker, label=label)
|
||||
ax.axhline(protocol["criteria"]["median_direct_proxy_traffic_ratio_max"],
|
||||
linestyle="--", color="green", label="median threshold")
|
||||
ax.axhline(protocol["criteria"]["each_pair_traffic_ratio_max"],
|
||||
linestyle=":", color="red", label="each pair threshold")
|
||||
finish(fig, ax, "Paired API traffic ratio (all available pairs)",
|
||||
"direct / proxy", "traffic-ratio")
|
||||
provenance = {
|
||||
"created_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"python": platform.python_version(), "pandas": pd.__version__,
|
||||
"matplotlib": matplotlib.__version__,
|
||||
"inputs_sha256": {f.name: sha256(f) for f in inputs + [campaign / "protocol.json"]},
|
||||
"plot_script_sha256": sha256(Path(__file__)),
|
||||
"requirements_sha256": sha256(Path(__file__).parent / "requirements.txt"),
|
||||
"criterion_status": status,
|
||||
"note": "All available runs shown; missing values omitted, never replaced with zero. "
|
||||
"Instrumentation validity and system failures must be read in metrics.json.",
|
||||
}
|
||||
(output / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n")
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("campaign", help="prepared campaign under root evidence/upload-comparison")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
print(create_plots(campaign_path(args.campaign)))
|
||||
except (OSError, ValueError, KeyError) as exc:
|
||||
parser.exit(1, "Cannot plot: check campaign path and run summarize first (" +
|
||||
type(exc).__name__ + ").\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user