Files
mtusi2026lab4-fork/experiments/tests/test_instruments.py

417 lines
16 KiB
Python

import copy
import hashlib
import json
import struct
import sys
import tempfile
import threading
import unittest
import zlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from common import EXPERIMENT, EVIDENCE, ROOT, save, load, ExperimentError
from driver import make_plan, safe_down, validate_compose
from fixtures import make_png
from telemetry import parse_snapshot
from analysis import percentile, decision, case_metrics
from client import Client
PROTOCOL = load(EXPERIMENT / "config/protocol.json")
class InstrumentTests(unittest.TestCase):
def test_evidence_is_in_repository_root(self):
self.assertEqual(EVIDENCE, ROOT / "evidence" / "upload-comparison")
def test_png_is_valid_deterministic_and_exact_size(self):
for width, height, size in [(16, 8, 4096), (32, 16, 8192)]:
data = make_png(width, height, size, 123)
self.assertEqual(data, make_png(width, height, size, 123))
self.assertEqual(len(data), size)
self.assertEqual(data[:8], b"\x89PNG\r\n\x1a\n")
chunks, offset = {}, 8
while offset < len(data):
length = struct.unpack(">I", data[offset : offset + 4])[0]
kind, payload = (
data[offset + 4 : offset + 8],
data[offset + 8 : offset + 8 + length],
)
crc = struct.unpack(
">I", data[offset + 8 + length : offset + 12 + length]
)[0]
self.assertEqual(crc, zlib.crc32(kind + payload) & 0xFFFFFFFF)
chunks[kind] = payload
offset += length + 12
self.assertEqual(offset, len(data))
self.assertEqual(struct.unpack(">II", chunks[b"IHDR"][:8]), (width, height))
self.assertEqual(
len(zlib.decompress(chunks[b"IDAT"])), height * (width * 3 + 1)
)
self.assertIn(b"npAD", chunks)
def test_all_actual_profile_headers_and_lengths(self):
for profile in PROTOCOL["profiles"]:
data = make_png(
profile["width"],
profile["height"],
profile["size_bytes"],
PROTOCOL["seed"],
)
self.assertEqual(len(data), profile["size_bytes"])
self.assertEqual(
struct.unpack(">II", data[16:24]), (profile["width"], profile["height"])
)
def test_plan_has_complete_counterbalanced_pairs(self):
plan = make_plan(PROTOCOL)
self.assertEqual(len(plan), 24)
self.assertEqual(plan, make_plan(PROTOCOL))
self.assertEqual(sum(p["mode"] == "proxy" for p in plan[::2]), 6)
self.assertEqual(len({p["directory"] for p in plan}), 24)
for profile in PROTOCOL["profiles"]:
for repetition in range(1, 4):
pair = [
c["mode"]
for c in plan
if c["profile"] == profile["name"] and c["repetition"] == repetition
]
self.assertEqual(sorted(pair), ["direct", "proxy"])
changed = copy.deepcopy(PROTOCOL)
changed["seed"] += 1
self.assertNotEqual(plan, make_plan(changed))
def test_counters_sum_all_non_loopback_interfaces(self):
text = """@memory.current
100
@memory.peak
200
@memory.max
536870912
@memory.events
oom 0
oom_kill 0
max 0
@cpu.stat
usage_usec 500
nr_throttled 1
throttled_usec 12
@net
Inter-| Receive | Transmit
lo: 900 0 0 0 0 0 0 0 800 0 0 0 0 0 0 0
eth0: 100 0 0 0 0 0 0 0 200 0 0 0 0 0 0 0
eth1: 300 0 0 0 0 0 0 0 400 0 0 0 0 0 0 0
"""
parsed = parse_snapshot(text)
self.assertEqual(parsed["rx_bytes"], 400)
self.assertEqual(parsed["tx_bytes"], 600)
self.assertEqual(parsed["cpu.stat"]["usage_usec"], 500)
def test_cleanup_cannot_target_another_campaign(self):
with tempfile.TemporaryDirectory() as folder:
campaign = Path(folder) / "series-formal-123"
campaign.mkdir()
save(campaign / "plan.json", make_plan(PROTOCOL))
with patch("driver.command") as command:
with self.assertRaises(ExperimentError):
safe_down(
campaign,
Path(folder),
"hl-upload-exp-another-series-1",
None,
False,
)
command.assert_not_called()
def test_compose_memory_values_may_be_strings(self):
services = {}
r = PROTOCOL["resources"]
for role in ("api", "db", "redis", "s3", "haproxy"):
services[role] = {
"cpus": r[role + "_cpus"],
"mem_limit": str(r[role + "_memory_bytes"]),
"memswap_limit": str(r[role + "_memory_bytes"]),
}
services["api"].update(
deploy={"replicas": 2},
tmpfs=["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"],
environment={
"GOMAXPROCS": "1",
"GOMEMLIMIT": "384MiB",
"DATABASE_MAX_CONNS": "10",
},
)
response = SimpleNamespace(
returncode=0, stdout=json.dumps({"services": services})
)
with patch("driver.subprocess.run", return_value=response):
validate_compose(PROTOCOL)
services["api"]["memswap_limit"] = "1073741824"
response.stdout = json.dumps({"services": services})
with patch("driver.subprocess.run", return_value=response):
with self.assertRaises(ExperimentError):
validate_compose(PROTOCOL)
class DecisionTests(unittest.TestCase):
def setUp(self):
self.plan = make_plan(PROTOCOL)
self.cases = [
{
**c,
"api_bytes_per_ready": 1100000 if c["mode"] == "direct" else 2000000,
"system_failures": [],
"instrumentation_issues": [],
}
for c in self.plan
]
def result(self, protocol=None):
return decision(protocol or PROTOCOL, self.cases, self.plan)
def test_declared_threshold_can_confirm(self):
self.assertEqual(
self.result()["criterion_status"], "confirmed_in_tested_conditions"
)
def test_bad_ratio_is_rejected(self):
for case in self.cases:
if case["mode"] == "direct":
case["api_bytes_per_ready"] = 1600000
self.assertEqual(self.result()["criterion_status"], "rejected")
def test_one_bad_pair_cannot_hide_behind_median(self):
next(c for c in self.cases if c["mode"] == "direct")["api_bytes_per_ready"] = (
1420000
)
self.assertEqual(self.result()["criterion_status"], "rejected")
def test_oom_is_not_discarded_when_sampler_also_failed(self):
self.cases[0]["system_failures"] = ["oom_event:api"]
self.cases[0]["instrumentation_issues"] = ["sampler_error"]
self.assertEqual(self.result()["criterion_status"], "rejected")
def test_broken_instruments_do_not_create_false_traffic_rejection(self):
direct = next(c for c in self.cases if c["mode"] == "direct")
direct["api_bytes_per_ready"] = 9000000
direct["instrumentation_issues"] = ["network_counter_reset:api"]
self.assertEqual(self.result()["criterion_status"], "inconclusive")
def test_missing_case_is_inconclusive(self):
self.cases.pop()
self.assertEqual(self.result()["criterion_status"], "inconclusive")
def test_pilot_never_confirms_the_hypothesis(self):
protocol = copy.deepcopy(PROTOCOL)
protocol["purpose"] = "pilot"
self.assertEqual(
self.result(protocol)["criterion_status"], "pilot_not_for_hypothesis"
)
def test_percentiles_use_nearest_rank(self):
self.assertEqual(percentile(list(range(1, 31)), 0.99), 30)
self.assertEqual(percentile(list(range(1, 31)), 0.50), 15)
self.assertIsNone(percentile([], 0.50))
def test_user_interruption_is_not_a_server_failure(self):
with tempfile.TemporaryDirectory() as folder:
path = Path(folder)
save(path / "case.json", {**self.plan[0]})
save(path / "outcome.json", {"status": "interrupted"})
(path / "requests.jsonl").write_text(
json.dumps({"ready": False, "error": "interrupted"}) + "\n"
)
result = case_metrics(path, PROTOCOL)
self.assertEqual(result["errors"], 0)
self.assertEqual(result["cancelled"], 1)
self.assertEqual(result["system_failures"], [])
def test_metrics_keep_window_ram_distinct_and_exclude_s3_from_api_traffic(self):
protocol = copy.deepcopy(PROTOCOL)
protocol.update(clients=2, waves=1, sample_interval_seconds=0.1)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder)
save(path / "case.json", self.plan[0])
save(path / "outcome.json", {"status": "completed"})
containers = [
{
"id": cid,
"hostname": cid,
"service": role,
"status": "running",
"started_at": "same",
"restarts": 0,
"oom_killed": False,
"nano_cpus": cpu,
}
for cid, role, cpu in [
("a", "api", 500000000),
("b", "api", 500000000),
("s", "s3", 1000000000),
]
]
save(path / "containers-before.json", containers)
save(path / "containers-after.json", containers)
def counters(received, sent, ns, peak):
return {
"rx_bytes": received,
"tx_bytes": sent,
"memory.current": 100,
"memory.peak": peak,
"memory.max": 536870912,
"memory.events": {"oom": 0, "oom_kill": 0, "max": 0},
"cpu.stat": {"usage_usec": ns // 10000},
"read_started_ns": ns,
"read_finished_ns": ns + 1,
}
ids = [c["id"] for c in containers]
m = {
"started_ns": 10000000000,
"finished_ns": 12000000000,
"generator_cpu_seconds": 0.1,
"idle_before": {cid: counters(0, 0, 0, 300) for cid in ids},
"idle_after": {cid: counters(0, 0, 5000000000, 300) for cid in ids},
"before": {cid: counters(0, 0, 10000000000, 300) for cid in ids},
"after": {
cid: counters(4000000, 4000000, 12000000000, 400) for cid in ids
},
}
save(path / "measurement.json", m)
requests = [
{
"wave": 0,
"ready": True,
"started_ns": 10100000000 + i * 1000000,
"finished_ns": 11900000000,
"attachment": {"id": cid, "instance": cid},
}
for i, cid in enumerate(("a", "b"))
]
(path / "requests.jsonl").write_text(
"".join(json.dumps(r) + "\n" for r in requests)
)
(path / "warmup.jsonl").write_text(
"".join(json.dumps(r) + "\n" for r in requests)
)
(path / "verification.jsonl").write_text(
"".join(
json.dumps({"id": cid, "verified": True}) + "\n"
for cid in ("a", "b")
)
)
(path / "samples.jsonl").write_text(
"".join(
json.dumps(
{
"container": cid,
"memory_current": 200,
"monotonic_ns": 10000000000 + i * 100000000,
}
)
+ "\n"
for cid in ids
for i in range(21)
)
)
result = case_metrics(path, protocol)
self.assertEqual(result["api_rx_tx_bytes"], 16000000)
self.assertEqual(result["api_bytes_per_ready"], 8000000)
self.assertEqual(result["containers"][0]["sampled_peak_window"], 200)
self.assertEqual(result["containers"][0]["peak_lifetime_after"], 400)
self.assertEqual(result["system_failures"], [])
self.assertEqual(result["instrumentation_issues"], [])
class TransportTests(unittest.TestCase):
def test_direct_stages_do_not_send_bearer_to_s3_or_leak_secrets_to_records(self):
observed, data = [], b"synthetic PNG transport fixture"
checksum = hashlib.sha256(data).hexdigest()
token, signed_field = "DO-NOT-PERSIST-TOKEN", "DO-NOT-PERSIST-SIGNATURE"
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def do_POST(self):
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
observed.append(
{
"path": self.path,
"authorization": self.headers.get("Authorization"),
"body": body,
}
)
if self.path.endswith("/attachment-uploads"):
response = {
"upload_id": "upload",
"upload": {
"url": f"http://127.0.0.1:{self.server.server_port}/bucket",
"fields": {
"key": "staging/direct/upload",
"policy": signed_field,
},
},
}
elif self.path == "/bucket":
self.send_response(204)
self.send_header("Content-Length", "0")
self.end_headers()
return
else:
response = {
"id": "attachment",
"status": "ready",
"size_bytes": len(data),
"sha256": checksum,
}
encoded = json.dumps(response).encode()
self.send_response(201)
self.send_header("Content-Length", str(len(encoded)))
self.send_header("X-Instance-Id", "api-test")
self.end_headers()
self.wfile.write(encoded)
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
protocol = copy.deepcopy(PROTOCOL)
protocol["ports"] = {"api": server.server_port, "s3": server.server_port}
client = Client(
f"http://127.0.0.1:{server.server_port}", protocol, threading.Event()
)
records = []
client.upload(
"direct",
{"token": token},
"chat",
data,
checksum,
records,
__import__("time").monotonic() + 10,
)
self.assertEqual(
[r["stage"] for r in records],
["client_sha256", "init", "s3_post", "complete"],
)
self.assertEqual(observed[0]["authorization"], "Bearer " + token)
self.assertIsNone(observed[1]["authorization"])
self.assertIn(data, observed[1]["body"])
self.assertEqual(observed[2]["authorization"], "Bearer " + token)
self.assertNotIn(token, json.dumps(records))
self.assertNotIn(signed_field, json.dumps(records))
finally:
server.shutdown()
server.server_close()
thread.join()
if __name__ == "__main__":
unittest.main()