417 lines
16 KiB
Python
417 lines
16 KiB
Python
import copy
|
|
import hashlib
|
|
import json
|
|
import struct
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
import zlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from common import EXPERIMENT, EVIDENCE, ROOT, save, load, ExperimentError
|
|
from driver import make_plan, safe_down, validate_compose
|
|
from fixtures import make_png
|
|
from telemetry import parse_snapshot
|
|
from analysis import percentile, decision, case_metrics
|
|
from client import Client
|
|
|
|
|
|
PROTOCOL = load(EXPERIMENT / "config/protocol.json")
|
|
|
|
|
|
class InstrumentTests(unittest.TestCase):
|
|
def test_evidence_is_in_repository_root(self):
|
|
self.assertEqual(EVIDENCE, ROOT / "evidence" / "upload-comparison")
|
|
|
|
def test_png_is_valid_deterministic_and_exact_size(self):
|
|
for width, height, size in [(16, 8, 4096), (32, 16, 8192)]:
|
|
data = make_png(width, height, size, 123)
|
|
self.assertEqual(data, make_png(width, height, size, 123))
|
|
self.assertEqual(len(data), size)
|
|
self.assertEqual(data[:8], b"\x89PNG\r\n\x1a\n")
|
|
chunks, offset = {}, 8
|
|
while offset < len(data):
|
|
length = struct.unpack(">I", data[offset : offset + 4])[0]
|
|
kind, payload = (
|
|
data[offset + 4 : offset + 8],
|
|
data[offset + 8 : offset + 8 + length],
|
|
)
|
|
crc = struct.unpack(
|
|
">I", data[offset + 8 + length : offset + 12 + length]
|
|
)[0]
|
|
self.assertEqual(crc, zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
|
chunks[kind] = payload
|
|
offset += length + 12
|
|
self.assertEqual(offset, len(data))
|
|
self.assertEqual(struct.unpack(">II", chunks[b"IHDR"][:8]), (width, height))
|
|
self.assertEqual(
|
|
len(zlib.decompress(chunks[b"IDAT"])), height * (width * 3 + 1)
|
|
)
|
|
self.assertIn(b"npAD", chunks)
|
|
|
|
def test_all_actual_profile_headers_and_lengths(self):
|
|
for profile in PROTOCOL["profiles"]:
|
|
data = make_png(
|
|
profile["width"],
|
|
profile["height"],
|
|
profile["size_bytes"],
|
|
PROTOCOL["seed"],
|
|
)
|
|
self.assertEqual(len(data), profile["size_bytes"])
|
|
self.assertEqual(
|
|
struct.unpack(">II", data[16:24]), (profile["width"], profile["height"])
|
|
)
|
|
|
|
def test_plan_has_complete_counterbalanced_pairs(self):
|
|
plan = make_plan(PROTOCOL)
|
|
self.assertEqual(len(plan), 24)
|
|
self.assertEqual(plan, make_plan(PROTOCOL))
|
|
self.assertEqual(sum(p["mode"] == "proxy" for p in plan[::2]), 6)
|
|
self.assertEqual(len({p["directory"] for p in plan}), 24)
|
|
for profile in PROTOCOL["profiles"]:
|
|
for repetition in range(1, 4):
|
|
pair = [
|
|
c["mode"]
|
|
for c in plan
|
|
if c["profile"] == profile["name"] and c["repetition"] == repetition
|
|
]
|
|
self.assertEqual(sorted(pair), ["direct", "proxy"])
|
|
changed = copy.deepcopy(PROTOCOL)
|
|
changed["seed"] += 1
|
|
self.assertNotEqual(plan, make_plan(changed))
|
|
|
|
def test_counters_sum_all_non_loopback_interfaces(self):
|
|
text = """@memory.current
|
|
100
|
|
@memory.peak
|
|
200
|
|
@memory.max
|
|
536870912
|
|
@memory.events
|
|
oom 0
|
|
oom_kill 0
|
|
max 0
|
|
@cpu.stat
|
|
usage_usec 500
|
|
nr_throttled 1
|
|
throttled_usec 12
|
|
@net
|
|
Inter-| Receive | Transmit
|
|
lo: 900 0 0 0 0 0 0 0 800 0 0 0 0 0 0 0
|
|
eth0: 100 0 0 0 0 0 0 0 200 0 0 0 0 0 0 0
|
|
eth1: 300 0 0 0 0 0 0 0 400 0 0 0 0 0 0 0
|
|
"""
|
|
parsed = parse_snapshot(text)
|
|
self.assertEqual(parsed["rx_bytes"], 400)
|
|
self.assertEqual(parsed["tx_bytes"], 600)
|
|
self.assertEqual(parsed["cpu.stat"]["usage_usec"], 500)
|
|
|
|
def test_cleanup_cannot_target_another_campaign(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
campaign = Path(folder) / "series-formal-123"
|
|
campaign.mkdir()
|
|
save(campaign / "plan.json", make_plan(PROTOCOL))
|
|
with patch("driver.command") as command:
|
|
with self.assertRaises(ExperimentError):
|
|
safe_down(
|
|
campaign,
|
|
Path(folder),
|
|
"hl-upload-exp-another-series-1",
|
|
None,
|
|
False,
|
|
)
|
|
command.assert_not_called()
|
|
|
|
def test_compose_memory_values_may_be_strings(self):
|
|
services = {}
|
|
r = PROTOCOL["resources"]
|
|
for role in ("api", "db", "redis", "s3", "haproxy"):
|
|
services[role] = {
|
|
"cpus": r[role + "_cpus"],
|
|
"mem_limit": str(r[role + "_memory_bytes"]),
|
|
"memswap_limit": str(r[role + "_memory_bytes"]),
|
|
}
|
|
services["api"].update(
|
|
deploy={"replicas": 2},
|
|
tmpfs=["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"],
|
|
environment={
|
|
"GOMAXPROCS": "1",
|
|
"GOMEMLIMIT": "384MiB",
|
|
"DATABASE_MAX_CONNS": "10",
|
|
},
|
|
)
|
|
response = SimpleNamespace(
|
|
returncode=0, stdout=json.dumps({"services": services})
|
|
)
|
|
with patch("driver.subprocess.run", return_value=response):
|
|
validate_compose(PROTOCOL)
|
|
services["api"]["memswap_limit"] = "1073741824"
|
|
response.stdout = json.dumps({"services": services})
|
|
with patch("driver.subprocess.run", return_value=response):
|
|
with self.assertRaises(ExperimentError):
|
|
validate_compose(PROTOCOL)
|
|
|
|
|
|
class DecisionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.plan = make_plan(PROTOCOL)
|
|
self.cases = [
|
|
{
|
|
**c,
|
|
"api_bytes_per_ready": 1100000 if c["mode"] == "direct" else 2000000,
|
|
"system_failures": [],
|
|
"instrumentation_issues": [],
|
|
}
|
|
for c in self.plan
|
|
]
|
|
|
|
def result(self, protocol=None):
|
|
return decision(protocol or PROTOCOL, self.cases, self.plan)
|
|
|
|
def test_declared_threshold_can_confirm(self):
|
|
self.assertEqual(
|
|
self.result()["criterion_status"], "confirmed_in_tested_conditions"
|
|
)
|
|
|
|
def test_bad_ratio_is_rejected(self):
|
|
for case in self.cases:
|
|
if case["mode"] == "direct":
|
|
case["api_bytes_per_ready"] = 1600000
|
|
self.assertEqual(self.result()["criterion_status"], "rejected")
|
|
|
|
def test_one_bad_pair_cannot_hide_behind_median(self):
|
|
next(c for c in self.cases if c["mode"] == "direct")["api_bytes_per_ready"] = (
|
|
1420000
|
|
)
|
|
self.assertEqual(self.result()["criterion_status"], "rejected")
|
|
|
|
def test_oom_is_not_discarded_when_sampler_also_failed(self):
|
|
self.cases[0]["system_failures"] = ["oom_event:api"]
|
|
self.cases[0]["instrumentation_issues"] = ["sampler_error"]
|
|
self.assertEqual(self.result()["criterion_status"], "rejected")
|
|
|
|
def test_broken_instruments_do_not_create_false_traffic_rejection(self):
|
|
direct = next(c for c in self.cases if c["mode"] == "direct")
|
|
direct["api_bytes_per_ready"] = 9000000
|
|
direct["instrumentation_issues"] = ["network_counter_reset:api"]
|
|
self.assertEqual(self.result()["criterion_status"], "inconclusive")
|
|
|
|
def test_missing_case_is_inconclusive(self):
|
|
self.cases.pop()
|
|
self.assertEqual(self.result()["criterion_status"], "inconclusive")
|
|
|
|
def test_pilot_never_confirms_the_hypothesis(self):
|
|
protocol = copy.deepcopy(PROTOCOL)
|
|
protocol["purpose"] = "pilot"
|
|
self.assertEqual(
|
|
self.result(protocol)["criterion_status"], "pilot_not_for_hypothesis"
|
|
)
|
|
|
|
def test_percentiles_use_nearest_rank(self):
|
|
self.assertEqual(percentile(list(range(1, 31)), 0.99), 30)
|
|
self.assertEqual(percentile(list(range(1, 31)), 0.50), 15)
|
|
self.assertIsNone(percentile([], 0.50))
|
|
|
|
def test_user_interruption_is_not_a_server_failure(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
path = Path(folder)
|
|
save(path / "case.json", {**self.plan[0]})
|
|
save(path / "outcome.json", {"status": "interrupted"})
|
|
(path / "requests.jsonl").write_text(
|
|
json.dumps({"ready": False, "error": "interrupted"}) + "\n"
|
|
)
|
|
result = case_metrics(path, PROTOCOL)
|
|
self.assertEqual(result["errors"], 0)
|
|
self.assertEqual(result["cancelled"], 1)
|
|
self.assertEqual(result["system_failures"], [])
|
|
|
|
def test_metrics_keep_window_ram_distinct_and_exclude_s3_from_api_traffic(self):
|
|
protocol = copy.deepcopy(PROTOCOL)
|
|
protocol.update(clients=2, waves=1, sample_interval_seconds=0.1)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
path = Path(folder)
|
|
save(path / "case.json", self.plan[0])
|
|
save(path / "outcome.json", {"status": "completed"})
|
|
containers = [
|
|
{
|
|
"id": cid,
|
|
"hostname": cid,
|
|
"service": role,
|
|
"status": "running",
|
|
"started_at": "same",
|
|
"restarts": 0,
|
|
"oom_killed": False,
|
|
"nano_cpus": cpu,
|
|
}
|
|
for cid, role, cpu in [
|
|
("a", "api", 500000000),
|
|
("b", "api", 500000000),
|
|
("s", "s3", 1000000000),
|
|
]
|
|
]
|
|
save(path / "containers-before.json", containers)
|
|
save(path / "containers-after.json", containers)
|
|
|
|
def counters(received, sent, ns, peak):
|
|
return {
|
|
"rx_bytes": received,
|
|
"tx_bytes": sent,
|
|
"memory.current": 100,
|
|
"memory.peak": peak,
|
|
"memory.max": 536870912,
|
|
"memory.events": {"oom": 0, "oom_kill": 0, "max": 0},
|
|
"cpu.stat": {"usage_usec": ns // 10000},
|
|
"read_started_ns": ns,
|
|
"read_finished_ns": ns + 1,
|
|
}
|
|
|
|
ids = [c["id"] for c in containers]
|
|
m = {
|
|
"started_ns": 10000000000,
|
|
"finished_ns": 12000000000,
|
|
"generator_cpu_seconds": 0.1,
|
|
"idle_before": {cid: counters(0, 0, 0, 300) for cid in ids},
|
|
"idle_after": {cid: counters(0, 0, 5000000000, 300) for cid in ids},
|
|
"before": {cid: counters(0, 0, 10000000000, 300) for cid in ids},
|
|
"after": {
|
|
cid: counters(4000000, 4000000, 12000000000, 400) for cid in ids
|
|
},
|
|
}
|
|
save(path / "measurement.json", m)
|
|
requests = [
|
|
{
|
|
"wave": 0,
|
|
"ready": True,
|
|
"started_ns": 10100000000 + i * 1000000,
|
|
"finished_ns": 11900000000,
|
|
"attachment": {"id": cid, "instance": cid},
|
|
}
|
|
for i, cid in enumerate(("a", "b"))
|
|
]
|
|
(path / "requests.jsonl").write_text(
|
|
"".join(json.dumps(r) + "\n" for r in requests)
|
|
)
|
|
(path / "warmup.jsonl").write_text(
|
|
"".join(json.dumps(r) + "\n" for r in requests)
|
|
)
|
|
(path / "verification.jsonl").write_text(
|
|
"".join(
|
|
json.dumps({"id": cid, "verified": True}) + "\n"
|
|
for cid in ("a", "b")
|
|
)
|
|
)
|
|
(path / "samples.jsonl").write_text(
|
|
"".join(
|
|
json.dumps(
|
|
{
|
|
"container": cid,
|
|
"memory_current": 200,
|
|
"monotonic_ns": 10000000000 + i * 100000000,
|
|
}
|
|
)
|
|
+ "\n"
|
|
for cid in ids
|
|
for i in range(21)
|
|
)
|
|
)
|
|
result = case_metrics(path, protocol)
|
|
self.assertEqual(result["api_rx_tx_bytes"], 16000000)
|
|
self.assertEqual(result["api_bytes_per_ready"], 8000000)
|
|
self.assertEqual(result["containers"][0]["sampled_peak_window"], 200)
|
|
self.assertEqual(result["containers"][0]["peak_lifetime_after"], 400)
|
|
self.assertEqual(result["system_failures"], [])
|
|
self.assertEqual(result["instrumentation_issues"], [])
|
|
|
|
|
|
class TransportTests(unittest.TestCase):
|
|
def test_direct_stages_do_not_send_bearer_to_s3_or_leak_secrets_to_records(self):
|
|
observed, data = [], b"synthetic PNG transport fixture"
|
|
checksum = hashlib.sha256(data).hexdigest()
|
|
token, signed_field = "DO-NOT-PERSIST-TOKEN", "DO-NOT-PERSIST-SIGNATURE"
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
observed.append(
|
|
{
|
|
"path": self.path,
|
|
"authorization": self.headers.get("Authorization"),
|
|
"body": body,
|
|
}
|
|
)
|
|
if self.path.endswith("/attachment-uploads"):
|
|
response = {
|
|
"upload_id": "upload",
|
|
"upload": {
|
|
"url": f"http://127.0.0.1:{self.server.server_port}/bucket",
|
|
"fields": {
|
|
"key": "staging/direct/upload",
|
|
"policy": signed_field,
|
|
},
|
|
},
|
|
}
|
|
elif self.path == "/bucket":
|
|
self.send_response(204)
|
|
self.send_header("Content-Length", "0")
|
|
self.end_headers()
|
|
return
|
|
else:
|
|
response = {
|
|
"id": "attachment",
|
|
"status": "ready",
|
|
"size_bytes": len(data),
|
|
"sha256": checksum,
|
|
}
|
|
encoded = json.dumps(response).encode()
|
|
self.send_response(201)
|
|
self.send_header("Content-Length", str(len(encoded)))
|
|
self.send_header("X-Instance-Id", "api-test")
|
|
self.end_headers()
|
|
self.wfile.write(encoded)
|
|
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
try:
|
|
protocol = copy.deepcopy(PROTOCOL)
|
|
protocol["ports"] = {"api": server.server_port, "s3": server.server_port}
|
|
client = Client(
|
|
f"http://127.0.0.1:{server.server_port}", protocol, threading.Event()
|
|
)
|
|
records = []
|
|
client.upload(
|
|
"direct",
|
|
{"token": token},
|
|
"chat",
|
|
data,
|
|
checksum,
|
|
records,
|
|
__import__("time").monotonic() + 10,
|
|
)
|
|
self.assertEqual(
|
|
[r["stage"] for r in records],
|
|
["client_sha256", "init", "s3_post", "complete"],
|
|
)
|
|
self.assertEqual(observed[0]["authorization"], "Bearer " + token)
|
|
self.assertIsNone(observed[1]["authorization"])
|
|
self.assertIn(data, observed[1]["body"])
|
|
self.assertEqual(observed[2]["authorization"], "Bearer " + token)
|
|
self.assertNotIn(token, json.dumps(records))
|
|
self.assertNotIn(signed_field, json.dumps(records))
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|