214 lines
8.1 KiB
Python
214 lines
8.1 KiB
Python
import hashlib
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
import driver
|
|
from analysis import json_lines
|
|
from client import FlowError
|
|
from common import (
|
|
ROOT,
|
|
ExperimentError,
|
|
check_seal,
|
|
digest,
|
|
load,
|
|
save,
|
|
source_manifest,
|
|
)
|
|
|
|
|
|
class OrchestrationTests(unittest.TestCase):
|
|
def test_source_snapshot_excludes_secrets_evidence_and_git(self):
|
|
manifest = source_manifest()
|
|
self.assertIn("src/backend/services/direct_upload.go", manifest)
|
|
self.assertIn("experiments/docs/HYPOTHESIS.md", manifest)
|
|
for name in manifest:
|
|
self.assertNotIn(
|
|
name.split("/")[0], (".git", ".memory", "secrets", "evidence")
|
|
)
|
|
self.assertNotEqual(Path(name).name, ".env")
|
|
|
|
def test_prepare_seals_protocol_before_builds_and_does_not_upload(self):
|
|
with tempfile.TemporaryDirectory(dir=ROOT) as folder:
|
|
evidence = Path(folder)
|
|
names = [
|
|
"src/go.mod",
|
|
"infra/minio.Dockerfile",
|
|
"experiments/src/analysis.py",
|
|
"experiments/src/common.py",
|
|
]
|
|
manifest = {name: digest(ROOT / name) for name in names}
|
|
|
|
def fake_command(args, *_args, **_kwargs):
|
|
if args[:2] == ["docker", "build"]:
|
|
campaign = next(evidence.iterdir())
|
|
self.assertTrue((campaign / "protocol.json").exists())
|
|
self.assertFalse((campaign / "cases").exists())
|
|
return ""
|
|
if args[:3] == ["docker", "image", "inspect"]:
|
|
return json.dumps(
|
|
[
|
|
{
|
|
"Id": "sha256:" + "a" * 64,
|
|
"RepoDigests": [],
|
|
"Os": "linux",
|
|
"Architecture": "amd64",
|
|
"RootFS": {"Layers": ["layer"]},
|
|
}
|
|
]
|
|
)
|
|
if args[0] == "git":
|
|
return "synthetic-git-metadata"
|
|
raise AssertionError("unexpected command in prepare")
|
|
|
|
with (
|
|
patch("driver.EVIDENCE", evidence),
|
|
patch("driver.preflight", return_value={"purpose": "fake-host"}),
|
|
patch("driver.source_manifest", return_value=manifest),
|
|
patch("driver.check_sources"),
|
|
patch("driver.command", side_effect=fake_command),
|
|
patch("driver.Client") as client,
|
|
patch("builtins.print"),
|
|
):
|
|
campaign = driver.prepare(pilot=True)
|
|
client.assert_not_called()
|
|
check_seal(campaign)
|
|
self.assertEqual(load(campaign / "prepared.json")["cases"], 4)
|
|
self.assertEqual(load(campaign / "protocol.json")["purpose"], "pilot")
|
|
self.assertFalse((campaign / "cases").exists())
|
|
protocol = load(campaign / "protocol.json")
|
|
protocol["clients"] += 1
|
|
save(campaign / "protocol.json", protocol)
|
|
with self.assertRaises(ExperimentError):
|
|
check_seal(campaign)
|
|
|
|
def test_failed_uploads_are_saved_and_verification_starts_after_counters_close(
|
|
self,
|
|
):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
campaign = Path(folder)
|
|
protocol = load(driver.EXPERIMENT / "config/protocol.json")
|
|
protocol.update(
|
|
clients=2,
|
|
waves=1,
|
|
warmup_clients=2,
|
|
settle_seconds=0,
|
|
idle_seconds=0.001,
|
|
)
|
|
save(campaign / "protocol.json", protocol)
|
|
save(campaign / "images.json", {"app": {"id": "app"}, "s3": {"id": "s3"}})
|
|
data = b"synthetic-fixture"
|
|
checksum = hashlib.sha256(data).hexdigest()
|
|
save(
|
|
campaign / "fixtures.json",
|
|
[{"name": "fake", "filename": "fake.png", "sha256": checksum}],
|
|
)
|
|
(campaign / "fixtures").mkdir()
|
|
(campaign / "fixtures/fake.png").write_bytes(data)
|
|
case = {
|
|
"directory": "01-fake",
|
|
"number": 1,
|
|
"profile": "fake",
|
|
"mode": "direct",
|
|
"size_bytes": len(data),
|
|
"repetition": 1,
|
|
}
|
|
containers = [
|
|
{
|
|
"id": cid,
|
|
"hostname": cid,
|
|
"service": role,
|
|
"image": image,
|
|
"memory_limit": 536870912,
|
|
"memory_and_swap_limit": 536870912,
|
|
"nano_cpus": 500000000,
|
|
"tmpfs": {"/tmp/uploads": "rw,noexec,nosuid,size=16m,mode=1777"},
|
|
}
|
|
for cid, role, image in [
|
|
("a", "api", "app"),
|
|
("b", "api", "app"),
|
|
("s", "s3", "s3"),
|
|
]
|
|
]
|
|
state = {"snapshots": 0, "sampler_closed": False, "uploads": 0}
|
|
mutex = threading.Lock()
|
|
|
|
class FakeClient:
|
|
def __init__(self, _base, p, _stop):
|
|
self.protocol = p
|
|
|
|
def bootstrap(self):
|
|
return [{"token": "secret-a"}, {"token": "secret-b"}], "chat"
|
|
|
|
def upload(
|
|
self, _mode, _actor, _chat, _data, _digest, records, _deadline
|
|
):
|
|
with mutex:
|
|
state["uploads"] += 1
|
|
index = state["uploads"]
|
|
if index == 3:
|
|
raise FlowError("http_503")
|
|
return {
|
|
"id": "attachment-" + str(index),
|
|
"instance": "a" if index % 2 else "b",
|
|
}
|
|
|
|
def verify(self, _actor, attachment, _size, _digest):
|
|
if state["snapshots"] != 4 or not state["sampler_closed"]:
|
|
raise AssertionError("verification contaminated measurement")
|
|
return {"id": attachment["id"], "verified": True}
|
|
|
|
class FakeSampler:
|
|
errors = []
|
|
|
|
def __init__(self, _containers, path, _interval):
|
|
path.write_text("")
|
|
|
|
def wait_ready(self):
|
|
pass
|
|
|
|
def close(self):
|
|
state["sampler_closed"] = True
|
|
|
|
def fake_snapshot(_containers):
|
|
state["snapshots"] += 1
|
|
return {c["id"]: {"sample": "fake"} for c in containers}
|
|
|
|
def fake_command(args, *_args, **_kwargs):
|
|
return "a\nb\ns" if "ps" in args else ""
|
|
|
|
with (
|
|
patch("driver.command", side_effect=fake_command),
|
|
patch("driver.inspect_containers", return_value=containers),
|
|
patch("driver.Client", FakeClient),
|
|
patch("driver.Sampler", FakeSampler),
|
|
patch("driver.snapshot", side_effect=fake_snapshot),
|
|
):
|
|
outcome = driver.run_case(
|
|
campaign,
|
|
case,
|
|
Path(folder),
|
|
"hl-upload-exp-fake-1",
|
|
threading.Event(),
|
|
)
|
|
self.assertEqual(outcome["status"], "completed")
|
|
directory = campaign / "cases/01-fake"
|
|
requests = json_lines(directory / "requests.jsonl")
|
|
self.assertEqual(len(requests), 2)
|
|
self.assertEqual(sum(r["ready"] for r in requests), 1)
|
|
self.assertEqual(
|
|
next(r for r in requests if not r["ready"])["error"], "http_503"
|
|
)
|
|
self.assertEqual(len(json_lines(directory / "verification.jsonl")), 1)
|
|
self.assertNotIn("secret-a", (directory / "requests.jsonl").read_text())
|
|
self.assertTrue((directory / "measurement.json").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|