Files
mtusi2026lab4/experiments/tests/test_orchestration.py
T

214 lines
8.1 KiB
Python

import hashlib
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import driver
from analysis import json_lines
from client import FlowError
from common import (
ROOT,
ExperimentError,
check_seal,
digest,
load,
save,
source_manifest,
)
class OrchestrationTests(unittest.TestCase):
def test_source_snapshot_excludes_secrets_evidence_and_git(self):
manifest = source_manifest()
self.assertIn("src/backend/services/direct_upload.go", manifest)
self.assertIn("experiments/docs/HYPOTHESIS.md", manifest)
for name in manifest:
self.assertNotIn(
name.split("/")[0], (".git", ".memory", "secrets", "evidence")
)
self.assertNotEqual(Path(name).name, ".env")
def test_prepare_seals_protocol_before_builds_and_does_not_upload(self):
with tempfile.TemporaryDirectory(dir=ROOT) as folder:
evidence = Path(folder)
names = [
"src/go.mod",
"infra/minio.Dockerfile",
"experiments/src/analysis.py",
"experiments/src/common.py",
]
manifest = {name: digest(ROOT / name) for name in names}
def fake_command(args, *_args, **_kwargs):
if args[:2] == ["docker", "build"]:
campaign = next(evidence.iterdir())
self.assertTrue((campaign / "protocol.json").exists())
self.assertFalse((campaign / "cases").exists())
return ""
if args[:3] == ["docker", "image", "inspect"]:
return json.dumps(
[
{
"Id": "sha256:" + "a" * 64,
"RepoDigests": [],
"Os": "linux",
"Architecture": "amd64",
"RootFS": {"Layers": ["layer"]},
}
]
)
if args[0] == "git":
return "synthetic-git-metadata"
raise AssertionError("unexpected command in prepare")
with (
patch("driver.EVIDENCE", evidence),
patch("driver.preflight", return_value={"purpose": "fake-host"}),
patch("driver.source_manifest", return_value=manifest),
patch("driver.check_sources"),
patch("driver.command", side_effect=fake_command),
patch("driver.Client") as client,
patch("builtins.print"),
):
campaign = driver.prepare(pilot=True)
client.assert_not_called()
check_seal(campaign)
self.assertEqual(load(campaign / "prepared.json")["cases"], 4)
self.assertEqual(load(campaign / "protocol.json")["purpose"], "pilot")
self.assertFalse((campaign / "cases").exists())
protocol = load(campaign / "protocol.json")
protocol["clients"] += 1
save(campaign / "protocol.json", protocol)
with self.assertRaises(ExperimentError):
check_seal(campaign)
def test_failed_uploads_are_saved_and_verification_starts_after_counters_close(
self,
):
with tempfile.TemporaryDirectory() as folder:
campaign = Path(folder)
protocol = load(driver.EXPERIMENT / "config/protocol.json")
protocol.update(
clients=2,
waves=1,
warmup_clients=2,
settle_seconds=0,
idle_seconds=0.001,
)
save(campaign / "protocol.json", protocol)
save(campaign / "images.json", {"app": {"id": "app"}, "s3": {"id": "s3"}})
data = b"synthetic-fixture"
checksum = hashlib.sha256(data).hexdigest()
save(
campaign / "fixtures.json",
[{"name": "fake", "filename": "fake.png", "sha256": checksum}],
)
(campaign / "fixtures").mkdir()
(campaign / "fixtures/fake.png").write_bytes(data)
case = {
"directory": "01-fake",
"number": 1,
"profile": "fake",
"mode": "direct",
"size_bytes": len(data),
"repetition": 1,
}
containers = [
{
"id": cid,
"hostname": cid,
"service": role,
"image": image,
"memory_limit": 536870912,
"memory_and_swap_limit": 536870912,
"nano_cpus": 500000000,
"tmpfs": {"/tmp/uploads": "rw,noexec,nosuid,size=16m,mode=1777"},
}
for cid, role, image in [
("a", "api", "app"),
("b", "api", "app"),
("s", "s3", "s3"),
]
]
state = {"snapshots": 0, "sampler_closed": False, "uploads": 0}
mutex = threading.Lock()
class FakeClient:
def __init__(self, _base, p, _stop):
self.protocol = p
def bootstrap(self):
return [{"token": "secret-a"}, {"token": "secret-b"}], "chat"
def upload(
self, _mode, _actor, _chat, _data, _digest, records, _deadline
):
with mutex:
state["uploads"] += 1
index = state["uploads"]
if index == 3:
raise FlowError("http_503")
return {
"id": "attachment-" + str(index),
"instance": "a" if index % 2 else "b",
}
def verify(self, _actor, attachment, _size, _digest):
if state["snapshots"] != 4 or not state["sampler_closed"]:
raise AssertionError("verification contaminated measurement")
return {"id": attachment["id"], "verified": True}
class FakeSampler:
errors = []
def __init__(self, _containers, path, _interval):
path.write_text("")
def wait_ready(self):
pass
def close(self):
state["sampler_closed"] = True
def fake_snapshot(_containers):
state["snapshots"] += 1
return {c["id"]: {"sample": "fake"} for c in containers}
def fake_command(args, *_args, **_kwargs):
return "a\nb\ns" if "ps" in args else ""
with (
patch("driver.command", side_effect=fake_command),
patch("driver.inspect_containers", return_value=containers),
patch("driver.Client", FakeClient),
patch("driver.Sampler", FakeSampler),
patch("driver.snapshot", side_effect=fake_snapshot),
):
outcome = driver.run_case(
campaign,
case,
Path(folder),
"hl-upload-exp-fake-1",
threading.Event(),
)
self.assertEqual(outcome["status"], "completed")
directory = campaign / "cases/01-fake"
requests = json_lines(directory / "requests.jsonl")
self.assertEqual(len(requests), 2)
self.assertEqual(sum(r["ready"] for r in requests), 1)
self.assertEqual(
next(r for r in requests if not r["ready"])["error"], "http_503"
)
self.assertEqual(len(json_lines(directory / "verification.jsonl")), 1)
self.assertNotIn("secret-a", (directory / "requests.jsonl").read_text())
self.assertTrue((directory / "measurement.json").exists())
if __name__ == "__main__":
unittest.main()