Files
mtusi2026lab5/experiments/tests/test_tools.py
T

226 lines
9.4 KiB
Python

"""Preparation checks with synthetic in-memory data; these do not run the series."""
import copy
import importlib.util
import json
import struct
import subprocess
import sys
import tempfile
import unittest
import zlib
from unittest.mock import patch
from pathlib import Path
EXPERIMENTS = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(EXPERIMENTS / "src"))
import support
spec = importlib.util.spec_from_file_location("experiment_cli", EXPERIMENTS / "run.py")
cli = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cli)
class ToolTests(unittest.TestCase):
def test_git_keeps_public_evidence_and_hides_credentials_after_migration(self):
rules = (support.ROOT / ".gitignore").read_text()
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
subprocess.run(["git", "init", "-q", str(root)], check=True)
exclude = root / ".git/info/exclude"
exclude.write_text("evidence/idempotency/\ncustom-local-rule/\n")
public = root / "evidence/idempotency/main-test/summary.json"
private = public.parent / ".private/r1/private.env"
private.parent.mkdir(parents=True)
public.write_text("{}")
private.write_text("POSTGRES_PASSWORD=test-only\n")
(root / ".gitignore").write_text(rules)
log = public.parent / "boot-r1.log"
log.write_text("setup log\n")
with patch.object(support, "ROOT", root):
support.ensure_local_ignore()
first = exclude.read_text()
support.ensure_local_ignore()
self.assertEqual(first, exclude.read_text())
files = support.source_files()
self.assertIn("custom-local-rule/", first)
self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(public)], cwd=root).returncode, 1)
self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(private)], cwd=root).returncode, 0)
self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(log)], cwd=root).returncode, 1)
self.assertFalse(any(name.startswith("evidence/") for name in files))
def test_outputs_belong_to_root_evidence(self):
self.assertEqual(cli.RESULTS, support.ROOT / "evidence/idempotency")
def test_png_is_reproducible_and_decodable(self):
profile = {"width": 17, "height": 9, "alpha": True}
first = support.png(profile, 2303)
self.assertEqual(first, support.png(profile, 2303))
self.assertNotEqual(first, support.png(profile, 2304))
offset = 8
compressed = []
while offset < len(first):
size = struct.unpack(">I", first[offset : offset + 4])[0]
kind = first[offset + 4 : offset + 8]
data = first[offset + 8 : offset + 8 + size]
self.assertEqual(
zlib.crc32(kind + data) & 0xFFFFFFFF,
struct.unpack(">I", first[offset + 8 + size : offset + 12 + size])[0],
)
if kind == b"IDAT":
compressed.append(data)
offset += 12 + size
self.assertEqual(len(zlib.decompress(b"".join(compressed))), 9 * (17 * 4 + 1))
def test_isolation_removes_shared_resource_names_and_secret_paths(self):
service = {
"build": {"context": "x"},
"volumes": [
{
"type": "bind",
"source": str(support.ROOT / "secrets/database_url"),
"target": "/run/secrets/database_url",
}
],
"environment": {},
}
services = {
name: copy.deepcopy(service)
for name in (
"api",
"worker",
"dispatcher",
"s3",
"haproxy",
"frontend",
"backend-tests",
)
}
source = {
"services": services,
"volumes": {"db": {"name": "highload-lab5_db"}},
"networks": {"data": {"name": "highload-lab5_data"}},
}
protocol = json.loads(support.PROTOCOL.read_text())
private = Path("/tmp/private-example")
result = support.isolated_config(
source, "lab5-idem-unit", private, protocol, "sha256:app", "sha256:s3"
)
self.assertNotIn("name", result["volumes"]["db"])
self.assertNotIn("name", result["networks"]["data"])
self.assertNotIn("frontend", result["services"])
self.assertEqual(result["services"]["worker"]["deploy"]["replicas"], 2)
for entry in result["services"].values():
self.assertNotIn("build", entry)
self.assertTrue(
all(
not mount.get("source", "").startswith(
str(support.ROOT / "secrets")
)
for mount in entry["volumes"]
)
)
def test_analyzer_does_not_call_partial_or_duplicate_matrix_success(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
protocol = json.loads(support.PROTOCOL.read_text())
support.dump(root / "protocol.json", protocol)
support.dump(root / "manifest.json", {"phase": "main"})
(root / "cases.jsonl").write_text("")
self.assertEqual(cli.analyze(root)["outcome"], "inconclusive")
row = {
"repetition": 1,
"profile": "rgb_small",
"scenario": "control",
"directory": "cases/test",
"status": "violation",
}
(root / "cases.jsonl").write_text(json.dumps(row) + "\n")
self.assertEqual(cli.analyze(root)["outcome"], "refuted")
support.dump(root / "manifest.json", {"phase": "pilot"})
self.assertEqual(
cli.analyze(root)["outcome"], "pilot_only_not_confirmation"
)
def test_private_credentials_never_enter_share_archive(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary) / "case"
root.mkdir()
(root / ".private/r1/secrets").mkdir(parents=True)
(root / ".private/r1/secrets/broker_password").write_text(
"sensitive-password-value-123"
)
(root / "safe.json").write_text("{}")
archive = support.archive_public(root)
import zipfile
with zipfile.ZipFile(archive) as value:
self.assertEqual(value.namelist(), ["safe.json"])
def test_analyzer_requires_unique_complete_cases_and_counts_oom(self):
# A tiny artificial protocol tests the policy, never supplies experiment data.
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
protocol = json.loads(support.PROTOCOL.read_text())
protocol.update(
repetitions=1, profiles=protocol["profiles"][:1], scenarios=["control"]
)
protocol["pass"]["cases"] = 1
support.dump(root / "protocol.json", protocol)
support.dump(
root / "manifest.json", {"phase": "main", "finished_utc": "unit-test"}
)
for filename in (
"seal/manifest.json",
"seal/source.zip",
"fixtures/manifest.json",
"docker-events-r1.jsonl",
"environment-r1.json",
"order-r1.json",
):
path = root / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("")
row = {
"repetition": 1,
"profile": "rgb_small",
"scenario": "control",
"directory": "cases/unit",
"status": "pass",
}
for filename in (
"events.jsonl",
"result.json",
"containers-before.json",
"containers-after.json",
"database-final.json",
"fault-markers.json",
):
path = root / row["directory"] / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("{}")
(root / "cases.jsonl").write_text(json.dumps(row) + "\n")
self.assertEqual(
cli.analyze(root)["outcome"], "supported_in_tested_conditions"
)
(root / "cases.jsonl").write_text((json.dumps(row) + "\n") * 2)
self.assertEqual(cli.analyze(root)["outcome"], "inconclusive")
(root / "cases.jsonl").write_text(json.dumps(row) + "\n")
(root / "docker-events-r1.jsonl").write_text('{"action":"oom"}\n')
self.assertEqual(cli.analyze(root)["outcome"], "refuted")
def test_archive_refuses_a_public_file_with_local_secret(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary) / "case"
(root / ".private/r1/secrets").mkdir(parents=True)
secret = "local-sensitive-password-value"
(root / ".private/r1/secrets/postgres_password").write_text(secret)
(root / "leak.log").write_text(secret)
with self.assertRaises(RuntimeError):
support.archive_public(root)
if __name__ == "__main__":
unittest.main()