226 lines
9.4 KiB
Python
226 lines
9.4 KiB
Python
"""Preparation checks with synthetic in-memory data; these do not run the series."""
|
|
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import zlib
|
|
from unittest.mock import patch
|
|
from pathlib import Path
|
|
|
|
EXPERIMENTS = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(EXPERIMENTS / "src"))
|
|
import support
|
|
|
|
spec = importlib.util.spec_from_file_location("experiment_cli", EXPERIMENTS / "run.py")
|
|
cli = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(cli)
|
|
|
|
|
|
class ToolTests(unittest.TestCase):
|
|
def test_git_keeps_public_evidence_and_hides_credentials_after_migration(self):
|
|
rules = (support.ROOT / ".gitignore").read_text()
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
subprocess.run(["git", "init", "-q", str(root)], check=True)
|
|
exclude = root / ".git/info/exclude"
|
|
exclude.write_text("evidence/idempotency/\ncustom-local-rule/\n")
|
|
public = root / "evidence/idempotency/main-test/summary.json"
|
|
private = public.parent / ".private/r1/private.env"
|
|
private.parent.mkdir(parents=True)
|
|
public.write_text("{}")
|
|
private.write_text("POSTGRES_PASSWORD=test-only\n")
|
|
(root / ".gitignore").write_text(rules)
|
|
log = public.parent / "boot-r1.log"
|
|
log.write_text("setup log\n")
|
|
with patch.object(support, "ROOT", root):
|
|
support.ensure_local_ignore()
|
|
first = exclude.read_text()
|
|
support.ensure_local_ignore()
|
|
self.assertEqual(first, exclude.read_text())
|
|
files = support.source_files()
|
|
self.assertIn("custom-local-rule/", first)
|
|
self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(public)], cwd=root).returncode, 1)
|
|
self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(private)], cwd=root).returncode, 0)
|
|
self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(log)], cwd=root).returncode, 1)
|
|
self.assertFalse(any(name.startswith("evidence/") for name in files))
|
|
|
|
def test_outputs_belong_to_root_evidence(self):
|
|
self.assertEqual(cli.RESULTS, support.ROOT / "evidence/idempotency")
|
|
|
|
def test_png_is_reproducible_and_decodable(self):
|
|
profile = {"width": 17, "height": 9, "alpha": True}
|
|
first = support.png(profile, 2303)
|
|
self.assertEqual(first, support.png(profile, 2303))
|
|
self.assertNotEqual(first, support.png(profile, 2304))
|
|
offset = 8
|
|
compressed = []
|
|
while offset < len(first):
|
|
size = struct.unpack(">I", first[offset : offset + 4])[0]
|
|
kind = first[offset + 4 : offset + 8]
|
|
data = first[offset + 8 : offset + 8 + size]
|
|
self.assertEqual(
|
|
zlib.crc32(kind + data) & 0xFFFFFFFF,
|
|
struct.unpack(">I", first[offset + 8 + size : offset + 12 + size])[0],
|
|
)
|
|
if kind == b"IDAT":
|
|
compressed.append(data)
|
|
offset += 12 + size
|
|
self.assertEqual(len(zlib.decompress(b"".join(compressed))), 9 * (17 * 4 + 1))
|
|
|
|
def test_isolation_removes_shared_resource_names_and_secret_paths(self):
|
|
service = {
|
|
"build": {"context": "x"},
|
|
"volumes": [
|
|
{
|
|
"type": "bind",
|
|
"source": str(support.ROOT / "secrets/database_url"),
|
|
"target": "/run/secrets/database_url",
|
|
}
|
|
],
|
|
"environment": {},
|
|
}
|
|
services = {
|
|
name: copy.deepcopy(service)
|
|
for name in (
|
|
"api",
|
|
"worker",
|
|
"dispatcher",
|
|
"s3",
|
|
"haproxy",
|
|
"frontend",
|
|
"backend-tests",
|
|
)
|
|
}
|
|
source = {
|
|
"services": services,
|
|
"volumes": {"db": {"name": "highload-lab5_db"}},
|
|
"networks": {"data": {"name": "highload-lab5_data"}},
|
|
}
|
|
protocol = json.loads(support.PROTOCOL.read_text())
|
|
private = Path("/tmp/private-example")
|
|
result = support.isolated_config(
|
|
source, "lab5-idem-unit", private, protocol, "sha256:app", "sha256:s3"
|
|
)
|
|
self.assertNotIn("name", result["volumes"]["db"])
|
|
self.assertNotIn("name", result["networks"]["data"])
|
|
self.assertNotIn("frontend", result["services"])
|
|
self.assertEqual(result["services"]["worker"]["deploy"]["replicas"], 2)
|
|
for entry in result["services"].values():
|
|
self.assertNotIn("build", entry)
|
|
self.assertTrue(
|
|
all(
|
|
not mount.get("source", "").startswith(
|
|
str(support.ROOT / "secrets")
|
|
)
|
|
for mount in entry["volumes"]
|
|
)
|
|
)
|
|
|
|
def test_analyzer_does_not_call_partial_or_duplicate_matrix_success(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
protocol = json.loads(support.PROTOCOL.read_text())
|
|
support.dump(root / "protocol.json", protocol)
|
|
support.dump(root / "manifest.json", {"phase": "main"})
|
|
(root / "cases.jsonl").write_text("")
|
|
self.assertEqual(cli.analyze(root)["outcome"], "inconclusive")
|
|
row = {
|
|
"repetition": 1,
|
|
"profile": "rgb_small",
|
|
"scenario": "control",
|
|
"directory": "cases/test",
|
|
"status": "violation",
|
|
}
|
|
(root / "cases.jsonl").write_text(json.dumps(row) + "\n")
|
|
self.assertEqual(cli.analyze(root)["outcome"], "refuted")
|
|
support.dump(root / "manifest.json", {"phase": "pilot"})
|
|
self.assertEqual(
|
|
cli.analyze(root)["outcome"], "pilot_only_not_confirmation"
|
|
)
|
|
|
|
def test_private_credentials_never_enter_share_archive(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary) / "case"
|
|
root.mkdir()
|
|
(root / ".private/r1/secrets").mkdir(parents=True)
|
|
(root / ".private/r1/secrets/broker_password").write_text(
|
|
"sensitive-password-value-123"
|
|
)
|
|
(root / "safe.json").write_text("{}")
|
|
archive = support.archive_public(root)
|
|
import zipfile
|
|
|
|
with zipfile.ZipFile(archive) as value:
|
|
self.assertEqual(value.namelist(), ["safe.json"])
|
|
|
|
def test_analyzer_requires_unique_complete_cases_and_counts_oom(self):
|
|
# A tiny artificial protocol tests the policy, never supplies experiment data.
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
protocol = json.loads(support.PROTOCOL.read_text())
|
|
protocol.update(
|
|
repetitions=1, profiles=protocol["profiles"][:1], scenarios=["control"]
|
|
)
|
|
protocol["pass"]["cases"] = 1
|
|
support.dump(root / "protocol.json", protocol)
|
|
support.dump(
|
|
root / "manifest.json", {"phase": "main", "finished_utc": "unit-test"}
|
|
)
|
|
for filename in (
|
|
"seal/manifest.json",
|
|
"seal/source.zip",
|
|
"fixtures/manifest.json",
|
|
"docker-events-r1.jsonl",
|
|
"environment-r1.json",
|
|
"order-r1.json",
|
|
):
|
|
path = root / filename
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("")
|
|
row = {
|
|
"repetition": 1,
|
|
"profile": "rgb_small",
|
|
"scenario": "control",
|
|
"directory": "cases/unit",
|
|
"status": "pass",
|
|
}
|
|
for filename in (
|
|
"events.jsonl",
|
|
"result.json",
|
|
"containers-before.json",
|
|
"containers-after.json",
|
|
"database-final.json",
|
|
"fault-markers.json",
|
|
):
|
|
path = root / row["directory"] / filename
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("{}")
|
|
(root / "cases.jsonl").write_text(json.dumps(row) + "\n")
|
|
self.assertEqual(
|
|
cli.analyze(root)["outcome"], "supported_in_tested_conditions"
|
|
)
|
|
(root / "cases.jsonl").write_text((json.dumps(row) + "\n") * 2)
|
|
self.assertEqual(cli.analyze(root)["outcome"], "inconclusive")
|
|
(root / "cases.jsonl").write_text(json.dumps(row) + "\n")
|
|
(root / "docker-events-r1.jsonl").write_text('{"action":"oom"}\n')
|
|
self.assertEqual(cli.analyze(root)["outcome"], "refuted")
|
|
|
|
def test_archive_refuses_a_public_file_with_local_secret(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary) / "case"
|
|
(root / ".private/r1/secrets").mkdir(parents=True)
|
|
secret = "local-sensitive-password-value"
|
|
(root / ".private/r1/secrets/postgres_password").write_text(secret)
|
|
(root / "leak.log").write_text(secret)
|
|
with self.assertRaises(RuntimeError):
|
|
support.archive_public(root)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|