Files
mtusi2026lab5/experiments/run.py
T

525 lines
20 KiB
Python

#!/usr/bin/env python3
"""Seal first; run pilot/main only on an explicit user command."""
import argparse
import datetime as dt
import json
import os
import platform
import random
import shutil
import socket
import subprocess
import sys
import time
import uuid
import zipfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
from support import (
PROTOCOL,
ROOT,
archive_public,
build_images,
digest,
dump,
ensure_local_ignore,
isolated_config,
png,
private_environment,
source_manifest,
verify_seal,
)
RESULTS = ROOT / "evidence/idempotency"
def stamp():
return (
dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+ "-"
+ uuid.uuid4().hex[:8]
)
def seal():
directory = RESULTS / "seals" / stamp()
directory.mkdir(parents=True, exist_ok=False)
files = source_manifest()
with zipfile.ZipFile(
directory / "source.zip", "x", zipfile.ZIP_DEFLATED
) as archive:
for name in files:
archive.write(ROOT / name, name)
value = {
"registered_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
"protocol_sha256": digest(PROTOCOL.read_bytes()),
"files": files,
"git_head": subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip(),
"git_dirty": bool(
subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT)
),
}
dump(directory / "manifest.json", value)
print(directory.relative_to(ROOT))
def analyze(run):
protocol = json.loads((run / "protocol.json").read_text())
manifest = json.loads((run / "manifest.json").read_text())
records = (
[json.loads(line) for line in (run / "cases.jsonl").read_text().splitlines()]
if (run / "cases.jsonl").exists()
else []
)
violations = sum(row["status"] == "violation" for row in records)
invalid = sum(row["status"] == "invalid" for row in records)
missing = []
for row in records:
for filename in (
"events.jsonl",
"result.json",
"containers-before.json",
"containers-after.json",
"database-final.json",
"fault-markers.json",
):
if not (run / row["directory"] / filename).is_file():
missing.append(row["directory"] + "/" + filename)
expected = (
protocol["pass"]["cases"]
if manifest["phase"] == "main"
else len(protocol["scenarios"])
)
repetitions = (
range(1, protocol["repetitions"] + 1) if manifest["phase"] == "main" else [1]
)
profiles = (
protocol["profiles"]
if manifest["phase"] == "main"
else protocol["profiles"][:1]
)
planned = {
(r, p["name"], s)
for r in repetitions
for p in profiles
for s in protocol["scenarios"]
}
for repetition in repetitions:
for name in (
"docker-events-r" + str(repetition) + ".jsonl",
"environment-r" + str(repetition) + ".json",
"order-r" + str(repetition) + ".json",
):
if not (run / name).is_file():
missing.append(name)
keys = [(row["repetition"], row["profile"], row["scenario"]) for row in records]
invalid_matrix = len(keys) != len(set(keys)) or set(keys) != planned
oom_events = 0
for path in run.glob("docker-events-r*.jsonl"):
for line in path.read_text().splitlines():
if json.loads(line).get("action") == "oom":
oom_events += 1
for filename in ("seal/manifest.json", "seal/source.zip", "fixtures/manifest.json"):
if not (run / filename).exists():
missing.append(filename)
if manifest["phase"] != "main":
outcome = "pilot_only_not_confirmation"
elif violations or oom_events:
outcome = "refuted"
elif (
len(records) != expected
or invalid_matrix
or invalid
or missing
or manifest.get("interrupted")
or not manifest.get("finished_utc")
):
outcome = "inconclusive"
else:
outcome = "supported_in_tested_conditions"
summary = {
"phase": manifest["phase"],
"expected": expected,
"completed": len(records),
"violations": violations,
"complete": len(records) == expected
and not invalid_matrix
and not invalid
and not missing
and not oom_events
and not manifest.get("interrupted")
and bool(manifest.get("finished_utc")),
"oom_events": oom_events,
"invalid": invalid,
"invalid_matrix": invalid_matrix,
"missing_artifacts": missing,
"outcome": outcome,
"groups": {},
}
for row in records:
key = row["scenario"] + "/" + row["profile"]
group = summary["groups"].setdefault(
key, {"cases": 0, "violations": 0, "invalid": 0, "elapsed_seconds": []}
)
group["cases"] += 1
group["violations"] += row["status"] == "violation"
group["invalid"] += row["status"] == "invalid"
if row.get("elapsed_seconds") is not None:
group["elapsed_seconds"].append(row["elapsed_seconds"])
import statistics
for group in summary["groups"].values():
times = group["elapsed_seconds"]
group["diagnostic_time"] = (
None
if not times
else {
"min": min(times),
"median": statistics.median(times),
"max": max(times),
}
)
dump(run / "summary.json", summary)
return summary
def ports_available(protocol):
for port in protocol["ports"].values():
with socket.socket() as probe:
probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
probe.bind(("127.0.0.1", port))
except OSError:
raise RuntimeError(
"Experiment port "
+ str(port)
+ " is occupied; do not stop unrelated services."
) from None
def run_series(seal_directory, phase):
if sys.platform != "linux":
raise RuntimeError("Run this command in WSL Ubuntu.")
import fcntl
from driver import Case, Failure, Stack, Violation
from events import LifecycleEvents
RESULTS.mkdir(parents=True, exist_ok=True)
lock = (RESULTS / ".run.lock").open("a")
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
frozen = verify_seal(seal_directory)
protocol = json.loads(PROTOCOL.read_text())
ports_available(protocol)
run = RESULTS / (phase + "-" + stamp())
run.mkdir(exist_ok=False)
shutil.copytree(seal_directory, run / "seal")
shutil.copytree(ROOT / "experiments/docs", run / "docs")
shutil.copy2(PROTOCOL, run / "protocol.json")
manifest = {
"phase": phase,
"started_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
"protocol_sha256": frozen["protocol_sha256"],
"python": sys.version,
"platform": platform.platform(),
"git_head": frozen["git_head"],
"source_dirty": frozen["git_dirty"],
"docker": subprocess.check_output(
["docker", "version", "--format", "{{.Server.Version}}"], text=True
).strip(),
"compose": subprocess.check_output(
["docker", "compose", "version", "--short"], text=True
).strip(),
"projects": [],
"interrupted": False,
}
dump(run / "manifest.json", manifest)
print("Results:", run, flush=True)
app_image, s3_image = build_images(run)
manifest["application_image"] = app_image
manifest["s3_image"] = s3_image
fixtures = run / "fixtures"
fixtures.mkdir()
fixture_data = {}
fixture_manifest = {}
for profile in protocol["profiles"]:
data = png(profile, protocol["seed"])
fixture_data[profile["name"]] = data
(fixtures / (profile["name"] + ".png")).write_bytes(data)
fixture_manifest[profile["name"]] = {
"sha256": digest(data),
"bytes": len(data),
**profile,
}
dump(fixtures / "manifest.json", fixture_manifest)
profiles = protocol["profiles"] if phase == "main" else protocol["profiles"][:1]
repeats = protocol["repetitions"] if phase == "main" else 1
baseline_images = None
records = (run / "cases.jsonl").open("x")
try:
for repetition in range(1, repeats + 1):
verify_seal(seal_directory)
ports_available(protocol)
project = "lab5-idem-" + run.name.lower() + "-r" + str(repetition)
private = run / ".private" / ("r" + str(repetition))
private.parent.mkdir(exist_ok=True)
os.chmod(private.parent, 0o700)
values = private_environment(private, protocol)
env = {**os.environ, **values}
base = json.loads(
subprocess.check_output(
[
"docker",
"compose",
"--env-file",
str(private / "private.env"),
"config",
"--format",
"json",
],
cwd=ROOT,
env=env,
text=True,
)
)
configuration = isolated_config(
base, project, private, protocol, app_image, s3_image
)
dump(private / "compose.json", configuration)
os.chmod(private / "compose.json", 0o600)
stack = Stack(project, private / "compose.json", private, protocol)
manifest["projects"].append(project)
dump(run / "manifest.json", manifest)
print("Starting isolated repeat", repetition, "of", repeats, flush=True)
collector = LifecycleEvents(
project, run / ("docker-events-r" + str(repetition) + ".jsonl")
)
try:
setup_deadline = time.monotonic() + 300
with (run / ("boot-r" + str(repetition) + ".log")).open("w") as output:
subprocess.run(
stack.prefix + ["up", "-d", "--wait"],
cwd=ROOT,
stdout=output,
stderr=subprocess.STDOUT,
check=True,
timeout=300,
)
stack.wait_broker_ready(timeout=max(0, setup_deadline-time.monotonic()))
initial = stack.containers()
dump(run / ("environment-r" + str(repetition) + ".json"), initial)
images = {row["service"]: row["image"] for row in initial}
if baseline_images is None:
baseline_images = images
elif images != baseline_images:
raise RuntimeError("Image IDs changed between repetitions")
warmup = Case(
stack,
protocol["profiles"][0],
"control",
fixture_data[protocol["profiles"][0]["name"]],
run / "warmup" / ("r" + str(repetition)),
)
try:
dump(warmup.directory / "result.json", warmup.execute())
except Exception as exc:
dump(warmup.directory / "result.json", {
"status": "invalid", "error_type": type(exc).__name__,
"error": str(exc), "attachment_id": warmup.attachment,
})
if warmup.attachment:
try:
dump(warmup.directory / "database-final.json", stack.snapshot(warmup.attachment))
except Exception:
pass
raise
finally:
warmup.close()
order = [
(profile, scenario)
for profile in profiles
for scenario in protocol["scenarios"]
]
random.Random(protocol["seed"] + repetition).shuffle(order)
dump(
run / ("order-r" + str(repetition) + ".json"),
[{"profile": p["name"], "scenario": s} for p, s in order],
)
for index, (profile, scenario) in enumerate(order, 1):
relative = (
"cases/r"
+ str(repetition)
+ "/"
+ str(index).zfill(2)
+ "-"
+ profile["name"]
+ "-"
+ scenario
)
case = Case(
stack,
profile,
scenario,
fixture_data[profile["name"]],
run / relative,
)
row = {
"repetition": repetition,
"profile": profile["name"],
"scenario": scenario,
"directory": relative,
"status": "pass",
}
recovered = True
try:
if (
stack.sql(
"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')"
)
!= "0"
or stack.queue_depth() != 0
):
raise RuntimeError("Non-quiescent initial state")
dump(
case.directory / "containers-before.json",
stack.containers(),
)
row.update(case.execute())
except KeyboardInterrupt:
row.update(
status="invalid",
error_type="KeyboardInterrupt",
error="Interrupted by user",
)
case.event("case_interrupted")
raise
except Exception as exc:
row.update(
status="violation"
if isinstance(exc, Violation)
or isinstance(exc, Failure)
and case.attachment is not None
else "invalid",
error_type=type(exc).__name__,
error=str(exc),
)
case.event(
"case_error", error_type=type(exc).__name__, error=str(exc)
)
try:
case.recover()
except Exception as recovery_error:
row["recovery_error_type"] = type(recovery_error).__name__
recovered = False
finally:
case.close()
try:
dump(
case.directory / "containers-after.json",
stack.containers(),
)
except Exception:
row["container_snapshot_missing"] = True
if case.attachment:
try:
dump(
case.directory / "database-final.json",
stack.snapshot(case.attachment),
)
except Exception:
row["database_snapshot_missing"] = True
markers = [
{"name": p.name, "mtime": p.stat().st_mtime}
for p in (private / "faults").glob(
"*" + case.attachment + "*"
)
]
dump(case.directory / "fault-markers.json", markers)
row.setdefault("elapsed_seconds", time.monotonic() - case.start)
dump(case.directory / "result.json", row)
records.write(json.dumps(row, ensure_ascii=False) + "\n")
records.flush()
if collector.failed:
raise RuntimeError(
"Docker lifecycle event stream ended unexpectedly"
)
print(
"Repeat",
repetition,
"case",
index,
profile["name"],
scenario,
row["status"],
flush=True,
)
if not recovered:
raise RuntimeError(
"Cannot restore quiescence; remaining cases not replaced"
)
finally:
# Remove this project's ephemeral networks before the next repeat.
# Named volumes retain the experiment data (no --volumes).
try:
subprocess.run(
stack.prefix + ["down", "--timeout", "30"],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=120,
check=True,
)
finally:
collector.close()
if collector.failed:
manifest["interrupted"] = True
except BaseException as exc:
manifest.update(interrupted=True, interruption_type=type(exc).__name__)
dump(
run / "interruption.json", {"type": type(exc).__name__, "message": str(exc)}
)
raise
finally:
records.close()
manifest["finished_utc"] = dt.datetime.now(dt.timezone.utc).isoformat()
dump(run / "manifest.json", manifest)
summary = analyze(run)
print("Outcome:", summary["outcome"], flush=True)
print("Return archive:", archive_public(run), flush=True)
fcntl.flock(lock, fcntl.LOCK_UN)
lock.close()
def main():
ensure_local_ignore()
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
commands.add_parser(
"seal", help="freeze source/protocol; no containers are started"
)
run = commands.add_parser("run")
run.add_argument("--phase", choices=("pilot", "main"), required=True)
run.add_argument("--seal", type=Path, required=True)
for name in ("analyze", "pack"):
command = commands.add_parser(name)
command.add_argument("directory", type=Path)
args = parser.parse_args()
if args.command == "seal":
seal()
elif args.command == "run":
run_series(args.seal.resolve(), args.phase)
elif args.command == "analyze":
print(
json.dumps(analyze(args.directory.resolve()), ensure_ascii=False, indent=2)
)
elif args.command == "pack":
print(archive_public(args.directory.resolve()))
if __name__ == "__main__":
main()