forked from mtusi2026/mtusi2026lab5
525 lines
20 KiB
Python
525 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Seal first; run pilot/main only on an explicit user command."""
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import platform
|
|
import random
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
|
from support import (
|
|
PROTOCOL,
|
|
ROOT,
|
|
archive_public,
|
|
build_images,
|
|
digest,
|
|
dump,
|
|
ensure_local_ignore,
|
|
isolated_config,
|
|
png,
|
|
private_environment,
|
|
source_manifest,
|
|
verify_seal,
|
|
)
|
|
|
|
RESULTS = ROOT / "evidence/idempotency"
|
|
|
|
|
|
def stamp():
|
|
return (
|
|
dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
+ "-"
|
|
+ uuid.uuid4().hex[:8]
|
|
)
|
|
|
|
|
|
def seal():
|
|
directory = RESULTS / "seals" / stamp()
|
|
directory.mkdir(parents=True, exist_ok=False)
|
|
files = source_manifest()
|
|
with zipfile.ZipFile(
|
|
directory / "source.zip", "x", zipfile.ZIP_DEFLATED
|
|
) as archive:
|
|
for name in files:
|
|
archive.write(ROOT / name, name)
|
|
value = {
|
|
"registered_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"protocol_sha256": digest(PROTOCOL.read_bytes()),
|
|
"files": files,
|
|
"git_head": subprocess.check_output(
|
|
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
|
).strip(),
|
|
"git_dirty": bool(
|
|
subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT)
|
|
),
|
|
}
|
|
dump(directory / "manifest.json", value)
|
|
print(directory.relative_to(ROOT))
|
|
|
|
|
|
def analyze(run):
|
|
protocol = json.loads((run / "protocol.json").read_text())
|
|
manifest = json.loads((run / "manifest.json").read_text())
|
|
records = (
|
|
[json.loads(line) for line in (run / "cases.jsonl").read_text().splitlines()]
|
|
if (run / "cases.jsonl").exists()
|
|
else []
|
|
)
|
|
violations = sum(row["status"] == "violation" for row in records)
|
|
invalid = sum(row["status"] == "invalid" for row in records)
|
|
missing = []
|
|
for row in records:
|
|
for filename in (
|
|
"events.jsonl",
|
|
"result.json",
|
|
"containers-before.json",
|
|
"containers-after.json",
|
|
"database-final.json",
|
|
"fault-markers.json",
|
|
):
|
|
if not (run / row["directory"] / filename).is_file():
|
|
missing.append(row["directory"] + "/" + filename)
|
|
expected = (
|
|
protocol["pass"]["cases"]
|
|
if manifest["phase"] == "main"
|
|
else len(protocol["scenarios"])
|
|
)
|
|
repetitions = (
|
|
range(1, protocol["repetitions"] + 1) if manifest["phase"] == "main" else [1]
|
|
)
|
|
profiles = (
|
|
protocol["profiles"]
|
|
if manifest["phase"] == "main"
|
|
else protocol["profiles"][:1]
|
|
)
|
|
planned = {
|
|
(r, p["name"], s)
|
|
for r in repetitions
|
|
for p in profiles
|
|
for s in protocol["scenarios"]
|
|
}
|
|
for repetition in repetitions:
|
|
for name in (
|
|
"docker-events-r" + str(repetition) + ".jsonl",
|
|
"environment-r" + str(repetition) + ".json",
|
|
"order-r" + str(repetition) + ".json",
|
|
):
|
|
if not (run / name).is_file():
|
|
missing.append(name)
|
|
keys = [(row["repetition"], row["profile"], row["scenario"]) for row in records]
|
|
invalid_matrix = len(keys) != len(set(keys)) or set(keys) != planned
|
|
oom_events = 0
|
|
for path in run.glob("docker-events-r*.jsonl"):
|
|
for line in path.read_text().splitlines():
|
|
if json.loads(line).get("action") == "oom":
|
|
oom_events += 1
|
|
for filename in ("seal/manifest.json", "seal/source.zip", "fixtures/manifest.json"):
|
|
if not (run / filename).exists():
|
|
missing.append(filename)
|
|
if manifest["phase"] != "main":
|
|
outcome = "pilot_only_not_confirmation"
|
|
elif violations or oom_events:
|
|
outcome = "refuted"
|
|
elif (
|
|
len(records) != expected
|
|
or invalid_matrix
|
|
or invalid
|
|
or missing
|
|
or manifest.get("interrupted")
|
|
or not manifest.get("finished_utc")
|
|
):
|
|
outcome = "inconclusive"
|
|
else:
|
|
outcome = "supported_in_tested_conditions"
|
|
summary = {
|
|
"phase": manifest["phase"],
|
|
"expected": expected,
|
|
"completed": len(records),
|
|
"violations": violations,
|
|
"complete": len(records) == expected
|
|
and not invalid_matrix
|
|
and not invalid
|
|
and not missing
|
|
and not oom_events
|
|
and not manifest.get("interrupted")
|
|
and bool(manifest.get("finished_utc")),
|
|
"oom_events": oom_events,
|
|
"invalid": invalid,
|
|
"invalid_matrix": invalid_matrix,
|
|
"missing_artifacts": missing,
|
|
"outcome": outcome,
|
|
"groups": {},
|
|
}
|
|
for row in records:
|
|
key = row["scenario"] + "/" + row["profile"]
|
|
group = summary["groups"].setdefault(
|
|
key, {"cases": 0, "violations": 0, "invalid": 0, "elapsed_seconds": []}
|
|
)
|
|
group["cases"] += 1
|
|
group["violations"] += row["status"] == "violation"
|
|
group["invalid"] += row["status"] == "invalid"
|
|
if row.get("elapsed_seconds") is not None:
|
|
group["elapsed_seconds"].append(row["elapsed_seconds"])
|
|
import statistics
|
|
|
|
for group in summary["groups"].values():
|
|
times = group["elapsed_seconds"]
|
|
group["diagnostic_time"] = (
|
|
None
|
|
if not times
|
|
else {
|
|
"min": min(times),
|
|
"median": statistics.median(times),
|
|
"max": max(times),
|
|
}
|
|
)
|
|
dump(run / "summary.json", summary)
|
|
return summary
|
|
|
|
|
|
def ports_available(protocol):
|
|
for port in protocol["ports"].values():
|
|
with socket.socket() as probe:
|
|
probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
try:
|
|
probe.bind(("127.0.0.1", port))
|
|
except OSError:
|
|
raise RuntimeError(
|
|
"Experiment port "
|
|
+ str(port)
|
|
+ " is occupied; do not stop unrelated services."
|
|
) from None
|
|
|
|
|
|
def run_series(seal_directory, phase):
|
|
if sys.platform != "linux":
|
|
raise RuntimeError("Run this command in WSL Ubuntu.")
|
|
import fcntl
|
|
|
|
from driver import Case, Failure, Stack, Violation
|
|
from events import LifecycleEvents
|
|
|
|
RESULTS.mkdir(parents=True, exist_ok=True)
|
|
lock = (RESULTS / ".run.lock").open("a")
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
frozen = verify_seal(seal_directory)
|
|
protocol = json.loads(PROTOCOL.read_text())
|
|
ports_available(protocol)
|
|
run = RESULTS / (phase + "-" + stamp())
|
|
run.mkdir(exist_ok=False)
|
|
shutil.copytree(seal_directory, run / "seal")
|
|
shutil.copytree(ROOT / "experiments/docs", run / "docs")
|
|
shutil.copy2(PROTOCOL, run / "protocol.json")
|
|
manifest = {
|
|
"phase": phase,
|
|
"started_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"protocol_sha256": frozen["protocol_sha256"],
|
|
"python": sys.version,
|
|
"platform": platform.platform(),
|
|
"git_head": frozen["git_head"],
|
|
"source_dirty": frozen["git_dirty"],
|
|
"docker": subprocess.check_output(
|
|
["docker", "version", "--format", "{{.Server.Version}}"], text=True
|
|
).strip(),
|
|
"compose": subprocess.check_output(
|
|
["docker", "compose", "version", "--short"], text=True
|
|
).strip(),
|
|
"projects": [],
|
|
"interrupted": False,
|
|
}
|
|
dump(run / "manifest.json", manifest)
|
|
print("Results:", run, flush=True)
|
|
app_image, s3_image = build_images(run)
|
|
manifest["application_image"] = app_image
|
|
manifest["s3_image"] = s3_image
|
|
fixtures = run / "fixtures"
|
|
fixtures.mkdir()
|
|
fixture_data = {}
|
|
fixture_manifest = {}
|
|
for profile in protocol["profiles"]:
|
|
data = png(profile, protocol["seed"])
|
|
fixture_data[profile["name"]] = data
|
|
(fixtures / (profile["name"] + ".png")).write_bytes(data)
|
|
fixture_manifest[profile["name"]] = {
|
|
"sha256": digest(data),
|
|
"bytes": len(data),
|
|
**profile,
|
|
}
|
|
dump(fixtures / "manifest.json", fixture_manifest)
|
|
profiles = protocol["profiles"] if phase == "main" else protocol["profiles"][:1]
|
|
repeats = protocol["repetitions"] if phase == "main" else 1
|
|
baseline_images = None
|
|
records = (run / "cases.jsonl").open("x")
|
|
try:
|
|
for repetition in range(1, repeats + 1):
|
|
verify_seal(seal_directory)
|
|
ports_available(protocol)
|
|
project = "lab5-idem-" + run.name.lower() + "-r" + str(repetition)
|
|
private = run / ".private" / ("r" + str(repetition))
|
|
private.parent.mkdir(exist_ok=True)
|
|
os.chmod(private.parent, 0o700)
|
|
values = private_environment(private, protocol)
|
|
env = {**os.environ, **values}
|
|
base = json.loads(
|
|
subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--env-file",
|
|
str(private / "private.env"),
|
|
"config",
|
|
"--format",
|
|
"json",
|
|
],
|
|
cwd=ROOT,
|
|
env=env,
|
|
text=True,
|
|
)
|
|
)
|
|
configuration = isolated_config(
|
|
base, project, private, protocol, app_image, s3_image
|
|
)
|
|
dump(private / "compose.json", configuration)
|
|
os.chmod(private / "compose.json", 0o600)
|
|
stack = Stack(project, private / "compose.json", private, protocol)
|
|
manifest["projects"].append(project)
|
|
dump(run / "manifest.json", manifest)
|
|
print("Starting isolated repeat", repetition, "of", repeats, flush=True)
|
|
collector = LifecycleEvents(
|
|
project, run / ("docker-events-r" + str(repetition) + ".jsonl")
|
|
)
|
|
try:
|
|
setup_deadline = time.monotonic() + 300
|
|
with (run / ("boot-r" + str(repetition) + ".log")).open("w") as output:
|
|
subprocess.run(
|
|
stack.prefix + ["up", "-d", "--wait"],
|
|
cwd=ROOT,
|
|
stdout=output,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
timeout=300,
|
|
)
|
|
stack.wait_broker_ready(timeout=max(0, setup_deadline-time.monotonic()))
|
|
initial = stack.containers()
|
|
dump(run / ("environment-r" + str(repetition) + ".json"), initial)
|
|
images = {row["service"]: row["image"] for row in initial}
|
|
if baseline_images is None:
|
|
baseline_images = images
|
|
elif images != baseline_images:
|
|
raise RuntimeError("Image IDs changed between repetitions")
|
|
warmup = Case(
|
|
stack,
|
|
protocol["profiles"][0],
|
|
"control",
|
|
fixture_data[protocol["profiles"][0]["name"]],
|
|
run / "warmup" / ("r" + str(repetition)),
|
|
)
|
|
try:
|
|
dump(warmup.directory / "result.json", warmup.execute())
|
|
except Exception as exc:
|
|
dump(warmup.directory / "result.json", {
|
|
"status": "invalid", "error_type": type(exc).__name__,
|
|
"error": str(exc), "attachment_id": warmup.attachment,
|
|
})
|
|
if warmup.attachment:
|
|
try:
|
|
dump(warmup.directory / "database-final.json", stack.snapshot(warmup.attachment))
|
|
except Exception:
|
|
pass
|
|
raise
|
|
finally:
|
|
warmup.close()
|
|
order = [
|
|
(profile, scenario)
|
|
for profile in profiles
|
|
for scenario in protocol["scenarios"]
|
|
]
|
|
random.Random(protocol["seed"] + repetition).shuffle(order)
|
|
dump(
|
|
run / ("order-r" + str(repetition) + ".json"),
|
|
[{"profile": p["name"], "scenario": s} for p, s in order],
|
|
)
|
|
for index, (profile, scenario) in enumerate(order, 1):
|
|
relative = (
|
|
"cases/r"
|
|
+ str(repetition)
|
|
+ "/"
|
|
+ str(index).zfill(2)
|
|
+ "-"
|
|
+ profile["name"]
|
|
+ "-"
|
|
+ scenario
|
|
)
|
|
case = Case(
|
|
stack,
|
|
profile,
|
|
scenario,
|
|
fixture_data[profile["name"]],
|
|
run / relative,
|
|
)
|
|
row = {
|
|
"repetition": repetition,
|
|
"profile": profile["name"],
|
|
"scenario": scenario,
|
|
"directory": relative,
|
|
"status": "pass",
|
|
}
|
|
recovered = True
|
|
try:
|
|
if (
|
|
stack.sql(
|
|
"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')"
|
|
)
|
|
!= "0"
|
|
or stack.queue_depth() != 0
|
|
):
|
|
raise RuntimeError("Non-quiescent initial state")
|
|
dump(
|
|
case.directory / "containers-before.json",
|
|
stack.containers(),
|
|
)
|
|
row.update(case.execute())
|
|
except KeyboardInterrupt:
|
|
row.update(
|
|
status="invalid",
|
|
error_type="KeyboardInterrupt",
|
|
error="Interrupted by user",
|
|
)
|
|
case.event("case_interrupted")
|
|
raise
|
|
except Exception as exc:
|
|
row.update(
|
|
status="violation"
|
|
if isinstance(exc, Violation)
|
|
or isinstance(exc, Failure)
|
|
and case.attachment is not None
|
|
else "invalid",
|
|
error_type=type(exc).__name__,
|
|
error=str(exc),
|
|
)
|
|
case.event(
|
|
"case_error", error_type=type(exc).__name__, error=str(exc)
|
|
)
|
|
try:
|
|
case.recover()
|
|
except Exception as recovery_error:
|
|
row["recovery_error_type"] = type(recovery_error).__name__
|
|
recovered = False
|
|
finally:
|
|
case.close()
|
|
try:
|
|
dump(
|
|
case.directory / "containers-after.json",
|
|
stack.containers(),
|
|
)
|
|
except Exception:
|
|
row["container_snapshot_missing"] = True
|
|
if case.attachment:
|
|
try:
|
|
dump(
|
|
case.directory / "database-final.json",
|
|
stack.snapshot(case.attachment),
|
|
)
|
|
except Exception:
|
|
row["database_snapshot_missing"] = True
|
|
markers = [
|
|
{"name": p.name, "mtime": p.stat().st_mtime}
|
|
for p in (private / "faults").glob(
|
|
"*" + case.attachment + "*"
|
|
)
|
|
]
|
|
dump(case.directory / "fault-markers.json", markers)
|
|
row.setdefault("elapsed_seconds", time.monotonic() - case.start)
|
|
dump(case.directory / "result.json", row)
|
|
records.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
records.flush()
|
|
if collector.failed:
|
|
raise RuntimeError(
|
|
"Docker lifecycle event stream ended unexpectedly"
|
|
)
|
|
print(
|
|
"Repeat",
|
|
repetition,
|
|
"case",
|
|
index,
|
|
profile["name"],
|
|
scenario,
|
|
row["status"],
|
|
flush=True,
|
|
)
|
|
if not recovered:
|
|
raise RuntimeError(
|
|
"Cannot restore quiescence; remaining cases not replaced"
|
|
)
|
|
finally:
|
|
# Remove this project's ephemeral networks before the next repeat.
|
|
# Named volumes retain the experiment data (no --volumes).
|
|
try:
|
|
subprocess.run(
|
|
stack.prefix + ["down", "--timeout", "30"],
|
|
cwd=ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=120,
|
|
check=True,
|
|
)
|
|
finally:
|
|
collector.close()
|
|
if collector.failed:
|
|
manifest["interrupted"] = True
|
|
except BaseException as exc:
|
|
manifest.update(interrupted=True, interruption_type=type(exc).__name__)
|
|
dump(
|
|
run / "interruption.json", {"type": type(exc).__name__, "message": str(exc)}
|
|
)
|
|
raise
|
|
finally:
|
|
records.close()
|
|
manifest["finished_utc"] = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
dump(run / "manifest.json", manifest)
|
|
summary = analyze(run)
|
|
print("Outcome:", summary["outcome"], flush=True)
|
|
print("Return archive:", archive_public(run), flush=True)
|
|
fcntl.flock(lock, fcntl.LOCK_UN)
|
|
lock.close()
|
|
|
|
|
|
def main():
|
|
ensure_local_ignore()
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
commands.add_parser(
|
|
"seal", help="freeze source/protocol; no containers are started"
|
|
)
|
|
run = commands.add_parser("run")
|
|
run.add_argument("--phase", choices=("pilot", "main"), required=True)
|
|
run.add_argument("--seal", type=Path, required=True)
|
|
for name in ("analyze", "pack"):
|
|
command = commands.add_parser(name)
|
|
command.add_argument("directory", type=Path)
|
|
args = parser.parse_args()
|
|
if args.command == "seal":
|
|
seal()
|
|
elif args.command == "run":
|
|
run_series(args.seal.resolve(), args.phase)
|
|
elif args.command == "analyze":
|
|
print(
|
|
json.dumps(analyze(args.directory.resolve()), ensure_ascii=False, indent=2)
|
|
)
|
|
elif args.command == "pack":
|
|
print(archive_public(args.directory.resolve()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|