#!/usr/bin/env python3 """Seal first; run pilot/main only on an explicit user command.""" import argparse import datetime as dt import json import os import platform import random import shutil import socket import subprocess import sys import time import uuid import zipfile from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent / "src")) from support import ( PROTOCOL, ROOT, archive_public, build_images, digest, dump, ensure_local_ignore, isolated_config, png, private_environment, source_manifest, verify_seal, ) RESULTS = ROOT / "evidence/idempotency" def stamp(): return ( dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] ) def seal(): directory = RESULTS / "seals" / stamp() directory.mkdir(parents=True, exist_ok=False) files = source_manifest() with zipfile.ZipFile( directory / "source.zip", "x", zipfile.ZIP_DEFLATED ) as archive: for name in files: archive.write(ROOT / name, name) value = { "registered_utc": dt.datetime.now(dt.timezone.utc).isoformat(), "protocol_sha256": digest(PROTOCOL.read_bytes()), "files": files, "git_head": subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True ).strip(), "git_dirty": bool( subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT) ), } dump(directory / "manifest.json", value) print(directory.relative_to(ROOT)) def analyze(run): protocol = json.loads((run / "protocol.json").read_text()) manifest = json.loads((run / "manifest.json").read_text()) records = ( [json.loads(line) for line in (run / "cases.jsonl").read_text().splitlines()] if (run / "cases.jsonl").exists() else [] ) violations = sum(row["status"] == "violation" for row in records) invalid = sum(row["status"] == "invalid" for row in records) missing = [] for row in records: for filename in ( "events.jsonl", "result.json", "containers-before.json", "containers-after.json", "database-final.json", "fault-markers.json", ): if not (run / row["directory"] / filename).is_file(): missing.append(row["directory"] + "/" + filename) expected = ( protocol["pass"]["cases"] if manifest["phase"] == "main" else len(protocol["scenarios"]) ) repetitions = ( range(1, protocol["repetitions"] + 1) if manifest["phase"] == "main" else [1] ) profiles = ( protocol["profiles"] if manifest["phase"] == "main" else protocol["profiles"][:1] ) planned = { (r, p["name"], s) for r in repetitions for p in profiles for s in protocol["scenarios"] } for repetition in repetitions: for name in ( "docker-events-r" + str(repetition) + ".jsonl", "environment-r" + str(repetition) + ".json", "order-r" + str(repetition) + ".json", ): if not (run / name).is_file(): missing.append(name) keys = [(row["repetition"], row["profile"], row["scenario"]) for row in records] invalid_matrix = len(keys) != len(set(keys)) or set(keys) != planned oom_events = 0 for path in run.glob("docker-events-r*.jsonl"): for line in path.read_text().splitlines(): if json.loads(line).get("action") == "oom": oom_events += 1 for filename in ("seal/manifest.json", "seal/source.zip", "fixtures/manifest.json"): if not (run / filename).exists(): missing.append(filename) if manifest["phase"] != "main": outcome = "pilot_only_not_confirmation" elif violations or oom_events: outcome = "refuted" elif ( len(records) != expected or invalid_matrix or invalid or missing or manifest.get("interrupted") or not manifest.get("finished_utc") ): outcome = "inconclusive" else: outcome = "supported_in_tested_conditions" summary = { "phase": manifest["phase"], "expected": expected, "completed": len(records), "violations": violations, "complete": len(records) == expected and not invalid_matrix and not invalid and not missing and not oom_events and not manifest.get("interrupted") and bool(manifest.get("finished_utc")), "oom_events": oom_events, "invalid": invalid, "invalid_matrix": invalid_matrix, "missing_artifacts": missing, "outcome": outcome, "groups": {}, } for row in records: key = row["scenario"] + "/" + row["profile"] group = summary["groups"].setdefault( key, {"cases": 0, "violations": 0, "invalid": 0, "elapsed_seconds": []} ) group["cases"] += 1 group["violations"] += row["status"] == "violation" group["invalid"] += row["status"] == "invalid" if row.get("elapsed_seconds") is not None: group["elapsed_seconds"].append(row["elapsed_seconds"]) import statistics for group in summary["groups"].values(): times = group["elapsed_seconds"] group["diagnostic_time"] = ( None if not times else { "min": min(times), "median": statistics.median(times), "max": max(times), } ) dump(run / "summary.json", summary) return summary def ports_available(protocol): for port in protocol["ports"].values(): with socket.socket() as probe: probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: probe.bind(("127.0.0.1", port)) except OSError: raise RuntimeError( "Experiment port " + str(port) + " is occupied; do not stop unrelated services." ) from None def run_series(seal_directory, phase): if sys.platform != "linux": raise RuntimeError("Run this command in WSL Ubuntu.") import fcntl from driver import Case, Failure, Stack, Violation from events import LifecycleEvents RESULTS.mkdir(parents=True, exist_ok=True) lock = (RESULTS / ".run.lock").open("a") fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) frozen = verify_seal(seal_directory) protocol = json.loads(PROTOCOL.read_text()) ports_available(protocol) run = RESULTS / (phase + "-" + stamp()) run.mkdir(exist_ok=False) shutil.copytree(seal_directory, run / "seal") shutil.copytree(ROOT / "experiments/docs", run / "docs") shutil.copy2(PROTOCOL, run / "protocol.json") manifest = { "phase": phase, "started_utc": dt.datetime.now(dt.timezone.utc).isoformat(), "protocol_sha256": frozen["protocol_sha256"], "python": sys.version, "platform": platform.platform(), "git_head": frozen["git_head"], "source_dirty": frozen["git_dirty"], "docker": subprocess.check_output( ["docker", "version", "--format", "{{.Server.Version}}"], text=True ).strip(), "compose": subprocess.check_output( ["docker", "compose", "version", "--short"], text=True ).strip(), "projects": [], "interrupted": False, } dump(run / "manifest.json", manifest) print("Results:", run, flush=True) app_image, s3_image = build_images(run) manifest["application_image"] = app_image manifest["s3_image"] = s3_image fixtures = run / "fixtures" fixtures.mkdir() fixture_data = {} fixture_manifest = {} for profile in protocol["profiles"]: data = png(profile, protocol["seed"]) fixture_data[profile["name"]] = data (fixtures / (profile["name"] + ".png")).write_bytes(data) fixture_manifest[profile["name"]] = { "sha256": digest(data), "bytes": len(data), **profile, } dump(fixtures / "manifest.json", fixture_manifest) profiles = protocol["profiles"] if phase == "main" else protocol["profiles"][:1] repeats = protocol["repetitions"] if phase == "main" else 1 baseline_images = None records = (run / "cases.jsonl").open("x") try: for repetition in range(1, repeats + 1): verify_seal(seal_directory) ports_available(protocol) project = "lab5-idem-" + run.name.lower() + "-r" + str(repetition) private = run / ".private" / ("r" + str(repetition)) private.parent.mkdir(exist_ok=True) os.chmod(private.parent, 0o700) values = private_environment(private, protocol) env = {**os.environ, **values} base = json.loads( subprocess.check_output( [ "docker", "compose", "--env-file", str(private / "private.env"), "config", "--format", "json", ], cwd=ROOT, env=env, text=True, ) ) configuration = isolated_config( base, project, private, protocol, app_image, s3_image ) dump(private / "compose.json", configuration) os.chmod(private / "compose.json", 0o600) stack = Stack(project, private / "compose.json", private, protocol) manifest["projects"].append(project) dump(run / "manifest.json", manifest) print("Starting isolated repeat", repetition, "of", repeats, flush=True) collector = LifecycleEvents( project, run / ("docker-events-r" + str(repetition) + ".jsonl") ) try: setup_deadline = time.monotonic() + 300 with (run / ("boot-r" + str(repetition) + ".log")).open("w") as output: subprocess.run( stack.prefix + ["up", "-d", "--wait"], cwd=ROOT, stdout=output, stderr=subprocess.STDOUT, check=True, timeout=300, ) stack.wait_broker_ready(timeout=max(0, setup_deadline-time.monotonic())) initial = stack.containers() dump(run / ("environment-r" + str(repetition) + ".json"), initial) images = {row["service"]: row["image"] for row in initial} if baseline_images is None: baseline_images = images elif images != baseline_images: raise RuntimeError("Image IDs changed between repetitions") warmup = Case( stack, protocol["profiles"][0], "control", fixture_data[protocol["profiles"][0]["name"]], run / "warmup" / ("r" + str(repetition)), ) try: dump(warmup.directory / "result.json", warmup.execute()) except Exception as exc: dump(warmup.directory / "result.json", { "status": "invalid", "error_type": type(exc).__name__, "error": str(exc), "attachment_id": warmup.attachment, }) if warmup.attachment: try: dump(warmup.directory / "database-final.json", stack.snapshot(warmup.attachment)) except Exception: pass raise finally: warmup.close() order = [ (profile, scenario) for profile in profiles for scenario in protocol["scenarios"] ] random.Random(protocol["seed"] + repetition).shuffle(order) dump( run / ("order-r" + str(repetition) + ".json"), [{"profile": p["name"], "scenario": s} for p, s in order], ) for index, (profile, scenario) in enumerate(order, 1): relative = ( "cases/r" + str(repetition) + "/" + str(index).zfill(2) + "-" + profile["name"] + "-" + scenario ) case = Case( stack, profile, scenario, fixture_data[profile["name"]], run / relative, ) row = { "repetition": repetition, "profile": profile["name"], "scenario": scenario, "directory": relative, "status": "pass", } recovered = True try: if ( stack.sql( "SELECT count(*) FROM attachments WHERE status IN ('queued','processing')" ) != "0" or stack.queue_depth() != 0 ): raise RuntimeError("Non-quiescent initial state") dump( case.directory / "containers-before.json", stack.containers(), ) row.update(case.execute()) except KeyboardInterrupt: row.update( status="invalid", error_type="KeyboardInterrupt", error="Interrupted by user", ) case.event("case_interrupted") raise except Exception as exc: row.update( status="violation" if isinstance(exc, Violation) or isinstance(exc, Failure) and case.attachment is not None else "invalid", error_type=type(exc).__name__, error=str(exc), ) case.event( "case_error", error_type=type(exc).__name__, error=str(exc) ) try: case.recover() except Exception as recovery_error: row["recovery_error_type"] = type(recovery_error).__name__ recovered = False finally: case.close() try: dump( case.directory / "containers-after.json", stack.containers(), ) except Exception: row["container_snapshot_missing"] = True if case.attachment: try: dump( case.directory / "database-final.json", stack.snapshot(case.attachment), ) except Exception: row["database_snapshot_missing"] = True markers = [ {"name": p.name, "mtime": p.stat().st_mtime} for p in (private / "faults").glob( "*" + case.attachment + "*" ) ] dump(case.directory / "fault-markers.json", markers) row.setdefault("elapsed_seconds", time.monotonic() - case.start) dump(case.directory / "result.json", row) records.write(json.dumps(row, ensure_ascii=False) + "\n") records.flush() if collector.failed: raise RuntimeError( "Docker lifecycle event stream ended unexpectedly" ) print( "Repeat", repetition, "case", index, profile["name"], scenario, row["status"], flush=True, ) if not recovered: raise RuntimeError( "Cannot restore quiescence; remaining cases not replaced" ) finally: # Remove this project's ephemeral networks before the next repeat. # Named volumes retain the experiment data (no --volumes). try: subprocess.run( stack.prefix + ["down", "--timeout", "30"], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=120, check=True, ) finally: collector.close() if collector.failed: manifest["interrupted"] = True except BaseException as exc: manifest.update(interrupted=True, interruption_type=type(exc).__name__) dump( run / "interruption.json", {"type": type(exc).__name__, "message": str(exc)} ) raise finally: records.close() manifest["finished_utc"] = dt.datetime.now(dt.timezone.utc).isoformat() dump(run / "manifest.json", manifest) summary = analyze(run) print("Outcome:", summary["outcome"], flush=True) print("Return archive:", archive_public(run), flush=True) fcntl.flock(lock, fcntl.LOCK_UN) lock.close() def main(): ensure_local_ignore() parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) commands.add_parser( "seal", help="freeze source/protocol; no containers are started" ) run = commands.add_parser("run") run.add_argument("--phase", choices=("pilot", "main"), required=True) run.add_argument("--seal", type=Path, required=True) for name in ("analyze", "pack"): command = commands.add_parser(name) command.add_argument("directory", type=Path) args = parser.parse_args() if args.command == "seal": seal() elif args.command == "run": run_series(args.seal.resolve(), args.phase) elif args.command == "analyze": print( json.dumps(analyze(args.directory.resolve()), ensure_ascii=False, indent=2) ) elif args.command == "pack": print(archive_public(args.directory.resolve())) if __name__ == "__main__": main()