From 228b510ff3c8cb2a48b08a9f16cdd514e007e6fc Mon Sep 17 00:00:00 2001 From: TAskMAster339 Date: Thu, 5 Nov 2026 23:46:33 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B0=D0=B2=D1=82=D0=BE=D0=BC=D0=B0=D1=82=D0=B8=D1=87?= =?UTF-8?q?=D0=B5=D1=81=D0=BA=D0=B8=D0=B9=20=D1=8D=D0=BA=D1=81=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=20=D0=B8=20=D0=B8=D0=B7?= =?UTF-8?q?=D0=BE=D0=BB=D1=8F=D1=86=D0=B8=D1=8E=20=D0=BF=D0=BE=D0=B2=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 7 +- compose.yml | 4 +- experiments/docs/PREPARATION.md | 43 ++ experiments/docs/RUNBOOK.md | 136 ++++ experiments/run.py | 524 ++++++++++++++++ experiments/src/driver.py | 693 +++++++++++++++++++++ experiments/src/events.py | 85 +++ experiments/src/support.py | 293 +++++++++ experiments/tests/test_broker_readiness.py | 62 ++ experiments/tests/test_tools.py | 225 +++++++ 10 files changed, 2069 insertions(+), 3 deletions(-) create mode 100644 experiments/docs/PREPARATION.md create mode 100644 experiments/docs/RUNBOOK.md create mode 100644 experiments/run.py create mode 100644 experiments/src/driver.py create mode 100644 experiments/src/events.py create mode 100644 experiments/src/support.py create mode 100644 experiments/tests/test_broker_readiness.py create mode 100644 experiments/tests/test_tools.py diff --git a/.gitignore b/.gitignore index b5a71896..18b8dae7 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,10 @@ coverage/ .DS_Store .idea/ .vscode/ -# Keep small, sanitized evidence summaries in evidence/ under version control. +# Keep sanitized experiment evidence under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +# Experiment evidence: keep public setup logs, exclude credentials and locks. +!evidence/**/*.log +evidence/**/.private/ +evidence/**/.run.lock diff --git a/compose.yml b/compose.yml index 7e063538..9b4815f4 100644 --- a/compose.yml +++ b/compose.yml @@ -172,9 +172,9 @@ services: - ./secrets/broker_user:/run/secrets/broker_user:ro - ./secrets/broker_password:/run/secrets/broker_password:ro healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_port_connectivity"] interval: 5s - timeout: 5s + timeout: 10s start_period: 20s retries: 30 worker: diff --git a/experiments/docs/PREPARATION.md b/experiments/docs/PREPARATION.md new file mode 100644 index 00000000..6a14b8b0 --- /dev/null +++ b/experiments/docs/PREPARATION.md @@ -0,0 +1,43 @@ +# Проверки инструментов и реализации + +## Инструменты эксперимента + +```sh +python3 -m unittest discover -s experiments/tests -v +``` + +Тесты проверяют воспроизводимость PNG и корректность их структуры, изоляцию +ресурсов Compose, размещение результатов в `evidence/idempotency`, правила +агрегации неполных и дублированных случаев, учёт OOM и исключение приватных +учётных данных из архива. + +Проверка конфигурации созданного повтора: + +```sh +docker compose -f evidence/idempotency/ИМЯ_СЕРИИ/.private/r1/compose.json config --quiet +``` + +В конфигурации должны отсутствовать общие с учебным стендом имена сетей и томов, +а пути файлов учётных данных должны вести в приватный каталог соответствующего повтора. + +## Приложение + +```sh +make check +docker compose --project-name highload-lab5 --profile test run --rm --build backend-tests +``` + +Backend-тесты включают проверку устаревшего lease token, публикации v2, запрета +понижения версии и повторного открытия старого задания. Тесты обработки изображений +проверяют геометрию, целостность исходника и различие результатов v1/v2. + +Сборка и статическая проверка в WSL: + +```sh +cd src +go build -o /tmp/highload-lab5-messenger ./backend +go vet ./backend/... +``` + +Сквозная проверка управляющего сценария выполняется пилотом. При дефекте инструмента +сохраняются артефакты пилота; после исправления создаётся новый снимок исходников. diff --git a/experiments/docs/RUNBOOK.md b/experiments/docs/RUNBOOK.md new file mode 100644 index 00000000..ec143a70 --- /dev/null +++ b/experiments/docs/RUNBOOK.md @@ -0,0 +1,136 @@ +# Запуск + +## Окружение + +Требования: WSL Ubuntu, Linux Docker Engine с интеграцией Docker Desktop, +Docker Compose с поддержкой `config --format json`, Python 3.9+ и Git. +При первой сборке нужен доступ к реестрам образов и пакетов. + +```sh +python3 -m unittest discover -s experiments/tests -v +``` + +Порты 18085 и 18900 должны быть свободны. Для стенда рекомендуется выделить +около 6 GiB памяти Docker и несколько GiB диска. Первая сборка и основная серия +могут занять более часа. Параллельные сборки, нагрузочные тесты и другие серии +меняют условия выполнения и должны быть остановлены на время измерений. + +До прогрева runner ожидает готовности приложения RabbitMQ и доступности списка +очередей. На запуск стенда и эту проверку отводится общий интервал 300 секунд. +Попытки записываются в `broker-startup.jsonl`, успешная проверка — в +`broker-readiness-r*.json`. Во время сценариев ошибка чтения очереди завершает +проверку и сохраняется в журнале команд. + +Эксперимент использует отдельный Compose project. Учебный `highload-lab5` не +перезапускается. При нехватке ресурсов его можно остановить отдельно, сохранив тома. + +## Снимок исходников + +```sh +experiment_seal=$(python3 experiments/run.py seal) +printf '%s\n' "$experiment_seal" +``` + +Команда создаёт каталог `evidence/idempotency/seals/...` с `manifest.json` и +`source.zip`. В manifest записаны SHA-256 файлов и протокола, Git HEAD и признак +незакоммиченных изменений. Контейнеры при этом не запускаются. + +Значение `experiment_seal` используется в следующих командах. В новом терминале +оно задаётся сохранённым путём: + +```sh +experiment_seal=evidence/idempotency/seals/ИМЯ_СНИМКА +``` + +Runner проверяет контрольные суммы перед серией и каждым повтором. После изменения +исходников требуется новый снимок; после изменения методики — также новая версия +протокола. Существующие данные серий сохраняются. + +## Пилот + +```sh +python3 experiments/run.py run --phase pilot --seal "$experiment_seal" +``` + +Пилот выполняет девять сценариев для `rgb_small` в одном повторе. В конце выводится +путь публичного архива. Условия перехода к основной серии в `summary.json`: + +```text +complete: true +completed: 9 +violations: 0 +invalid: 0 +missing_artifacts: [] +outcome: pilot_only_not_confirmation +``` + +При ошибке пилота основная серия откладывается до диагностики. Данные пилота +хранятся отдельно и не включаются в основную выборку. + +## Основная серия + +```sh +python3 experiments/run.py run --phase main --seal "$experiment_seal" +``` + +Runner выполняет 135 случаев. Каждый из пяти повторов получает новые тома, +сети и учётные данные. Порядок случаев и seed записываются в результаты. +После каждого повтора `docker compose down` удаляет его контейнеры и сети. +Именованные тома с данными сохраняются; параметр `--volumes` не используется. + +Изменение кода во время серии или повторный запуск только неудачных случаев +нарушают сопоставимость данных. Исправленная реализация проверяется отдельной +серией с новым снимком исходников. + +## Результаты + +Все файлы размещаются в `evidence/idempotency/`: + +| Путь внутри серии | Содержимое | +| --- | --- | +| `docs/`, `protocol.json`, `seal/` | Методика, параметры и исходники | +| `fixtures/` | Исходные PNG и контрольные суммы | +| `cases.jsonl`, `cases/` | Исходы случаев, наблюдения API, состояния БД, аудит и варианты изображений | +| `environment-r*.json`, `order-r*.json` | Образы, ресурсы и порядок выполнения | +| `docker-events-r*.jsonl` | События контейнеров, включая OOM и завершения | +| `command-errors.jsonl` | Ошибки команд с маскированием учётных данных | +| `summary.json` | Сводка по сценариям и профилям | +| `.private/` | Учётные данные и рабочие конфигурации стенда | + +Архив для анализа создаётся рядом с каталогом серии: +`evidence/idempotency/main-...-public.zip` или `pilot-...-public.zip`. +В него не включается `.private`. Для передачи результатов используется этот архив; +полный каталог серии содержит пароли и строки подключения. + +Публичные результаты и журналы запуска доступны Git. Корневой `.gitignore` +исключает только `.private/` и служебный `.run.lock` внутри `evidence/`. +Данные `evidence/` не включаются в новые снимки исходников. + +Повторная агрегация без запуска стенда: + +```sh +python3 experiments/run.py analyze evidence/idempotency/main-ИМЯ_СЕРИИ +``` + +Создание архива после прерванного запуска: + +```sh +python3 experiments/run.py pack evidence/idempotency/main-ИМЯ_СЕРИИ +``` + +`pack` не перезаписывает существующий архив. Значения итогового поля `outcome` +описаны в [HYPOTHESIS.md](HYPOTHESIS.md). + +## Прерывание и остановка + +При Ctrl+C сохраняются собранные данные и признак прерванной серии. +После kill -9 или сбоя host автоматическая остановка может не выполниться. +Имена созданных project находятся в `manifest.json`. Остановка выбранного повтора: + +```sh +docker compose --project-name ИМЯ_ИЗ_MANIFEST \ + -f evidence/idempotency/ИМЯ_СЕРИИ/.private/r1/compose.json down +``` + +Тома сохраняются до завершения анализа. Глобальные команды `docker stop` и +`docker prune` для управления экспериментом не используются. diff --git a/experiments/run.py b/experiments/run.py new file mode 100644 index 00000000..5aad5f8c --- /dev/null +++ b/experiments/run.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Seal first; run pilot/main only on an explicit user command.""" + +import argparse +import datetime as dt +import json +import os +import platform +import random +import shutil +import socket +import subprocess +import sys +import time +import uuid +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent / "src")) +from support import ( + PROTOCOL, + ROOT, + archive_public, + build_images, + digest, + dump, + ensure_local_ignore, + isolated_config, + png, + private_environment, + source_manifest, + verify_seal, +) + +RESULTS = ROOT / "evidence/idempotency" + + +def stamp(): + return ( + dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + "-" + + uuid.uuid4().hex[:8] + ) + + +def seal(): + directory = RESULTS / "seals" / stamp() + directory.mkdir(parents=True, exist_ok=False) + files = source_manifest() + with zipfile.ZipFile( + directory / "source.zip", "x", zipfile.ZIP_DEFLATED + ) as archive: + for name in files: + archive.write(ROOT / name, name) + value = { + "registered_utc": dt.datetime.now(dt.timezone.utc).isoformat(), + "protocol_sha256": digest(PROTOCOL.read_bytes()), + "files": files, + "git_head": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + "git_dirty": bool( + subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT) + ), + } + dump(directory / "manifest.json", value) + print(directory.relative_to(ROOT)) + + +def analyze(run): + protocol = json.loads((run / "protocol.json").read_text()) + manifest = json.loads((run / "manifest.json").read_text()) + records = ( + [json.loads(line) for line in (run / "cases.jsonl").read_text().splitlines()] + if (run / "cases.jsonl").exists() + else [] + ) + violations = sum(row["status"] == "violation" for row in records) + invalid = sum(row["status"] == "invalid" for row in records) + missing = [] + for row in records: + for filename in ( + "events.jsonl", + "result.json", + "containers-before.json", + "containers-after.json", + "database-final.json", + "fault-markers.json", + ): + if not (run / row["directory"] / filename).is_file(): + missing.append(row["directory"] + "/" + filename) + expected = ( + protocol["pass"]["cases"] + if manifest["phase"] == "main" + else len(protocol["scenarios"]) + ) + repetitions = ( + range(1, protocol["repetitions"] + 1) if manifest["phase"] == "main" else [1] + ) + profiles = ( + protocol["profiles"] + if manifest["phase"] == "main" + else protocol["profiles"][:1] + ) + planned = { + (r, p["name"], s) + for r in repetitions + for p in profiles + for s in protocol["scenarios"] + } + for repetition in repetitions: + for name in ( + "docker-events-r" + str(repetition) + ".jsonl", + "environment-r" + str(repetition) + ".json", + "order-r" + str(repetition) + ".json", + ): + if not (run / name).is_file(): + missing.append(name) + keys = [(row["repetition"], row["profile"], row["scenario"]) for row in records] + invalid_matrix = len(keys) != len(set(keys)) or set(keys) != planned + oom_events = 0 + for path in run.glob("docker-events-r*.jsonl"): + for line in path.read_text().splitlines(): + if json.loads(line).get("action") == "oom": + oom_events += 1 + for filename in ("seal/manifest.json", "seal/source.zip", "fixtures/manifest.json"): + if not (run / filename).exists(): + missing.append(filename) + if manifest["phase"] != "main": + outcome = "pilot_only_not_confirmation" + elif violations or oom_events: + outcome = "refuted" + elif ( + len(records) != expected + or invalid_matrix + or invalid + or missing + or manifest.get("interrupted") + or not manifest.get("finished_utc") + ): + outcome = "inconclusive" + else: + outcome = "supported_in_tested_conditions" + summary = { + "phase": manifest["phase"], + "expected": expected, + "completed": len(records), + "violations": violations, + "complete": len(records) == expected + and not invalid_matrix + and not invalid + and not missing + and not oom_events + and not manifest.get("interrupted") + and bool(manifest.get("finished_utc")), + "oom_events": oom_events, + "invalid": invalid, + "invalid_matrix": invalid_matrix, + "missing_artifacts": missing, + "outcome": outcome, + "groups": {}, + } + for row in records: + key = row["scenario"] + "/" + row["profile"] + group = summary["groups"].setdefault( + key, {"cases": 0, "violations": 0, "invalid": 0, "elapsed_seconds": []} + ) + group["cases"] += 1 + group["violations"] += row["status"] == "violation" + group["invalid"] += row["status"] == "invalid" + if row.get("elapsed_seconds") is not None: + group["elapsed_seconds"].append(row["elapsed_seconds"]) + import statistics + + for group in summary["groups"].values(): + times = group["elapsed_seconds"] + group["diagnostic_time"] = ( + None + if not times + else { + "min": min(times), + "median": statistics.median(times), + "max": max(times), + } + ) + dump(run / "summary.json", summary) + return summary + + +def ports_available(protocol): + for port in protocol["ports"].values(): + with socket.socket() as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", port)) + except OSError: + raise RuntimeError( + "Experiment port " + + str(port) + + " is occupied; do not stop unrelated services." + ) from None + + +def run_series(seal_directory, phase): + if sys.platform != "linux": + raise RuntimeError("Run this command in WSL Ubuntu.") + import fcntl + + from driver import Case, Failure, Stack, Violation + from events import LifecycleEvents + + RESULTS.mkdir(parents=True, exist_ok=True) + lock = (RESULTS / ".run.lock").open("a") + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + frozen = verify_seal(seal_directory) + protocol = json.loads(PROTOCOL.read_text()) + ports_available(protocol) + run = RESULTS / (phase + "-" + stamp()) + run.mkdir(exist_ok=False) + shutil.copytree(seal_directory, run / "seal") + shutil.copytree(ROOT / "experiments/docs", run / "docs") + shutil.copy2(PROTOCOL, run / "protocol.json") + manifest = { + "phase": phase, + "started_utc": dt.datetime.now(dt.timezone.utc).isoformat(), + "protocol_sha256": frozen["protocol_sha256"], + "python": sys.version, + "platform": platform.platform(), + "git_head": frozen["git_head"], + "source_dirty": frozen["git_dirty"], + "docker": subprocess.check_output( + ["docker", "version", "--format", "{{.Server.Version}}"], text=True + ).strip(), + "compose": subprocess.check_output( + ["docker", "compose", "version", "--short"], text=True + ).strip(), + "projects": [], + "interrupted": False, + } + dump(run / "manifest.json", manifest) + print("Results:", run, flush=True) + app_image, s3_image = build_images(run) + manifest["application_image"] = app_image + manifest["s3_image"] = s3_image + fixtures = run / "fixtures" + fixtures.mkdir() + fixture_data = {} + fixture_manifest = {} + for profile in protocol["profiles"]: + data = png(profile, protocol["seed"]) + fixture_data[profile["name"]] = data + (fixtures / (profile["name"] + ".png")).write_bytes(data) + fixture_manifest[profile["name"]] = { + "sha256": digest(data), + "bytes": len(data), + **profile, + } + dump(fixtures / "manifest.json", fixture_manifest) + profiles = protocol["profiles"] if phase == "main" else protocol["profiles"][:1] + repeats = protocol["repetitions"] if phase == "main" else 1 + baseline_images = None + records = (run / "cases.jsonl").open("x") + try: + for repetition in range(1, repeats + 1): + verify_seal(seal_directory) + ports_available(protocol) + project = "lab5-idem-" + run.name.lower() + "-r" + str(repetition) + private = run / ".private" / ("r" + str(repetition)) + private.parent.mkdir(exist_ok=True) + os.chmod(private.parent, 0o700) + values = private_environment(private, protocol) + env = {**os.environ, **values} + base = json.loads( + subprocess.check_output( + [ + "docker", + "compose", + "--env-file", + str(private / "private.env"), + "config", + "--format", + "json", + ], + cwd=ROOT, + env=env, + text=True, + ) + ) + configuration = isolated_config( + base, project, private, protocol, app_image, s3_image + ) + dump(private / "compose.json", configuration) + os.chmod(private / "compose.json", 0o600) + stack = Stack(project, private / "compose.json", private, protocol) + manifest["projects"].append(project) + dump(run / "manifest.json", manifest) + print("Starting isolated repeat", repetition, "of", repeats, flush=True) + collector = LifecycleEvents( + project, run / ("docker-events-r" + str(repetition) + ".jsonl") + ) + try: + setup_deadline = time.monotonic() + 300 + with (run / ("boot-r" + str(repetition) + ".log")).open("w") as output: + subprocess.run( + stack.prefix + ["up", "-d", "--wait"], + cwd=ROOT, + stdout=output, + stderr=subprocess.STDOUT, + check=True, + timeout=300, + ) + stack.wait_broker_ready(timeout=max(0, setup_deadline-time.monotonic())) + initial = stack.containers() + dump(run / ("environment-r" + str(repetition) + ".json"), initial) + images = {row["service"]: row["image"] for row in initial} + if baseline_images is None: + baseline_images = images + elif images != baseline_images: + raise RuntimeError("Image IDs changed between repetitions") + warmup = Case( + stack, + protocol["profiles"][0], + "control", + fixture_data[protocol["profiles"][0]["name"]], + run / "warmup" / ("r" + str(repetition)), + ) + try: + dump(warmup.directory / "result.json", warmup.execute()) + except Exception as exc: + dump(warmup.directory / "result.json", { + "status": "invalid", "error_type": type(exc).__name__, + "error": str(exc), "attachment_id": warmup.attachment, + }) + if warmup.attachment: + try: + dump(warmup.directory / "database-final.json", stack.snapshot(warmup.attachment)) + except Exception: + pass + raise + finally: + warmup.close() + order = [ + (profile, scenario) + for profile in profiles + for scenario in protocol["scenarios"] + ] + random.Random(protocol["seed"] + repetition).shuffle(order) + dump( + run / ("order-r" + str(repetition) + ".json"), + [{"profile": p["name"], "scenario": s} for p, s in order], + ) + for index, (profile, scenario) in enumerate(order, 1): + relative = ( + "cases/r" + + str(repetition) + + "/" + + str(index).zfill(2) + + "-" + + profile["name"] + + "-" + + scenario + ) + case = Case( + stack, + profile, + scenario, + fixture_data[profile["name"]], + run / relative, + ) + row = { + "repetition": repetition, + "profile": profile["name"], + "scenario": scenario, + "directory": relative, + "status": "pass", + } + recovered = True + try: + if ( + stack.sql( + "SELECT count(*) FROM attachments WHERE status IN ('queued','processing')" + ) + != "0" + or stack.queue_depth() != 0 + ): + raise RuntimeError("Non-quiescent initial state") + dump( + case.directory / "containers-before.json", + stack.containers(), + ) + row.update(case.execute()) + except KeyboardInterrupt: + row.update( + status="invalid", + error_type="KeyboardInterrupt", + error="Interrupted by user", + ) + case.event("case_interrupted") + raise + except Exception as exc: + row.update( + status="violation" + if isinstance(exc, Violation) + or isinstance(exc, Failure) + and case.attachment is not None + else "invalid", + error_type=type(exc).__name__, + error=str(exc), + ) + case.event( + "case_error", error_type=type(exc).__name__, error=str(exc) + ) + try: + case.recover() + except Exception as recovery_error: + row["recovery_error_type"] = type(recovery_error).__name__ + recovered = False + finally: + case.close() + try: + dump( + case.directory / "containers-after.json", + stack.containers(), + ) + except Exception: + row["container_snapshot_missing"] = True + if case.attachment: + try: + dump( + case.directory / "database-final.json", + stack.snapshot(case.attachment), + ) + except Exception: + row["database_snapshot_missing"] = True + markers = [ + {"name": p.name, "mtime": p.stat().st_mtime} + for p in (private / "faults").glob( + "*" + case.attachment + "*" + ) + ] + dump(case.directory / "fault-markers.json", markers) + row.setdefault("elapsed_seconds", time.monotonic() - case.start) + dump(case.directory / "result.json", row) + records.write(json.dumps(row, ensure_ascii=False) + "\n") + records.flush() + if collector.failed: + raise RuntimeError( + "Docker lifecycle event stream ended unexpectedly" + ) + print( + "Repeat", + repetition, + "case", + index, + profile["name"], + scenario, + row["status"], + flush=True, + ) + if not recovered: + raise RuntimeError( + "Cannot restore quiescence; remaining cases not replaced" + ) + finally: + # Remove this project's ephemeral networks before the next repeat. + # Named volumes retain the experiment data (no --volumes). + try: + subprocess.run( + stack.prefix + ["down", "--timeout", "30"], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=120, + check=True, + ) + finally: + collector.close() + if collector.failed: + manifest["interrupted"] = True + except BaseException as exc: + manifest.update(interrupted=True, interruption_type=type(exc).__name__) + dump( + run / "interruption.json", {"type": type(exc).__name__, "message": str(exc)} + ) + raise + finally: + records.close() + manifest["finished_utc"] = dt.datetime.now(dt.timezone.utc).isoformat() + dump(run / "manifest.json", manifest) + summary = analyze(run) + print("Outcome:", summary["outcome"], flush=True) + print("Return archive:", archive_public(run), flush=True) + fcntl.flock(lock, fcntl.LOCK_UN) + lock.close() + + +def main(): + ensure_local_ignore() + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser( + "seal", help="freeze source/protocol; no containers are started" + ) + run = commands.add_parser("run") + run.add_argument("--phase", choices=("pilot", "main"), required=True) + run.add_argument("--seal", type=Path, required=True) + for name in ("analyze", "pack"): + command = commands.add_parser(name) + command.add_argument("directory", type=Path) + args = parser.parse_args() + if args.command == "seal": + seal() + elif args.command == "run": + run_series(args.seal.resolve(), args.phase) + elif args.command == "analyze": + print( + json.dumps(analyze(args.directory.resolve()), ensure_ascii=False, indent=2) + ) + elif args.command == "pack": + print(archive_public(args.directory.resolve())) + + +if __name__ == "__main__": + main() diff --git a/experiments/src/driver.py b/experiments/src/driver.py new file mode 100644 index 00000000..86671ffb --- /dev/null +++ b/experiments/src/driver.py @@ -0,0 +1,693 @@ +"""Controlled delivery/fault cases. Called only by an explicit experiment run.""" + +import datetime as dt +import hashlib +import hmac +import json +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from pathlib import Path + +from support import ROOT, digest, dump + +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure +from image_probe import jpeg_size + + +class Violation(AssertionError): + pass + + +class HarnessError(RuntimeError): + pass + + +def require(condition, message): + if not condition: + raise Violation(message) + + +class Stack: + def __init__(self, project, config, private, protocol): + self.project, self.config, self.private, self.protocol = ( + project, + config, + private, + protocol, + ) + self.prefix = [ + "docker", + "compose", + "--project-name", + project, + "-f", + str(config), + ] + + def call(self, *args): + result = subprocess.run( + self.prefix + list(args), + cwd=ROOT, + text=True, + capture_output=True, + timeout=60, + ) + if result.returncode: + error = result.stderr + for path in (self.private / "secrets").iterdir(): + value = path.read_text().strip() + if len(value) >= 12: + error = error.replace(value, "[REDACTED]") + with (self.private.parent.parent / "command-errors.jsonl").open( + "a" + ) as output: + output.write( + json.dumps( + { + "project": self.project, + "args": args, + "exit_code": result.returncode, + "stderr": error, + } + ) + + "\n" + ) + raise HarnessError("Compose command failed; see command-errors.jsonl") + return result.stdout.strip() + + def wait_broker_ready(self, timeout=60): + """Wait only during setup; runtime queue failures remain fatal.""" + deadline = time.monotonic() + timeout + args = ["exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", + "name", "messages_ready", "messages_unacknowledged", + "--formatter", "json", "--timeout", "5"] + attempts = 0 + while time.monotonic() < deadline: + attempts += 1 + try: + result = subprocess.run( + self.prefix + args, cwd=ROOT, text=True, capture_output=True, + timeout=min(15, max(0.1, deadline-time.monotonic())), + ) + if result.returncode == 0: + rows = json.loads(result.stdout) + if not isinstance(rows, list): + raise HarnessError("Unexpected RabbitMQ queue response") + dump(self.private.parent.parent / ("broker-readiness-"+self.private.name+".json"), { + "project": self.project, "attempts": attempts, + "ready": True, + }) + return + error = result.stderr + # Auth, command syntax and stopped-container errors are not startup races. + if "requires the 'rabbit' app to be running" not in error: + raise HarnessError("RabbitMQ readiness probe failed: " + error.strip()) + except subprocess.TimeoutExpired: + error = "RabbitMQ startup probe timed out" + with (self.private.parent.parent / "broker-startup.jsonl").open("a") as output: + output.write(json.dumps({"project": self.project, "attempt": attempts, + "stderr": error}) + "\n") + time.sleep(min(1, max(0, deadline-time.monotonic()))) + raise HarnessError("RabbitMQ did not finish startup before the setup deadline") + + def sql(self, query): + return self.call( + "exec", + "-T", + "db", + "sh", + "-c", + 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', + "sh", + query, + ) + + def workers(self): + return self.call("ps", "-aq", "worker").splitlines() + + def worker_action(self, action, ids): + for ident in ids: + owner = subprocess.check_output( + [ + "docker", + "inspect", + "--format", + '{{index .Config.Labels "com.docker.compose.project"}}', + ident, + ], + text=True, + ).strip() + if owner != self.project: + raise HarnessError("Container belongs to another project") + if ids: + subprocess.run( + [ + "docker", + action, + *(["--time", "5"] if action == "stop" else []), + *ids, + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def queue_depth(self): + rows = json.loads( + self.call( + "exec", + "-T", + "broker", + "rabbitmqctl", + "-q", + "list_queues", + "name", + "messages_ready", + "messages_unacknowledged", + "--formatter", + "json", + ) + ) + return sum( + row["messages_ready"] + row["messages_unacknowledged"] + for row in rows + if row["name"] == "attachments.v1" + ) + + def operator(self, *args): + running = self.call("ps", "--status", "running", "-q", "dispatcher") + mode = ( + ["exec", "-T", "dispatcher"] + if running + else ["run", "--rm", "--no-deps", "-T", "dispatcher"] + ) + return self.call(*mode, "./messenger", *args) + + def admin(self, *args): + return json.loads(self.operator("jobs", *args)) + + def snapshot(self, attachment): + result = self.sql( + "SELECT json_build_object('desired',a.pipeline_version,'published',a.published_pipeline_version,'status',a.status,'jobs',(SELECT json_agg(json_build_object('job_id',j.id,'version',j.pipeline_version,'attempts',j.attempts,'superseded',j.superseded_at,'lease_until',j.lease_until) ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id),'attempts_audit',(SELECT COALESCE(json_agg(json_build_object('job_id',t.job_id,'generation',t.generation,'attempt',t.attempt,'started_at',t.started_at,'finished_at',t.finished_at,'error_code',t.error_code,'next_attempt_at',t.next_attempt_at) ORDER BY t.started_at),'[]') FROM attachment_job_attempts t JOIN attachment_jobs x ON x.id=t.job_id WHERE x.attachment_id=a.id),'publications',(SELECT COALESCE(json_agg(json_build_object('id',p.id,'version',p.pipeline_version,'variants',p.variants,'published_at',p.published_at) ORDER BY p.id),'[]') FROM attachment_publications p WHERE p.attachment_id=a.id)) FROM attachments a WHERE a.id='" + + attachment + + "'" + ) + return json.loads(result) + + def messages(self, attachment): + return json.loads( + self.sql( + "SELECT json_agg(json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',pipeline_version,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) ORDER BY pipeline_version) FROM attachment_jobs WHERE attachment_id='" + + attachment + + "'" + ) + ) + + def publish(self, job): + self.operator("broker-publish", json.dumps(job)) + + def containers(self): + ids = self.call("ps", "-aq").splitlines() + if not ids: + return [] + # Never save Config.Env: it contains the experiment DB password. + rows = json.loads( + subprocess.check_output(["docker", "inspect", *ids], text=True) + ) + return [ + { + "id": row["Id"], + "image": row["Image"], + "service": row["Config"]["Labels"].get("com.docker.compose.service"), + "state": { + k: row["State"].get(k) + for k in ( + "Status", + "ExitCode", + "OOMKilled", + "StartedAt", + "FinishedAt", + ) + }, + "restarts": row["RestartCount"], + "memory": row["HostConfig"]["Memory"], + "nano_cpus": row["HostConfig"]["NanoCpus"], + } + for row in rows + ] + + def s3(self, method, key="", query=None): + secret_dir = self.private / "secrets" + access = (secret_dir / "s3_api_access_key_id").read_text().strip() + secret = (secret_dir / "s3_api_secret_access_key").read_text().strip() + host = "127.0.0.1:" + str(self.protocol["ports"]["s3"]) + uri = "/messenger-attachments/" + key + query_string = urllib.parse.urlencode( + sorted((query or {}).items()), quote_via=urllib.parse.quote + ) + now = dt.datetime.now(dt.timezone.utc) + timestamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d") + payload = digest(b"") + signed = "host;x-amz-content-sha256;x-amz-date" + headers = ( + "host:" + + host + + "\nx-amz-content-sha256:" + + payload + + "\nx-amz-date:" + + timestamp + + "\n" + ) + canonical = "\n".join([method, uri, query_string, headers, signed, payload]) + scope = day + "/us-east-1/s3/aws4_request" + text = ( + "AWS4-HMAC-SHA256\n" + + timestamp + + "\n" + + scope + + "\n" + + digest(canonical.encode()) + ) + signing = ("AWS4" + secret).encode() + for part in (day, "us-east-1", "s3", "aws4_request"): + signing = hmac.new(signing, part.encode(), hashlib.sha256).digest() + signature = hmac.new(signing, text.encode(), hashlib.sha256).hexdigest() + request = urllib.request.Request( + "http://" + host + uri + ("?" + query_string if query_string else ""), + method=method, + headers={ + "x-amz-date": timestamp, + "x-amz-content-sha256": payload, + "Authorization": "AWS4-HMAC-SHA256 Credential=" + + access + + "/" + + scope + + ", SignedHeaders=" + + signed + + ", Signature=" + + signature, + }, + ) + try: + response = urllib.request.urlopen(request, timeout=15) + except urllib.error.HTTPError as exc: + response = exc + with response: + code, body = response.code, response.read() + if code in (401, 403): + raise HarnessError("S3 measurement identity could not authenticate") + return code, body + + +def webp_size(data): + require(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "invalid WebP container") + offset = 12 + dimensions = None + while offset + 8 <= len(data): + kind = data[offset : offset + 4] + length = int.from_bytes(data[offset + 4 : offset + 8], "little") + body = data[offset + 8 : offset + 8 + length] + require(kind not in (b"EXIF", b"XMP ", b"ICCP"), "WebP metadata leaked") + if kind == b"VP8 ": + dimensions = ( + int.from_bytes(body[6:8], "little") & 16383, + int.from_bytes(body[8:10], "little") & 16383, + ) + if kind == b"VP8L": + bits = int.from_bytes(body[1:5], "little") + dimensions = ((bits & 16383) + 1, ((bits >> 14) & 16383) + 1) + if kind == b"VP8X": + dimensions = ( + int.from_bytes(body[4:7], "little") + 1, + int.from_bytes(body[7:10], "little") + 1, + ) + offset += 8 + length + (length % 2) + require(dimensions is not None, "WebP dimensions absent") + return dimensions + + +class Case: + def __init__(self, stack, profile, scenario, data, directory): + self.s, self.profile, self.scenario, self.data, self.directory = ( + stack, + profile, + scenario, + data, + directory, + ) + directory.mkdir(parents=True) + self.events = (directory / "events.jsonl").open("x") + self.start = time.monotonic() + self.last = None + self.gaps = [] + self.attachment = None + self.verifications = 0 + self.c = Client("http://127.0.0.1:" + str(stack.protocol["ports"]["api"])) + + def event(self, kind, **values): + now = time.monotonic() + record = { + "utc": dt.datetime.now(dt.timezone.utc).isoformat(), + "elapsed": now - self.start, + "kind": kind, + **values, + } + self.events.write(json.dumps(record, ensure_ascii=False) + "\n") + self.events.flush() + + def wait(self, fn, timeout=None, invariant=True): + deadline = time.monotonic() + ( + timeout or self.s.protocol["case_timeout_seconds"] + ) + while time.monotonic() < deadline: + value = fn() + if value and time.monotonic() <= deadline: + return value + time.sleep(self.s.protocol["poll_interval_seconds"]) + if invariant: + raise Violation("completion/quiescence deadline exceeded") + raise HarnessError("required fault/precondition did not occur before deadline") + + def observe(self): + now = time.monotonic() + if self.last is not None and now - self.last > 2: + self.gaps.append(now - self.last) + self.last = now + value = self.c.call( + "GET", + "/api/v1/attachments/" + self.attachment, + actor=self.alice, + schema="Attachment", + )["json"] + snapshot = self.s.snapshot(self.attachment) + self.event("observation", api=value, database=snapshot) + return value, snapshot + + def ready(self, version): + def check(): + value, snapshot = self.observe() + require(value["status"] != "failed", "valid fixture reached failed") + return ( + (value, snapshot) + if value["status"] == "ready" and snapshot["published"] == version + else None + ) + + return self.wait(check) + + def arm(self, point): + marker = self.s.private / "faults" / (point + "." + self.attachment) + marker.write_text("experiment " + self.scenario + "\n") + self.event("arm", point=point) + return Path(str(marker) + ".hit") + + def upload(self): + self.alice, self.bob, self.eve, chat = self.c.bootstrap() + value = self.c.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=fixture.png", + 202, + actor=self.alice, + raw=self.data, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + self.attachment = value["id"] + require( + value["sha256"] == digest(self.data), "accepted original checksum mismatch" + ) + self.event("accepted_202", attachment_id=self.attachment) + require( + len(self.s.messages(self.attachment)) == 1, + "accepted upload has no unique durable job", + ) + + def upgrade(self): + require( + self.s.admin( + "upgrade", self.attachment, "2", "Protocol v1 " + self.scenario + )["changed"], + "upgrade intent was not persisted", + ) + self.event("upgrade_accepted", version=2) + + def publish(self, job, count): + for _ in range(count): + self.s.publish(job) + self.event( + "explicit_delivery", + job_id=job["job_id"], + version=job["pipeline_version"], + count=count, + ) + + def verify(self, version): + value, snapshot = self.ready(version) + self.verifications += 1 + observation = "verify-" + str(self.verifications).zfill(3) + "-v" + str(version) + objects = self.directory / "objects" / observation + objects.mkdir(parents=True) + require( + snapshot["desired"] == version, "desired version differs from publication" + ) + original = self.c.call( + "GET", "/api/v1/attachments/" + self.attachment + "/content", actor=self.bob + )["bytes"] + require(original == self.data, "original changed") + require( + len(value["variants"]) == 3 + and {v["name"] for v in value["variants"]} + == {"thumbnail", "crop", "optimized"}, + "incomplete or duplicate variant set", + ) + manifest = {} + for item in value["variants"]: + name = item["name"] + key = ( + "variants/" + + self.attachment + + "/v" + + str(version) + + "/" + + name + + (".webp" if name == "optimized" else ".jpg") + ) + response = self.c.call( + "GET", + "/api/v1/attachments/" + self.attachment + "/content?variant=" + name, + actor=self.bob, + ) + require( + digest(response["bytes"]) == item["sha256"] + and len(response["bytes"]) == item["size_bytes"], + "variant checksum mismatch", + ) + require( + response["headers"].get_content_type() == item["content_type"], + "variant MIME mismatch", + ) + try: + dimensions = ( + webp_size(response["bytes"]) + if name == "optimized" + else jpeg_size(response["bytes"]) + ) + except Failure as exc: + raise Violation("actual variant decode failed") from exc + w, h = self.profile["width"], self.profile["height"] + limit = 1280 if name == "optimized" else 256 + expected = ( + (128, 128) + if name == "crop" + else (min(w, limit), max(1, h * limit // w)) + if w >= h and w > limit + else (max(1, w * limit // h), limit) + if h > w and h > limit + else (w, h) + ) + require( + dimensions == expected == (item["width"], item["height"]), + "actual variant dimensions mismatch", + ) + code, body = self.s.s3("GET", key) + require( + code == 200 and body == response["bytes"], + "API exposes a different version than S3", + ) + ( + objects / (name + (".webp" if name == "optimized" else ".jpg")) + ).write_bytes(body) + manifest[key] = digest(body) + code, body = self.s.s3( + "GET", + query={ + "list-type": "2", + "prefix": "variants/" + self.attachment + "/v" + str(version) + "/", + }, + ) + require(code == 200, "cannot list variant set") + require( + len(ET.fromstring(body).findall("{*}Contents")) == 3, + "extra/missing object set", + ) + history = snapshot["publications"] + versions = [p["version"] for p in history] + require( + versions == sorted(versions) and len(versions) == len(set(versions)), + "publication duplicated or rolled back", + ) + require(versions[-1] == version, "publication history missing active version") + require( + all( + j["version"] == version or j["superseded"] is not None + for j in snapshot["jobs"] + ), + "older accepted job has no durable disposition", + ) + dump(self.directory / (observation + "-manifest.json"), manifest) + return manifest, snapshot + + def execute(self): + self.s.worker_action("stop", self.s.workers()) + self.s.call("stop", "dispatcher") + self.upload() + workers = self.s.workers() + require(len(workers) == 2, "two workers are required by protocol") + scenario = self.scenario + version = 1 + first = None + if scenario == "commit_publish_gap": + marker = self.arm("after_commit_before_publish") + if scenario != "reorder": + self.s.call("start", "dispatcher") + if scenario == "commit_publish_gap": + self.wait(marker.exists, invariant=False) + if scenario == "duplicates": + self.publish( + self.s.messages(self.attachment)[0], + self.s.protocol["duplicates_before_ready"], + ) + if scenario.startswith("crash_"): + point = { + "crash_before_effect": "before_effect", + "crash_after_effect": "after_object_before_commit", + "crash_before_ack": "after_commit_before_ack", + }[scenario] + marker = self.arm(point) + self.s.worker_action("start", [workers[0]]) + self.wait(marker.exists, invariant=False) + self.event("fault_hit", point=point) + elif scenario == "reorder": + # No consumer is running; broker order is deliberately v2 then old v1. + self.upgrade() + jobs = self.s.messages(self.attachment) + self.publish(jobs[1], 1) + self.publish(jobs[0], 1) + version = 2 + self.s.worker_action("start", workers) + elif scenario == "stale_lease": + marker = self.arm("after_claim") + self.s.worker_action("start", [workers[0]]) + self.wait(marker.exists, invariant=False) + self.s.call("start", "dispatcher") + self.upgrade() + self.s.worker_action("start", [workers[1]]) + first, _ = self.verify(2) + version = 2 + release = ( + self.s.private + / "faults" + / ("after_claim." + self.attachment + ".release") + ) + release.write_text("release\n") + self.event("release_old_lease") + self.wait( + Path(str(release) + ".observed").exists, timeout=5, invariant=False + ) + self.wait( + ( + self.s.private + / "faults" + / ("stale_rejected." + self.attachment + ".observed") + ).exists, + timeout=5, + invariant=False, + ) + else: + self.s.worker_action("start", workers) + if scenario == "old_after_v2": + old_manifest, _ = self.verify(1) + old = self.s.messages(self.attachment)[0] + self.upgrade() + version = 2 + first, _ = self.verify(2) + if not any( + old_manifest.get(key.replace("/v2/", "/v1/")) != value + for key, value in first.items() + ): + raise HarnessError( + "Version fingerprints are indistinguishable for this fixture" + ) + self.publish(old, self.s.protocol["duplicates_after_ready"]) + for key, checksum in old_manifest.items(): + code, body = self.s.s3("GET", key) + require( + code == 200 and digest(body) == checksum, + "v1 immutable object changed", + ) + if scenario == "duplicates": + first, before = self.verify(1) + self.publish( + self.s.messages(self.attachment)[0], + self.s.protocol["duplicates_after_ready"], + ) + if scenario != "reorder": + self.s.call("start", "dispatcher") + self.s.worker_action("start", workers) + self.wait(lambda: self.s.queue_depth() == 0) + # Let an already-delivered stale attempt leave its pause and hit the fence. + if scenario == "stale_lease": + time.sleep(1) + final, snapshot = self.verify(version) + if first is not None: + require(first == final, "old delivery changed published bytes") + if scenario in ("duplicates", "old_after_v2"): + require( + all(j["attempts"] == 1 for j in snapshot["jobs"]), + "terminal duplicate started another attempt", + ) + if scenario == "reorder": + self.s.call("start", "dispatcher") + self.event("case_complete", version=version) + return { + "attachment_id": self.attachment, + "version": version, + "snapshot": snapshot, + "manifest": final, + "observation_gaps": self.gaps, + "elapsed_seconds": time.monotonic() - self.start, + } + + def recover(self): + for marker in (self.s.private / "faults").glob("after_claim.*.hit"): + Path(str(marker)[:-4] + ".release").write_text("cleanup release\n") + self.s.call("start", "dispatcher") + self.s.worker_action("start", self.s.workers()) + self.wait( + lambda: ( + self.s.sql( + "SELECT count(*) FROM attachments WHERE status IN ('queued','processing')" + ) + == "0" + and self.s.queue_depth() == 0 + ) + ) + + def close(self): + self.events.close() diff --git a/experiments/src/events.py b/experiments/src/events.py new file mode 100644 index 00000000..e99e32a1 --- /dev/null +++ b/experiments/src/events.py @@ -0,0 +1,85 @@ +"""Whitelisted Docker lifecycle events; never serialize environment or exec args.""" + +import json +import subprocess +import threading + + +class LifecycleEvents: + def __init__(self, project, path): + self.closing = False + self.oom = 0 + self.failed = False + self.path = path + self.process = subprocess.Popen( + [ + "docker", + "events", + "--filter", + "label=com.docker.compose.project=" + project, + "--format", + "{{json .}}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + self.thread = threading.Thread(target=self.read, daemon=True) + self.thread.start() + + def read(self): + try: + with self.path.open("x") as output: + for line in self.process.stdout: + try: + event = json.loads(line) + except json.JSONDecodeError: + if self.closing: + break + raise + action = event.get("Action", "") + if event.get("Type") != "container" or action not in ( + "create", + "start", + "stop", + "kill", + "die", + "restart", + "destroy", + "oom", + "pause", + "unpause", + ): + continue + actor = event.get("Actor", {}) + attributes = actor.get("Attributes", {}) + value = { + "time_nano": event.get("timeNano"), + "action": action, + "container_id": actor.get("ID"), + "service": attributes.get("com.docker.compose.service"), + "exit_code": attributes.get("exitCode"), + "signal": attributes.get("signal"), + } + if action == "oom": + self.oom += 1 + output.write(json.dumps(value) + "\n") + output.flush() + except Exception: + self.failed = True + finally: + if not self.closing: + self.failed = True + + def close(self): + self.closing = True + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + self.thread.join(timeout=5) + if self.thread.is_alive(): + self.failed = True diff --git a/experiments/src/support.py b/experiments/src/support.py new file mode 100644 index 00000000..46de9711 --- /dev/null +++ b/experiments/src/support.py @@ -0,0 +1,293 @@ +"""Deterministic inputs, source sealing and private isolated Compose configuration.""" + +import hashlib +import json +import os +import secrets +import struct +import subprocess +import zipfile +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PROTOCOL = ROOT / "experiments/config/protocol.json" + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def ensure_local_ignore(): + path = Path( + subprocess.check_output( + ["git", "rev-parse", "--git-path", "info/exclude"], cwd=ROOT, text=True + ).strip() + ) + if not path.is_absolute(): + path = ROOT / path + text = path.read_text() if path.exists() else "" + # Migrate the old blanket exclusion; public results belong in Git. + lines = [line for line in text.splitlines() if line.strip() != "evidence/idempotency/"] + for pattern in ("experiments/results/", "evidence/idempotency/**/.private/", "evidence/idempotency/.run.lock"): + if pattern not in lines: + lines.append(pattern) + updated = "\n".join(lines).rstrip() + "\n" + if updated != text: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(updated) + + +def dump(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + +def source_files(): + names = ( + subprocess.check_output( + ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + cwd=ROOT, + ) + .decode() + .split("\0") + ) + return sorted( + { + name + for name in names + if name + and not name.startswith( + ( + "experiments/results/", + "evidence/", + ".memory/", + "secrets/", + ".git/", + ) + ) + and (ROOT / name).is_file() + } + ) + + +def source_manifest(): + return {name: digest((ROOT / name).read_bytes()) for name in source_files()} + + +def verify_seal(seal): + value = json.loads((seal / "manifest.json").read_text()) + if value["files"] != source_manifest(): + raise RuntimeError("Source changed after seal; create a new seal/series.") + if value["protocol_sha256"] != digest(PROTOCOL.read_bytes()): + raise RuntimeError("Protocol changed after seal.") + return value + + +def png(profile, seed): + w, h = profile["width"], profile["height"] + encoder = zlib.compressobj(6) + parts = [] + for y in range(h): + row = bytearray([0]) + for x in range(w): + v = (x * 17 + y * 31 + seed + (x * y) % 251) & 255 + row.extend( + ( + v, + (v * 13 + x) & 255, + (v * 7 + y) & 255, + ((x + y) * 11) & 255 if profile["alpha"] else 255, + ) + ) + parts.append(encoder.compress(row)) + parts.append(encoder.flush()) + + def chunk(kind, data): + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 6, 0, 0, 0)) + + chunk(b"IDAT", b"".join(parts)) + + chunk(b"IEND", b"") + ) + + +def private_environment(directory, protocol): + directory.mkdir(parents=True, exist_ok=False) + os.chmod(directory, 0o700) + secret_dir = directory / "secrets" + secret_dir.mkdir() + password = secrets.token_hex(24) + values = { + "POSTGRES_DB": "messenger", + "POSTGRES_USER": "messenger", + "POSTGRES_PASSWORD": password, + "APP_HOST": "0.0.0.0", + "APP_PORT": "8080", + "APP_ENV": "experiment", + "API_REPLICAS": "2", + "API_GOMAXPROCS": "1", + "SESSION_TTL_SECONDS": "7200", + "IDEMPOTENCY_TTL_SECONDS": "86400", + "S3_ENDPOINT": "http://s3:9000", + "S3_PUBLIC_ENDPOINT": "http://127.0.0.1:" + str(protocol["ports"]["s3"]), + "S3_BUCKET": "messenger-attachments", + "S3_REGION": "us-east-1", + "DATABASE_URL_FILE": "/run/secrets/database_url", + "CACHE_URL_FILE": "/run/secrets/cache_url", + "BROKER_URL_FILE": "/run/secrets/broker_url", + "S3_ACCESS_KEY_ID_FILE": "/run/secrets/s3_access_key_id", + "S3_SECRET_ACCESS_KEY_FILE": "/run/secrets/s3_secret_access_key", + "JOB_TIMEOUT_SECONDS": str(protocol["job_timeout_seconds"]), + "JOB_MAX_ATTEMPTS": str(protocol["job_max_attempts"]), + "WORKER_MAX_PIXELS": "8000000", + "UPLOAD_CLEANUP_GRACE_SECONDS": "600", + } + credentials = { + "postgres_password": password, + "database_url": "postgres://messenger:" + + password + + "@db:5432/messenger?sslmode=disable", + "cache_url": "redis://redis:6379/0", + "s3_access_key_id": secrets.token_hex(12), + "s3_secret_access_key": secrets.token_hex(32), + "s3_api_access_key_id": secrets.token_hex(12), + "s3_api_secret_access_key": secrets.token_hex(32), + "broker_user": "messenger", + "broker_password": secrets.token_hex(32), + } + credentials["broker_url"] = ( + "amqp://messenger:" + credentials["broker_password"] + "@broker:5672/%2F" + ) + for name, value in credentials.items(): + (secret_dir / name).write_text(value + "\n") + os.chmod(secret_dir / name, 0o644) + env_file = directory / "private.env" + env_file.write_text("".join(k + "=" + v + "\n" for k, v in values.items())) + os.chmod(env_file, 0o600) + faults = directory / "faults" + faults.mkdir() + os.chmod(faults, 0o777) + return values + + +def isolated_config(base, project, directory, protocol, app_image, s3_image): + # config --format json resolves names: remove them to avoid sharing lab volumes. + base["name"] = project + for group in ("volumes", "networks"): + for value in base.get(group, {}).values(): + if value.get("external"): + raise RuntimeError("External resources are forbidden in experiment.") + value.pop("name", None) + services = base["services"] + for name in ("frontend", "backend-tests"): + services.pop(name, None) + for name, service in services.items(): + if "build" in service: + service.pop("build") + service["image"] = s3_image if name == "s3" else app_image + for mount in service.get("volumes", []): + if ( + mount.get("type") == "bind" + and Path(mount["source"]).parent == ROOT / "secrets" + ): + mount["source"] = str( + directory / "secrets" / Path(mount["source"]).name + ) + services["s3"]["ports"] = [ + { + "target": 9000, + "published": str(protocol["ports"]["s3"]), + "host_ip": "127.0.0.1", + "protocol": "tcp", + } + ] + services["haproxy"]["ports"] = [ + { + "target": 8080, + "published": str(protocol["ports"]["api"]), + "host_ip": "127.0.0.1", + "protocol": "tcp", + } + ] + services["worker"].setdefault("deploy", {})["replicas"] = protocol["workers"] + for name in ("worker", "dispatcher"): + services[name]["environment"]["PIPELINE_FAULT_DIR"] = "/run/pipeline-faults" + services[name].setdefault("volumes", []).append( + { + "type": "bind", + "source": str(directory / "faults"), + "target": "/run/pipeline-faults", + } + ) + return base + + +def build_images(public): + # Only builds images; does not start a container or collect an experiment case. + subprocess.run( + ["docker", "build", "-t", "lab5-idem-app:prepared", str(ROOT / "src")], + check=True, + stdout=(public / "app-build.log").open("w"), + stderr=subprocess.STDOUT, + ) + subprocess.run( + [ + "docker", + "build", + "-f", + str(ROOT / "infra/minio.Dockerfile"), + "-t", + "lab5-idem-s3:prepared", + str(ROOT / "infra"), + ], + check=True, + stdout=(public / "s3-build.log").open("w"), + stderr=subprocess.STDOUT, + ) + + def image(tag): + return subprocess.check_output( + ["docker", "image", "inspect", "--format", "{{.Id}}", tag], text=True + ).strip() + + return image("lab5-idem-app:prepared"), image("lab5-idem-s3:prepared") + + +def archive_public(run): + target = run.parent / (run.name + "-public.zip") + secret_values = [] + for directory in (run / ".private").glob("*/secrets"): + for path in directory.iterdir(): + if any( + word in path.name + for word in ( + "password", + "secret", + "access_key", + "database_url", + "broker_url", + ) + ): + value = path.read_bytes().strip() + if len(value) >= 16: + secret_values.append(value) + with zipfile.ZipFile(target, "x", zipfile.ZIP_DEFLATED) as archive: + for path in sorted(run.rglob("*")): + relative = path.relative_to(run) + if path.is_file() and ".private" not in relative.parts: + if any(value in path.read_bytes() for value in secret_values): + raise RuntimeError( + "Secret detected; do not share archive: " + str(relative) + ) + archive.write(path, relative) + return target diff --git a/experiments/tests/test_broker_readiness.py b/experiments/tests/test_broker_readiness.py new file mode 100644 index 00000000..b698a6e7 --- /dev/null +++ b/experiments/tests/test_broker_readiness.py @@ -0,0 +1,62 @@ +"""Startup races must not be hidden during experimental cases.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from driver import Stack, HarnessError + + +class BrokerReadinessTests(unittest.TestCase): + def make_stack(self, root): + private = root / ".private" / "r1" + (private / "secrets").mkdir(parents=True) + return Stack("lab5-idem-unit", root / "config.json", private, {}) + + def test_booting_application_is_retried_before_warmup(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + stack = self.make_stack(root) + responses = [ + subprocess.CompletedProcess([], 64, "", "this command requires the 'rabbit' app to be running"), + subprocess.CompletedProcess([], 0, "[]", ""), + ] + with patch("driver.subprocess.run", side_effect=responses) as run, patch("driver.time.sleep"): + stack.wait_broker_ready(timeout=10) + self.assertEqual(run.call_count, 2) + self.assertEqual(json.loads((root / "broker-readiness-r1.json").read_text())["attempts"], 2) + self.assertEqual(len((root / "broker-startup.jsonl").read_text().splitlines()), 1) + + def test_nonstartup_errors_fail_without_retry(self): + with tempfile.TemporaryDirectory() as temporary: + stack = self.make_stack(Path(temporary)) + response = subprocess.CompletedProcess([], 70, "", "Authentication failed") + with patch("driver.subprocess.run", return_value=response) as run: + with self.assertRaises(HarnessError): + stack.wait_broker_ready(timeout=10) + self.assertEqual(run.call_count, 1) + + def test_runtime_queue_probe_does_not_swallow_startup_or_outage(self): + with tempfile.TemporaryDirectory() as temporary: + stack = self.make_stack(Path(temporary)) + response = subprocess.CompletedProcess([], 64, "", "requires the 'rabbit' app to be running") + with patch("driver.subprocess.run", return_value=response) as run: + with self.assertRaises(HarnessError): + stack.queue_depth() + self.assertEqual(run.call_count, 1) + + def test_expired_setup_budget_does_not_start_a_probe(self): + with tempfile.TemporaryDirectory() as temporary: + stack = self.make_stack(Path(temporary)) + with patch("driver.subprocess.run") as run: + with self.assertRaises(HarnessError): + stack.wait_broker_ready(timeout=0) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/tests/test_tools.py b/experiments/tests/test_tools.py new file mode 100644 index 00000000..db115eba --- /dev/null +++ b/experiments/tests/test_tools.py @@ -0,0 +1,225 @@ +"""Preparation checks with synthetic in-memory data; these do not run the series.""" + +import copy +import importlib.util +import json +import struct +import subprocess +import sys +import tempfile +import unittest +import zlib +from unittest.mock import patch +from pathlib import Path + +EXPERIMENTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(EXPERIMENTS / "src")) +import support + +spec = importlib.util.spec_from_file_location("experiment_cli", EXPERIMENTS / "run.py") +cli = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cli) + + +class ToolTests(unittest.TestCase): + def test_git_keeps_public_evidence_and_hides_credentials_after_migration(self): + rules = (support.ROOT / ".gitignore").read_text() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(["git", "init", "-q", str(root)], check=True) + exclude = root / ".git/info/exclude" + exclude.write_text("evidence/idempotency/\ncustom-local-rule/\n") + public = root / "evidence/idempotency/main-test/summary.json" + private = public.parent / ".private/r1/private.env" + private.parent.mkdir(parents=True) + public.write_text("{}") + private.write_text("POSTGRES_PASSWORD=test-only\n") + (root / ".gitignore").write_text(rules) + log = public.parent / "boot-r1.log" + log.write_text("setup log\n") + with patch.object(support, "ROOT", root): + support.ensure_local_ignore() + first = exclude.read_text() + support.ensure_local_ignore() + self.assertEqual(first, exclude.read_text()) + files = support.source_files() + self.assertIn("custom-local-rule/", first) + self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(public)], cwd=root).returncode, 1) + self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(private)], cwd=root).returncode, 0) + self.assertEqual(subprocess.run(["git", "check-ignore", "-q", str(log)], cwd=root).returncode, 1) + self.assertFalse(any(name.startswith("evidence/") for name in files)) + + def test_outputs_belong_to_root_evidence(self): + self.assertEqual(cli.RESULTS, support.ROOT / "evidence/idempotency") + + def test_png_is_reproducible_and_decodable(self): + profile = {"width": 17, "height": 9, "alpha": True} + first = support.png(profile, 2303) + self.assertEqual(first, support.png(profile, 2303)) + self.assertNotEqual(first, support.png(profile, 2304)) + offset = 8 + compressed = [] + while offset < len(first): + size = struct.unpack(">I", first[offset : offset + 4])[0] + kind = first[offset + 4 : offset + 8] + data = first[offset + 8 : offset + 8 + size] + self.assertEqual( + zlib.crc32(kind + data) & 0xFFFFFFFF, + struct.unpack(">I", first[offset + 8 + size : offset + 12 + size])[0], + ) + if kind == b"IDAT": + compressed.append(data) + offset += 12 + size + self.assertEqual(len(zlib.decompress(b"".join(compressed))), 9 * (17 * 4 + 1)) + + def test_isolation_removes_shared_resource_names_and_secret_paths(self): + service = { + "build": {"context": "x"}, + "volumes": [ + { + "type": "bind", + "source": str(support.ROOT / "secrets/database_url"), + "target": "/run/secrets/database_url", + } + ], + "environment": {}, + } + services = { + name: copy.deepcopy(service) + for name in ( + "api", + "worker", + "dispatcher", + "s3", + "haproxy", + "frontend", + "backend-tests", + ) + } + source = { + "services": services, + "volumes": {"db": {"name": "highload-lab5_db"}}, + "networks": {"data": {"name": "highload-lab5_data"}}, + } + protocol = json.loads(support.PROTOCOL.read_text()) + private = Path("/tmp/private-example") + result = support.isolated_config( + source, "lab5-idem-unit", private, protocol, "sha256:app", "sha256:s3" + ) + self.assertNotIn("name", result["volumes"]["db"]) + self.assertNotIn("name", result["networks"]["data"]) + self.assertNotIn("frontend", result["services"]) + self.assertEqual(result["services"]["worker"]["deploy"]["replicas"], 2) + for entry in result["services"].values(): + self.assertNotIn("build", entry) + self.assertTrue( + all( + not mount.get("source", "").startswith( + str(support.ROOT / "secrets") + ) + for mount in entry["volumes"] + ) + ) + + def test_analyzer_does_not_call_partial_or_duplicate_matrix_success(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + protocol = json.loads(support.PROTOCOL.read_text()) + support.dump(root / "protocol.json", protocol) + support.dump(root / "manifest.json", {"phase": "main"}) + (root / "cases.jsonl").write_text("") + self.assertEqual(cli.analyze(root)["outcome"], "inconclusive") + row = { + "repetition": 1, + "profile": "rgb_small", + "scenario": "control", + "directory": "cases/test", + "status": "violation", + } + (root / "cases.jsonl").write_text(json.dumps(row) + "\n") + self.assertEqual(cli.analyze(root)["outcome"], "refuted") + support.dump(root / "manifest.json", {"phase": "pilot"}) + self.assertEqual( + cli.analyze(root)["outcome"], "pilot_only_not_confirmation" + ) + + def test_private_credentials_never_enter_share_archive(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "case" + root.mkdir() + (root / ".private/r1/secrets").mkdir(parents=True) + (root / ".private/r1/secrets/broker_password").write_text( + "sensitive-password-value-123" + ) + (root / "safe.json").write_text("{}") + archive = support.archive_public(root) + import zipfile + + with zipfile.ZipFile(archive) as value: + self.assertEqual(value.namelist(), ["safe.json"]) + + def test_analyzer_requires_unique_complete_cases_and_counts_oom(self): + # A tiny artificial protocol tests the policy, never supplies experiment data. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + protocol = json.loads(support.PROTOCOL.read_text()) + protocol.update( + repetitions=1, profiles=protocol["profiles"][:1], scenarios=["control"] + ) + protocol["pass"]["cases"] = 1 + support.dump(root / "protocol.json", protocol) + support.dump( + root / "manifest.json", {"phase": "main", "finished_utc": "unit-test"} + ) + for filename in ( + "seal/manifest.json", + "seal/source.zip", + "fixtures/manifest.json", + "docker-events-r1.jsonl", + "environment-r1.json", + "order-r1.json", + ): + path = root / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + row = { + "repetition": 1, + "profile": "rgb_small", + "scenario": "control", + "directory": "cases/unit", + "status": "pass", + } + for filename in ( + "events.jsonl", + "result.json", + "containers-before.json", + "containers-after.json", + "database-final.json", + "fault-markers.json", + ): + path = root / row["directory"] / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}") + (root / "cases.jsonl").write_text(json.dumps(row) + "\n") + self.assertEqual( + cli.analyze(root)["outcome"], "supported_in_tested_conditions" + ) + (root / "cases.jsonl").write_text((json.dumps(row) + "\n") * 2) + self.assertEqual(cli.analyze(root)["outcome"], "inconclusive") + (root / "cases.jsonl").write_text(json.dumps(row) + "\n") + (root / "docker-events-r1.jsonl").write_text('{"action":"oom"}\n') + self.assertEqual(cli.analyze(root)["outcome"], "refuted") + + def test_archive_refuses_a_public_file_with_local_secret(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "case" + (root / ".private/r1/secrets").mkdir(parents=True) + secret = "local-sensitive-password-value" + (root / ".private/r1/secrets/postgres_password").write_text(secret) + (root / "leak.log").write_text(secret) + with self.assertRaises(RuntimeError): + support.archive_public(root) + + +if __name__ == "__main__": + unittest.main()