#!/usr/bin/env python3 """Verify Redis-only restart and generate sanitized Track 1 summaries. The verifier creates short-lived users and chat, retains bearer tokens only in memory, and writes sanitized output. Redis eviction is narrowly scoped to ``messages:*`` keys; it never invokes FLUSHALL or removes volumes. Recorded results and the evidence index live in evidence/README.md and evidence/track1-cache-consistency.md; the main report is REPORT.md. """ import argparse import json import subprocess import sys import time from pathlib import Path from track1_cache_benchmark import ( create_chat, create_user, history, login, post_message, request_json, require_object, ) def compose_command(args: argparse.Namespace, *parts: str) -> list[str]: command = ["docker", "compose"] if args.compose_file: command.extend(["-f", args.compose_file]) if args.env_file: command.extend(["--env-file", args.env_file]) command.extend(parts) return command def run_compose(args: argparse.Namespace, *parts: str) -> subprocess.CompletedProcess[str]: # Compose diagnostics may include environment-derived URLs; do not emit them. return subprocess.run(compose_command(args, *parts), check=True, capture_output=True, text=True) def evict_message_cache(args: argparse.Namespace) -> int: scan = run_compose(args, "exec", "-T", "redis", "redis-cli", "--scan", "--pattern", "messages:*") keys = [line.strip() for line in scan.stdout.splitlines() if line.strip()] for offset in range(0, len(keys), 100): run_compose(args, "exec", "-T", "redis", "redis-cli", "DEL", *keys[offset : offset + 100]) return len(keys) def wait_ready(base_url: str, timeout: float, attempts: int = 30) -> None: for _ in range(attempts): try: if isinstance(request_json(base_url, "GET", "/health/ready", timeout=timeout), dict): return except Exception: pass time.sleep(1) raise RuntimeError("API did not become ready after Redis operation") def verify(args: argparse.Namespace) -> dict[str, object]: suffix = f"verify_{int(time.time())}" password = "Track1-verify-password" owner_name, owner_id = create_user(args.base_url, suffix, "owner", password, args.timeout) reader_name, reader_id = create_user(args.base_url, suffix, "reader", password, args.timeout) owner_token = login(args.base_url, owner_name, password, args.timeout) reader_token = login(args.base_url, reader_name, password, args.timeout) chat_id = create_chat(args.base_url, owner_token, [owner_id, reader_id], "track1-restart", args.timeout) post_message(args.base_url, owner_token, chat_id, "track1 restart evidence", args.timeout) before_me = require_object(request_json(args.base_url, "GET", "/api/v1/users/me", token=reader_token, timeout=args.timeout), "users/me") before_history = history(args.base_url, reader_token, chat_id, args.timeout) cache_keys_deleted = 0 restarted = False redis_stopped = False after_me: dict[str, object] | None = None after_history: dict[str, object] | None = None try: if args.stop_start: run_compose(args, "stop", "redis") redis_stopped = True run_compose(args, "start", "redis") redis_stopped = False restarted = True wait_ready(args.base_url, args.timeout) elif args.restart_redis: run_compose(args, "restart", "redis") restarted = True wait_ready(args.base_url, args.timeout) if args.evict_cache: cache_keys_deleted = evict_message_cache(args) after_me = require_object(request_json(args.base_url, "GET", "/api/v1/users/me", token=reader_token, timeout=args.timeout), "users/me after restart") after_history = history(args.base_url, reader_token, chat_id, args.timeout) finally: if redis_stopped: try: run_compose(args, "start", "redis") except subprocess.CalledProcessError: pass if after_me is None or after_history is None: raise RuntimeError("verification did not obtain post-operation responses") users_equal = before_me == after_me before_items = before_history.get("items") after_items = after_history.get("items") history_equal = before_items == after_items if not users_equal or not history_equal: raise RuntimeError("same-token users/me or history changed after Redis operation") return {"schema_version": 1, "restart_redis": restarted, "cache_eviction": {"requested": args.evict_cache, "message_keys_deleted": cache_keys_deleted}, "same_token_users_me": users_equal, "history_equal": history_equal, "history_item_count": len(after_items) if isinstance(after_items, list) else 0} def read_raw(path: str) -> dict[str, object]: value = json.loads(Path(path).read_text(encoding="utf-8")) if not isinstance(value, dict): raise RuntimeError(f"raw benchmark is not an object: {path}") return value def sanitize_runtime(runtime: object) -> dict[str, object]: if not isinstance(runtime, dict): return {} result: dict[str, object] = {} for key in ("python", "platform", "supplied", "environment"): value = runtime.get(key) if key == "supplied" and isinstance(value, dict): result[key] = {str(k): str(v) for k, v in value.items() if "token" not in str(k).lower() and "pass" not in str(k).lower()} elif key == "environment" and isinstance(value, dict): result[key] = {str(k): str(v) for k, v in value.items() if "token" not in str(k).lower() and "pass" not in str(k).lower() and "url" not in str(k).lower()} elif isinstance(value, (str, int, float, bool)): result[key] = value return result def summarize(args: argparse.Namespace) -> dict[str, object]: naive = read_raw(args.naive_raw) protected = read_raw(args.protected_raw) def view(raw: dict[str, object]) -> dict[str, object]: workload = raw.get("workload", {}) totals = raw.get("totals", {}) return {"mode": raw.get("mode"), "workload": workload, "totals": totals, "repetitions": raw.get("repetitions", []), "runtime": sanitize_runtime(raw.get("runtime"))} return {"schema_version": 1, "source": "sanitized from two raw benchmark JSON files", "naive": view(naive), "protected": view(protected), "comparison": {"hypothesis": "naive cold misses issue more history SQL queries than protected lock-fill", "result_fields": ["totals.metrics_delta.history_sql", "totals.metrics_delta.cache_miss", "totals.metrics_delta.cache_lock_wait", "repetitions[].latency_ms"], "interpretation": "Use observed fields above; this generator does not claim results before execution."}} def render_summary(summary: dict[str, object]) -> str: lines = ["# Track 1 cache consistency evidence", "", str(summary["source"]), "", "## Observed benchmark data", "", "| Mode | History SQL | Cache misses | Lock waits |", "|---|---:|---:|---:|"] for mode in ("naive", "protected"): value = summary.get(mode, {}) totals = value.get("totals", {}) if isinstance(value, dict) else {} metrics = totals.get("metrics_delta", {}) if isinstance(totals, dict) else {} lines.append(f"| {mode} | {metrics.get('history_sql', 'n/a')} | {metrics.get('cache_miss', 'n/a')} | {metrics.get('cache_lock_wait', 'n/a')} |") lines.extend(["", "Per-repetition p50/p95/p99 successful latency and error counts remain in raw JSON.", "Interpret the hypothesis only after both runs complete; this generator makes no inference.", ""]) return "\n".join(lines) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--output", required=True) parser.add_argument("--compose-file", default=None) parser.add_argument("--env-file", default=None) parser.add_argument("--restart-redis", action="store_true", help="restart only the Redis Compose service") parser.add_argument("--stop-start", action="store_true", help="start Redis in finally after an explicit stop") parser.add_argument("--evict-cache", action="store_true", help="DEL only messages:* Redis keys") parser.add_argument("--timeout", type=float, default=30.0) parser.add_argument("--summary", action="store_true", help="generate summary from two raw benchmark files") parser.add_argument("--naive-raw", help="naive raw JSON for --summary") parser.add_argument("--protected-raw", help="protected raw JSON for --summary") return parser.parse_args() def main() -> None: args = parse_args() if args.summary: if not args.naive_raw or not args.protected_raw: raise RuntimeError("--summary requires --naive-raw and --protected-raw") result = summarize(args) else: result = verify(args) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) if args.summary and output.suffix.lower() == ".md": output.write_text(render_summary(result), encoding="utf-8") else: output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(f"wrote {output}") if __name__ == "__main__": try: main() except (OSError, RuntimeError, subprocess.CalledProcessError) as error: print(f"verification failed: {error.__class__.__name__}", file=sys.stderr) raise SystemExit(2)