forked from mtusi2026/mtusi2026lab2
208 lines
9.7 KiB
Python
Executable File
208 lines
9.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Verify Redis-only restart and generate sanitized Track 1 summaries.
|
|
|
|
The verifier creates short-lived users and chat, retains bearer tokens only in
|
|
memory, and writes sanitized output. Redis eviction is narrowly scoped to
|
|
``messages:*`` keys; it never invokes FLUSHALL or removes volumes.
|
|
|
|
Recorded results and the evidence index live in evidence/README.md and
|
|
evidence/track1-cache-consistency.md; the main report is REPORT.md.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from track1_cache_benchmark import (
|
|
create_chat,
|
|
create_user,
|
|
history,
|
|
login,
|
|
post_message,
|
|
request_json,
|
|
require_object,
|
|
)
|
|
|
|
|
|
def compose_command(args: argparse.Namespace, *parts: str) -> list[str]:
|
|
command = ["docker", "compose"]
|
|
if args.compose_file:
|
|
command.extend(["-f", args.compose_file])
|
|
if args.env_file:
|
|
command.extend(["--env-file", args.env_file])
|
|
command.extend(parts)
|
|
return command
|
|
|
|
|
|
def run_compose(args: argparse.Namespace, *parts: str) -> subprocess.CompletedProcess[str]:
|
|
# Compose diagnostics may include environment-derived URLs; do not emit them.
|
|
return subprocess.run(compose_command(args, *parts), check=True, capture_output=True, text=True)
|
|
|
|
|
|
def evict_message_cache(args: argparse.Namespace) -> int:
|
|
scan = run_compose(args, "exec", "-T", "redis", "redis-cli", "--scan", "--pattern", "messages:*")
|
|
keys = [line.strip() for line in scan.stdout.splitlines() if line.strip()]
|
|
for offset in range(0, len(keys), 100):
|
|
run_compose(args, "exec", "-T", "redis", "redis-cli", "DEL", *keys[offset : offset + 100])
|
|
return len(keys)
|
|
|
|
|
|
def wait_ready(base_url: str, timeout: float, attempts: int = 30) -> None:
|
|
for _ in range(attempts):
|
|
try:
|
|
if isinstance(request_json(base_url, "GET", "/health/ready", timeout=timeout), dict):
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(1)
|
|
raise RuntimeError("API did not become ready after Redis operation")
|
|
|
|
|
|
def verify(args: argparse.Namespace) -> dict[str, object]:
|
|
suffix = f"verify_{int(time.time())}"
|
|
password = "Track1-verify-password"
|
|
owner_name, owner_id = create_user(args.base_url, suffix, "owner", password, args.timeout)
|
|
reader_name, reader_id = create_user(args.base_url, suffix, "reader", password, args.timeout)
|
|
owner_token = login(args.base_url, owner_name, password, args.timeout)
|
|
reader_token = login(args.base_url, reader_name, password, args.timeout)
|
|
chat_id = create_chat(args.base_url, owner_token, [owner_id, reader_id], "track1-restart", args.timeout)
|
|
post_message(args.base_url, owner_token, chat_id, "track1 restart evidence", args.timeout)
|
|
before_me = require_object(request_json(args.base_url, "GET", "/api/v1/users/me",
|
|
token=reader_token, timeout=args.timeout), "users/me")
|
|
before_history = history(args.base_url, reader_token, chat_id, args.timeout)
|
|
cache_keys_deleted = 0
|
|
restarted = False
|
|
redis_stopped = False
|
|
after_me: dict[str, object] | None = None
|
|
after_history: dict[str, object] | None = None
|
|
try:
|
|
if args.stop_start:
|
|
run_compose(args, "stop", "redis")
|
|
redis_stopped = True
|
|
run_compose(args, "start", "redis")
|
|
redis_stopped = False
|
|
restarted = True
|
|
wait_ready(args.base_url, args.timeout)
|
|
elif args.restart_redis:
|
|
run_compose(args, "restart", "redis")
|
|
restarted = True
|
|
wait_ready(args.base_url, args.timeout)
|
|
if args.evict_cache:
|
|
cache_keys_deleted = evict_message_cache(args)
|
|
after_me = require_object(request_json(args.base_url, "GET", "/api/v1/users/me",
|
|
token=reader_token, timeout=args.timeout), "users/me after restart")
|
|
after_history = history(args.base_url, reader_token, chat_id, args.timeout)
|
|
finally:
|
|
if redis_stopped:
|
|
try:
|
|
run_compose(args, "start", "redis")
|
|
except subprocess.CalledProcessError:
|
|
pass
|
|
if after_me is None or after_history is None:
|
|
raise RuntimeError("verification did not obtain post-operation responses")
|
|
users_equal = before_me == after_me
|
|
before_items = before_history.get("items")
|
|
after_items = after_history.get("items")
|
|
history_equal = before_items == after_items
|
|
if not users_equal or not history_equal:
|
|
raise RuntimeError("same-token users/me or history changed after Redis operation")
|
|
return {"schema_version": 1, "restart_redis": restarted,
|
|
"cache_eviction": {"requested": args.evict_cache, "message_keys_deleted": cache_keys_deleted},
|
|
"same_token_users_me": users_equal, "history_equal": history_equal,
|
|
"history_item_count": len(after_items) if isinstance(after_items, list) else 0}
|
|
|
|
|
|
def read_raw(path: str) -> dict[str, object]:
|
|
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise RuntimeError(f"raw benchmark is not an object: {path}")
|
|
return value
|
|
|
|
|
|
def sanitize_runtime(runtime: object) -> dict[str, object]:
|
|
if not isinstance(runtime, dict):
|
|
return {}
|
|
result: dict[str, object] = {}
|
|
for key in ("python", "platform", "supplied", "environment"):
|
|
value = runtime.get(key)
|
|
if key == "supplied" and isinstance(value, dict):
|
|
result[key] = {str(k): str(v) for k, v in value.items() if "token" not in str(k).lower() and "pass" not in str(k).lower()}
|
|
elif key == "environment" and isinstance(value, dict):
|
|
result[key] = {str(k): str(v) for k, v in value.items() if "token" not in str(k).lower() and "pass" not in str(k).lower() and "url" not in str(k).lower()}
|
|
elif isinstance(value, (str, int, float, bool)):
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def summarize(args: argparse.Namespace) -> dict[str, object]:
|
|
naive = read_raw(args.naive_raw)
|
|
protected = read_raw(args.protected_raw)
|
|
def view(raw: dict[str, object]) -> dict[str, object]:
|
|
workload = raw.get("workload", {})
|
|
totals = raw.get("totals", {})
|
|
return {"mode": raw.get("mode"), "workload": workload, "totals": totals,
|
|
"repetitions": raw.get("repetitions", []), "runtime": sanitize_runtime(raw.get("runtime"))}
|
|
return {"schema_version": 1, "source": "sanitized from two raw benchmark JSON files",
|
|
"naive": view(naive), "protected": view(protected),
|
|
"comparison": {"hypothesis": "naive cold misses issue more history SQL queries than protected lock-fill",
|
|
"result_fields": ["totals.metrics_delta.history_sql", "totals.metrics_delta.cache_miss",
|
|
"totals.metrics_delta.cache_lock_wait", "repetitions[].latency_ms"],
|
|
"interpretation": "Use observed fields above; this generator does not claim results before execution."}}
|
|
|
|
def render_summary(summary: dict[str, object]) -> str:
|
|
lines = ["# Track 1 cache consistency evidence", "", str(summary["source"]), "",
|
|
"## Observed benchmark data", "", "| Mode | History SQL | Cache misses | Lock waits |",
|
|
"|---|---:|---:|---:|"]
|
|
for mode in ("naive", "protected"):
|
|
value = summary.get(mode, {})
|
|
totals = value.get("totals", {}) if isinstance(value, dict) else {}
|
|
metrics = totals.get("metrics_delta", {}) if isinstance(totals, dict) else {}
|
|
lines.append(f"| {mode} | {metrics.get('history_sql', 'n/a')} | {metrics.get('cache_miss', 'n/a')} | {metrics.get('cache_lock_wait', 'n/a')} |")
|
|
lines.extend(["", "Per-repetition p50/p95/p99 successful latency and error counts remain in raw JSON.",
|
|
"Interpret the hypothesis only after both runs complete; this generator makes no inference.", ""])
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--output", required=True)
|
|
parser.add_argument("--compose-file", default=None)
|
|
parser.add_argument("--env-file", default=None)
|
|
parser.add_argument("--restart-redis", action="store_true", help="restart only the Redis Compose service")
|
|
parser.add_argument("--stop-start", action="store_true", help="start Redis in finally after an explicit stop")
|
|
parser.add_argument("--evict-cache", action="store_true", help="DEL only messages:* Redis keys")
|
|
parser.add_argument("--timeout", type=float, default=30.0)
|
|
parser.add_argument("--summary", action="store_true", help="generate summary from two raw benchmark files")
|
|
parser.add_argument("--naive-raw", help="naive raw JSON for --summary")
|
|
parser.add_argument("--protected-raw", help="protected raw JSON for --summary")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
if args.summary:
|
|
if not args.naive_raw or not args.protected_raw:
|
|
raise RuntimeError("--summary requires --naive-raw and --protected-raw")
|
|
result = summarize(args)
|
|
else:
|
|
result = verify(args)
|
|
output = Path(args.output)
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
if args.summary and output.suffix.lower() == ".md":
|
|
output.write_text(render_summary(result), encoding="utf-8")
|
|
else:
|
|
output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(f"wrote {output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (OSError, RuntimeError, subprocess.CalledProcessError) as error:
|
|
print(f"verification failed: {error.__class__.__name__}", file=sys.stderr)
|
|
raise SystemExit(2)
|