Files
mtusi2026lab2/scripts/track1_verify.py
T
2026-10-09 15:48:03 +03:00

208 lines
9.7 KiB
Python
Executable File

#!/usr/bin/env python3
"""Verify Redis-only restart and generate sanitized Track 1 summaries.
The verifier creates short-lived users and chat, retains bearer tokens only in
memory, and writes sanitized output. Redis eviction is narrowly scoped to
``messages:*`` keys; it never invokes FLUSHALL or removes volumes.
Recorded results and the evidence index live in evidence/README.md and
evidence/track1-cache-consistency.md; the main report is REPORT.md.
"""
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
from track1_cache_benchmark import (
create_chat,
create_user,
history,
login,
post_message,
request_json,
require_object,
)
def compose_command(args: argparse.Namespace, *parts: str) -> list[str]:
command = ["docker", "compose"]
if args.compose_file:
command.extend(["-f", args.compose_file])
if args.env_file:
command.extend(["--env-file", args.env_file])
command.extend(parts)
return command
def run_compose(args: argparse.Namespace, *parts: str) -> subprocess.CompletedProcess[str]:
# Compose diagnostics may include environment-derived URLs; do not emit them.
return subprocess.run(compose_command(args, *parts), check=True, capture_output=True, text=True)
def evict_message_cache(args: argparse.Namespace) -> int:
scan = run_compose(args, "exec", "-T", "redis", "redis-cli", "--scan", "--pattern", "messages:*")
keys = [line.strip() for line in scan.stdout.splitlines() if line.strip()]
for offset in range(0, len(keys), 100):
run_compose(args, "exec", "-T", "redis", "redis-cli", "DEL", *keys[offset : offset + 100])
return len(keys)
def wait_ready(base_url: str, timeout: float, attempts: int = 30) -> None:
for _ in range(attempts):
try:
if isinstance(request_json(base_url, "GET", "/health/ready", timeout=timeout), dict):
return
except Exception:
pass
time.sleep(1)
raise RuntimeError("API did not become ready after Redis operation")
def verify(args: argparse.Namespace) -> dict[str, object]:
suffix = f"verify_{int(time.time())}"
password = "Track1-verify-password"
owner_name, owner_id = create_user(args.base_url, suffix, "owner", password, args.timeout)
reader_name, reader_id = create_user(args.base_url, suffix, "reader", password, args.timeout)
owner_token = login(args.base_url, owner_name, password, args.timeout)
reader_token = login(args.base_url, reader_name, password, args.timeout)
chat_id = create_chat(args.base_url, owner_token, [owner_id, reader_id], "track1-restart", args.timeout)
post_message(args.base_url, owner_token, chat_id, "track1 restart evidence", args.timeout)
before_me = require_object(request_json(args.base_url, "GET", "/api/v1/users/me",
token=reader_token, timeout=args.timeout), "users/me")
before_history = history(args.base_url, reader_token, chat_id, args.timeout)
cache_keys_deleted = 0
restarted = False
redis_stopped = False
after_me: dict[str, object] | None = None
after_history: dict[str, object] | None = None
try:
if args.stop_start:
run_compose(args, "stop", "redis")
redis_stopped = True
run_compose(args, "start", "redis")
redis_stopped = False
restarted = True
wait_ready(args.base_url, args.timeout)
elif args.restart_redis:
run_compose(args, "restart", "redis")
restarted = True
wait_ready(args.base_url, args.timeout)
if args.evict_cache:
cache_keys_deleted = evict_message_cache(args)
after_me = require_object(request_json(args.base_url, "GET", "/api/v1/users/me",
token=reader_token, timeout=args.timeout), "users/me after restart")
after_history = history(args.base_url, reader_token, chat_id, args.timeout)
finally:
if redis_stopped:
try:
run_compose(args, "start", "redis")
except subprocess.CalledProcessError:
pass
if after_me is None or after_history is None:
raise RuntimeError("verification did not obtain post-operation responses")
users_equal = before_me == after_me
before_items = before_history.get("items")
after_items = after_history.get("items")
history_equal = before_items == after_items
if not users_equal or not history_equal:
raise RuntimeError("same-token users/me or history changed after Redis operation")
return {"schema_version": 1, "restart_redis": restarted,
"cache_eviction": {"requested": args.evict_cache, "message_keys_deleted": cache_keys_deleted},
"same_token_users_me": users_equal, "history_equal": history_equal,
"history_item_count": len(after_items) if isinstance(after_items, list) else 0}
def read_raw(path: str) -> dict[str, object]:
value = json.loads(Path(path).read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise RuntimeError(f"raw benchmark is not an object: {path}")
return value
def sanitize_runtime(runtime: object) -> dict[str, object]:
if not isinstance(runtime, dict):
return {}
result: dict[str, object] = {}
for key in ("python", "platform", "supplied", "environment"):
value = runtime.get(key)
if key == "supplied" and isinstance(value, dict):
result[key] = {str(k): str(v) for k, v in value.items() if "token" not in str(k).lower() and "pass" not in str(k).lower()}
elif key == "environment" and isinstance(value, dict):
result[key] = {str(k): str(v) for k, v in value.items() if "token" not in str(k).lower() and "pass" not in str(k).lower() and "url" not in str(k).lower()}
elif isinstance(value, (str, int, float, bool)):
result[key] = value
return result
def summarize(args: argparse.Namespace) -> dict[str, object]:
naive = read_raw(args.naive_raw)
protected = read_raw(args.protected_raw)
def view(raw: dict[str, object]) -> dict[str, object]:
workload = raw.get("workload", {})
totals = raw.get("totals", {})
return {"mode": raw.get("mode"), "workload": workload, "totals": totals,
"repetitions": raw.get("repetitions", []), "runtime": sanitize_runtime(raw.get("runtime"))}
return {"schema_version": 1, "source": "sanitized from two raw benchmark JSON files",
"naive": view(naive), "protected": view(protected),
"comparison": {"hypothesis": "naive cold misses issue more history SQL queries than protected lock-fill",
"result_fields": ["totals.metrics_delta.history_sql", "totals.metrics_delta.cache_miss",
"totals.metrics_delta.cache_lock_wait", "repetitions[].latency_ms"],
"interpretation": "Use observed fields above; this generator does not claim results before execution."}}
def render_summary(summary: dict[str, object]) -> str:
lines = ["# Track 1 cache consistency evidence", "", str(summary["source"]), "",
"## Observed benchmark data", "", "| Mode | History SQL | Cache misses | Lock waits |",
"|---|---:|---:|---:|"]
for mode in ("naive", "protected"):
value = summary.get(mode, {})
totals = value.get("totals", {}) if isinstance(value, dict) else {}
metrics = totals.get("metrics_delta", {}) if isinstance(totals, dict) else {}
lines.append(f"| {mode} | {metrics.get('history_sql', 'n/a')} | {metrics.get('cache_miss', 'n/a')} | {metrics.get('cache_lock_wait', 'n/a')} |")
lines.extend(["", "Per-repetition p50/p95/p99 successful latency and error counts remain in raw JSON.",
"Interpret the hypothesis only after both runs complete; this generator makes no inference.", ""])
return "\n".join(lines)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--output", required=True)
parser.add_argument("--compose-file", default=None)
parser.add_argument("--env-file", default=None)
parser.add_argument("--restart-redis", action="store_true", help="restart only the Redis Compose service")
parser.add_argument("--stop-start", action="store_true", help="start Redis in finally after an explicit stop")
parser.add_argument("--evict-cache", action="store_true", help="DEL only messages:* Redis keys")
parser.add_argument("--timeout", type=float, default=30.0)
parser.add_argument("--summary", action="store_true", help="generate summary from two raw benchmark files")
parser.add_argument("--naive-raw", help="naive raw JSON for --summary")
parser.add_argument("--protected-raw", help="protected raw JSON for --summary")
return parser.parse_args()
def main() -> None:
args = parse_args()
if args.summary:
if not args.naive_raw or not args.protected_raw:
raise RuntimeError("--summary requires --naive-raw and --protected-raw")
result = summarize(args)
else:
result = verify(args)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
if args.summary and output.suffix.lower() == ".md":
output.write_text(render_summary(result), encoding="utf-8")
else:
output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"wrote {output}")
if __name__ == "__main__":
try:
main()
except (OSError, RuntimeError, subprocess.CalledProcessError) as error:
print(f"verification failed: {error.__class__.__name__}", file=sys.stderr)
raise SystemExit(2)