#!/usr/bin/env python3 """Offline consistency check: recorded summaries vs independent raw samples.""" from collections import Counter import json import math from pathlib import Path import subprocess ROOT = Path(__file__).resolve().parents[1] def verify(record, folder, name): lines = (folder / (name + '.jsonl')).read_text(encoding='utf-8').splitlines() samples = [json.loads(line) for line in lines] successes = [s['latency_ms'] for s in samples if s['status'] == 201] assert len(successes) == record['successful_requests'], name + ': success count' assert all(s['instance'] for s in samples if s['status'] == 201), name + ': missing instance header' if 'replicas' in record: assert len({s['instance'] for s in samples if s['status'] == 201}) == record['replicas'], name + ': replica count' assert len(samples) - len(successes) == record['failed_requests'], name + ': error count' assert dict(Counter(str(s['status']) for s in samples)) == record['status_counts'], name + ': status counts' assert dict(Counter(s['instance'] for s in samples)) == record['instance_counts'], name + ': instance counts' assert abs(len(successes) / record['elapsed_seconds'] - record['successful_rps']) < .05, name + ': RPS' ordered = sorted(successes) for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]: value = ordered[max(0, math.ceil(len(ordered) * p) - 1)] assert abs(value - record['successful_latency_ms'][label]) < .0011, name + ': percentile ' + label resources = json.loads((folder / (name + '.resources.json')).read_text(encoding='utf-8')) assert resources['dataset_after']['messages'] >= resources['dataset_before']['messages'], name + ': dataset' assert any(s['containers'] for s in resources['resource_samples']), name + ': no resource samples' def main(): immutable = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'contracts', 'tests', 'Makefile', '.github/workflows/contracts.yml'] diff = subprocess.check_output(['git', 'diff', '0ab3a34', '--', *immutable], cwd=ROOT) assert not diff, 'Skeleton contract, rubric or public tests changed' records = json.loads((ROOT / 'evidence/scaling/summary.json').read_text(encoding='utf-8')) assert len({r['name'] for r in records}) == len(records), 'Duplicate run name' for record in records: verify(record, ROOT / 'evidence/scaling', record['name']) rolling = json.loads((ROOT / 'evidence/rolling.json').read_text(encoding='utf-8')) rolling_raw = ROOT / rolling.get('raw_prefix', 'evidence/scaling/rolling-update') verify(rolling['load'], rolling_raw.parent, rolling_raw.name) first, last = rolling['events'][0]['containers'], rolling['events'][-1]['containers'] old = {c['id'] for c in first if c['service'] == 'api'} new = {c['id'] for c in last if c['service'] == 'api'} assert len(old) == len(new) == 2 and not old & new, 'Rolling did not replace both original APIs' for name in ('REPORT.md', 'RESEARCH.md', 'RUNBOOK.md', 'DEFENSE.md', 'PR_DESCRIPTION.md', '.github/pull_request_template.md'): assert 'TODO' not in (ROOT / name).read_text(encoding='utf-8'), name + ': unfilled placeholder' print('PASS: %d measured runs + rolling raw samples match counts, RPS and percentiles; ' 'resource evidence present; both API IDs replaced; contract/public tests unchanged.' % len(records)) if __name__ == '__main__': main()