Files
mtusi2026lab3/scripts/check_evidence.py

58 lines
3.4 KiB
Python

#!/usr/bin/env python3
"""Offline consistency check: recorded summaries vs independent raw samples."""
from collections import Counter
import json
import math
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def verify(record, folder, name):
lines = (folder / (name + '.jsonl')).read_text(encoding='utf-8').splitlines()
samples = [json.loads(line) for line in lines]
successes = [s['latency_ms'] for s in samples if s['status'] == 201]
assert len(successes) == record['successful_requests'], name + ': success count'
assert all(s['instance'] for s in samples if s['status'] == 201), name + ': missing instance header'
if 'replicas' in record:
assert len({s['instance'] for s in samples if s['status'] == 201}) == record['replicas'], name + ': replica count'
assert len(samples) - len(successes) == record['failed_requests'], name + ': error count'
assert dict(Counter(str(s['status']) for s in samples)) == record['status_counts'], name + ': status counts'
assert dict(Counter(s['instance'] for s in samples)) == record['instance_counts'], name + ': instance counts'
assert abs(len(successes) / record['elapsed_seconds'] - record['successful_rps']) < .05, name + ': RPS'
ordered = sorted(successes)
for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]:
value = ordered[max(0, math.ceil(len(ordered) * p) - 1)]
assert abs(value - record['successful_latency_ms'][label]) < .0011, name + ': percentile ' + label
resources = json.loads((folder / (name + '.resources.json')).read_text(encoding='utf-8'))
assert resources['dataset_after']['messages'] >= resources['dataset_before']['messages'], name + ': dataset'
assert any(s['containers'] for s in resources['resource_samples']), name + ': no resource samples'
def main():
immutable = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'contracts', 'tests', 'Makefile',
'.github/workflows/contracts.yml']
diff = subprocess.check_output(['git', 'diff', '0ab3a34', '--', *immutable], cwd=ROOT)
assert not diff, 'Skeleton contract, rubric or public tests changed'
records = json.loads((ROOT / 'evidence/scaling/summary.json').read_text(encoding='utf-8'))
assert len({r['name'] for r in records}) == len(records), 'Duplicate run name'
for record in records:
verify(record, ROOT / 'evidence/scaling', record['name'])
rolling = json.loads((ROOT / 'evidence/rolling.json').read_text(encoding='utf-8'))
rolling_raw = ROOT / rolling.get('raw_prefix', 'evidence/scaling/rolling-update')
verify(rolling['load'], rolling_raw.parent, rolling_raw.name)
first, last = rolling['events'][0]['containers'], rolling['events'][-1]['containers']
old = {c['id'] for c in first if c['service'] == 'api'}
new = {c['id'] for c in last if c['service'] == 'api'}
assert len(old) == len(new) == 2 and not old & new, 'Rolling did not replace both original APIs'
for name in ('REPORT.md', 'RESEARCH.md', 'RUNBOOK.md', 'DEFENSE.md', 'PR_DESCRIPTION.md',
'.github/pull_request_template.md'):
assert 'TODO' not in (ROOT / name).read_text(encoding='utf-8'), name + ': unfilled placeholder'
print('PASS: %d measured runs + rolling raw samples match counts, RPS and percentiles; '
'resource evidence present; both API IDs replaced; contract/public tests unchanged.' % len(records))
if __name__ == '__main__':
main()