forked from mtusi2026/mtusi2026lab3
58 lines
3.4 KiB
Python
58 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Offline consistency check: recorded summaries vs independent raw samples."""
|
|
from collections import Counter
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def verify(record, folder, name):
|
|
lines = (folder / (name + '.jsonl')).read_text(encoding='utf-8').splitlines()
|
|
samples = [json.loads(line) for line in lines]
|
|
successes = [s['latency_ms'] for s in samples if s['status'] == 201]
|
|
assert len(successes) == record['successful_requests'], name + ': success count'
|
|
assert all(s['instance'] for s in samples if s['status'] == 201), name + ': missing instance header'
|
|
if 'replicas' in record:
|
|
assert len({s['instance'] for s in samples if s['status'] == 201}) == record['replicas'], name + ': replica count'
|
|
assert len(samples) - len(successes) == record['failed_requests'], name + ': error count'
|
|
assert dict(Counter(str(s['status']) for s in samples)) == record['status_counts'], name + ': status counts'
|
|
assert dict(Counter(s['instance'] for s in samples)) == record['instance_counts'], name + ': instance counts'
|
|
assert abs(len(successes) / record['elapsed_seconds'] - record['successful_rps']) < .05, name + ': RPS'
|
|
ordered = sorted(successes)
|
|
for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]:
|
|
value = ordered[max(0, math.ceil(len(ordered) * p) - 1)]
|
|
assert abs(value - record['successful_latency_ms'][label]) < .0011, name + ': percentile ' + label
|
|
resources = json.loads((folder / (name + '.resources.json')).read_text(encoding='utf-8'))
|
|
assert resources['dataset_after']['messages'] >= resources['dataset_before']['messages'], name + ': dataset'
|
|
assert any(s['containers'] for s in resources['resource_samples']), name + ': no resource samples'
|
|
|
|
|
|
def main():
|
|
immutable = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'contracts', 'tests', 'Makefile',
|
|
'.github/workflows/contracts.yml']
|
|
diff = subprocess.check_output(['git', 'diff', '0ab3a34', '--', *immutable], cwd=ROOT)
|
|
assert not diff, 'Skeleton contract, rubric or public tests changed'
|
|
records = json.loads((ROOT / 'evidence/scaling/summary.json').read_text(encoding='utf-8'))
|
|
assert len({r['name'] for r in records}) == len(records), 'Duplicate run name'
|
|
for record in records:
|
|
verify(record, ROOT / 'evidence/scaling', record['name'])
|
|
rolling = json.loads((ROOT / 'evidence/rolling.json').read_text(encoding='utf-8'))
|
|
rolling_raw = ROOT / rolling.get('raw_prefix', 'evidence/scaling/rolling-update')
|
|
verify(rolling['load'], rolling_raw.parent, rolling_raw.name)
|
|
first, last = rolling['events'][0]['containers'], rolling['events'][-1]['containers']
|
|
old = {c['id'] for c in first if c['service'] == 'api'}
|
|
new = {c['id'] for c in last if c['service'] == 'api'}
|
|
assert len(old) == len(new) == 2 and not old & new, 'Rolling did not replace both original APIs'
|
|
for name in ('REPORT.md', 'RESEARCH.md', 'RUNBOOK.md', 'DEFENSE.md', 'PR_DESCRIPTION.md',
|
|
'.github/pull_request_template.md'):
|
|
assert 'TODO' not in (ROOT / name).read_text(encoding='utf-8'), name + ': unfilled placeholder'
|
|
print('PASS: %d measured runs + rolling raw samples match counts, RPS and percentiles; '
|
|
'resource evidence present; both API IDs replaced; contract/public tests unchanged.' % len(records))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|