Files
mtusi2026lab6/scripts/lab6/measurement.py

338 lines
14 KiB
Python

"""Pure measurement logic for lab6 track 1 (no I/O): statistics, counter
deltas, cohort classification, run validity and the p25/full selection rules.
Everything here is unit-tested in test_measurement.py."""
import math
MODES = {
'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER_ARG': '1'},
'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1'},
'p25': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.25'},
'p10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.1'},
}
SAMPLER_RATIO = {'off': None, 'full': 1.0, 'p25': 0.25, 'p10': 0.1}
ORDERS = (('off', 'full', 'p25', 'p10'), ('p25', 'off', 'p10', 'full'),
('p10', 'p25', 'full', 'off'))
START_LAG_LIMIT_MS = 50.0
COMPLETION_LIMIT = 0.99
SPAN_RATIO_LIMIT = 0.5
LATENCY_RATIO_LIMIT = 1.10
# ----------------------------------------------------------------- statistics
def nearest_rank(values, q):
"""Nearest-rank quantile: sorted[ceil(n*q)-1] (same as tests/load.py)."""
if not values:
return None
ordered = sorted(values)
return ordered[max(0, math.ceil(len(ordered) * q) - 1)]
def median(values):
values = [v for v in values if v is not None]
if not values:
return None
ordered = sorted(values)
n = len(ordered)
mid = n // 2
return ordered[mid] if n % 2 else (ordered[mid - 1] + ordered[mid]) / 2
def spread(values):
values = [v for v in values if v is not None]
if not values:
return {'min': None, 'median': None, 'max': None, 'n': 0}
return {'min': min(values), 'median': median(values), 'max': max(values),
'n': len(values)}
def relative(value, reference):
if value is None or reference in (None, 0):
return None
return value / reference - 1.0
def miss_all_probability(ratio, n):
"""(1-r)^N: chance that independent root sampling keeps none of N events.
A model assumption, not a measurement."""
return (1.0 - ratio) ** n
# ---------------------------------------------------------------------- load
ERROR_OUTCOMES = ('timeout', 'transport_error', 'protocol_error')
def slot_completed(slot):
return slot['outcome'] == 'http'
def summarize_load(slots, rate):
"""slots: one record per *planned* slot, including client_dropped ones.
Dropped and late slots stay in the denominator; p95 includes error
responses (it is not a latency of successes only)."""
scheduled = len(slots)
done = [s for s in slots if slot_completed(s)]
dropped = sum(1 for s in slots if s['outcome'] == 'client_dropped')
failures = {k: sum(1 for s in slots if s['outcome'] == k)
for k in ERROR_OUTCOMES}
http_5xx = sum(1 for s in done if s['status'] >= 500)
http_4xx = sum(1 for s in done if 400 <= s['status'] < 500)
successes = [s for s in done if 200 <= s['status'] < 300]
started = [s for s in slots if s.get('start_lag_ms') is not None]
lag = [s['start_lag_ms'] for s in started]
sched_all = [s['scheduled_ms'] for s in done]
service_all = [s['service_ms'] for s in done]
return {
'rate_rps': rate,
'scheduled': scheduled,
'completed': len(done),
'completed_ratio': (len(done) / scheduled) if scheduled else None,
'client_dropped': dropped,
'transport_failures': failures,
'http_5xx': http_5xx,
'http_4xx': http_4xx,
'http_2xx': len(successes),
'start_lag_ms_p95': nearest_rank(lag, 0.95),
'start_lag_ms_max': max(lag) if lag else None,
'scheduled_ms': {q: nearest_rank(sched_all, v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
'service_ms': {q: nearest_rank(service_all, v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
'success_scheduled_ms': {q: nearest_rank(
[s['scheduled_ms'] for s in successes], v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
'success_service_ms': {q: nearest_rank(
[s['service_ms'] for s in successes], v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
}
def load_errors(load):
"""HTTP/protocol errors among delivered healthy traffic."""
return (load['http_5xx'] + sum(load['transport_failures'].values()))
def delivered_stream(load):
"""Generator delivered the planned stream (protocol validity)."""
reasons = []
if load['client_dropped']:
reasons.append('client_dropped=%d' % load['client_dropped'])
lag = load['start_lag_ms_p95']
if lag is None or lag > START_LAG_LIMIT_MS:
reasons.append('start_lag_p95=%s' % lag)
return reasons
# ------------------------------------------------------------------ counters
def counter_delta(start, end, failure_only=False):
"""-> (delta, reason). Never turns a missing positive series into zero and
never reports a reset as a negative loss."""
if start is None and end is None:
if failure_only:
return 0.0, 'absent_until_first_error'
return None, 'missing_series'
if start is None:
if failure_only:
return end, 'series_created_in_window'
return None, 'missing_start'
if end is None:
return None, 'missing_end'
if end < start:
return None, 'counter_reset'
return end - start, None
# ------------------------------------------------------------------- cohorts
def classify_record(mode, record):
"""Classify one cohort request.
record: {'stdout': None | {'trace_id','span_id','trace_sampled'} (the keys
are absent when the SDK is off or no span context), 'loki': True/False/None
(None: backend observation incomplete), 'span': True/False/None}.
Returns (trace_class, log_class)."""
stdout = record.get('stdout')
if stdout is None:
return 'unmatched_stdout', 'incomplete_observation'
if mode == 'off':
if any(k in stdout for k in ('trace_id', 'span_id', 'trace_sampled')):
return 'ids_present_with_sdk_off', 'not_applicable'
return 'not_applicable', 'not_applicable'
sampled = stdout.get('trace_sampled')
if not stdout.get('trace_id') or not stdout.get('span_id') or \
not isinstance(sampled, bool):
return 'unmatched_stdout', 'incomplete_observation'
if record.get('loki') is None:
log_class = 'incomplete_observation'
elif record['loki']:
log_class = 'exported'
else:
log_class = 'missing_exported_logs'
if sampled is False:
return 'intentional_unsampled', log_class
if record.get('span') is None:
return 'incomplete_observation', log_class
return ('sampled_present' if record['span'] else 'sampled_missing'), log_class
def summarize_cohort(mode, records):
counts = {'n': len(records)}
for record in records:
trace_class, log_class = classify_record(mode, record)
counts[trace_class] = counts.get(trace_class, 0) + 1
counts['log_' + log_class] = counts.get('log_' + log_class, 0) + 1
counts['retained_traces'] = counts.get('sampled_present', 0)
return counts
def export_loss(items_failed_delta, send_failed, enqueue_failed):
"""Telemetry export losses; a false trace_sampled flag is NOT a loss."""
parts = {'app_export_failed_items': items_failed_delta,
'collector_send_failed': send_failed,
'collector_enqueue_failed': enqueue_failed}
known = [v for v in parts.values() if isinstance(v, (int, float))]
return {'parts': parts, 'total_known': sum(known),
'loss': sum(known) > 0}
# ------------------------------------------------------------------ validity
def evaluate_run(summary):
"""-> (valid, invalid_reasons, configuration_failures).
Invalid = the protocol was not executed as specified; configuration
failures are valid adverse outcomes and stay in the matrix."""
invalid = list(summary.get('measurement_defects', []))
failures = []
load = summary['load']
invalid += delivered_stream(load)
if load['completed_ratio'] is None or load['completed_ratio'] < COMPLETION_LIMIT:
invalid.append('completed_ratio=%s' % load['completed_ratio'])
if summary.get('effective_mode_ok') is False:
invalid.append('effective_env_mismatch')
if summary.get('image_ids_ok') is False:
invalid.append('image_changed')
if summary.get('foreign_faults'):
invalid.append('foreign_faults')
for name, cohort in (summary.get('cohorts') or {}).items():
if cohort.get('invalid'):
invalid.append('cohort_invalid:%s' % name)
# Counter resets are only explained by an observed restart/OOM.
for name, item in (summary.get('ingest') or {}).items():
if isinstance(item, dict) and item.get('reason') == 'counter_reset' \
and not summary.get('restarts'):
invalid.append('unexplained_counter_reset:%s' % name)
# --- adverse outcomes of the configuration (valid)
if load_errors(load):
failures.append('healthy_errors=%d' % load_errors(load))
if summary.get('not_ready_at_deadline'):
failures.append('not_ready_at_deadline=%d' % summary['not_ready_at_deadline'])
if summary.get('integrity_errors'):
failures.append('healthy_errors_integrity=%d' % summary['integrity_errors'])
if summary.get('restarts'):
failures.append('restart_or_oom')
loss = summary.get('export_failures') or {}
if loss.get('loss'):
failures.append('export_loss')
for name, cohort in (summary.get('cohorts') or {}).items():
if cohort.get('sampled_missing'):
failures.append('sampled_missing:%s=%d' % (name, cohort['sampled_missing']))
if cohort.get('log_missing_exported_logs'):
failures.append('missing_exported_logs:%s=%d' % (
name, cohort['log_missing_exported_logs']))
return (not invalid), invalid, failures
def matrix_entry(summary):
"""Every protocol-run stays in the matrix, valid-adverse included."""
return {
'mode': summary['mode'], 'repeat': summary.get('repeat'),
'attempt': summary.get('attempt', 1), 'valid': summary['valid'],
'invalid_reasons': summary['invalid_reasons'],
'configuration_failures': summary['configuration_failures'],
'spans_per_operation': summary.get('spans_per_operation'),
'scheduled_p95_ms': (summary['load']['scheduled_ms'] or {}).get('p95'),
'dir': summary.get('dir'),
}
# ---------------------------------------------------------------- selection
def _clean(run):
return (run['valid'] and not any(
f.startswith(('healthy_errors', 'not_ready', 'restart', 'export_loss'))
for f in run['configuration_failures']))
def _retained(run, name):
return (run['cohorts'].get(name) or {}).get('retained_traces', 0)
def _no_missing(run):
for cohort in run['cohorts'].values():
if cohort.get('sampled_missing') or cohort.get('log_missing_exported_logs') \
or cohort.get('log_incomplete_observation') \
or cohort.get('unmatched_stdout') or cohort.get('incomplete_observation'):
return False
return True
def full_reference(full_runs):
"""Frozen reference: medians of the three full runs; None if any datum is
unmeasurable (then p25 cannot be declared a winner)."""
spans = [r.get('spans_per_operation') for r in full_runs]
p95 = [r['load']['scheduled_ms']['p95'] for r in full_runs]
ok = (len(full_runs) == 3 and all(r['valid'] for r in full_runs) and
all(v is not None for v in spans + p95))
return {'complete': ok, 'spans_per_operation_median': median(spans),
'scheduled_p95_ms_median': median(p95),
'run_dirs': [r.get('dir') for r in full_runs]}
def select_p25(full_runs, p25_runs):
ref = full_reference(full_runs)
checks = {'full_reference_complete': ref['complete']}
if not ref['complete'] or len(p25_runs) != 3:
checks['p25_runs'] = len(p25_runs) == 3
return {'passed': False, 'checks': checks, 'reference': ref}
checks['all_valid_clean'] = all(_clean(r) for r in p25_runs)
checks['retained_error_and_slow_each_run'] = all(
_retained(r, 'error') >= 1 and _retained(r, 'slow') >= 1 for r in p25_runs)
checks['no_missing_spans_or_logs'] = all(_no_missing(r) for r in p25_runs)
spans = median([r.get('spans_per_operation') for r in p25_runs])
checks['spans_median_le_50pct_full'] = (
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
p95 = median([r['load']['scheduled_ms']['p95'] for r in p25_runs])
checks['scheduled_p95_le_110pct_full'] = (
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
return {'passed': all(checks.values()), 'checks': checks, 'reference': ref,
'p25_spans_per_operation_median': spans,
'p25_scheduled_p95_ms_median': p95}
def validate_single(run, mode, ref):
"""One fresh run against the FROZEN full reference (never a 3-run median
of the validation itself)."""
checks = {'protocol_valid': run['valid'],
'zero_errors_notready_loss': _clean(run)}
if mode == 'p25':
checks['complete_stdout_and_logs'] = _no_missing(run) and all(
(run['cohorts'].get(n) or {}).get('n') == 40 for n in ('error', 'slow'))
checks['retained_each_class'] = (
_retained(run, 'error') >= 1 and _retained(run, 'slow') >= 1)
spans = run.get('spans_per_operation')
checks['spans_le_50pct_frozen_full'] = (
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
p95 = run['load']['scheduled_ms']['p95']
checks['scheduled_p95_le_110pct_frozen_full'] = (
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
else:
checks['complete_stdout_and_logs'] = _no_missing(run)
checks['all_80_server_spans'] = all(
(run['cohorts'].get(n) or {}).get('sampled_present') == 40
for n in ('error', 'slow'))
return {'passed': all(checks.values()), 'checks': checks, 'mode': mode}