forked from mtusi2026/mtusi2026lab6
338 lines
14 KiB
Python
338 lines
14 KiB
Python
"""Pure measurement logic for lab6 track 1 (no I/O): statistics, counter
|
|
deltas, cohort classification, run validity and the p25/full selection rules.
|
|
Everything here is unit-tested in test_measurement.py."""
|
|
import math
|
|
|
|
MODES = {
|
|
'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER_ARG': '1'},
|
|
'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1'},
|
|
'p25': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.25'},
|
|
'p10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.1'},
|
|
}
|
|
SAMPLER_RATIO = {'off': None, 'full': 1.0, 'p25': 0.25, 'p10': 0.1}
|
|
ORDERS = (('off', 'full', 'p25', 'p10'), ('p25', 'off', 'p10', 'full'),
|
|
('p10', 'p25', 'full', 'off'))
|
|
START_LAG_LIMIT_MS = 50.0
|
|
COMPLETION_LIMIT = 0.99
|
|
SPAN_RATIO_LIMIT = 0.5
|
|
LATENCY_RATIO_LIMIT = 1.10
|
|
|
|
|
|
# ----------------------------------------------------------------- statistics
|
|
|
|
def nearest_rank(values, q):
|
|
"""Nearest-rank quantile: sorted[ceil(n*q)-1] (same as tests/load.py)."""
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
return ordered[max(0, math.ceil(len(ordered) * q) - 1)]
|
|
|
|
|
|
def median(values):
|
|
values = [v for v in values if v is not None]
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
n = len(ordered)
|
|
mid = n // 2
|
|
return ordered[mid] if n % 2 else (ordered[mid - 1] + ordered[mid]) / 2
|
|
|
|
|
|
def spread(values):
|
|
values = [v for v in values if v is not None]
|
|
if not values:
|
|
return {'min': None, 'median': None, 'max': None, 'n': 0}
|
|
return {'min': min(values), 'median': median(values), 'max': max(values),
|
|
'n': len(values)}
|
|
|
|
|
|
def relative(value, reference):
|
|
if value is None or reference in (None, 0):
|
|
return None
|
|
return value / reference - 1.0
|
|
|
|
|
|
def miss_all_probability(ratio, n):
|
|
"""(1-r)^N: chance that independent root sampling keeps none of N events.
|
|
A model assumption, not a measurement."""
|
|
return (1.0 - ratio) ** n
|
|
|
|
|
|
# ---------------------------------------------------------------------- load
|
|
|
|
ERROR_OUTCOMES = ('timeout', 'transport_error', 'protocol_error')
|
|
|
|
|
|
def slot_completed(slot):
|
|
return slot['outcome'] == 'http'
|
|
|
|
|
|
def summarize_load(slots, rate):
|
|
"""slots: one record per *planned* slot, including client_dropped ones.
|
|
|
|
Dropped and late slots stay in the denominator; p95 includes error
|
|
responses (it is not a latency of successes only)."""
|
|
scheduled = len(slots)
|
|
done = [s for s in slots if slot_completed(s)]
|
|
dropped = sum(1 for s in slots if s['outcome'] == 'client_dropped')
|
|
failures = {k: sum(1 for s in slots if s['outcome'] == k)
|
|
for k in ERROR_OUTCOMES}
|
|
http_5xx = sum(1 for s in done if s['status'] >= 500)
|
|
http_4xx = sum(1 for s in done if 400 <= s['status'] < 500)
|
|
successes = [s for s in done if 200 <= s['status'] < 300]
|
|
started = [s for s in slots if s.get('start_lag_ms') is not None]
|
|
lag = [s['start_lag_ms'] for s in started]
|
|
sched_all = [s['scheduled_ms'] for s in done]
|
|
service_all = [s['service_ms'] for s in done]
|
|
return {
|
|
'rate_rps': rate,
|
|
'scheduled': scheduled,
|
|
'completed': len(done),
|
|
'completed_ratio': (len(done) / scheduled) if scheduled else None,
|
|
'client_dropped': dropped,
|
|
'transport_failures': failures,
|
|
'http_5xx': http_5xx,
|
|
'http_4xx': http_4xx,
|
|
'http_2xx': len(successes),
|
|
'start_lag_ms_p95': nearest_rank(lag, 0.95),
|
|
'start_lag_ms_max': max(lag) if lag else None,
|
|
'scheduled_ms': {q: nearest_rank(sched_all, v) for q, v in
|
|
(('p50', .5), ('p95', .95), ('p99', .99))},
|
|
'service_ms': {q: nearest_rank(service_all, v) for q, v in
|
|
(('p50', .5), ('p95', .95), ('p99', .99))},
|
|
'success_scheduled_ms': {q: nearest_rank(
|
|
[s['scheduled_ms'] for s in successes], v) for q, v in
|
|
(('p50', .5), ('p95', .95), ('p99', .99))},
|
|
'success_service_ms': {q: nearest_rank(
|
|
[s['service_ms'] for s in successes], v) for q, v in
|
|
(('p50', .5), ('p95', .95), ('p99', .99))},
|
|
}
|
|
|
|
|
|
def load_errors(load):
|
|
"""HTTP/protocol errors among delivered healthy traffic."""
|
|
return (load['http_5xx'] + sum(load['transport_failures'].values()))
|
|
|
|
|
|
def delivered_stream(load):
|
|
"""Generator delivered the planned stream (protocol validity)."""
|
|
reasons = []
|
|
if load['client_dropped']:
|
|
reasons.append('client_dropped=%d' % load['client_dropped'])
|
|
lag = load['start_lag_ms_p95']
|
|
if lag is None or lag > START_LAG_LIMIT_MS:
|
|
reasons.append('start_lag_p95=%s' % lag)
|
|
return reasons
|
|
|
|
|
|
# ------------------------------------------------------------------ counters
|
|
|
|
def counter_delta(start, end, failure_only=False):
|
|
"""-> (delta, reason). Never turns a missing positive series into zero and
|
|
never reports a reset as a negative loss."""
|
|
if start is None and end is None:
|
|
if failure_only:
|
|
return 0.0, 'absent_until_first_error'
|
|
return None, 'missing_series'
|
|
if start is None:
|
|
if failure_only:
|
|
return end, 'series_created_in_window'
|
|
return None, 'missing_start'
|
|
if end is None:
|
|
return None, 'missing_end'
|
|
if end < start:
|
|
return None, 'counter_reset'
|
|
return end - start, None
|
|
|
|
|
|
# ------------------------------------------------------------------- cohorts
|
|
|
|
def classify_record(mode, record):
|
|
"""Classify one cohort request.
|
|
|
|
record: {'stdout': None | {'trace_id','span_id','trace_sampled'} (the keys
|
|
are absent when the SDK is off or no span context), 'loki': True/False/None
|
|
(None: backend observation incomplete), 'span': True/False/None}.
|
|
Returns (trace_class, log_class)."""
|
|
stdout = record.get('stdout')
|
|
if stdout is None:
|
|
return 'unmatched_stdout', 'incomplete_observation'
|
|
if mode == 'off':
|
|
if any(k in stdout for k in ('trace_id', 'span_id', 'trace_sampled')):
|
|
return 'ids_present_with_sdk_off', 'not_applicable'
|
|
return 'not_applicable', 'not_applicable'
|
|
sampled = stdout.get('trace_sampled')
|
|
if not stdout.get('trace_id') or not stdout.get('span_id') or \
|
|
not isinstance(sampled, bool):
|
|
return 'unmatched_stdout', 'incomplete_observation'
|
|
if record.get('loki') is None:
|
|
log_class = 'incomplete_observation'
|
|
elif record['loki']:
|
|
log_class = 'exported'
|
|
else:
|
|
log_class = 'missing_exported_logs'
|
|
if sampled is False:
|
|
return 'intentional_unsampled', log_class
|
|
if record.get('span') is None:
|
|
return 'incomplete_observation', log_class
|
|
return ('sampled_present' if record['span'] else 'sampled_missing'), log_class
|
|
|
|
|
|
def summarize_cohort(mode, records):
|
|
counts = {'n': len(records)}
|
|
for record in records:
|
|
trace_class, log_class = classify_record(mode, record)
|
|
counts[trace_class] = counts.get(trace_class, 0) + 1
|
|
counts['log_' + log_class] = counts.get('log_' + log_class, 0) + 1
|
|
counts['retained_traces'] = counts.get('sampled_present', 0)
|
|
return counts
|
|
|
|
|
|
def export_loss(items_failed_delta, send_failed, enqueue_failed):
|
|
"""Telemetry export losses; a false trace_sampled flag is NOT a loss."""
|
|
parts = {'app_export_failed_items': items_failed_delta,
|
|
'collector_send_failed': send_failed,
|
|
'collector_enqueue_failed': enqueue_failed}
|
|
known = [v for v in parts.values() if isinstance(v, (int, float))]
|
|
return {'parts': parts, 'total_known': sum(known),
|
|
'loss': sum(known) > 0}
|
|
|
|
|
|
# ------------------------------------------------------------------ validity
|
|
|
|
def evaluate_run(summary):
|
|
"""-> (valid, invalid_reasons, configuration_failures).
|
|
|
|
Invalid = the protocol was not executed as specified; configuration
|
|
failures are valid adverse outcomes and stay in the matrix."""
|
|
invalid = list(summary.get('measurement_defects', []))
|
|
failures = []
|
|
load = summary['load']
|
|
invalid += delivered_stream(load)
|
|
if load['completed_ratio'] is None or load['completed_ratio'] < COMPLETION_LIMIT:
|
|
invalid.append('completed_ratio=%s' % load['completed_ratio'])
|
|
if summary.get('effective_mode_ok') is False:
|
|
invalid.append('effective_env_mismatch')
|
|
if summary.get('image_ids_ok') is False:
|
|
invalid.append('image_changed')
|
|
if summary.get('foreign_faults'):
|
|
invalid.append('foreign_faults')
|
|
for name, cohort in (summary.get('cohorts') or {}).items():
|
|
if cohort.get('invalid'):
|
|
invalid.append('cohort_invalid:%s' % name)
|
|
# Counter resets are only explained by an observed restart/OOM.
|
|
for name, item in (summary.get('ingest') or {}).items():
|
|
if isinstance(item, dict) and item.get('reason') == 'counter_reset' \
|
|
and not summary.get('restarts'):
|
|
invalid.append('unexplained_counter_reset:%s' % name)
|
|
# --- adverse outcomes of the configuration (valid)
|
|
if load_errors(load):
|
|
failures.append('healthy_errors=%d' % load_errors(load))
|
|
if summary.get('not_ready_at_deadline'):
|
|
failures.append('not_ready_at_deadline=%d' % summary['not_ready_at_deadline'])
|
|
if summary.get('integrity_errors'):
|
|
failures.append('healthy_errors_integrity=%d' % summary['integrity_errors'])
|
|
if summary.get('restarts'):
|
|
failures.append('restart_or_oom')
|
|
loss = summary.get('export_failures') or {}
|
|
if loss.get('loss'):
|
|
failures.append('export_loss')
|
|
for name, cohort in (summary.get('cohorts') or {}).items():
|
|
if cohort.get('sampled_missing'):
|
|
failures.append('sampled_missing:%s=%d' % (name, cohort['sampled_missing']))
|
|
if cohort.get('log_missing_exported_logs'):
|
|
failures.append('missing_exported_logs:%s=%d' % (
|
|
name, cohort['log_missing_exported_logs']))
|
|
return (not invalid), invalid, failures
|
|
|
|
|
|
def matrix_entry(summary):
|
|
"""Every protocol-run stays in the matrix, valid-adverse included."""
|
|
return {
|
|
'mode': summary['mode'], 'repeat': summary.get('repeat'),
|
|
'attempt': summary.get('attempt', 1), 'valid': summary['valid'],
|
|
'invalid_reasons': summary['invalid_reasons'],
|
|
'configuration_failures': summary['configuration_failures'],
|
|
'spans_per_operation': summary.get('spans_per_operation'),
|
|
'scheduled_p95_ms': (summary['load']['scheduled_ms'] or {}).get('p95'),
|
|
'dir': summary.get('dir'),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------- selection
|
|
|
|
def _clean(run):
|
|
return (run['valid'] and not any(
|
|
f.startswith(('healthy_errors', 'not_ready', 'restart', 'export_loss'))
|
|
for f in run['configuration_failures']))
|
|
|
|
|
|
def _retained(run, name):
|
|
return (run['cohorts'].get(name) or {}).get('retained_traces', 0)
|
|
|
|
|
|
def _no_missing(run):
|
|
for cohort in run['cohorts'].values():
|
|
if cohort.get('sampled_missing') or cohort.get('log_missing_exported_logs') \
|
|
or cohort.get('log_incomplete_observation') \
|
|
or cohort.get('unmatched_stdout') or cohort.get('incomplete_observation'):
|
|
return False
|
|
return True
|
|
|
|
|
|
def full_reference(full_runs):
|
|
"""Frozen reference: medians of the three full runs; None if any datum is
|
|
unmeasurable (then p25 cannot be declared a winner)."""
|
|
spans = [r.get('spans_per_operation') for r in full_runs]
|
|
p95 = [r['load']['scheduled_ms']['p95'] for r in full_runs]
|
|
ok = (len(full_runs) == 3 and all(r['valid'] for r in full_runs) and
|
|
all(v is not None for v in spans + p95))
|
|
return {'complete': ok, 'spans_per_operation_median': median(spans),
|
|
'scheduled_p95_ms_median': median(p95),
|
|
'run_dirs': [r.get('dir') for r in full_runs]}
|
|
|
|
|
|
def select_p25(full_runs, p25_runs):
|
|
ref = full_reference(full_runs)
|
|
checks = {'full_reference_complete': ref['complete']}
|
|
if not ref['complete'] or len(p25_runs) != 3:
|
|
checks['p25_runs'] = len(p25_runs) == 3
|
|
return {'passed': False, 'checks': checks, 'reference': ref}
|
|
checks['all_valid_clean'] = all(_clean(r) for r in p25_runs)
|
|
checks['retained_error_and_slow_each_run'] = all(
|
|
_retained(r, 'error') >= 1 and _retained(r, 'slow') >= 1 for r in p25_runs)
|
|
checks['no_missing_spans_or_logs'] = all(_no_missing(r) for r in p25_runs)
|
|
spans = median([r.get('spans_per_operation') for r in p25_runs])
|
|
checks['spans_median_le_50pct_full'] = (
|
|
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
|
|
p95 = median([r['load']['scheduled_ms']['p95'] for r in p25_runs])
|
|
checks['scheduled_p95_le_110pct_full'] = (
|
|
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
|
|
return {'passed': all(checks.values()), 'checks': checks, 'reference': ref,
|
|
'p25_spans_per_operation_median': spans,
|
|
'p25_scheduled_p95_ms_median': p95}
|
|
|
|
|
|
def validate_single(run, mode, ref):
|
|
"""One fresh run against the FROZEN full reference (never a 3-run median
|
|
of the validation itself)."""
|
|
checks = {'protocol_valid': run['valid'],
|
|
'zero_errors_notready_loss': _clean(run)}
|
|
if mode == 'p25':
|
|
checks['complete_stdout_and_logs'] = _no_missing(run) and all(
|
|
(run['cohorts'].get(n) or {}).get('n') == 40 for n in ('error', 'slow'))
|
|
checks['retained_each_class'] = (
|
|
_retained(run, 'error') >= 1 and _retained(run, 'slow') >= 1)
|
|
spans = run.get('spans_per_operation')
|
|
checks['spans_le_50pct_frozen_full'] = (
|
|
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
|
|
p95 = run['load']['scheduled_ms']['p95']
|
|
checks['scheduled_p95_le_110pct_frozen_full'] = (
|
|
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
|
|
else:
|
|
checks['complete_stdout_and_logs'] = _no_missing(run)
|
|
checks['all_80_server_spans'] = all(
|
|
(run['cohorts'].get(n) or {}).get('sampled_present') == 40
|
|
for n in ('error', 'slow'))
|
|
return {'passed': all(checks.values()), 'checks': checks, 'mode': mode}
|