"""Pure measurement logic for lab6 track 1 (no I/O): statistics, counter deltas, cohort classification, run validity and the p25/full selection rules. Everything here is unit-tested in test_measurement.py.""" import math MODES = { 'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER_ARG': '1'}, 'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1'}, 'p25': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.25'}, 'p10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.1'}, } SAMPLER_RATIO = {'off': None, 'full': 1.0, 'p25': 0.25, 'p10': 0.1} ORDERS = (('off', 'full', 'p25', 'p10'), ('p25', 'off', 'p10', 'full'), ('p10', 'p25', 'full', 'off')) START_LAG_LIMIT_MS = 50.0 COMPLETION_LIMIT = 0.99 SPAN_RATIO_LIMIT = 0.5 LATENCY_RATIO_LIMIT = 1.10 # ----------------------------------------------------------------- statistics def nearest_rank(values, q): """Nearest-rank quantile: sorted[ceil(n*q)-1] (same as tests/load.py).""" if not values: return None ordered = sorted(values) return ordered[max(0, math.ceil(len(ordered) * q) - 1)] def median(values): values = [v for v in values if v is not None] if not values: return None ordered = sorted(values) n = len(ordered) mid = n // 2 return ordered[mid] if n % 2 else (ordered[mid - 1] + ordered[mid]) / 2 def spread(values): values = [v for v in values if v is not None] if not values: return {'min': None, 'median': None, 'max': None, 'n': 0} return {'min': min(values), 'median': median(values), 'max': max(values), 'n': len(values)} def relative(value, reference): if value is None or reference in (None, 0): return None return value / reference - 1.0 def miss_all_probability(ratio, n): """(1-r)^N: chance that independent root sampling keeps none of N events. A model assumption, not a measurement.""" return (1.0 - ratio) ** n # ---------------------------------------------------------------------- load ERROR_OUTCOMES = ('timeout', 'transport_error', 'protocol_error') def slot_completed(slot): return slot['outcome'] == 'http' def summarize_load(slots, rate): """slots: one record per *planned* slot, including client_dropped ones. Dropped and late slots stay in the denominator; p95 includes error responses (it is not a latency of successes only).""" scheduled = len(slots) done = [s for s in slots if slot_completed(s)] dropped = sum(1 for s in slots if s['outcome'] == 'client_dropped') failures = {k: sum(1 for s in slots if s['outcome'] == k) for k in ERROR_OUTCOMES} http_5xx = sum(1 for s in done if s['status'] >= 500) http_4xx = sum(1 for s in done if 400 <= s['status'] < 500) successes = [s for s in done if 200 <= s['status'] < 300] started = [s for s in slots if s.get('start_lag_ms') is not None] lag = [s['start_lag_ms'] for s in started] sched_all = [s['scheduled_ms'] for s in done] service_all = [s['service_ms'] for s in done] return { 'rate_rps': rate, 'scheduled': scheduled, 'completed': len(done), 'completed_ratio': (len(done) / scheduled) if scheduled else None, 'client_dropped': dropped, 'transport_failures': failures, 'http_5xx': http_5xx, 'http_4xx': http_4xx, 'http_2xx': len(successes), 'start_lag_ms_p95': nearest_rank(lag, 0.95), 'start_lag_ms_max': max(lag) if lag else None, 'scheduled_ms': {q: nearest_rank(sched_all, v) for q, v in (('p50', .5), ('p95', .95), ('p99', .99))}, 'service_ms': {q: nearest_rank(service_all, v) for q, v in (('p50', .5), ('p95', .95), ('p99', .99))}, 'success_scheduled_ms': {q: nearest_rank( [s['scheduled_ms'] for s in successes], v) for q, v in (('p50', .5), ('p95', .95), ('p99', .99))}, 'success_service_ms': {q: nearest_rank( [s['service_ms'] for s in successes], v) for q, v in (('p50', .5), ('p95', .95), ('p99', .99))}, } def load_errors(load): """HTTP/protocol errors among delivered healthy traffic.""" return (load['http_5xx'] + sum(load['transport_failures'].values())) def delivered_stream(load): """Generator delivered the planned stream (protocol validity).""" reasons = [] if load['client_dropped']: reasons.append('client_dropped=%d' % load['client_dropped']) lag = load['start_lag_ms_p95'] if lag is None or lag > START_LAG_LIMIT_MS: reasons.append('start_lag_p95=%s' % lag) return reasons # ------------------------------------------------------------------ counters def counter_delta(start, end, failure_only=False): """-> (delta, reason). Never turns a missing positive series into zero and never reports a reset as a negative loss.""" if start is None and end is None: if failure_only: return 0.0, 'absent_until_first_error' return None, 'missing_series' if start is None: if failure_only: return end, 'series_created_in_window' return None, 'missing_start' if end is None: return None, 'missing_end' if end < start: return None, 'counter_reset' return end - start, None # ------------------------------------------------------------------- cohorts def classify_record(mode, record): """Classify one cohort request. record: {'stdout': None | {'trace_id','span_id','trace_sampled'} (the keys are absent when the SDK is off or no span context), 'loki': True/False/None (None: backend observation incomplete), 'span': True/False/None}. Returns (trace_class, log_class).""" stdout = record.get('stdout') if stdout is None: return 'unmatched_stdout', 'incomplete_observation' if mode == 'off': if any(k in stdout for k in ('trace_id', 'span_id', 'trace_sampled')): return 'ids_present_with_sdk_off', 'not_applicable' return 'not_applicable', 'not_applicable' sampled = stdout.get('trace_sampled') if not stdout.get('trace_id') or not stdout.get('span_id') or \ not isinstance(sampled, bool): return 'unmatched_stdout', 'incomplete_observation' if record.get('loki') is None: log_class = 'incomplete_observation' elif record['loki']: log_class = 'exported' else: log_class = 'missing_exported_logs' if sampled is False: return 'intentional_unsampled', log_class if record.get('span') is None: return 'incomplete_observation', log_class return ('sampled_present' if record['span'] else 'sampled_missing'), log_class def summarize_cohort(mode, records): counts = {'n': len(records)} for record in records: trace_class, log_class = classify_record(mode, record) counts[trace_class] = counts.get(trace_class, 0) + 1 counts['log_' + log_class] = counts.get('log_' + log_class, 0) + 1 counts['retained_traces'] = counts.get('sampled_present', 0) return counts def export_loss(items_failed_delta, send_failed, enqueue_failed): """Telemetry export losses; a false trace_sampled flag is NOT a loss.""" parts = {'app_export_failed_items': items_failed_delta, 'collector_send_failed': send_failed, 'collector_enqueue_failed': enqueue_failed} known = [v for v in parts.values() if isinstance(v, (int, float))] return {'parts': parts, 'total_known': sum(known), 'loss': sum(known) > 0} # ------------------------------------------------------------------ validity def evaluate_run(summary): """-> (valid, invalid_reasons, configuration_failures). Invalid = the protocol was not executed as specified; configuration failures are valid adverse outcomes and stay in the matrix.""" invalid = list(summary.get('measurement_defects', [])) failures = [] load = summary['load'] invalid += delivered_stream(load) if load['completed_ratio'] is None or load['completed_ratio'] < COMPLETION_LIMIT: invalid.append('completed_ratio=%s' % load['completed_ratio']) if summary.get('effective_mode_ok') is False: invalid.append('effective_env_mismatch') if summary.get('image_ids_ok') is False: invalid.append('image_changed') if summary.get('foreign_faults'): invalid.append('foreign_faults') for name, cohort in (summary.get('cohorts') or {}).items(): if cohort.get('invalid'): invalid.append('cohort_invalid:%s' % name) # Counter resets are only explained by an observed restart/OOM. for name, item in (summary.get('ingest') or {}).items(): if isinstance(item, dict) and item.get('reason') == 'counter_reset' \ and not summary.get('restarts'): invalid.append('unexplained_counter_reset:%s' % name) # --- adverse outcomes of the configuration (valid) if load_errors(load): failures.append('healthy_errors=%d' % load_errors(load)) if summary.get('not_ready_at_deadline'): failures.append('not_ready_at_deadline=%d' % summary['not_ready_at_deadline']) if summary.get('integrity_errors'): failures.append('healthy_errors_integrity=%d' % summary['integrity_errors']) if summary.get('restarts'): failures.append('restart_or_oom') loss = summary.get('export_failures') or {} if loss.get('loss'): failures.append('export_loss') for name, cohort in (summary.get('cohorts') or {}).items(): if cohort.get('sampled_missing'): failures.append('sampled_missing:%s=%d' % (name, cohort['sampled_missing'])) if cohort.get('log_missing_exported_logs'): failures.append('missing_exported_logs:%s=%d' % ( name, cohort['log_missing_exported_logs'])) return (not invalid), invalid, failures def matrix_entry(summary): """Every protocol-run stays in the matrix, valid-adverse included.""" return { 'mode': summary['mode'], 'repeat': summary.get('repeat'), 'attempt': summary.get('attempt', 1), 'valid': summary['valid'], 'invalid_reasons': summary['invalid_reasons'], 'configuration_failures': summary['configuration_failures'], 'spans_per_operation': summary.get('spans_per_operation'), 'scheduled_p95_ms': (summary['load']['scheduled_ms'] or {}).get('p95'), 'dir': summary.get('dir'), } # ---------------------------------------------------------------- selection def _clean(run): return (run['valid'] and not any( f.startswith(('healthy_errors', 'not_ready', 'restart', 'export_loss')) for f in run['configuration_failures'])) def _retained(run, name): return (run['cohorts'].get(name) or {}).get('retained_traces', 0) def _no_missing(run): for cohort in run['cohorts'].values(): if cohort.get('sampled_missing') or cohort.get('log_missing_exported_logs') \ or cohort.get('log_incomplete_observation') \ or cohort.get('unmatched_stdout') or cohort.get('incomplete_observation'): return False return True def full_reference(full_runs): """Frozen reference: medians of the three full runs; None if any datum is unmeasurable (then p25 cannot be declared a winner).""" spans = [r.get('spans_per_operation') for r in full_runs] p95 = [r['load']['scheduled_ms']['p95'] for r in full_runs] ok = (len(full_runs) == 3 and all(r['valid'] for r in full_runs) and all(v is not None for v in spans + p95)) return {'complete': ok, 'spans_per_operation_median': median(spans), 'scheduled_p95_ms_median': median(p95), 'run_dirs': [r.get('dir') for r in full_runs]} def select_p25(full_runs, p25_runs): ref = full_reference(full_runs) checks = {'full_reference_complete': ref['complete']} if not ref['complete'] or len(p25_runs) != 3: checks['p25_runs'] = len(p25_runs) == 3 return {'passed': False, 'checks': checks, 'reference': ref} checks['all_valid_clean'] = all(_clean(r) for r in p25_runs) checks['retained_error_and_slow_each_run'] = all( _retained(r, 'error') >= 1 and _retained(r, 'slow') >= 1 for r in p25_runs) checks['no_missing_spans_or_logs'] = all(_no_missing(r) for r in p25_runs) spans = median([r.get('spans_per_operation') for r in p25_runs]) checks['spans_median_le_50pct_full'] = ( spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median']) p95 = median([r['load']['scheduled_ms']['p95'] for r in p25_runs]) checks['scheduled_p95_le_110pct_full'] = ( p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median']) return {'passed': all(checks.values()), 'checks': checks, 'reference': ref, 'p25_spans_per_operation_median': spans, 'p25_scheduled_p95_ms_median': p95} def validate_single(run, mode, ref): """One fresh run against the FROZEN full reference (never a 3-run median of the validation itself).""" checks = {'protocol_valid': run['valid'], 'zero_errors_notready_loss': _clean(run)} if mode == 'p25': checks['complete_stdout_and_logs'] = _no_missing(run) and all( (run['cohorts'].get(n) or {}).get('n') == 40 for n in ('error', 'slow')) checks['retained_each_class'] = ( _retained(run, 'error') >= 1 and _retained(run, 'slow') >= 1) spans = run.get('spans_per_operation') checks['spans_le_50pct_frozen_full'] = ( spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median']) p95 = run['load']['scheduled_ms']['p95'] checks['scheduled_p95_le_110pct_frozen_full'] = ( p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median']) else: checks['complete_stdout_and_logs'] = _no_missing(run) checks['all_80_server_spans'] = all( (run['cohorts'].get(n) or {}).get('sampled_present') == 40 for n in ('error', 'slow')) return {'passed': all(checks.values()), 'checks': checks, 'mode': mode}