#!/usr/bin/env python3 """Track 1: fixed-rate, equal-sized trials with measured sampling retention.""" import argparse from concurrent.futures import ThreadPoolExecutor import datetime import json import math import os import subprocess import sys import threading import time import uuid from observability_common import ROOT, compose, get_json, prom, ready, save sys.path.insert(0, str(ROOT / 'tests')) from client import Client def percentile(values, p): return round(sorted(values)[max(0, math.ceil(len(values) * p) - 1)] * 1000, 3) if values else None def snapshot(): queries = { 'exported_spans': 'sum(otelcol_exporter_sent_spans_total)', 'failed_spans': 'sum(otelcol_exporter_send_failed_spans_total)', 'accepted_spans': 'sum(otelcol_receiver_accepted_spans_total)', 'refused_spans': 'sum(otelcol_receiver_refused_spans_total)', 'tempo_ingested_bytes': 'sum(tempo_distributor_bytes_received_total)', 'loki_ingested_bytes': 'sum(loki_distributor_bytes_received_total)', 'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)', } result = {} for key, query in queries.items(): rows = prom(query) result[key] = sum(float(x['value'][1]) for x in rows) if rows else None return result def samples(stop, output): while not stop.is_set(): result = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{json .}}'], capture_output=True, text=True) for line in result.stdout.splitlines(): row = json.loads(line) if row['Name'].startswith('mtusi2026lab6-'): output.append({'at': time.time(), **{k: row[k] for k in ('Name', 'CPUPerc', 'MemUsage')}}) stop.wait(1) def configure(disabled, ratio): # Compose interpolation only; no secrets or .env files are read. os.environ['OTEL_SDK_DISABLED'] = str(disabled).lower() os.environ['OTEL_TRACES_SAMPLER_ARG'] = str(ratio) compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker') ready() time.sleep(8) def trial(name, disabled, ratio, args, repetition): configure(disabled, ratio) c = Client(args.base_url) actor = c.user('research') chat = c.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'Synthetic research', 'member_ids': [actor['id']]}, schema='Chat')['json'] # Equal warmup before every trial, including telemetry off. for _ in range(20): c.call('GET', '/api/v1/users/me', actor=actor, schema='User') time.sleep(6) before = snapshot() stop, resource = threading.Event(), [] sampler = threading.Thread(target=samples, args=(stop, resource), daemon=True) sampler.start() known, latency, errors, uploads = [], [], [], [] lock = threading.Lock() started = time.monotonic() def request(index): scheduled = started + index / args.rate time.sleep(max(0, scheduled - time.monotonic())) client = Client(args.base_url) trace_id = None headers = {} begin = time.monotonic() outcome, kind = 'ok', 'read' try: if index % 20 == 0: kind = 'upload' headers['Content-Type'] = 'image/png' response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', 202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers=headers, schema='Attachment') att = response['json'] with lock: uploads.append(att) elif index % 20 == 1: kind = 'client_error' response = client.call('GET', '/api/v1/users/me', 401, headers=headers) elif index % 5 == 0: kind = 'write' headers['Idempotency-Key'] = uuid.uuid4().hex response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=actor, body={'text': 'synthetic fixed-size benchmark message'}, headers=headers, schema='Message') else: response = client.call('GET', '/api/v1/users/me', actor=actor, headers=headers, schema='User') trace_id = response['headers'].get('X-Trace-Id') except Exception as exc: outcome = type(exc).__name__ with lock: elapsed_request = time.monotonic() - begin latency.append(elapsed_request) known.append({'trace_id': trace_id, 'kind': kind, 'outcome': outcome, 'elapsed_seconds': elapsed_request}) if outcome != 'ok': errors.append(outcome) with ThreadPoolExecutor(max_workers=args.concurrency) as pool: list(pool.map(request, range(args.requests))) elapsed = time.monotonic() - started for att in uploads: c.wait_ready(actor, att) stop.set() sampler.join(timeout=10) # Flush SDK batches, Collector and backend scrape before taking counters. time.sleep(12) after = snapshot() retained = {kind: {'sent': 0, 'found': 0} for kind in ('read', 'write', 'upload', 'client_error')} slow = {'definition': 'slowest 5 percent of this trial by measured client latency', 'sent': 0, 'found': 0} slow_ids = {id(x) for x in sorted(known, key=lambda x: x['elapsed_seconds'], reverse=True)[:max(1, math.ceil(.05*len(known)))]} for row in known: retained[row['kind']]['sent'] += 1 is_slow = id(row) in slow_ids slow['sent'] += int(is_slow) try: trace = get_json('http://127.0.0.1:13200/api/traces/' + (row['trace_id'] or 'missing')) if trace.get('batches') or trace.get('resourceSpans'): retained[row['kind']]['found'] += 1 slow['found'] += int(is_slow) except Exception: pass return {'configuration': name, 'repetition': repetition, 'disabled': disabled, 'sampling_ratio': ratio, 'requests': args.requests, 'target_rps': args.rate, 'concurrency_limit': args.concurrency, 'elapsed_seconds': round(elapsed, 3), 'successful_rps': round((args.requests-len(errors))/elapsed, 3), 'unexpected_errors': len(errors), 'error_types': errors, 'all_request_latency_ms': {f'p{int(p*100)}': percentile(latency, p) for p in (.5, .95, .99)}, 'ready_uploads': len(uploads), 'counter_before': before, 'counter_after': after, 'counter_delta': {k: after[k]-before[k] if after[k] is not None and before[k] is not None else None for k in before}, 'trace_retention': retained, 'slow_request_trace_retention': slow, 'resources': resource} def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--base-url', default='http://127.0.0.1:18086') p.add_argument('--requests', type=int, default=120) p.add_argument('--rate', type=float, default=8) p.add_argument('--concurrency', type=int, default=4) p.add_argument('--repetitions', type=int, default=2) args = p.parse_args() if args.requests < 20 or args.rate <= 0 or args.concurrency < 1 or args.repetitions < 1: p.error('requests>=20, rate>0, concurrency>=1, repetitions>=1 required') result = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'method': 'Fixed-rate schedule, fixed request count/mix; 20 warmups; SDK and backend flush; sampled IDs queried from Tempo.', 'trials': []} try: configs = [('off', True, 1), ('full', False, 1), ('ten_percent', False, .1), ('balanced', False, .5)] for rep in range(args.repetitions): # Counterbalance order on second repetition. for name, disabled, ratio in (configs if rep % 2 == 0 else list(reversed(configs))): print('Running', name, 'repetition', rep + 1, flush=True) row = trial(name, disabled, ratio, args, rep + 1) result['trials'].append(row) save('evidence/lab6-research.json', result) print(json.dumps({k: row[k] for k in ('configuration', 'unexpected_errors', 'all_request_latency_ms', 'trace_retention')}), flush=True) assert row['unexpected_errors'] == 0, 'Unexpected business errors during research' found = sum(x['found'] for x in row['trace_retention'].values()) if name == 'off': assert found == 0, 'SDK off still exported request traces' elif name == 'full': assert found == args.requests, 'Full sampling lost request traces' else: assert 0 < found < args.requests, 'Sampling setting had no observed effect' result['status'] = 'PASS' finally: configure(False, 1) result['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() save('evidence/lab6-research.json', result) if __name__ == '__main__': main()