#!/usr/bin/env python3 """Track 1: fixed-count attachment benchmark; stdlib only, no credentials in evidence. Creates synthetic durable data. Never deletes volumes or changes business code. Always restores SDK on / Collector 100% when the experiment finishes or fails. """ import argparse import base64 from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import hashlib import json import math import os from pathlib import Path import platform import re import shlex import signal import statistics import subprocess import sys import threading import time import urllib.error import urllib.parse import urllib.request import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure VARIANTS = {'A': ('Off', True, 'collector.yaml'), 'B': ('On, SDK 100% / Collector 100%', False, 'collector.yaml'), 'C': ('On, SDK 100% / Collector 10%', False, 'collector-sampling10.yaml')} def percentile(values, quantile): return sorted(values)[max(0, math.ceil(len(values) * quantile) - 1)] def memory_mib(text): number, unit = re.fullmatch(r'([\d.]+)\s*([kKMGT]?i?B)', text.strip()).groups() factors = {'B': 1, 'kB': 1000, 'MB': 1000**2, 'GB': 1000**3, 'TB': 1000**4, 'KiB': 1024, 'MiB': 1024**2, 'GiB': 1024**3, 'TiB': 1024**4} return float(number) * factors[unit] / 1024**2 def parse_metrics(text): result = {} for line in text.splitlines(): if not line or line.startswith('#'): continue name = line.split('{', 1)[0].split(' ', 1)[0] result[name] = result.get(name, 0) + float(line.rsplit(' ', 1)[1]) return result class Bench: def __init__(self, args): self.args = args self.compose = shlex.split(args.compose) self.docker = shlex.split(args.docker) self.output = ROOT / args.output self.output.mkdir(parents=True, exist_ok=True) self.content = (ROOT / 'tests/fixtures/sample.png').read_bytes() self.report = {'started_utc': datetime.now(timezone.utc).isoformat(), 'method': {'requests': args.requests, 'concurrency': args.concurrency, 'repetitions': args.repetitions, 'warmup': args.warmup, 'fixture_bytes': len(self.content), 'fixture_sha256': hashlib.sha256(self.content).hexdigest(), 'stats_source': 'docker stats stream; 100% CPU = one logical core', 'cpu_ram_window': 'uploads plus complete worker drain, excluding final export wait', 'percentile': 'nearest rank; HTTP upload acceptance, client wall clock', 'config_sha256': {name: hashlib.sha256((ROOT/'infra/otel'/name).read_bytes()).hexdigest() for name in ('collector.yaml', 'collector-sampling10.yaml')}, 'order': 'ABC / BCA / CAB (rotated each repetition)'}, 'host': {'machine': platform.machine(), 'logical_cpus': os.cpu_count(), 'cpu_model': next((line.split(':',1)[1].strip() for line in Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'unknown'), 'memory_kib': int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')))}, 'trials': []} def run(self, cmd, *, env=None, data=None): completed = subprocess.run(cmd, cwd=ROOT, env=env, input=data, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if completed.returncode: # Subprocess output can contain connection credentials; do not forward it. raise RuntimeError('Command failed: ' + Path(cmd[0]).name + ' ' + cmd[1] + ' (exit ' + str(completed.returncode) + ')') return completed.stdout def compose_run(self, *args, **kwargs): return self.run(self.compose + list(args), **kwargs) def sql(self, sql): return self.compose_run('exec', '-T', 'db', 'sh', '-c', 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tA', data=sql+'\n').strip() def wait_jobs(self, chat=None, expected=None): deadline = time.monotonic() + self.args.drain_timeout while time.monotonic() < deadline: if chat: uuid.UUID(chat) # Only synthetic UUIDs may enter the query. row = self.sql("SELECT count(*),count(*) FILTER (WHERE j.status='ready')," "count(*) FILTER (WHERE j.status='failed') FROM jobs j JOIN attachments a " "ON a.id=j.attachment_id WHERE a.chat_id='" + chat + "'::uuid;") total, ready, failed = map(int, row.split('|')) if failed: raise RuntimeError('Worker produced failed benchmark jobs') if total == expected and ready == expected: return elif int(self.sql("SELECT count(*) FROM jobs WHERE status IN ('queued','processing');")) == 0: return time.sleep(1) raise RuntimeError('Worker backlog did not drain; evidence is incomplete') def configure(self, variant): _, disabled, filename = VARIANTS[variant] env = dict(os.environ, OTEL_SDK_DISABLED=str(disabled).lower(), OTEL_COLLECTOR_CONFIG=str(ROOT / 'infra/otel' / filename)) self.compose_run('config', '--quiet', env=env) self.compose_run('up', '-d', '--no-deps', '--force-recreate', '--wait', '--wait-timeout', '120', 'otel-collector', 'api', 'worker', env=env) self.compose_run('exec', '-T', 'otel-collector', '/otelcol-contrib', 'validate', '--config=/etc/otelcol/config.yaml', env=env) deadline = time.monotonic() + 60 while True: try: Client(self.args.base_url).call('GET', '/health/ready', schema='Health') self.collector() return except (Failure, RuntimeError): if time.monotonic() >= deadline: raise RuntimeError('API/Collector did not become ready') from None time.sleep(1) def collector(self): return parse_metrics(self.compose_run('exec', '-T', '--index', '1', 'api', 'curl', '-fsS', '--max-time', '5', 'http://otel-collector:8888/metrics')) def upload(self, count, actor, chat): barrier = threading.Barrier(self.args.concurrency) def work(slot): client = Client(self.args.base_url, timeout=60) results = [] barrier.wait() for index in range(slot, count, self.args.concurrency): started = time.monotonic() try: # No supplied traceparent: the SDK's AlwaysOn root policy is exercised. client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', expected=202, actor=actor, raw=self.content, headers={'Content-Type': 'image/png'}, schema='Attachment') results.append((time.monotonic()-started, True)) except Failure: results.append((time.monotonic()-started, False)) return results start = time.monotonic() with ThreadPoolExecutor(max_workers=self.args.concurrency) as pool: rows = [r for batch in pool.map(work, range(self.args.concurrency)) for r in batch] return rows, time.monotonic()-start def sample_stats(self, ids, stop, rows, errors): process = subprocess.Popen(self.docker + ['stats', '--format', '{{json .}}'] + list(ids), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) def terminate(): stop.wait() process.terminate() closer = threading.Thread(target=terminate, daemon=True) closer.start() try: for line in process.stdout: clean = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', line).strip() if not clean: continue if stop.is_set() and not clean.endswith('}'): continue obj = json.loads(clean) ident = obj['ID'] if ident not in ids: ident = next((key for key in ids if key.startswith(ident)), '') if ident: rows.append({'t': time.monotonic(), 'container': ident, 'service': ids[ident], 'cpu_percent': float(obj['CPUPerc'].rstrip('%')), 'ram_mib': memory_mib(obj['MemUsage'].split('/')[0])}) if not stop.is_set(): errors.append('docker stats stopped unexpectedly') except Exception as error: errors.append(type(error).__name__ + ': ' + str(error)) finally: stop.set() process.terminate() process.wait(timeout=10) closer.join(timeout=10) def trial(self, variant, repetition): print(f'Preparing {variant}, repetition {repetition}', flush=True) self.wait_jobs() self.configure(variant) client = Client(self.args.base_url) alice, _, _, chat = client.bootstrap() warm, _ = self.upload(self.args.warmup, alice, chat) if not all(ok for _, ok in warm): raise RuntimeError('Warm-up HTTP errors') self.wait_jobs(chat['id'], self.args.warmup) time.sleep(8) # SDK reader/processor + Collector batch settle before baseline. ids = {} for service in ('api', 'worker'): for ident in self.compose_run('ps', '-q', service).split(): ids[ident] = service self.report['container_limits'] = {service: self.run(self.docker + ['inspect', '--format', '{{.Config.Image}}|{{.Image}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}', ident]).strip() for ident, service in ids.items()} if sum(s == 'api' for s in ids.values()) != 2 or sum(s == 'worker' for s in ids.values()) != 1: raise RuntimeError('Benchmark requires two API replicas and one worker') rows, errors, stop = [], [], threading.Event() sampler = threading.Thread(target=self.sample_stats, args=(ids, stop, rows, errors)) sampler.start() try: time.sleep(3) before = self.collector() start = time.monotonic() measurements, http_seconds = self.upload(self.args.requests, alice, chat) if any(not ok for _, ok in measurements): raise RuntimeError('HTTP upload errors invalidate the comparison') self.wait_jobs(chat['id'], self.args.warmup+self.args.requests) end = time.monotonic() finally: stop.set() sampler.join(timeout=15) if sampler.is_alive() or errors: raise RuntimeError('docker stats sampling failed: ' + ', '.join(errors)) # Export is asynchronous. Wait for SDK batches and for backend queues to empty. time.sleep(3) after = self.collector() for _ in range(10): if after.get('otelcol_exporter_queue_size', 0) == 0: break time.sleep(1) after = self.collector() metric_seconds = time.monotonic()-start def delta(name): return after.get(name, 0)-before.get(name, 0) samples = [r for r in rows if start <= r['t'] <= end] resource = {} for service in ('api', 'worker'): per_container = [] for ident, kind in ids.items(): if kind != service: continue series = [r for r in samples if r['container'] == ident] if len(series) < 3: raise RuntimeError('Too few docker stats samples; use more requests') per_container.append({'cpu_percent_mean': statistics.mean(r['cpu_percent'] for r in series), 'ram_mib_mean': statistics.mean(r['ram_mib'] for r in series), 'ram_mib_max': max(r['ram_mib'] for r in series), 'samples': len(series)}) resource[service] = {key: sum(r[key] for r in per_container) for key in ('cpu_percent_mean', 'ram_mib_mean', 'ram_mib_max', 'samples')} resource[service]['cpu_seconds_per_job_estimate'] = resource[service]['cpu_percent_mean']/100*(end-start)/self.args.requests good = [seconds*1000 for seconds, ok in measurements if ok] failures = sum(not ok for _, ok in measurements) counters = {name: delta(name) for name in ( 'otelcol_receiver_accepted_spans', 'otelcol_exporter_sent_spans', 'otelcol_receiver_refused_spans', 'otelcol_exporter_send_failed_spans', 'otelcol_receiver_accepted_log_records', 'otelcol_receiver_accepted_metric_points')} if failures or not good or counters['otelcol_receiver_refused_spans'] or counters['otelcol_exporter_send_failed_spans']: raise RuntimeError('HTTP or Collector failures invalidate the comparison') if variant == 'A' and counters['otelcol_receiver_accepted_spans'] != 0: raise RuntimeError('Telemetry Off still exported spans') retention = counters['otelcol_exporter_sent_spans']/max(counters['otelcol_receiver_accepted_spans'], 1) if variant == 'B' and not 0.95 <= retention <= 1.05: raise RuntimeError('100% trace export did not match receiver ingest') if variant == 'C' and not 0.04 <= retention <= 0.18: raise RuntimeError('Collector sampling did not reduce export to approximately 10%') result = {'variant': variant, 'repetition': repetition, 'completed_utc': datetime.now(timezone.utc).isoformat(), 'successful_uploads': len(good), 'http_errors': failures, 'http_seconds': http_seconds, 'pipeline_seconds': end-start, 'collector_window_seconds': metric_seconds, 'rps': len(good)/http_seconds, 'latency_mean_ms': statistics.mean(good), 'latency_p50_ms': percentile(good, .5), 'latency_p99_ms': percentile(good, .99), 'latencies_ms': good, 'resources': resource, 'collector_deltas': counters, 'ingest_spans_per_second': counters['otelcol_receiver_accepted_spans']/metric_seconds, 'tempo_spans_per_second': counters['otelcol_exporter_sent_spans']/metric_seconds, 'trace_retention': retention, 'stats_samples': [{**r, 't': r['t']-start} for r in samples]} self.report['trials'].append(result) self.save() print(f'{variant}/{repetition}: {result["rps"]:.2f} RPS, p99 {result["latency_p99_ms"]:.2f} ms, ' f'accepted {counters["otelcol_receiver_accepted_spans"]:.0f}, exported {counters["otelcol_exporter_sent_spans"]:.0f}', flush=True) def save(self): (self.output / 'track1-benchmark.json').write_text(json.dumps(self.report, indent=2)+'\n') def write_summary(self): def med(variant, getter): return statistics.median(getter(t) for t in self.report['trials'] if t['variant'] == variant) lines = ['# Track 1: Цена наблюдаемости', '', 'Результат: **' + self.report.get('result', 'INCOMPLETE') + '**. Начало UTC: ' + self.report['started_utc'] + '.', '', f"По {self.args.repetitions} прогона каждого варианта, {self.args.requests} POST uploads, concurrency {self.args.concurrency}, warm-up {self.args.warmup} uploads.", 'Порядок ABC / BCA / CAB. PNG: ' + str(len(self.content)) + ' bytes. Каждый job дошёл до ready; очередь между прогонами пуста.', 'Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).', 'RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.', 'CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.', '', '| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |', '|---|---:|---:|---:|---:|---:|---:|---:|---:|'] for v in 'ABC': values = [med(v, lambda t: t[k]) for k in ('rps', 'latency_p50_ms', 'latency_p99_ms')] api = [med(v, lambda t: t['resources']['api'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')] worker = [med(v, lambda t: t['resources']['worker'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')] lines.append(f'| {v}: {VARIANTS[v][0]} | {values[0]:.2f} | {values[1]:.2f} | {values[2]:.2f} | {api[0]:.2f}% | {api[1]:.2f} | {worker[0]:.2f}% | {worker[1]:.2f} | {med(v, lambda t:t["ingest_spans_per_second"]):.2f} |') lines += ['', '| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |', '|---|---:|---:|---:|---:|---:|---:|---:|'] for v in 'ABC': lines.append(f'| {v} | {med(v, lambda t:t["latency_mean_ms"]):.2f} | {med(v, lambda t:t["pipeline_seconds"]):.2f} | {med(v, lambda t:t["collector_deltas"]["otelcol_receiver_accepted_spans"]):.0f} | {med(v, lambda t:t["collector_deltas"]["otelcol_exporter_sent_spans"]):.0f} | {med(v, lambda t:t["tempo_spans_per_second"]):.2f} | {1000*med(v, lambda t:t["resources"]["api"]["cpu_seconds_per_job_estimate"]):.2f} | {1000*med(v, lambda t:t["resources"]["worker"]["cpu_seconds_per_job_estimate"]):.2f} |') lines += ['', '*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.', 'Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.', 'В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.', '', '| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |', '|---|---:|---:|---:|---:|---:|---:|'] for t in self.report['trials']: lines.append(f'| {t["variant"]}/{t["repetition"]} | {t["rps"]:.2f} | {t["latency_mean_ms"]:.2f} | {t["latency_p99_ms"]:.2f} | {t["pipeline_seconds"]:.2f} | {t["resources"]["api"]["samples"]} / {t["resources"]["worker"]["samples"]} | {t["collector_deltas"]["otelcol_receiver_refused_spans"]:.0f} / {t["collector_deltas"]["otelcol_exporter_send_failed_spans"]:.0f} |') if all('sdk_diagnostic_audit' in t for t in self.report['trials']): starts = sorted({t['sdk_diagnostic_audit']['warning_error_start'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_start'] is not None}) deltas = sorted({t['sdk_diagnostic_audit']['warning_error_delta'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_delta'] is not None}) drops = self.report.get('sdk_diagnostic_audit', {}).get('span_queue_drop_max_during_experiment') lines += ['', f'SDK audit: delta warning/error diagnostics в on-окнах: {deltas}; значения перед началом окон: {starts}. Startup/reconfiguration не входит в steady-state замер.', f'Максимум telemetry_sdk_dropped_spans в интервале эксперимента: {drops}. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.'] probe = self.report.get('error_probe', {}) lines += ['', 'Отдельный synthetic ERROR probe (вне замеров API): ' + json.dumps(probe, ensure_ascii=False) + '.', '', '## Воспроизведение', '', '```sh', 'python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3', '```', '', 'Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.', 'Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.', 'В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.', '', '## Ограничения', '', 'Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.', 'Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.', 'Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.', 'Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.'] (self.output/'track1-benchmark.md').write_text('\n'.join(lines)+'\n') def audit_sdk(self): # Read the same externally scraped diagnostics for each trial's own SDK instances. def query(expr, timestamp): url = self.args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr, 'time': timestamp}) with urllib.request.urlopen(url, timeout=15) as response: rows = json.load(response)['data']['result'] return float(rows[0]['value'][1]) if rows else None end = 0 for trial in self.report['trials']: ids = '|'.join(sorted({s['container'][:12] for s in trial['stats_samples']})) end = datetime.fromisoformat(trial['completed_utc']).timestamp() start = end-trial['collector_window_seconds'] expr = 'sum(telemetry_sdk_errors{service_instance_id=~"' + ids + '"})' a, b = query(expr, start), query(expr, end) trial['sdk_diagnostic_audit'] = {'source': 'Prometheus instant queries at trial start/end, restricted to trial container hostnames', 'warning_error_start': a, 'warning_error_end': b, 'warning_error_delta': b-a if a is not None and b is not None else None} seconds = math.ceil(end-datetime.fromisoformat(self.report['started_utc']).timestamp()) self.report['sdk_diagnostic_audit'] = { 'span_queue_drop_max_during_experiment': query(f'max(max_over_time(telemetry_sdk_dropped_spans[{seconds}s]))', end), 'note': 'Warning/error diagnostics are not counts of lost spans. Compare per-trial deltas; sampling discard is sent/accepted.'} self.save() if any(t['sdk_diagnostic_audit']['warning_error_delta'] not in (None, 0) for t in self.report['trials']): raise RuntimeError('SDK diagnostics increased during measured windows') def probe_errors(self): # Separate from measured trials: synthetic ERROR spans verify that probability sampling is not error-aware. self.configure('C') traces = [uuid.uuid4().hex for _ in range(100)] now = time.time_ns() spans = [{'traceId': ident, 'spanId': '0123456789abcdef', 'name': 'track1.synthetic-error-probe', 'kind': 2, 'startTimeUnixNano': str(now), 'endTimeUnixNano': str(now+1000000), 'status': {'code': 2}, 'attributes': [{'key': 'http.response.status_code', 'value': {'intValue': '503'}}]} for ident in traces] data = json.dumps({'resourceSpans': [{'resource': {'attributes': [{'key': 'service.name', 'value': {'stringValue': 'track1-sampling-probe'}}]}, 'scopeSpans': [{'spans': spans}]}]}) self.compose_run('exec', '-T', '--index', '1', 'api', 'curl', '-fsS', '--max-time', '10', '-H', 'Content-Type: application/json', '--data-binary', '@-', 'http://otel-collector:4318/v1/traces', data=data) time.sleep(5) auth = base64.b64encode((os.environ.get('GRAFANA_ADMIN_USER', 'admin')+':'+(ROOT/'secrets/grafana_admin_password').read_text().strip()).encode()).decode() def retained(ident): request = urllib.request.Request(self.args.grafana_url+'/api/datasources/proxy/uid/tempo/api/traces/'+ident, headers={'Authorization': 'Basic '+auth}) try: with urllib.request.urlopen(request, timeout=15) as response: return response.status == 200 except urllib.error.HTTPError as error: if error.code == 404: return False raise RuntimeError('Tempo probe query failed') from None with ThreadPoolExecutor(max_workers=4) as pool: found = sum(pool.map(retained, traces)) if not 0 < found < len(traces): raise RuntimeError('Error sampling probe failed') self.report['error_probe'] = {'type': 'synthetic ERROR spans, separate from upload benchmark', 'sent': len(traces), 'found_in_tempo': found, 'missing_in_tempo': len(traces)-found} self.save() print(f'Error probe: {found}/100 ERROR traces retained', flush=True) def self_test(): assert re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', '\x1b[?25l\x1b[H{}\x1b[?25h') == '{}' assert percentile([1, 2, 3, 4], .5) == 2 assert percentile(list(range(1, 101)), .99) == 99 assert memory_mib('1024KiB') == 1 assert memory_mib('1GiB') == 1024 assert abs(memory_mib('1048576B')-1) < 1e-9 assert parse_metrics('# comment\nx{a="b"} 2\nx{a="c"} 3\ny 4\n') == {'x': 5, 'y': 4} base = (ROOT/'infra/otel/collector.yaml').read_text() sampled = (ROOT/'infra/otel/collector-sampling10.yaml').read_text() expected = base.replace(' batch:\n', ' probabilistic_sampler/track1:\n hash_seed: 22\n sampling_percentage: 10\n batch:\n', 1) expected = expected.replace('processors: [memory_limiter, batch]\n exporters: [otlphttp/tempo]', 'processors: [memory_limiter, probabilistic_sampler/track1, batch]\n exporters: [otlphttp/tempo]') assert sampled.split('\n', 1)[1] == expected, 'Sampling must be the only Collector configuration difference' assert 'sampling_percentage: 10' in sampled assert sampled.count('processors: [memory_limiter, probabilistic_sampler/track1, batch]') == 1 assert sampled.count('processors: [memory_limiter, batch]') == base.count('processors: [memory_limiter, batch]')-1 print('PASS: percentile, docker memory units, metric sums, trace-only sampler configuration') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--requests', type=int, default=1000) parser.add_argument('--concurrency', type=int, default=4) parser.add_argument('--repetitions', type=int, default=3) parser.add_argument('--warmup', type=int, default=20) parser.add_argument('--drain-timeout', type=int, default=600) parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--prometheus-url', default='http://localhost:9090') parser.add_argument('--grafana-url', default='http://localhost:3000') parser.add_argument('--compose', default=os.environ.get('COMPOSE', 'docker compose')) parser.add_argument('--docker', default=os.environ.get('DOCKER', 'docker')) parser.add_argument('--output', default='evidence') parser.add_argument('--self-test', action='store_true') args = parser.parse_args() if args.self_test: return self_test() if not (args.requests >= 100 and 1 <= args.concurrency <= 16 and 1 <= args.repetitions <= 10 and args.warmup >= args.concurrency): parser.error('requests >=100, concurrency 1..16, repetitions 1..10, warmup >= concurrency required') def interrupted(signum, frame): raise KeyboardInterrupt('Benchmark interrupted; restoring baseline') signal.signal(signal.SIGTERM, interrupted) bench = Bench(args) try: for repetition in range(args.repetitions): order = 'ABC'[repetition % 3:] + 'ABC'[:repetition % 3] for variant in order: bench.trial(variant, repetition+1) bench.audit_sdk() bench.probe_errors() bench.report['result'] = 'PASS' finally: print('Restoring SDK on / Collector 100%', flush=True) bench.configure('B') bench.report['restored'] = 'SDK enabled, default Collector configuration' bench.save() bench.write_summary() print('PASS: all trials complete; raw sanitized evidence saved', flush=True) if __name__ == '__main__': main()