Files
mtusi2026lab6/scripts/benchmark_telemetry.py
T
2026-11-16 18:47:37 +03:00

462 lines
30 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Track 1: fixed-count attachment benchmark; stdlib only, no credentials in evidence.
Creates synthetic durable data. Never deletes volumes or changes business code.
Always restores SDK on / Collector 100% when the experiment finishes or fails.
"""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import re
import shlex
import signal
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
VARIANTS = {'A': ('Off', True, 'collector.yaml'),
'B': ('On, SDK 100% / Collector 100%', False, 'collector.yaml'),
'C': ('On, SDK 100% / Collector 10%', False, 'collector-sampling10.yaml')}
def percentile(values, quantile):
return sorted(values)[max(0, math.ceil(len(values) * quantile) - 1)]
def memory_mib(text):
number, unit = re.fullmatch(r'([\d.]+)\s*([kKMGT]?i?B)', text.strip()).groups()
factors = {'B': 1, 'kB': 1000, 'MB': 1000**2, 'GB': 1000**3, 'TB': 1000**4,
'KiB': 1024, 'MiB': 1024**2, 'GiB': 1024**3, 'TiB': 1024**4}
return float(number) * factors[unit] / 1024**2
def parse_metrics(text):
result = {}
for line in text.splitlines():
if not line or line.startswith('#'):
continue
name = line.split('{', 1)[0].split(' ', 1)[0]
result[name] = result.get(name, 0) + float(line.rsplit(' ', 1)[1])
return result
class Bench:
def __init__(self, args):
self.args = args
self.compose = shlex.split(args.compose)
self.docker = shlex.split(args.docker)
self.output = ROOT / args.output
self.output.mkdir(parents=True, exist_ok=True)
self.content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
self.report = {'started_utc': datetime.now(timezone.utc).isoformat(),
'method': {'requests': args.requests, 'concurrency': args.concurrency,
'repetitions': args.repetitions, 'warmup': args.warmup,
'fixture_bytes': len(self.content),
'fixture_sha256': hashlib.sha256(self.content).hexdigest(),
'stats_source': 'docker stats stream; 100% CPU = one logical core',
'cpu_ram_window': 'uploads plus complete worker drain, excluding final export wait',
'percentile': 'nearest rank; HTTP upload acceptance, client wall clock',
'config_sha256': {name: hashlib.sha256((ROOT/'infra/otel'/name).read_bytes()).hexdigest() for name in ('collector.yaml', 'collector-sampling10.yaml')},
'order': 'ABC / BCA / CAB (rotated each repetition)'},
'host': {'machine': platform.machine(), 'logical_cpus': os.cpu_count(),
'cpu_model': next((line.split(':',1)[1].strip() for line in Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'unknown'),
'memory_kib': int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')))},
'trials': []}
def run(self, cmd, *, env=None, data=None):
completed = subprocess.run(cmd, cwd=ROOT, env=env, input=data, text=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if completed.returncode:
# Subprocess output can contain connection credentials; do not forward it.
raise RuntimeError('Command failed: ' + Path(cmd[0]).name + ' ' + cmd[1] +
' (exit ' + str(completed.returncode) + ')')
return completed.stdout
def compose_run(self, *args, **kwargs):
return self.run(self.compose + list(args), **kwargs)
def sql(self, sql):
return self.compose_run('exec', '-T', 'db', 'sh', '-c',
'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tA', data=sql+'\n').strip()
def wait_jobs(self, chat=None, expected=None):
deadline = time.monotonic() + self.args.drain_timeout
while time.monotonic() < deadline:
if chat:
uuid.UUID(chat) # Only synthetic UUIDs may enter the query.
row = self.sql("SELECT count(*),count(*) FILTER (WHERE j.status='ready'),"
"count(*) FILTER (WHERE j.status='failed') FROM jobs j JOIN attachments a "
"ON a.id=j.attachment_id WHERE a.chat_id='" + chat + "'::uuid;")
total, ready, failed = map(int, row.split('|'))
if failed:
raise RuntimeError('Worker produced failed benchmark jobs')
if total == expected and ready == expected:
return
elif int(self.sql("SELECT count(*) FROM jobs WHERE status IN ('queued','processing');")) == 0:
return
time.sleep(1)
raise RuntimeError('Worker backlog did not drain; evidence is incomplete')
def configure(self, variant):
_, disabled, filename = VARIANTS[variant]
env = dict(os.environ, OTEL_SDK_DISABLED=str(disabled).lower(),
OTEL_COLLECTOR_CONFIG=str(ROOT / 'infra/otel' / filename))
self.compose_run('config', '--quiet', env=env)
self.compose_run('up', '-d', '--no-deps', '--force-recreate', '--wait',
'--wait-timeout', '120', 'otel-collector', 'api', 'worker', env=env)
self.compose_run('exec', '-T', 'otel-collector', '/otelcol-contrib', 'validate',
'--config=/etc/otelcol/config.yaml', env=env)
deadline = time.monotonic() + 60
while True:
try:
Client(self.args.base_url).call('GET', '/health/ready', schema='Health')
self.collector()
return
except (Failure, RuntimeError):
if time.monotonic() >= deadline:
raise RuntimeError('API/Collector did not become ready') from None
time.sleep(1)
def collector(self):
return parse_metrics(self.compose_run('exec', '-T', '--index', '1', 'api',
'curl', '-fsS', '--max-time', '5', 'http://otel-collector:8888/metrics'))
def upload(self, count, actor, chat):
barrier = threading.Barrier(self.args.concurrency)
def work(slot):
client = Client(self.args.base_url, timeout=60)
results = []
barrier.wait()
for index in range(slot, count, self.args.concurrency):
started = time.monotonic()
try:
# No supplied traceparent: the SDK's AlwaysOn root policy is exercised.
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
expected=202, actor=actor, raw=self.content,
headers={'Content-Type': 'image/png'}, schema='Attachment')
results.append((time.monotonic()-started, True))
except Failure:
results.append((time.monotonic()-started, False))
return results
start = time.monotonic()
with ThreadPoolExecutor(max_workers=self.args.concurrency) as pool:
rows = [r for batch in pool.map(work, range(self.args.concurrency)) for r in batch]
return rows, time.monotonic()-start
def sample_stats(self, ids, stop, rows, errors):
process = subprocess.Popen(self.docker + ['stats', '--format', '{{json .}}'] + list(ids),
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
def terminate():
stop.wait()
process.terminate()
closer = threading.Thread(target=terminate, daemon=True)
closer.start()
try:
for line in process.stdout:
clean = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', line).strip()
if not clean:
continue
if stop.is_set() and not clean.endswith('}'):
continue
obj = json.loads(clean)
ident = obj['ID']
if ident not in ids:
ident = next((key for key in ids if key.startswith(ident)), '')
if ident:
rows.append({'t': time.monotonic(), 'container': ident, 'service': ids[ident],
'cpu_percent': float(obj['CPUPerc'].rstrip('%')),
'ram_mib': memory_mib(obj['MemUsage'].split('/')[0])})
if not stop.is_set():
errors.append('docker stats stopped unexpectedly')
except Exception as error:
errors.append(type(error).__name__ + ': ' + str(error))
finally:
stop.set()
process.terminate()
process.wait(timeout=10)
closer.join(timeout=10)
def trial(self, variant, repetition):
print(f'Preparing {variant}, repetition {repetition}', flush=True)
self.wait_jobs()
self.configure(variant)
client = Client(self.args.base_url)
alice, _, _, chat = client.bootstrap()
warm, _ = self.upload(self.args.warmup, alice, chat)
if not all(ok for _, ok in warm):
raise RuntimeError('Warm-up HTTP errors')
self.wait_jobs(chat['id'], self.args.warmup)
time.sleep(8) # SDK reader/processor + Collector batch settle before baseline.
ids = {}
for service in ('api', 'worker'):
for ident in self.compose_run('ps', '-q', service).split():
ids[ident] = service
self.report['container_limits'] = {service: self.run(self.docker + ['inspect', '--format', '{{.Config.Image}}|{{.Image}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}', ident]).strip() for ident, service in ids.items()}
if sum(s == 'api' for s in ids.values()) != 2 or sum(s == 'worker' for s in ids.values()) != 1:
raise RuntimeError('Benchmark requires two API replicas and one worker')
rows, errors, stop = [], [], threading.Event()
sampler = threading.Thread(target=self.sample_stats, args=(ids, stop, rows, errors))
sampler.start()
try:
time.sleep(3)
before = self.collector()
start = time.monotonic()
measurements, http_seconds = self.upload(self.args.requests, alice, chat)
if any(not ok for _, ok in measurements):
raise RuntimeError('HTTP upload errors invalidate the comparison')
self.wait_jobs(chat['id'], self.args.warmup+self.args.requests)
end = time.monotonic()
finally:
stop.set()
sampler.join(timeout=15)
if sampler.is_alive() or errors:
raise RuntimeError('docker stats sampling failed: ' + ', '.join(errors))
# Export is asynchronous. Wait for SDK batches and for backend queues to empty.
time.sleep(3)
after = self.collector()
for _ in range(10):
if after.get('otelcol_exporter_queue_size', 0) == 0:
break
time.sleep(1)
after = self.collector()
metric_seconds = time.monotonic()-start
def delta(name):
return after.get(name, 0)-before.get(name, 0)
samples = [r for r in rows if start <= r['t'] <= end]
resource = {}
for service in ('api', 'worker'):
per_container = []
for ident, kind in ids.items():
if kind != service:
continue
series = [r for r in samples if r['container'] == ident]
if len(series) < 3:
raise RuntimeError('Too few docker stats samples; use more requests')
per_container.append({'cpu_percent_mean': statistics.mean(r['cpu_percent'] for r in series),
'ram_mib_mean': statistics.mean(r['ram_mib'] for r in series),
'ram_mib_max': max(r['ram_mib'] for r in series), 'samples': len(series)})
resource[service] = {key: sum(r[key] for r in per_container)
for key in ('cpu_percent_mean', 'ram_mib_mean', 'ram_mib_max', 'samples')}
resource[service]['cpu_seconds_per_job_estimate'] = resource[service]['cpu_percent_mean']/100*(end-start)/self.args.requests
good = [seconds*1000 for seconds, ok in measurements if ok]
failures = sum(not ok for _, ok in measurements)
counters = {name: delta(name) for name in (
'otelcol_receiver_accepted_spans', 'otelcol_exporter_sent_spans',
'otelcol_receiver_refused_spans', 'otelcol_exporter_send_failed_spans',
'otelcol_receiver_accepted_log_records', 'otelcol_receiver_accepted_metric_points')}
if failures or not good or counters['otelcol_receiver_refused_spans'] or counters['otelcol_exporter_send_failed_spans']:
raise RuntimeError('HTTP or Collector failures invalidate the comparison')
if variant == 'A' and counters['otelcol_receiver_accepted_spans'] != 0:
raise RuntimeError('Telemetry Off still exported spans')
retention = counters['otelcol_exporter_sent_spans']/max(counters['otelcol_receiver_accepted_spans'], 1)
if variant == 'B' and not 0.95 <= retention <= 1.05:
raise RuntimeError('100% trace export did not match receiver ingest')
if variant == 'C' and not 0.04 <= retention <= 0.18:
raise RuntimeError('Collector sampling did not reduce export to approximately 10%')
result = {'variant': variant, 'repetition': repetition, 'completed_utc': datetime.now(timezone.utc).isoformat(),
'successful_uploads': len(good), 'http_errors': failures, 'http_seconds': http_seconds,
'pipeline_seconds': end-start, 'collector_window_seconds': metric_seconds,
'rps': len(good)/http_seconds, 'latency_mean_ms': statistics.mean(good),
'latency_p50_ms': percentile(good, .5), 'latency_p99_ms': percentile(good, .99),
'latencies_ms': good,
'resources': resource, 'collector_deltas': counters,
'ingest_spans_per_second': counters['otelcol_receiver_accepted_spans']/metric_seconds,
'tempo_spans_per_second': counters['otelcol_exporter_sent_spans']/metric_seconds,
'trace_retention': retention, 'stats_samples': [{**r, 't': r['t']-start} for r in samples]}
self.report['trials'].append(result)
self.save()
print(f'{variant}/{repetition}: {result["rps"]:.2f} RPS, p99 {result["latency_p99_ms"]:.2f} ms, '
f'accepted {counters["otelcol_receiver_accepted_spans"]:.0f}, exported {counters["otelcol_exporter_sent_spans"]:.0f}', flush=True)
def save(self):
(self.output / 'track1-benchmark.json').write_text(json.dumps(self.report, indent=2)+'\n')
def write_summary(self):
def med(variant, getter):
return statistics.median(getter(t) for t in self.report['trials'] if t['variant'] == variant)
lines = ['# Track 1: Цена наблюдаемости', '',
'Результат: **' + self.report.get('result', 'INCOMPLETE') + '**. Начало UTC: ' + self.report['started_utc'] + '.', '',
f"По {self.args.repetitions} прогона каждого варианта, {self.args.requests} POST uploads, concurrency {self.args.concurrency}, warm-up {self.args.warmup} uploads.",
'Порядок ABC / BCA / CAB. PNG: ' + str(len(self.content)) + ' bytes. Каждый job дошёл до ready; очередь между прогонами пуста.',
'Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).',
'RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.',
'CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.', '',
'| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |',
'|---|---:|---:|---:|---:|---:|---:|---:|---:|']
for v in 'ABC':
values = [med(v, lambda t: t[k]) for k in ('rps', 'latency_p50_ms', 'latency_p99_ms')]
api = [med(v, lambda t: t['resources']['api'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
worker = [med(v, lambda t: t['resources']['worker'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
lines.append(f'| {v}: {VARIANTS[v][0]} | {values[0]:.2f} | {values[1]:.2f} | {values[2]:.2f} | {api[0]:.2f}% | {api[1]:.2f} | {worker[0]:.2f}% | {worker[1]:.2f} | {med(v, lambda t:t["ingest_spans_per_second"]):.2f} |')
lines += ['', '| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |',
'|---|---:|---:|---:|---:|---:|---:|---:|']
for v in 'ABC':
lines.append(f'| {v} | {med(v, lambda t:t["latency_mean_ms"]):.2f} | {med(v, lambda t:t["pipeline_seconds"]):.2f} | {med(v, lambda t:t["collector_deltas"]["otelcol_receiver_accepted_spans"]):.0f} | {med(v, lambda t:t["collector_deltas"]["otelcol_exporter_sent_spans"]):.0f} | {med(v, lambda t:t["tempo_spans_per_second"]):.2f} | {1000*med(v, lambda t:t["resources"]["api"]["cpu_seconds_per_job_estimate"]):.2f} | {1000*med(v, lambda t:t["resources"]["worker"]["cpu_seconds_per_job_estimate"]):.2f} |')
lines += ['', '*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.',
'Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.',
'В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.', '',
'| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |',
'|---|---:|---:|---:|---:|---:|---:|']
for t in self.report['trials']:
lines.append(f'| {t["variant"]}/{t["repetition"]} | {t["rps"]:.2f} | {t["latency_mean_ms"]:.2f} | {t["latency_p99_ms"]:.2f} | {t["pipeline_seconds"]:.2f} | {t["resources"]["api"]["samples"]} / {t["resources"]["worker"]["samples"]} | {t["collector_deltas"]["otelcol_receiver_refused_spans"]:.0f} / {t["collector_deltas"]["otelcol_exporter_send_failed_spans"]:.0f} |')
if all('sdk_diagnostic_audit' in t for t in self.report['trials']):
starts = sorted({t['sdk_diagnostic_audit']['warning_error_start'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_start'] is not None})
deltas = sorted({t['sdk_diagnostic_audit']['warning_error_delta'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_delta'] is not None})
drops = self.report.get('sdk_diagnostic_audit', {}).get('span_queue_drop_max_during_experiment')
lines += ['', f'SDK audit: delta warning/error diagnostics в on-окнах: {deltas}; значения перед началом окон: {starts}. Startup/reconfiguration не входит в steady-state замер.',
f'Максимум telemetry_sdk_dropped_spans в интервале эксперимента: {drops}. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.']
probe = self.report.get('error_probe', {})
lines += ['', 'Отдельный synthetic ERROR probe (вне замеров API): ' + json.dumps(probe, ensure_ascii=False) + '.', '',
'## Воспроизведение', '', '```sh',
'python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3', '```', '',
'Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.',
'Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.',
'В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.', '',
'## Ограничения', '',
'Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.',
'Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.',
'Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.',
'Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.']
(self.output/'track1-benchmark.md').write_text('\n'.join(lines)+'\n')
def audit_sdk(self):
# Read the same externally scraped diagnostics for each trial's own SDK instances.
def query(expr, timestamp):
url = self.args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr, 'time': timestamp})
with urllib.request.urlopen(url, timeout=15) as response:
rows = json.load(response)['data']['result']
return float(rows[0]['value'][1]) if rows else None
end = 0
for trial in self.report['trials']:
ids = '|'.join(sorted({s['container'][:12] for s in trial['stats_samples']}))
end = datetime.fromisoformat(trial['completed_utc']).timestamp()
start = end-trial['collector_window_seconds']
expr = 'sum(telemetry_sdk_errors{service_instance_id=~"' + ids + '"})'
a, b = query(expr, start), query(expr, end)
trial['sdk_diagnostic_audit'] = {'source': 'Prometheus instant queries at trial start/end, restricted to trial container hostnames',
'warning_error_start': a, 'warning_error_end': b,
'warning_error_delta': b-a if a is not None and b is not None else None}
seconds = math.ceil(end-datetime.fromisoformat(self.report['started_utc']).timestamp())
self.report['sdk_diagnostic_audit'] = {
'span_queue_drop_max_during_experiment': query(f'max(max_over_time(telemetry_sdk_dropped_spans[{seconds}s]))', end),
'note': 'Warning/error diagnostics are not counts of lost spans. Compare per-trial deltas; sampling discard is sent/accepted.'}
self.save()
if any(t['sdk_diagnostic_audit']['warning_error_delta'] not in (None, 0) for t in self.report['trials']):
raise RuntimeError('SDK diagnostics increased during measured windows')
def probe_errors(self):
# Separate from measured trials: synthetic ERROR spans verify that probability sampling is not error-aware.
self.configure('C')
traces = [uuid.uuid4().hex for _ in range(100)]
now = time.time_ns()
spans = [{'traceId': ident, 'spanId': '0123456789abcdef', 'name': 'track1.synthetic-error-probe',
'kind': 2, 'startTimeUnixNano': str(now), 'endTimeUnixNano': str(now+1000000),
'status': {'code': 2}, 'attributes': [{'key': 'http.response.status_code', 'value': {'intValue': '503'}}]}
for ident in traces]
data = json.dumps({'resourceSpans': [{'resource': {'attributes': [{'key': 'service.name',
'value': {'stringValue': 'track1-sampling-probe'}}]}, 'scopeSpans': [{'spans': spans}]}]})
self.compose_run('exec', '-T', '--index', '1', 'api', 'curl', '-fsS', '--max-time', '10',
'-H', 'Content-Type: application/json', '--data-binary', '@-',
'http://otel-collector:4318/v1/traces', data=data)
time.sleep(5)
auth = base64.b64encode((os.environ.get('GRAFANA_ADMIN_USER', 'admin')+':'+(ROOT/'secrets/grafana_admin_password').read_text().strip()).encode()).decode()
def retained(ident):
request = urllib.request.Request(self.args.grafana_url+'/api/datasources/proxy/uid/tempo/api/traces/'+ident,
headers={'Authorization': 'Basic '+auth})
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status == 200
except urllib.error.HTTPError as error:
if error.code == 404:
return False
raise RuntimeError('Tempo probe query failed') from None
with ThreadPoolExecutor(max_workers=4) as pool:
found = sum(pool.map(retained, traces))
if not 0 < found < len(traces):
raise RuntimeError('Error sampling probe failed')
self.report['error_probe'] = {'type': 'synthetic ERROR spans, separate from upload benchmark',
'sent': len(traces), 'found_in_tempo': found, 'missing_in_tempo': len(traces)-found}
self.save()
print(f'Error probe: {found}/100 ERROR traces retained', flush=True)
def self_test():
assert re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', '\x1b[?25l\x1b[H{}\x1b[?25h') == '{}'
assert percentile([1, 2, 3, 4], .5) == 2
assert percentile(list(range(1, 101)), .99) == 99
assert memory_mib('1024KiB') == 1
assert memory_mib('1GiB') == 1024
assert abs(memory_mib('1048576B')-1) < 1e-9
assert parse_metrics('# comment\nx{a="b"} 2\nx{a="c"} 3\ny 4\n') == {'x': 5, 'y': 4}
base = (ROOT/'infra/otel/collector.yaml').read_text()
sampled = (ROOT/'infra/otel/collector-sampling10.yaml').read_text()
expected = base.replace(' batch:\n', ' probabilistic_sampler/track1:\n hash_seed: 22\n sampling_percentage: 10\n batch:\n', 1)
expected = expected.replace('processors: [memory_limiter, batch]\n exporters: [otlphttp/tempo]',
'processors: [memory_limiter, probabilistic_sampler/track1, batch]\n exporters: [otlphttp/tempo]')
assert sampled.split('\n', 1)[1] == expected, 'Sampling must be the only Collector configuration difference'
assert 'sampling_percentage: 10' in sampled
assert sampled.count('processors: [memory_limiter, probabilistic_sampler/track1, batch]') == 1
assert sampled.count('processors: [memory_limiter, batch]') == base.count('processors: [memory_limiter, batch]')-1
print('PASS: percentile, docker memory units, metric sums, trace-only sampler configuration')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--requests', type=int, default=1000)
parser.add_argument('--concurrency', type=int, default=4)
parser.add_argument('--repetitions', type=int, default=3)
parser.add_argument('--warmup', type=int, default=20)
parser.add_argument('--drain-timeout', type=int, default=600)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--compose', default=os.environ.get('COMPOSE', 'docker compose'))
parser.add_argument('--docker', default=os.environ.get('DOCKER', 'docker'))
parser.add_argument('--output', default='evidence')
parser.add_argument('--self-test', action='store_true')
args = parser.parse_args()
if args.self_test:
return self_test()
if not (args.requests >= 100 and 1 <= args.concurrency <= 16 and 1 <= args.repetitions <= 10 and args.warmup >= args.concurrency):
parser.error('requests >=100, concurrency 1..16, repetitions 1..10, warmup >= concurrency required')
def interrupted(signum, frame):
raise KeyboardInterrupt('Benchmark interrupted; restoring baseline')
signal.signal(signal.SIGTERM, interrupted)
bench = Bench(args)
try:
for repetition in range(args.repetitions):
order = 'ABC'[repetition % 3:] + 'ABC'[:repetition % 3]
for variant in order:
bench.trial(variant, repetition+1)
bench.audit_sdk()
bench.probe_errors()
bench.report['result'] = 'PASS'
finally:
print('Restoring SDK on / Collector 100%', flush=True)
bench.configure('B')
bench.report['restored'] = 'SDK enabled, default Collector configuration'
bench.save()
bench.write_summary()
print('PASS: all trials complete; raw sanitized evidence saved', flush=True)
if __name__ == '__main__':
main()