forked from mtusi2026/mtusi2026lab6
462 lines
30 KiB
Python
462 lines
30 KiB
Python
#!/usr/bin/env python3
|
||
"""Track 1: fixed-count attachment benchmark; stdlib only, no credentials in evidence.
|
||
|
||
Creates synthetic durable data. Never deletes volumes or changes business code.
|
||
Always restores SDK on / Collector 100% when the experiment finishes or fails.
|
||
"""
|
||
import argparse
|
||
import base64
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from datetime import datetime, timezone
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import os
|
||
from pathlib import Path
|
||
import platform
|
||
import re
|
||
import shlex
|
||
import signal
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / 'tests'))
|
||
from client import Client, Failure
|
||
|
||
VARIANTS = {'A': ('Off', True, 'collector.yaml'),
|
||
'B': ('On, SDK 100% / Collector 100%', False, 'collector.yaml'),
|
||
'C': ('On, SDK 100% / Collector 10%', False, 'collector-sampling10.yaml')}
|
||
|
||
|
||
def percentile(values, quantile):
|
||
return sorted(values)[max(0, math.ceil(len(values) * quantile) - 1)]
|
||
|
||
|
||
def memory_mib(text):
|
||
number, unit = re.fullmatch(r'([\d.]+)\s*([kKMGT]?i?B)', text.strip()).groups()
|
||
factors = {'B': 1, 'kB': 1000, 'MB': 1000**2, 'GB': 1000**3, 'TB': 1000**4,
|
||
'KiB': 1024, 'MiB': 1024**2, 'GiB': 1024**3, 'TiB': 1024**4}
|
||
return float(number) * factors[unit] / 1024**2
|
||
|
||
|
||
def parse_metrics(text):
|
||
result = {}
|
||
for line in text.splitlines():
|
||
if not line or line.startswith('#'):
|
||
continue
|
||
name = line.split('{', 1)[0].split(' ', 1)[0]
|
||
result[name] = result.get(name, 0) + float(line.rsplit(' ', 1)[1])
|
||
return result
|
||
|
||
|
||
class Bench:
|
||
def __init__(self, args):
|
||
self.args = args
|
||
self.compose = shlex.split(args.compose)
|
||
self.docker = shlex.split(args.docker)
|
||
self.output = ROOT / args.output
|
||
self.output.mkdir(parents=True, exist_ok=True)
|
||
self.content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||
self.report = {'started_utc': datetime.now(timezone.utc).isoformat(),
|
||
'method': {'requests': args.requests, 'concurrency': args.concurrency,
|
||
'repetitions': args.repetitions, 'warmup': args.warmup,
|
||
'fixture_bytes': len(self.content),
|
||
'fixture_sha256': hashlib.sha256(self.content).hexdigest(),
|
||
'stats_source': 'docker stats stream; 100% CPU = one logical core',
|
||
'cpu_ram_window': 'uploads plus complete worker drain, excluding final export wait',
|
||
'percentile': 'nearest rank; HTTP upload acceptance, client wall clock',
|
||
'config_sha256': {name: hashlib.sha256((ROOT/'infra/otel'/name).read_bytes()).hexdigest() for name in ('collector.yaml', 'collector-sampling10.yaml')},
|
||
'order': 'ABC / BCA / CAB (rotated each repetition)'},
|
||
'host': {'machine': platform.machine(), 'logical_cpus': os.cpu_count(),
|
||
'cpu_model': next((line.split(':',1)[1].strip() for line in Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'unknown'),
|
||
'memory_kib': int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')))},
|
||
'trials': []}
|
||
|
||
def run(self, cmd, *, env=None, data=None):
|
||
completed = subprocess.run(cmd, cwd=ROOT, env=env, input=data, text=True,
|
||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||
if completed.returncode:
|
||
# Subprocess output can contain connection credentials; do not forward it.
|
||
raise RuntimeError('Command failed: ' + Path(cmd[0]).name + ' ' + cmd[1] +
|
||
' (exit ' + str(completed.returncode) + ')')
|
||
return completed.stdout
|
||
|
||
def compose_run(self, *args, **kwargs):
|
||
return self.run(self.compose + list(args), **kwargs)
|
||
|
||
def sql(self, sql):
|
||
return self.compose_run('exec', '-T', 'db', 'sh', '-c',
|
||
'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tA', data=sql+'\n').strip()
|
||
|
||
def wait_jobs(self, chat=None, expected=None):
|
||
deadline = time.monotonic() + self.args.drain_timeout
|
||
while time.monotonic() < deadline:
|
||
if chat:
|
||
uuid.UUID(chat) # Only synthetic UUIDs may enter the query.
|
||
row = self.sql("SELECT count(*),count(*) FILTER (WHERE j.status='ready'),"
|
||
"count(*) FILTER (WHERE j.status='failed') FROM jobs j JOIN attachments a "
|
||
"ON a.id=j.attachment_id WHERE a.chat_id='" + chat + "'::uuid;")
|
||
total, ready, failed = map(int, row.split('|'))
|
||
if failed:
|
||
raise RuntimeError('Worker produced failed benchmark jobs')
|
||
if total == expected and ready == expected:
|
||
return
|
||
elif int(self.sql("SELECT count(*) FROM jobs WHERE status IN ('queued','processing');")) == 0:
|
||
return
|
||
time.sleep(1)
|
||
raise RuntimeError('Worker backlog did not drain; evidence is incomplete')
|
||
|
||
def configure(self, variant):
|
||
_, disabled, filename = VARIANTS[variant]
|
||
env = dict(os.environ, OTEL_SDK_DISABLED=str(disabled).lower(),
|
||
OTEL_COLLECTOR_CONFIG=str(ROOT / 'infra/otel' / filename))
|
||
self.compose_run('config', '--quiet', env=env)
|
||
self.compose_run('up', '-d', '--no-deps', '--force-recreate', '--wait',
|
||
'--wait-timeout', '120', 'otel-collector', 'api', 'worker', env=env)
|
||
self.compose_run('exec', '-T', 'otel-collector', '/otelcol-contrib', 'validate',
|
||
'--config=/etc/otelcol/config.yaml', env=env)
|
||
deadline = time.monotonic() + 60
|
||
while True:
|
||
try:
|
||
Client(self.args.base_url).call('GET', '/health/ready', schema='Health')
|
||
self.collector()
|
||
return
|
||
except (Failure, RuntimeError):
|
||
if time.monotonic() >= deadline:
|
||
raise RuntimeError('API/Collector did not become ready') from None
|
||
time.sleep(1)
|
||
|
||
def collector(self):
|
||
return parse_metrics(self.compose_run('exec', '-T', '--index', '1', 'api',
|
||
'curl', '-fsS', '--max-time', '5', 'http://otel-collector:8888/metrics'))
|
||
|
||
def upload(self, count, actor, chat):
|
||
barrier = threading.Barrier(self.args.concurrency)
|
||
def work(slot):
|
||
client = Client(self.args.base_url, timeout=60)
|
||
results = []
|
||
barrier.wait()
|
||
for index in range(slot, count, self.args.concurrency):
|
||
started = time.monotonic()
|
||
try:
|
||
# No supplied traceparent: the SDK's AlwaysOn root policy is exercised.
|
||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
||
expected=202, actor=actor, raw=self.content,
|
||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||
results.append((time.monotonic()-started, True))
|
||
except Failure:
|
||
results.append((time.monotonic()-started, False))
|
||
return results
|
||
start = time.monotonic()
|
||
with ThreadPoolExecutor(max_workers=self.args.concurrency) as pool:
|
||
rows = [r for batch in pool.map(work, range(self.args.concurrency)) for r in batch]
|
||
return rows, time.monotonic()-start
|
||
|
||
def sample_stats(self, ids, stop, rows, errors):
|
||
process = subprocess.Popen(self.docker + ['stats', '--format', '{{json .}}'] + list(ids),
|
||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
|
||
def terminate():
|
||
stop.wait()
|
||
process.terminate()
|
||
closer = threading.Thread(target=terminate, daemon=True)
|
||
closer.start()
|
||
try:
|
||
for line in process.stdout:
|
||
clean = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', line).strip()
|
||
if not clean:
|
||
continue
|
||
if stop.is_set() and not clean.endswith('}'):
|
||
continue
|
||
obj = json.loads(clean)
|
||
ident = obj['ID']
|
||
if ident not in ids:
|
||
ident = next((key for key in ids if key.startswith(ident)), '')
|
||
if ident:
|
||
rows.append({'t': time.monotonic(), 'container': ident, 'service': ids[ident],
|
||
'cpu_percent': float(obj['CPUPerc'].rstrip('%')),
|
||
'ram_mib': memory_mib(obj['MemUsage'].split('/')[0])})
|
||
if not stop.is_set():
|
||
errors.append('docker stats stopped unexpectedly')
|
||
except Exception as error:
|
||
errors.append(type(error).__name__ + ': ' + str(error))
|
||
finally:
|
||
stop.set()
|
||
process.terminate()
|
||
process.wait(timeout=10)
|
||
closer.join(timeout=10)
|
||
|
||
def trial(self, variant, repetition):
|
||
print(f'Preparing {variant}, repetition {repetition}', flush=True)
|
||
self.wait_jobs()
|
||
self.configure(variant)
|
||
client = Client(self.args.base_url)
|
||
alice, _, _, chat = client.bootstrap()
|
||
warm, _ = self.upload(self.args.warmup, alice, chat)
|
||
if not all(ok for _, ok in warm):
|
||
raise RuntimeError('Warm-up HTTP errors')
|
||
self.wait_jobs(chat['id'], self.args.warmup)
|
||
time.sleep(8) # SDK reader/processor + Collector batch settle before baseline.
|
||
ids = {}
|
||
for service in ('api', 'worker'):
|
||
for ident in self.compose_run('ps', '-q', service).split():
|
||
ids[ident] = service
|
||
self.report['container_limits'] = {service: self.run(self.docker + ['inspect', '--format', '{{.Config.Image}}|{{.Image}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}', ident]).strip() for ident, service in ids.items()}
|
||
if sum(s == 'api' for s in ids.values()) != 2 or sum(s == 'worker' for s in ids.values()) != 1:
|
||
raise RuntimeError('Benchmark requires two API replicas and one worker')
|
||
rows, errors, stop = [], [], threading.Event()
|
||
sampler = threading.Thread(target=self.sample_stats, args=(ids, stop, rows, errors))
|
||
sampler.start()
|
||
try:
|
||
time.sleep(3)
|
||
before = self.collector()
|
||
start = time.monotonic()
|
||
measurements, http_seconds = self.upload(self.args.requests, alice, chat)
|
||
if any(not ok for _, ok in measurements):
|
||
raise RuntimeError('HTTP upload errors invalidate the comparison')
|
||
self.wait_jobs(chat['id'], self.args.warmup+self.args.requests)
|
||
end = time.monotonic()
|
||
finally:
|
||
stop.set()
|
||
sampler.join(timeout=15)
|
||
if sampler.is_alive() or errors:
|
||
raise RuntimeError('docker stats sampling failed: ' + ', '.join(errors))
|
||
# Export is asynchronous. Wait for SDK batches and for backend queues to empty.
|
||
time.sleep(3)
|
||
after = self.collector()
|
||
for _ in range(10):
|
||
if after.get('otelcol_exporter_queue_size', 0) == 0:
|
||
break
|
||
time.sleep(1)
|
||
after = self.collector()
|
||
metric_seconds = time.monotonic()-start
|
||
def delta(name):
|
||
return after.get(name, 0)-before.get(name, 0)
|
||
samples = [r for r in rows if start <= r['t'] <= end]
|
||
resource = {}
|
||
for service in ('api', 'worker'):
|
||
per_container = []
|
||
for ident, kind in ids.items():
|
||
if kind != service:
|
||
continue
|
||
series = [r for r in samples if r['container'] == ident]
|
||
if len(series) < 3:
|
||
raise RuntimeError('Too few docker stats samples; use more requests')
|
||
per_container.append({'cpu_percent_mean': statistics.mean(r['cpu_percent'] for r in series),
|
||
'ram_mib_mean': statistics.mean(r['ram_mib'] for r in series),
|
||
'ram_mib_max': max(r['ram_mib'] for r in series), 'samples': len(series)})
|
||
resource[service] = {key: sum(r[key] for r in per_container)
|
||
for key in ('cpu_percent_mean', 'ram_mib_mean', 'ram_mib_max', 'samples')}
|
||
resource[service]['cpu_seconds_per_job_estimate'] = resource[service]['cpu_percent_mean']/100*(end-start)/self.args.requests
|
||
good = [seconds*1000 for seconds, ok in measurements if ok]
|
||
failures = sum(not ok for _, ok in measurements)
|
||
counters = {name: delta(name) for name in (
|
||
'otelcol_receiver_accepted_spans', 'otelcol_exporter_sent_spans',
|
||
'otelcol_receiver_refused_spans', 'otelcol_exporter_send_failed_spans',
|
||
'otelcol_receiver_accepted_log_records', 'otelcol_receiver_accepted_metric_points')}
|
||
if failures or not good or counters['otelcol_receiver_refused_spans'] or counters['otelcol_exporter_send_failed_spans']:
|
||
raise RuntimeError('HTTP or Collector failures invalidate the comparison')
|
||
if variant == 'A' and counters['otelcol_receiver_accepted_spans'] != 0:
|
||
raise RuntimeError('Telemetry Off still exported spans')
|
||
retention = counters['otelcol_exporter_sent_spans']/max(counters['otelcol_receiver_accepted_spans'], 1)
|
||
if variant == 'B' and not 0.95 <= retention <= 1.05:
|
||
raise RuntimeError('100% trace export did not match receiver ingest')
|
||
if variant == 'C' and not 0.04 <= retention <= 0.18:
|
||
raise RuntimeError('Collector sampling did not reduce export to approximately 10%')
|
||
result = {'variant': variant, 'repetition': repetition, 'completed_utc': datetime.now(timezone.utc).isoformat(),
|
||
'successful_uploads': len(good), 'http_errors': failures, 'http_seconds': http_seconds,
|
||
'pipeline_seconds': end-start, 'collector_window_seconds': metric_seconds,
|
||
'rps': len(good)/http_seconds, 'latency_mean_ms': statistics.mean(good),
|
||
'latency_p50_ms': percentile(good, .5), 'latency_p99_ms': percentile(good, .99),
|
||
'latencies_ms': good,
|
||
'resources': resource, 'collector_deltas': counters,
|
||
'ingest_spans_per_second': counters['otelcol_receiver_accepted_spans']/metric_seconds,
|
||
'tempo_spans_per_second': counters['otelcol_exporter_sent_spans']/metric_seconds,
|
||
'trace_retention': retention, 'stats_samples': [{**r, 't': r['t']-start} for r in samples]}
|
||
self.report['trials'].append(result)
|
||
self.save()
|
||
print(f'{variant}/{repetition}: {result["rps"]:.2f} RPS, p99 {result["latency_p99_ms"]:.2f} ms, '
|
||
f'accepted {counters["otelcol_receiver_accepted_spans"]:.0f}, exported {counters["otelcol_exporter_sent_spans"]:.0f}', flush=True)
|
||
|
||
def save(self):
|
||
(self.output / 'track1-benchmark.json').write_text(json.dumps(self.report, indent=2)+'\n')
|
||
|
||
def write_summary(self):
|
||
def med(variant, getter):
|
||
return statistics.median(getter(t) for t in self.report['trials'] if t['variant'] == variant)
|
||
lines = ['# Track 1: Цена наблюдаемости', '',
|
||
'Результат: **' + self.report.get('result', 'INCOMPLETE') + '**. Начало UTC: ' + self.report['started_utc'] + '.', '',
|
||
f"По {self.args.repetitions} прогона каждого варианта, {self.args.requests} POST uploads, concurrency {self.args.concurrency}, warm-up {self.args.warmup} uploads.",
|
||
'Порядок ABC / BCA / CAB. PNG: ' + str(len(self.content)) + ' bytes. Каждый job дошёл до ready; очередь между прогонами пуста.',
|
||
'Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).',
|
||
'RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.',
|
||
'CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.', '',
|
||
'| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |',
|
||
'|---|---:|---:|---:|---:|---:|---:|---:|---:|']
|
||
for v in 'ABC':
|
||
values = [med(v, lambda t: t[k]) for k in ('rps', 'latency_p50_ms', 'latency_p99_ms')]
|
||
api = [med(v, lambda t: t['resources']['api'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
|
||
worker = [med(v, lambda t: t['resources']['worker'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
|
||
lines.append(f'| {v}: {VARIANTS[v][0]} | {values[0]:.2f} | {values[1]:.2f} | {values[2]:.2f} | {api[0]:.2f}% | {api[1]:.2f} | {worker[0]:.2f}% | {worker[1]:.2f} | {med(v, lambda t:t["ingest_spans_per_second"]):.2f} |')
|
||
lines += ['', '| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |',
|
||
'|---|---:|---:|---:|---:|---:|---:|---:|']
|
||
for v in 'ABC':
|
||
lines.append(f'| {v} | {med(v, lambda t:t["latency_mean_ms"]):.2f} | {med(v, lambda t:t["pipeline_seconds"]):.2f} | {med(v, lambda t:t["collector_deltas"]["otelcol_receiver_accepted_spans"]):.0f} | {med(v, lambda t:t["collector_deltas"]["otelcol_exporter_sent_spans"]):.0f} | {med(v, lambda t:t["tempo_spans_per_second"]):.2f} | {1000*med(v, lambda t:t["resources"]["api"]["cpu_seconds_per_job_estimate"]):.2f} | {1000*med(v, lambda t:t["resources"]["worker"]["cpu_seconds_per_job_estimate"]):.2f} |')
|
||
lines += ['', '*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.',
|
||
'Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.',
|
||
'В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.', '',
|
||
'| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |',
|
||
'|---|---:|---:|---:|---:|---:|---:|']
|
||
for t in self.report['trials']:
|
||
lines.append(f'| {t["variant"]}/{t["repetition"]} | {t["rps"]:.2f} | {t["latency_mean_ms"]:.2f} | {t["latency_p99_ms"]:.2f} | {t["pipeline_seconds"]:.2f} | {t["resources"]["api"]["samples"]} / {t["resources"]["worker"]["samples"]} | {t["collector_deltas"]["otelcol_receiver_refused_spans"]:.0f} / {t["collector_deltas"]["otelcol_exporter_send_failed_spans"]:.0f} |')
|
||
if all('sdk_diagnostic_audit' in t for t in self.report['trials']):
|
||
starts = sorted({t['sdk_diagnostic_audit']['warning_error_start'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_start'] is not None})
|
||
deltas = sorted({t['sdk_diagnostic_audit']['warning_error_delta'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_delta'] is not None})
|
||
drops = self.report.get('sdk_diagnostic_audit', {}).get('span_queue_drop_max_during_experiment')
|
||
lines += ['', f'SDK audit: delta warning/error diagnostics в on-окнах: {deltas}; значения перед началом окон: {starts}. Startup/reconfiguration не входит в steady-state замер.',
|
||
f'Максимум telemetry_sdk_dropped_spans в интервале эксперимента: {drops}. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.']
|
||
probe = self.report.get('error_probe', {})
|
||
lines += ['', 'Отдельный synthetic ERROR probe (вне замеров API): ' + json.dumps(probe, ensure_ascii=False) + '.', '',
|
||
'## Воспроизведение', '', '```sh',
|
||
'python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3', '```', '',
|
||
'Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.',
|
||
'Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.',
|
||
'В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.', '',
|
||
'## Ограничения', '',
|
||
'Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.',
|
||
'Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.',
|
||
'Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.',
|
||
'Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.']
|
||
(self.output/'track1-benchmark.md').write_text('\n'.join(lines)+'\n')
|
||
|
||
def audit_sdk(self):
|
||
# Read the same externally scraped diagnostics for each trial's own SDK instances.
|
||
def query(expr, timestamp):
|
||
url = self.args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr, 'time': timestamp})
|
||
with urllib.request.urlopen(url, timeout=15) as response:
|
||
rows = json.load(response)['data']['result']
|
||
return float(rows[0]['value'][1]) if rows else None
|
||
end = 0
|
||
for trial in self.report['trials']:
|
||
ids = '|'.join(sorted({s['container'][:12] for s in trial['stats_samples']}))
|
||
end = datetime.fromisoformat(trial['completed_utc']).timestamp()
|
||
start = end-trial['collector_window_seconds']
|
||
expr = 'sum(telemetry_sdk_errors{service_instance_id=~"' + ids + '"})'
|
||
a, b = query(expr, start), query(expr, end)
|
||
trial['sdk_diagnostic_audit'] = {'source': 'Prometheus instant queries at trial start/end, restricted to trial container hostnames',
|
||
'warning_error_start': a, 'warning_error_end': b,
|
||
'warning_error_delta': b-a if a is not None and b is not None else None}
|
||
seconds = math.ceil(end-datetime.fromisoformat(self.report['started_utc']).timestamp())
|
||
self.report['sdk_diagnostic_audit'] = {
|
||
'span_queue_drop_max_during_experiment': query(f'max(max_over_time(telemetry_sdk_dropped_spans[{seconds}s]))', end),
|
||
'note': 'Warning/error diagnostics are not counts of lost spans. Compare per-trial deltas; sampling discard is sent/accepted.'}
|
||
self.save()
|
||
if any(t['sdk_diagnostic_audit']['warning_error_delta'] not in (None, 0) for t in self.report['trials']):
|
||
raise RuntimeError('SDK diagnostics increased during measured windows')
|
||
|
||
def probe_errors(self):
|
||
# Separate from measured trials: synthetic ERROR spans verify that probability sampling is not error-aware.
|
||
self.configure('C')
|
||
traces = [uuid.uuid4().hex for _ in range(100)]
|
||
now = time.time_ns()
|
||
spans = [{'traceId': ident, 'spanId': '0123456789abcdef', 'name': 'track1.synthetic-error-probe',
|
||
'kind': 2, 'startTimeUnixNano': str(now), 'endTimeUnixNano': str(now+1000000),
|
||
'status': {'code': 2}, 'attributes': [{'key': 'http.response.status_code', 'value': {'intValue': '503'}}]}
|
||
for ident in traces]
|
||
data = json.dumps({'resourceSpans': [{'resource': {'attributes': [{'key': 'service.name',
|
||
'value': {'stringValue': 'track1-sampling-probe'}}]}, 'scopeSpans': [{'spans': spans}]}]})
|
||
self.compose_run('exec', '-T', '--index', '1', 'api', 'curl', '-fsS', '--max-time', '10',
|
||
'-H', 'Content-Type: application/json', '--data-binary', '@-',
|
||
'http://otel-collector:4318/v1/traces', data=data)
|
||
time.sleep(5)
|
||
auth = base64.b64encode((os.environ.get('GRAFANA_ADMIN_USER', 'admin')+':'+(ROOT/'secrets/grafana_admin_password').read_text().strip()).encode()).decode()
|
||
def retained(ident):
|
||
request = urllib.request.Request(self.args.grafana_url+'/api/datasources/proxy/uid/tempo/api/traces/'+ident,
|
||
headers={'Authorization': 'Basic '+auth})
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=15) as response:
|
||
return response.status == 200
|
||
except urllib.error.HTTPError as error:
|
||
if error.code == 404:
|
||
return False
|
||
raise RuntimeError('Tempo probe query failed') from None
|
||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||
found = sum(pool.map(retained, traces))
|
||
if not 0 < found < len(traces):
|
||
raise RuntimeError('Error sampling probe failed')
|
||
self.report['error_probe'] = {'type': 'synthetic ERROR spans, separate from upload benchmark',
|
||
'sent': len(traces), 'found_in_tempo': found, 'missing_in_tempo': len(traces)-found}
|
||
self.save()
|
||
print(f'Error probe: {found}/100 ERROR traces retained', flush=True)
|
||
|
||
|
||
def self_test():
|
||
assert re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', '\x1b[?25l\x1b[H{}\x1b[?25h') == '{}'
|
||
assert percentile([1, 2, 3, 4], .5) == 2
|
||
assert percentile(list(range(1, 101)), .99) == 99
|
||
assert memory_mib('1024KiB') == 1
|
||
assert memory_mib('1GiB') == 1024
|
||
assert abs(memory_mib('1048576B')-1) < 1e-9
|
||
assert parse_metrics('# comment\nx{a="b"} 2\nx{a="c"} 3\ny 4\n') == {'x': 5, 'y': 4}
|
||
base = (ROOT/'infra/otel/collector.yaml').read_text()
|
||
sampled = (ROOT/'infra/otel/collector-sampling10.yaml').read_text()
|
||
expected = base.replace(' batch:\n', ' probabilistic_sampler/track1:\n hash_seed: 22\n sampling_percentage: 10\n batch:\n', 1)
|
||
expected = expected.replace('processors: [memory_limiter, batch]\n exporters: [otlphttp/tempo]',
|
||
'processors: [memory_limiter, probabilistic_sampler/track1, batch]\n exporters: [otlphttp/tempo]')
|
||
assert sampled.split('\n', 1)[1] == expected, 'Sampling must be the only Collector configuration difference'
|
||
assert 'sampling_percentage: 10' in sampled
|
||
assert sampled.count('processors: [memory_limiter, probabilistic_sampler/track1, batch]') == 1
|
||
assert sampled.count('processors: [memory_limiter, batch]') == base.count('processors: [memory_limiter, batch]')-1
|
||
print('PASS: percentile, docker memory units, metric sums, trace-only sampler configuration')
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--requests', type=int, default=1000)
|
||
parser.add_argument('--concurrency', type=int, default=4)
|
||
parser.add_argument('--repetitions', type=int, default=3)
|
||
parser.add_argument('--warmup', type=int, default=20)
|
||
parser.add_argument('--drain-timeout', type=int, default=600)
|
||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
||
parser.add_argument('--compose', default=os.environ.get('COMPOSE', 'docker compose'))
|
||
parser.add_argument('--docker', default=os.environ.get('DOCKER', 'docker'))
|
||
parser.add_argument('--output', default='evidence')
|
||
parser.add_argument('--self-test', action='store_true')
|
||
args = parser.parse_args()
|
||
if args.self_test:
|
||
return self_test()
|
||
if not (args.requests >= 100 and 1 <= args.concurrency <= 16 and 1 <= args.repetitions <= 10 and args.warmup >= args.concurrency):
|
||
parser.error('requests >=100, concurrency 1..16, repetitions 1..10, warmup >= concurrency required')
|
||
def interrupted(signum, frame):
|
||
raise KeyboardInterrupt('Benchmark interrupted; restoring baseline')
|
||
signal.signal(signal.SIGTERM, interrupted)
|
||
bench = Bench(args)
|
||
try:
|
||
for repetition in range(args.repetitions):
|
||
order = 'ABC'[repetition % 3:] + 'ABC'[:repetition % 3]
|
||
for variant in order:
|
||
bench.trial(variant, repetition+1)
|
||
bench.audit_sdk()
|
||
bench.probe_errors()
|
||
bench.report['result'] = 'PASS'
|
||
finally:
|
||
print('Restoring SDK on / Collector 100%', flush=True)
|
||
bench.configure('B')
|
||
bench.report['restored'] = 'SDK enabled, default Collector configuration'
|
||
bench.save()
|
||
bench.write_summary()
|
||
print('PASS: all trials complete; raw sanitized evidence saved', flush=True)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|