forked from mtusi2026/mtusi2026lab6
190 lines
8.9 KiB
Python
190 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 1: fixed-rate, equal-sized trials with measured sampling retention."""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import datetime
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
from observability_common import ROOT, compose, get_json, prom, ready, save
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
|
|
def percentile(values, p):
|
|
return round(sorted(values)[max(0, math.ceil(len(values) * p) - 1)] * 1000, 3) if values else None
|
|
|
|
|
|
def snapshot():
|
|
queries = {
|
|
'exported_spans': 'sum(otelcol_exporter_sent_spans_total)',
|
|
'failed_spans': 'sum(otelcol_exporter_send_failed_spans_total)',
|
|
'accepted_spans': 'sum(otelcol_receiver_accepted_spans_total)',
|
|
'refused_spans': 'sum(otelcol_receiver_refused_spans_total)',
|
|
'tempo_ingested_bytes': 'sum(tempo_distributor_bytes_received_total)',
|
|
'loki_ingested_bytes': 'sum(loki_distributor_bytes_received_total)',
|
|
'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)',
|
|
}
|
|
result = {}
|
|
for key, query in queries.items():
|
|
rows = prom(query)
|
|
result[key] = sum(float(x['value'][1]) for x in rows) if rows else None
|
|
return result
|
|
|
|
|
|
def samples(stop, output):
|
|
while not stop.is_set():
|
|
result = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{json .}}'], capture_output=True, text=True)
|
|
for line in result.stdout.splitlines():
|
|
row = json.loads(line)
|
|
if row['Name'].startswith('mtusi2026lab6-'):
|
|
output.append({'at': time.time(), **{k: row[k] for k in ('Name', 'CPUPerc', 'MemUsage')}})
|
|
stop.wait(1)
|
|
|
|
|
|
def configure(disabled, ratio):
|
|
# Compose interpolation only; no secrets or .env files are read.
|
|
os.environ['OTEL_SDK_DISABLED'] = str(disabled).lower()
|
|
os.environ['OTEL_TRACES_SAMPLER_ARG'] = str(ratio)
|
|
compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker')
|
|
ready()
|
|
time.sleep(8)
|
|
|
|
|
|
def trial(name, disabled, ratio, args, repetition):
|
|
configure(disabled, ratio)
|
|
c = Client(args.base_url)
|
|
actor = c.user('research')
|
|
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Synthetic research', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
# Equal warmup before every trial, including telemetry off.
|
|
for _ in range(20):
|
|
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
time.sleep(6)
|
|
before = snapshot()
|
|
stop, resource = threading.Event(), []
|
|
sampler = threading.Thread(target=samples, args=(stop, resource), daemon=True)
|
|
sampler.start()
|
|
known, latency, errors, uploads = [], [], [], []
|
|
lock = threading.Lock()
|
|
started = time.monotonic()
|
|
|
|
def request(index):
|
|
scheduled = started + index / args.rate
|
|
time.sleep(max(0, scheduled - time.monotonic()))
|
|
client = Client(args.base_url)
|
|
trace_id = None
|
|
headers = {}
|
|
begin = time.monotonic()
|
|
outcome, kind = 'ok', 'read'
|
|
try:
|
|
if index % 20 == 0:
|
|
kind = 'upload'
|
|
headers['Content-Type'] = 'image/png'
|
|
response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png',
|
|
202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
|
|
headers=headers, schema='Attachment')
|
|
att = response['json']
|
|
with lock:
|
|
uploads.append(att)
|
|
elif index % 20 == 1:
|
|
kind = 'client_error'
|
|
response = client.call('GET', '/api/v1/users/me', 401, headers=headers)
|
|
elif index % 5 == 0:
|
|
kind = 'write'
|
|
headers['Idempotency-Key'] = uuid.uuid4().hex
|
|
response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=actor,
|
|
body={'text': 'synthetic fixed-size benchmark message'}, headers=headers, schema='Message')
|
|
else:
|
|
response = client.call('GET', '/api/v1/users/me', actor=actor, headers=headers, schema='User')
|
|
trace_id = response['headers'].get('X-Trace-Id')
|
|
except Exception as exc:
|
|
outcome = type(exc).__name__
|
|
with lock:
|
|
elapsed_request = time.monotonic() - begin
|
|
latency.append(elapsed_request)
|
|
known.append({'trace_id': trace_id, 'kind': kind, 'outcome': outcome, 'elapsed_seconds': elapsed_request})
|
|
if outcome != 'ok':
|
|
errors.append(outcome)
|
|
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
list(pool.map(request, range(args.requests)))
|
|
elapsed = time.monotonic() - started
|
|
for att in uploads:
|
|
c.wait_ready(actor, att)
|
|
stop.set()
|
|
sampler.join(timeout=10)
|
|
# Flush SDK batches, Collector and backend scrape before taking counters.
|
|
time.sleep(12)
|
|
after = snapshot()
|
|
retained = {kind: {'sent': 0, 'found': 0} for kind in ('read', 'write', 'upload', 'client_error')}
|
|
slow = {'definition': 'slowest 5 percent of this trial by measured client latency', 'sent': 0, 'found': 0}
|
|
slow_ids = {id(x) for x in sorted(known, key=lambda x: x['elapsed_seconds'], reverse=True)[:max(1, math.ceil(.05*len(known)))]}
|
|
for row in known:
|
|
retained[row['kind']]['sent'] += 1
|
|
is_slow = id(row) in slow_ids
|
|
slow['sent'] += int(is_slow)
|
|
try:
|
|
trace = get_json('http://127.0.0.1:13200/api/traces/' + (row['trace_id'] or 'missing'))
|
|
if trace.get('batches') or trace.get('resourceSpans'):
|
|
retained[row['kind']]['found'] += 1
|
|
slow['found'] += int(is_slow)
|
|
except Exception:
|
|
pass
|
|
return {'configuration': name, 'repetition': repetition, 'disabled': disabled, 'sampling_ratio': ratio,
|
|
'requests': args.requests, 'target_rps': args.rate, 'concurrency_limit': args.concurrency,
|
|
'elapsed_seconds': round(elapsed, 3), 'successful_rps': round((args.requests-len(errors))/elapsed, 3),
|
|
'unexpected_errors': len(errors), 'error_types': errors,
|
|
'all_request_latency_ms': {f'p{int(p*100)}': percentile(latency, p) for p in (.5, .95, .99)},
|
|
'ready_uploads': len(uploads), 'counter_before': before, 'counter_after': after,
|
|
'counter_delta': {k: after[k]-before[k] if after[k] is not None and before[k] is not None else None for k in before},
|
|
'trace_retention': retained, 'slow_request_trace_retention': slow, 'resources': resource}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--base-url', default='http://127.0.0.1:18086')
|
|
p.add_argument('--requests', type=int, default=120)
|
|
p.add_argument('--rate', type=float, default=8)
|
|
p.add_argument('--concurrency', type=int, default=4)
|
|
p.add_argument('--repetitions', type=int, default=2)
|
|
args = p.parse_args()
|
|
if args.requests < 20 or args.rate <= 0 or args.concurrency < 1 or args.repetitions < 1:
|
|
p.error('requests>=20, rate>0, concurrency>=1, repetitions>=1 required')
|
|
result = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
'method': 'Fixed-rate schedule, fixed request count/mix; 20 warmups; SDK and backend flush; sampled IDs queried from Tempo.',
|
|
'trials': []}
|
|
try:
|
|
configs = [('off', True, 1), ('full', False, 1), ('ten_percent', False, .1), ('balanced', False, .5)]
|
|
for rep in range(args.repetitions):
|
|
# Counterbalance order on second repetition.
|
|
for name, disabled, ratio in (configs if rep % 2 == 0 else list(reversed(configs))):
|
|
print('Running', name, 'repetition', rep + 1, flush=True)
|
|
row = trial(name, disabled, ratio, args, rep + 1)
|
|
result['trials'].append(row)
|
|
save('evidence/lab6-research.json', result)
|
|
print(json.dumps({k: row[k] for k in ('configuration', 'unexpected_errors', 'all_request_latency_ms', 'trace_retention')}), flush=True)
|
|
assert row['unexpected_errors'] == 0, 'Unexpected business errors during research'
|
|
found = sum(x['found'] for x in row['trace_retention'].values())
|
|
if name == 'off':
|
|
assert found == 0, 'SDK off still exported request traces'
|
|
elif name == 'full':
|
|
assert found == args.requests, 'Full sampling lost request traces'
|
|
else:
|
|
assert 0 < found < args.requests, 'Sampling setting had no observed effect'
|
|
result['status'] = 'PASS'
|
|
finally:
|
|
configure(False, 1)
|
|
result['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
save('evidence/lab6-research.json', result)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|