Files
mtusi2026lab6/scripts/observability_experiment.py
2026-11-18 21:37:25 +03:00

190 lines
8.9 KiB
Python

#!/usr/bin/env python3
"""Track 1: fixed-rate, equal-sized trials with measured sampling retention."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import datetime
import json
import math
import os
import subprocess
import sys
import threading
import time
import uuid
from observability_common import ROOT, compose, get_json, prom, ready, save
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
def percentile(values, p):
return round(sorted(values)[max(0, math.ceil(len(values) * p) - 1)] * 1000, 3) if values else None
def snapshot():
queries = {
'exported_spans': 'sum(otelcol_exporter_sent_spans_total)',
'failed_spans': 'sum(otelcol_exporter_send_failed_spans_total)',
'accepted_spans': 'sum(otelcol_receiver_accepted_spans_total)',
'refused_spans': 'sum(otelcol_receiver_refused_spans_total)',
'tempo_ingested_bytes': 'sum(tempo_distributor_bytes_received_total)',
'loki_ingested_bytes': 'sum(loki_distributor_bytes_received_total)',
'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)',
}
result = {}
for key, query in queries.items():
rows = prom(query)
result[key] = sum(float(x['value'][1]) for x in rows) if rows else None
return result
def samples(stop, output):
while not stop.is_set():
result = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{json .}}'], capture_output=True, text=True)
for line in result.stdout.splitlines():
row = json.loads(line)
if row['Name'].startswith('mtusi2026lab6-'):
output.append({'at': time.time(), **{k: row[k] for k in ('Name', 'CPUPerc', 'MemUsage')}})
stop.wait(1)
def configure(disabled, ratio):
# Compose interpolation only; no secrets or .env files are read.
os.environ['OTEL_SDK_DISABLED'] = str(disabled).lower()
os.environ['OTEL_TRACES_SAMPLER_ARG'] = str(ratio)
compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker')
ready()
time.sleep(8)
def trial(name, disabled, ratio, args, repetition):
configure(disabled, ratio)
c = Client(args.base_url)
actor = c.user('research')
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Synthetic research', 'member_ids': [actor['id']]}, schema='Chat')['json']
# Equal warmup before every trial, including telemetry off.
for _ in range(20):
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
time.sleep(6)
before = snapshot()
stop, resource = threading.Event(), []
sampler = threading.Thread(target=samples, args=(stop, resource), daemon=True)
sampler.start()
known, latency, errors, uploads = [], [], [], []
lock = threading.Lock()
started = time.monotonic()
def request(index):
scheduled = started + index / args.rate
time.sleep(max(0, scheduled - time.monotonic()))
client = Client(args.base_url)
trace_id = None
headers = {}
begin = time.monotonic()
outcome, kind = 'ok', 'read'
try:
if index % 20 == 0:
kind = 'upload'
headers['Content-Type'] = 'image/png'
response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png',
202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
headers=headers, schema='Attachment')
att = response['json']
with lock:
uploads.append(att)
elif index % 20 == 1:
kind = 'client_error'
response = client.call('GET', '/api/v1/users/me', 401, headers=headers)
elif index % 5 == 0:
kind = 'write'
headers['Idempotency-Key'] = uuid.uuid4().hex
response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=actor,
body={'text': 'synthetic fixed-size benchmark message'}, headers=headers, schema='Message')
else:
response = client.call('GET', '/api/v1/users/me', actor=actor, headers=headers, schema='User')
trace_id = response['headers'].get('X-Trace-Id')
except Exception as exc:
outcome = type(exc).__name__
with lock:
elapsed_request = time.monotonic() - begin
latency.append(elapsed_request)
known.append({'trace_id': trace_id, 'kind': kind, 'outcome': outcome, 'elapsed_seconds': elapsed_request})
if outcome != 'ok':
errors.append(outcome)
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
list(pool.map(request, range(args.requests)))
elapsed = time.monotonic() - started
for att in uploads:
c.wait_ready(actor, att)
stop.set()
sampler.join(timeout=10)
# Flush SDK batches, Collector and backend scrape before taking counters.
time.sleep(12)
after = snapshot()
retained = {kind: {'sent': 0, 'found': 0} for kind in ('read', 'write', 'upload', 'client_error')}
slow = {'definition': 'slowest 5 percent of this trial by measured client latency', 'sent': 0, 'found': 0}
slow_ids = {id(x) for x in sorted(known, key=lambda x: x['elapsed_seconds'], reverse=True)[:max(1, math.ceil(.05*len(known)))]}
for row in known:
retained[row['kind']]['sent'] += 1
is_slow = id(row) in slow_ids
slow['sent'] += int(is_slow)
try:
trace = get_json('http://127.0.0.1:13200/api/traces/' + (row['trace_id'] or 'missing'))
if trace.get('batches') or trace.get('resourceSpans'):
retained[row['kind']]['found'] += 1
slow['found'] += int(is_slow)
except Exception:
pass
return {'configuration': name, 'repetition': repetition, 'disabled': disabled, 'sampling_ratio': ratio,
'requests': args.requests, 'target_rps': args.rate, 'concurrency_limit': args.concurrency,
'elapsed_seconds': round(elapsed, 3), 'successful_rps': round((args.requests-len(errors))/elapsed, 3),
'unexpected_errors': len(errors), 'error_types': errors,
'all_request_latency_ms': {f'p{int(p*100)}': percentile(latency, p) for p in (.5, .95, .99)},
'ready_uploads': len(uploads), 'counter_before': before, 'counter_after': after,
'counter_delta': {k: after[k]-before[k] if after[k] is not None and before[k] is not None else None for k in before},
'trace_retention': retained, 'slow_request_trace_retention': slow, 'resources': resource}
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--base-url', default='http://127.0.0.1:18086')
p.add_argument('--requests', type=int, default=120)
p.add_argument('--rate', type=float, default=8)
p.add_argument('--concurrency', type=int, default=4)
p.add_argument('--repetitions', type=int, default=2)
args = p.parse_args()
if args.requests < 20 or args.rate <= 0 or args.concurrency < 1 or args.repetitions < 1:
p.error('requests>=20, rate>0, concurrency>=1, repetitions>=1 required')
result = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'method': 'Fixed-rate schedule, fixed request count/mix; 20 warmups; SDK and backend flush; sampled IDs queried from Tempo.',
'trials': []}
try:
configs = [('off', True, 1), ('full', False, 1), ('ten_percent', False, .1), ('balanced', False, .5)]
for rep in range(args.repetitions):
# Counterbalance order on second repetition.
for name, disabled, ratio in (configs if rep % 2 == 0 else list(reversed(configs))):
print('Running', name, 'repetition', rep + 1, flush=True)
row = trial(name, disabled, ratio, args, rep + 1)
result['trials'].append(row)
save('evidence/lab6-research.json', result)
print(json.dumps({k: row[k] for k in ('configuration', 'unexpected_errors', 'all_request_latency_ms', 'trace_retention')}), flush=True)
assert row['unexpected_errors'] == 0, 'Unexpected business errors during research'
found = sum(x['found'] for x in row['trace_retention'].values())
if name == 'off':
assert found == 0, 'SDK off still exported request traces'
elif name == 'full':
assert found == args.requests, 'Full sampling lost request traces'
else:
assert 0 < found < args.requests, 'Sampling setting had no observed effect'
result['status'] = 'PASS'
finally:
configure(False, 1)
result['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
save('evidence/lab6-research.json', result)
if __name__ == '__main__':
main()