Files
mtusi2026lab6/student-tests/telemetry_cost.py
T

223 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 1 lab6: цена наблюдаемости.
Конфигурации (одинаковая нагрузка, одинаковые лимиты контейнеров):
off OTEL_SDK_DISABLED=true — SDK не экспортирует ничего (stdout-логи остаются)
full трассы 100% (parentbased_always_on), Collector пропускает всё
head10 head sampling в SDK: parentbased_traceidratio 0.1
tail SDK 100% → Collector tail_sampling: ошибки + медленные (≥250 ms) + 10% остальных
(infra/otel/collector-tail.yaml) — предлагаемая сбалансированная конфигурация
Прогон: пересоздание api/relay/worker/collector с конфигурацией → прогрев 10 s →
45 s closed-loop POST сообщений (4 клиента) с двумя внедрёнными инцидентами
(LOCK TABLE messages на 6.5 s: часть запросов падает по таймауту 5 s = ошибки,
часть ждёт = редкие медленные). Каждый запрос несёт X-Request-Id; после прогона по
Tempo проверяется, сохранилась ли трасса каждой ошибки, каждого медленного и
выборки обычных запросов. Ресурсы — cgroup контейнеров (cpu.stat, memory.peak),
объём ingest — счётчики Collector/Tempo/Loki/Prometheus. 3 повтора, порядок
конфигураций чередуется.
"""
import json
import random
import statistics
import subprocess
import time
import uuid
from pathlib import Path
import labutil # noqa: F401 (добавляет tests/ в sys.path)
from client import Client, arguments, check, run
from labutil import ROOT, compose
from loadgen import Load
import obsutil as o
CONFIGS = {
'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER': 'parentbased_always_on', 'OTEL_TRACES_SAMPLER_ARG': '1.0',
'COLLECTOR_CONFIG': 'collector.yaml'},
'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER': 'parentbased_always_on', 'OTEL_TRACES_SAMPLER_ARG': '1.0',
'COLLECTOR_CONFIG': 'collector.yaml'},
'head10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER': 'parentbased_traceidratio', 'OTEL_TRACES_SAMPLER_ARG': '0.1',
'COLLECTOR_CONFIG': 'collector.yaml'},
'tail': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER': 'parentbased_always_on', 'OTEL_TRACES_SAMPLER_ARG': '1.0',
'COLLECTOR_CONFIG': 'collector-tail.yaml'},
}
COUNTERS = {
'spans_received': 'sum({__name__=~"otelcol_receiver_accepted_spans(_total)?"})',
'spans_exported': 'sum({__name__=~"otelcol_exporter_sent_spans(_total)?"})',
'log_records_received': 'sum({__name__=~"otelcol_receiver_accepted_log_records(_total)?"})',
'metric_points_received': 'sum({__name__=~"otelcol_receiver_accepted_metric_points(_total)?"})',
'tempo_bytes': 'sum(tempo_distributor_bytes_received_total)',
'loki_bytes': 'sum(loki_distributor_bytes_received_total)',
'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)',
}
DURATION, WARMUP, CONC = 45, 10, 4
LOCKS = (12, 28) # секунда прогона, когда начинается блокировка
LOCK_SECONDS = 6.5
SLOW = 0.25
def apply(name):
env = dict(CONFIGS[name])
compose('up', '-d', '--no-deps', '--wait', 'otel-collector', 'api', 'relay', 'worker', env=env, timeout=300)
def wait_api(c, timeout=120):
end = time.time() + timeout
while time.time() < end:
try:
c.call('GET', '/health/ready')
return
except Exception:
time.sleep(1)
raise RuntimeError('api not ready')
def counters():
return {k: o.prom_value(q, default=0.0) for k, q in COUNTERS.items()}
def containers(service):
return compose('ps', '-q', service).stdout.split()
def lock(seconds):
return subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c',
'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%s); COMMIT;' % seconds],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def traced(rid, since):
return bool(o.tempo_search('{span.request_id="%s"}' % rid, since, limit=1))
def one_run(c, name, actor, chat, rnd):
apply(name)
wait_api(c)
warm =Load(c.base, None, actor, chat, concurrency=CONC, timeout=10, prefix='warm-%s' % name).start()
time.sleep(WARMUP)
warm.stop()
time.sleep(6)
api = containers('api')
col = containers('otel-collector')
cpu0 = {cid: o.cpu_usec(cid) for cid in api + col}
c0 = counters()
t_start = time.time()
load = Load(c.base, None, actor, chat, concurrency=CONC, timeout=10, prefix='tc-%s-%d' % (name, rnd)).start()
windows, procs = [], []
for at in LOCKS:
time.sleep(max(0, t_start + at - time.time()))
procs.append(lock(LOCK_SECONDS))
windows.append((time.time(), time.time() + LOCK_SECONDS + 5.5))
time.sleep(max(0, t_start + DURATION - time.time()))
load.stop()
for p in procs:
p.wait(timeout=60)
elapsed = time.time() - t_start
cpu1 = {cid: o.cpu_usec(cid) for cid in api + col}
time.sleep(20) # tail decision_wait 10 s + batch + экспорт метрик + scrape
c1 = counters()
s = load.summary(exclude=windows)
recs = load.records
errors = [r for r in recs if r[3] != 201]
slow = [r for r in recs if r[3] == 201 and r[2] >= SLOW]
normal = [r for r in recs if r[3] == 201 and r[2] < SLOW]
random.seed(rnd * 7 + len(name))
sample = random.sample(normal, min(40, len(normal)))
kept = {}
if name != 'off':
for label, group in (('errors', errors[:40]), ('slow', slow[:40]), ('normal_sample', sample)):
kept[label] = [sum(traced(r[0], t_start - 5) for r in group), len(group)]
else:
kept = {'errors': [0, len(errors)], 'slow': [0, len(slow)], 'normal_sample': [0, len(sample)]}
def cores(ids):
tot = sum((cpu1[i] - cpu0[i]) for i in ids if cpu0.get(i) is not None and cpu1.get(i) is not None)
return round(tot / 1e6 / elapsed, 3)
ok = s['ok'] or 1
delta = {k: round(c1[k] - c0[k]) for k in COUNTERS}
return {
'config': name, 'round': rnd, 'load': s, 'slow_requests': len(slow), 'error_requests': len(errors),
'traces_kept': kept,
'api_cpu_cores': cores(api), 'api_cpu_ms_per_request': round(sum(
(cpu1[i] - cpu0[i]) for i in api if cpu0.get(i) is not None) / 1e3 / ok, 3),
'api_mem_peak_mib': [round((o.mem_bytes(i, 'memory.peak') or 0) / 2**20, 1) for i in api],
'api_mem_now_mib': [round((o.mem_bytes(i) or 0) / 2**20, 1) for i in api],
'collector_cpu_cores': cores(col),
'collector_mem_peak_mib': [round((o.mem_bytes(i, 'memory.peak') or 0) / 2**20, 1) for i in col],
'ingest_delta': delta,
'ingest_per_1000_requests': {k: round(v * 1000 / ok, 1) for k, v in delta.items()},
}
def aggregate(runs):
out = {}
for name in CONFIGS:
rs = [r for r in runs if r['config'] == name]
if not rs:
continue
med = lambda f: round(statistics.median(f(r) for r in rs), 3)
def kept(label):
k = sum(r['traces_kept'][label][0] for r in rs)
n = sum(r['traces_kept'][label][1] for r in rs)
return {'kept': k, 'total': n, 'ratio': round(k / n, 3) if n else None}
out[name] = {
'runs': len(rs),
'ok_rps_median': med(lambda r: r['load']['ok_rps']),
'p50_ms_median': med(lambda r: r['load']['latency_ms_outside_faults']['p50']),
'p95_ms_median': med(lambda r: r['load']['latency_ms_outside_faults']['p95']),
'p99_ms_median': med(lambda r: r['load']['latency_ms_outside_faults']['p99']),
'errors_total': sum(r['error_requests'] for r in rs),
'slow_total': sum(r['slow_requests'] for r in rs),
'api_cpu_cores_median': med(lambda r: r['api_cpu_cores']),
'api_cpu_ms_per_request_median': med(lambda r: r['api_cpu_ms_per_request']),
'api_mem_peak_mib_median': med(lambda r: max(r['api_mem_peak_mib'] or [0])),
'collector_cpu_cores_median': med(lambda r: r['collector_cpu_cores']),
'collector_mem_peak_mib_median': med(lambda r: max(r['collector_mem_peak_mib'] or [0])),
'spans_received_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['spans_received']),
'spans_exported_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['spans_exported']),
'tempo_kib_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['tempo_bytes'] / 1024),
'loki_kib_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['loki_bytes'] / 1024),
'log_records_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['log_records_received']),
'metric_points_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['metric_points_received']),
'traces_kept_errors': kept('errors'), 'traces_kept_slow': kept('slow'),
'traces_kept_normal_sample': kept('normal_sample'),
}
return out
def main():
def configure(p):
p.add_argument('--rounds', type=int, default=3)
p.add_argument('--output', default=None)
args = arguments('Lab6 track 1: cost of observability', configure)
c = Client(args.base_url, args.ca_file, timeout=30)
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
order = list(CONFIGS)
runs = []
try:
for rnd in range(args.rounds):
seq = order[rnd % len(order):] + order[:rnd % len(order)]
for name in seq:
r = one_run(c, name, alice, chat, rnd)
runs.append(r)
print(json.dumps({k: r[k] for k in ('config', 'round', 'traces_kept', 'api_cpu_cores')}, ensure_ascii=False),
json.dumps(r['load']['latency_ms_outside_faults']), flush=True)
finally:
apply('full')
result = {'method': {'duration_s': DURATION, 'warmup_s': WARMUP, 'concurrency': CONC, 'locks_at_s': LOCKS,
'lock_seconds': LOCK_SECONDS, 'slow_threshold_s': SLOW, 'rounds': args.rounds,
'handler_timeout_s': 5},
'summary': aggregate(runs), 'runs': runs}
text = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + '\n', encoding='utf-8')
print(json.dumps(result['summary'], ensure_ascii=False, indent=2))
check(len(runs) == args.rounds * len(CONFIGS), 'not all runs completed')
print('PASS: telemetry cost experiment')
if __name__ == '__main__':
run(main)