forked from mtusi2026/mtusi2026lab6
223 lines
11 KiB
Python
223 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Трек 1 lab6: цена наблюдаемости.
|
||
|
||
Конфигурации (одинаковая нагрузка, одинаковые лимиты контейнеров):
|
||
off OTEL_SDK_DISABLED=true — SDK не экспортирует ничего (stdout-логи остаются)
|
||
full трассы 100% (parentbased_always_on), Collector пропускает всё
|
||
head10 head sampling в SDK: parentbased_traceidratio 0.1
|
||
tail SDK 100% → Collector tail_sampling: ошибки + медленные (≥250 ms) + 10% остальных
|
||
(infra/otel/collector-tail.yaml) — предлагаемая сбалансированная конфигурация
|
||
|
||
Прогон: пересоздание api/relay/worker/collector с конфигурацией → прогрев 10 s →
|
||
45 s closed-loop POST сообщений (4 клиента) с двумя внедрёнными инцидентами
|
||
(LOCK TABLE messages на 6.5 s: часть запросов падает по таймауту 5 s = ошибки,
|
||
часть ждёт = редкие медленные). Каждый запрос несёт X-Request-Id; после прогона по
|
||
Tempo проверяется, сохранилась ли трасса каждой ошибки, каждого медленного и
|
||
выборки обычных запросов. Ресурсы — cgroup контейнеров (cpu.stat, memory.peak),
|
||
объём ingest — счётчики Collector/Tempo/Loki/Prometheus. 3 повтора, порядок
|
||
конфигураций чередуется.
|
||
"""
|
||
import json
|
||
import random
|
||
import statistics
|
||
import subprocess
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
import labutil # noqa: F401 (добавляет tests/ в sys.path)
|
||
from client import Client, arguments, check, run
|
||
from labutil import ROOT, compose
|
||
from loadgen import Load
|
||
import obsutil as o
|
||
|
||
CONFIGS = {
|
||
'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER': 'parentbased_always_on', 'OTEL_TRACES_SAMPLER_ARG': '1.0',
|
||
'COLLECTOR_CONFIG': 'collector.yaml'},
|
||
'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER': 'parentbased_always_on', 'OTEL_TRACES_SAMPLER_ARG': '1.0',
|
||
'COLLECTOR_CONFIG': 'collector.yaml'},
|
||
'head10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER': 'parentbased_traceidratio', 'OTEL_TRACES_SAMPLER_ARG': '0.1',
|
||
'COLLECTOR_CONFIG': 'collector.yaml'},
|
||
'tail': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER': 'parentbased_always_on', 'OTEL_TRACES_SAMPLER_ARG': '1.0',
|
||
'COLLECTOR_CONFIG': 'collector-tail.yaml'},
|
||
}
|
||
COUNTERS = {
|
||
'spans_received': 'sum({__name__=~"otelcol_receiver_accepted_spans(_total)?"})',
|
||
'spans_exported': 'sum({__name__=~"otelcol_exporter_sent_spans(_total)?"})',
|
||
'log_records_received': 'sum({__name__=~"otelcol_receiver_accepted_log_records(_total)?"})',
|
||
'metric_points_received': 'sum({__name__=~"otelcol_receiver_accepted_metric_points(_total)?"})',
|
||
'tempo_bytes': 'sum(tempo_distributor_bytes_received_total)',
|
||
'loki_bytes': 'sum(loki_distributor_bytes_received_total)',
|
||
'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)',
|
||
}
|
||
DURATION, WARMUP, CONC = 45, 10, 4
|
||
LOCKS = (12, 28) # секунда прогона, когда начинается блокировка
|
||
LOCK_SECONDS = 6.5
|
||
SLOW = 0.25
|
||
|
||
|
||
def apply(name):
|
||
env = dict(CONFIGS[name])
|
||
compose('up', '-d', '--no-deps', '--wait', 'otel-collector', 'api', 'relay', 'worker', env=env, timeout=300)
|
||
|
||
|
||
def wait_api(c, timeout=120):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
try:
|
||
c.call('GET', '/health/ready')
|
||
return
|
||
except Exception:
|
||
time.sleep(1)
|
||
raise RuntimeError('api not ready')
|
||
|
||
|
||
def counters():
|
||
return {k: o.prom_value(q, default=0.0) for k, q in COUNTERS.items()}
|
||
|
||
|
||
def containers(service):
|
||
return compose('ps', '-q', service).stdout.split()
|
||
|
||
|
||
def lock(seconds):
|
||
return subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c',
|
||
'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%s); COMMIT;' % seconds],
|
||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
|
||
|
||
def traced(rid, since):
|
||
return bool(o.tempo_search('{span.request_id="%s"}' % rid, since, limit=1))
|
||
|
||
|
||
def one_run(c, name, actor, chat, rnd):
|
||
apply(name)
|
||
wait_api(c)
|
||
warm =Load(c.base, None, actor, chat, concurrency=CONC, timeout=10, prefix='warm-%s' % name).start()
|
||
time.sleep(WARMUP)
|
||
warm.stop()
|
||
time.sleep(6)
|
||
api = containers('api')
|
||
col = containers('otel-collector')
|
||
cpu0 = {cid: o.cpu_usec(cid) for cid in api + col}
|
||
c0 = counters()
|
||
t_start = time.time()
|
||
load = Load(c.base, None, actor, chat, concurrency=CONC, timeout=10, prefix='tc-%s-%d' % (name, rnd)).start()
|
||
windows, procs = [], []
|
||
for at in LOCKS:
|
||
time.sleep(max(0, t_start + at - time.time()))
|
||
procs.append(lock(LOCK_SECONDS))
|
||
windows.append((time.time(), time.time() + LOCK_SECONDS + 5.5))
|
||
time.sleep(max(0, t_start + DURATION - time.time()))
|
||
load.stop()
|
||
for p in procs:
|
||
p.wait(timeout=60)
|
||
elapsed = time.time() - t_start
|
||
cpu1 = {cid: o.cpu_usec(cid) for cid in api + col}
|
||
time.sleep(20) # tail decision_wait 10 s + batch + экспорт метрик + scrape
|
||
c1 = counters()
|
||
s = load.summary(exclude=windows)
|
||
recs = load.records
|
||
errors = [r for r in recs if r[3] != 201]
|
||
slow = [r for r in recs if r[3] == 201 and r[2] >= SLOW]
|
||
normal = [r for r in recs if r[3] == 201 and r[2] < SLOW]
|
||
random.seed(rnd * 7 + len(name))
|
||
sample = random.sample(normal, min(40, len(normal)))
|
||
kept = {}
|
||
if name != 'off':
|
||
for label, group in (('errors', errors[:40]), ('slow', slow[:40]), ('normal_sample', sample)):
|
||
kept[label] = [sum(traced(r[0], t_start - 5) for r in group), len(group)]
|
||
else:
|
||
kept = {'errors': [0, len(errors)], 'slow': [0, len(slow)], 'normal_sample': [0, len(sample)]}
|
||
|
||
def cores(ids):
|
||
tot = sum((cpu1[i] - cpu0[i]) for i in ids if cpu0.get(i) is not None and cpu1.get(i) is not None)
|
||
return round(tot / 1e6 / elapsed, 3)
|
||
ok = s['ok'] or 1
|
||
delta = {k: round(c1[k] - c0[k]) for k in COUNTERS}
|
||
return {
|
||
'config': name, 'round': rnd, 'load': s, 'slow_requests': len(slow), 'error_requests': len(errors),
|
||
'traces_kept': kept,
|
||
'api_cpu_cores': cores(api), 'api_cpu_ms_per_request': round(sum(
|
||
(cpu1[i] - cpu0[i]) for i in api if cpu0.get(i) is not None) / 1e3 / ok, 3),
|
||
'api_mem_peak_mib': [round((o.mem_bytes(i, 'memory.peak') or 0) / 2**20, 1) for i in api],
|
||
'api_mem_now_mib': [round((o.mem_bytes(i) or 0) / 2**20, 1) for i in api],
|
||
'collector_cpu_cores': cores(col),
|
||
'collector_mem_peak_mib': [round((o.mem_bytes(i, 'memory.peak') or 0) / 2**20, 1) for i in col],
|
||
'ingest_delta': delta,
|
||
'ingest_per_1000_requests': {k: round(v * 1000 / ok, 1) for k, v in delta.items()},
|
||
}
|
||
|
||
|
||
def aggregate(runs):
|
||
out = {}
|
||
for name in CONFIGS:
|
||
rs = [r for r in runs if r['config'] == name]
|
||
if not rs:
|
||
continue
|
||
med = lambda f: round(statistics.median(f(r) for r in rs), 3)
|
||
|
||
def kept(label):
|
||
k = sum(r['traces_kept'][label][0] for r in rs)
|
||
n = sum(r['traces_kept'][label][1] for r in rs)
|
||
return {'kept': k, 'total': n, 'ratio': round(k / n, 3) if n else None}
|
||
out[name] = {
|
||
'runs': len(rs),
|
||
'ok_rps_median': med(lambda r: r['load']['ok_rps']),
|
||
'p50_ms_median': med(lambda r: r['load']['latency_ms_outside_faults']['p50']),
|
||
'p95_ms_median': med(lambda r: r['load']['latency_ms_outside_faults']['p95']),
|
||
'p99_ms_median': med(lambda r: r['load']['latency_ms_outside_faults']['p99']),
|
||
'errors_total': sum(r['error_requests'] for r in rs),
|
||
'slow_total': sum(r['slow_requests'] for r in rs),
|
||
'api_cpu_cores_median': med(lambda r: r['api_cpu_cores']),
|
||
'api_cpu_ms_per_request_median': med(lambda r: r['api_cpu_ms_per_request']),
|
||
'api_mem_peak_mib_median': med(lambda r: max(r['api_mem_peak_mib'] or [0])),
|
||
'collector_cpu_cores_median': med(lambda r: r['collector_cpu_cores']),
|
||
'collector_mem_peak_mib_median': med(lambda r: max(r['collector_mem_peak_mib'] or [0])),
|
||
'spans_received_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['spans_received']),
|
||
'spans_exported_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['spans_exported']),
|
||
'tempo_kib_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['tempo_bytes'] / 1024),
|
||
'loki_kib_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['loki_bytes'] / 1024),
|
||
'log_records_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['log_records_received']),
|
||
'metric_points_per_1000_req': med(lambda r: r['ingest_per_1000_requests']['metric_points_received']),
|
||
'traces_kept_errors': kept('errors'), 'traces_kept_slow': kept('slow'),
|
||
'traces_kept_normal_sample': kept('normal_sample'),
|
||
}
|
||
return out
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('--rounds', type=int, default=3)
|
||
p.add_argument('--output', default=None)
|
||
args = arguments('Lab6 track 1: cost of observability', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||
o.wait_grafana()
|
||
alice, bob, _, chat = c.bootstrap()
|
||
order = list(CONFIGS)
|
||
runs = []
|
||
try:
|
||
for rnd in range(args.rounds):
|
||
seq = order[rnd % len(order):] + order[:rnd % len(order)]
|
||
for name in seq:
|
||
r = one_run(c, name, alice, chat, rnd)
|
||
runs.append(r)
|
||
print(json.dumps({k: r[k] for k in ('config', 'round', 'traces_kept', 'api_cpu_cores')}, ensure_ascii=False),
|
||
json.dumps(r['load']['latency_ms_outside_faults']), flush=True)
|
||
finally:
|
||
apply('full')
|
||
result = {'method': {'duration_s': DURATION, 'warmup_s': WARMUP, 'concurrency': CONC, 'locks_at_s': LOCKS,
|
||
'lock_seconds': LOCK_SECONDS, 'slow_threshold_s': SLOW, 'rounds': args.rounds,
|
||
'handler_timeout_s': 5},
|
||
'summary': aggregate(runs), 'runs': runs}
|
||
text = json.dumps(result, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
Path(args.output).write_text(text + '\n', encoding='utf-8')
|
||
print(json.dumps(result['summary'], ensure_ascii=False, indent=2))
|
||
check(len(runs) == args.rounds * len(CONFIGS), 'not all runs completed')
|
||
print('PASS: telemetry cost experiment')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|