forked from mtusi2026/mtusi2026lab6
218 lines
11 KiB
Python
218 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Трек 1 (цена наблюдаемости), lab6.
|
|
|
|
Профили при ОДИНАКОВОЙ нагрузке (8 процессов-клиентов, 64 одновременных, 90 % POST сообщений / 10 % GET, 30 с после прогрева):
|
|
off — OTEL_ENABLED=0 (SDK не создаётся, экспорта нет)
|
|
on100 — телеметрия включена, head-sampling 100 % трасс
|
|
head10 — head-sampling 10 % (ParentBased + TraceIdRatio в SDK)
|
|
tail — SDK отправляет 100 %, Collector оставляет errors + медленные (>500 мс) + 10 % остальных (tail_sampling)
|
|
Во время замера внедряются отказы: 2 % запросов дают 500 и 2 % задерживаются на 0.6 с (одинаково для всех профилей) — это «редкие
|
|
ошибки и медленные запросы», судьбу которых в Tempo мы сравниваем.
|
|
Метрики: RPS, p50/p95/p99, CPU/RSS API и Collector (docker stats), объём телеметрии (принятые Collector спаны/логи/точки метрик),
|
|
потери (отказы приёма Collector), размер хранилищ Tempo/Loki, доля сохранённых ошибочных и медленных трасс.
|
|
python research/obs_cost.py --out evidence
|
|
"""
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import multiprocessing as mp
|
|
import os
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
import loadgen # noqa: E402
|
|
|
|
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
|
BASE = "http://127.0.0.1:8080"
|
|
PROFILES = {
|
|
"off": {"OTEL_ENABLED": "0", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"},
|
|
"on100": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"},
|
|
"head10": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"},
|
|
"tail": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector-tail.yaml"},
|
|
# то же, что on100, но большая очередь/пакет экспорта SDK: проверка гипотезы «часть спанов теряется очередью SDK»
|
|
"on100_bigq": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml",
|
|
"OTEL_QUEUE_SIZE": "65536", "OTEL_BATCH_SIZE": "2048"},
|
|
}
|
|
|
|
|
|
def sh(*args, env=None):
|
|
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
|
|
|
|
|
|
def dc(*args, env=None):
|
|
return sh("docker", "compose", "--env-file", ".env", *args, env=env)
|
|
|
|
|
|
def net_get(url):
|
|
return dc("exec", "-T", "prometheus", "wget", "-q", "-O", "-", url)
|
|
|
|
|
|
def prom_value(query):
|
|
try:
|
|
r = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query)))["data"]["result"]
|
|
return float(r[0]["value"][1]) if r else None
|
|
except (ValueError, KeyError):
|
|
return None
|
|
|
|
|
|
def apply_profile(profile, faults=True):
|
|
env = dict(PROFILES[profile], API_FAULT_INJECTION="http500:0.02,slow:0.02:0.6" if faults else "")
|
|
dc("up", "-d", "--no-deps", "--force-recreate", "otel-collector", env=env)
|
|
dc("up", "-d", "--no-deps", "--force-recreate", "api", "worker", env=env)
|
|
deadline = time.time() + 90
|
|
while time.time() < deadline:
|
|
ids = dc("ps", "-q", "api").split()
|
|
if len(ids) == 2 and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids):
|
|
break
|
|
time.sleep(1)
|
|
time.sleep(8)
|
|
|
|
|
|
def child(args):
|
|
stand_data, seconds, concurrency, seed, t0 = args
|
|
import random
|
|
random.seed(seed)
|
|
stand = loadgen.Stand(BASE)
|
|
stand.users, stand.chats = stand_data["users"], stand_data["chats"]
|
|
rec = loadgen.Recorder()
|
|
rec.t0 = t0
|
|
|
|
async def go():
|
|
await loadgen.run_load(stand, rec, seconds, concurrency)
|
|
asyncio.run(go())
|
|
return rec.events
|
|
|
|
|
|
class Sampler(threading.Thread):
|
|
def __init__(self):
|
|
super().__init__(daemon=True)
|
|
self.rows, self.stop = [], threading.Event()
|
|
|
|
def run(self):
|
|
while not self.stop.is_set():
|
|
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.CPUPerc}} {{.MemUsage}}"], capture_output=True, text=True).stdout.splitlines()
|
|
self.rows.append((time.time(), [l.split() for l in out if "mtusi-lab6" in l]))
|
|
|
|
def summary(self, t0, t1):
|
|
agg = {}
|
|
for ts, rows in self.rows:
|
|
if not t0 <= ts <= t1:
|
|
continue
|
|
for r in rows:
|
|
svc = r[0].replace("mtusi-lab6-", "").rsplit("-", 1)[0]
|
|
mem = float(r[2].replace("MiB", "").replace("GiB", "")) * (1024 if "GiB" in r[2] else 1)
|
|
agg.setdefault(svc, {"cpu": [], "mem": []})
|
|
agg[svc]["cpu"].append(float(r[1].rstrip("%")))
|
|
agg[svc]["mem"].append(mem)
|
|
return {k: {"avg_cpu_percent": round(statistics.mean(v["cpu"]), 1), "peak_mem_mib": round(max(v["mem"]), 1)} for k, v in agg.items()}
|
|
|
|
|
|
def volume_bytes(volume):
|
|
out = sh("docker", "run", "--rm", "-v", "%s:/v:ro" % volume, "alpine:3.20", "du", "-sk", "/v")
|
|
try:
|
|
return int(out.split()[0]) * 1024
|
|
except (ValueError, IndexError):
|
|
return None
|
|
|
|
|
|
def counter(name):
|
|
return prom_value("sum(%s)" % name) or 0.0
|
|
|
|
|
|
def tempo_count(traceql, since, limit=5000):
|
|
url = "http://tempo:3200/api/search?q=%s&limit=%d&start=%d&end=%d" % (urllib.parse.quote(traceql), limit, int(since), int(time.time()) + 5)
|
|
try:
|
|
return len(json.loads(net_get(url)).get("traces", []))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def run_profile(profile, args, stand_data):
|
|
apply_profile(profile)
|
|
# прогрев
|
|
t0 = time.time()
|
|
with mp.get_context("spawn").Pool(args.procs) as pool:
|
|
pool.map(child, [(stand_data, 5, args.concurrency // args.procs, i, t0) for i in range(args.procs)])
|
|
time.sleep(3)
|
|
before = {k: counter(k) for k in ("otelcol_receiver_accepted_spans", "otelcol_receiver_accepted_log_records",
|
|
"otelcol_receiver_accepted_metric_points", "otelcol_receiver_refused_spans",
|
|
"otelcol_exporter_sent_spans", "otelcol_exporter_send_failed_spans")}
|
|
vol_before = {"tempo": volume_bytes("mtusi-lab6_tempodata"), "loki": volume_bytes("mtusi-lab6_lokidata")}
|
|
sampler = Sampler()
|
|
sampler.start()
|
|
t_start = time.time()
|
|
with mp.get_context("spawn").Pool(args.procs) as pool:
|
|
results = pool.map(child, [(stand_data, args.seconds, args.concurrency // args.procs, 100 + i, t_start) for i in range(args.procs)])
|
|
t_end = time.time()
|
|
sampler.stop.set()
|
|
rec = loadgen.Recorder()
|
|
rec.t0 = t_start
|
|
for events in results:
|
|
rec.events.extend(events)
|
|
post = loadgen.summarize(rec, 0, args.seconds, "POST")
|
|
time.sleep(args.flush) # дать SDK/Collector/хранилищам дописать данные
|
|
after = {k: counter(k) for k in before}
|
|
vol_after = {"tempo": volume_bytes("mtusi-lab6_tempodata"), "loki": volume_bytes("mtusi-lab6_lokidata")}
|
|
# «внедрённые» отказы считаем на стороне клиента (счётчики Prometheus нестабильны при пересоздании реплик):
|
|
# ответы 500 и ответы медленнее 500 мс (внедряется задержка 0.6 с; фоновые запросы быстрее)
|
|
posts = [e for e in rec.events if e[1] == "POST"]
|
|
injected_err = sum(1 for e in posts if e[2] == 500)
|
|
injected_slow = sum(1 for e in posts if e[2] == 201 and e[3] > 500)
|
|
kept_err = tempo_count('{ status = error && resource.service.name = "messenger-api" && name =~ "HTTP.*" }', t_start - 5)
|
|
kept_slow = tempo_count('{ resource.service.name = "messenger-api" && name =~ "HTTP.*" && duration > 500ms }', t_start - 5)
|
|
kept_all = tempo_count('{ resource.service.name = "messenger-api" && name =~ "HTTP.*" }', t_start - 5)
|
|
out = {"profile": profile, "post": post, "docker_stats": sampler.summary(t_start + 2, t_end),
|
|
"collector_delta": {k.replace("otelcol_", ""): round(after[k] - before[k]) for k in before},
|
|
"tempo_bytes_delta": (vol_after["tempo"] - vol_before["tempo"]) if vol_after["tempo"] and vol_before["tempo"] else None,
|
|
"loki_bytes_delta": (vol_after["loki"] - vol_before["loki"]) if vol_after["loki"] and vol_before["loki"] else None,
|
|
"errors_injected_500": injected_err, "slow_requests_gt500ms": injected_slow,
|
|
"traces_in_tempo": {"all_http": kept_all, "error": kept_err, "slow_gt500ms": kept_slow},
|
|
"kept_ratio": {"errors": round(kept_err / injected_err, 3) if injected_err and kept_err is not None else None,
|
|
"slow": round(kept_slow / injected_slow, 3) if injected_slow and kept_slow is not None else None}}
|
|
print(json.dumps(out, ensure_ascii=False))
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--out", default="evidence")
|
|
ap.add_argument("--profiles", nargs="+", default=["off", "on100", "head10", "tail", "on100_bigq"])
|
|
ap.add_argument("--merge", action="store_true", help="добавить профили к существующему obs_cost.json")
|
|
ap.add_argument("--seconds", type=int, default=30)
|
|
ap.add_argument("--concurrency", type=int, default=64)
|
|
ap.add_argument("--procs", type=int, default=8)
|
|
ap.add_argument("--flush", type=int, default=25)
|
|
ap.add_argument("--repeat", type=int, default=2)
|
|
args = ap.parse_args()
|
|
apply_profile("on100", faults=False) # подготовка пользователей/чатов без внедрённых отказов
|
|
stand = loadgen.Stand(BASE, users=40, chats=40)
|
|
asyncio.run(stand.setup())
|
|
stand_data = {"users": stand.users, "chats": stand.chats}
|
|
results = []
|
|
for profile in args.profiles:
|
|
runs = [run_profile(profile, args, stand_data) for _ in range(args.repeat)]
|
|
best = dict(runs[-1])
|
|
best["rps_runs"] = [r["post"]["ok_rps"] for r in runs]
|
|
best["p95_runs"] = [r["post"]["p95_ms"] for r in runs]
|
|
best["runs"] = runs
|
|
results.append(best)
|
|
path = os.path.join(args.out, "obs_cost.json")
|
|
if args.merge and os.path.exists(path):
|
|
old = json.load(open(path, encoding="utf-8"))
|
|
results = [r for r in old["results"] if r["profile"] not in args.profiles] + results
|
|
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "faults": "http500:0.02,slow:0.02:0.6", "results": results}
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(report, fh, ensure_ascii=False, indent=2)
|
|
apply_profile("on100", faults=False)
|
|
print("saved")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
mp.freeze_support()
|
|
main()
|