Files
mtusi2026lab6/scripts/telemetry_cost.py
T

185 lines
5.9 KiB
Python

#!/usr/bin/env python3
"""Fixed offered-load comparison. Stops only named services in the current lab6 project."""
import argparse
import concurrent.futures
import json
import math
import os
import re
import subprocess
import time
from pathlib import Path
from urllib.request import urlopen
from observe import Client, compose, prom, until
MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25"))
COUNTERS = {
"accepted_spans": "sum(otelcol_receiver_accepted_spans_total)",
"sent_spans": "sum(otelcol_exporter_sent_spans_total)",
"failed_spans": "sum(otelcol_exporter_send_failed_spans_total)",
"accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)",
"accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)",
}
def configure(enabled, ratio):
compose(
"up",
"-d",
"--no-build",
"--no-deps",
"--force-recreate",
"api",
"worker",
"relay",
env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def ready():
try:
with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response:
return response.status == 200
except OSError:
return False
until(ready)
def snapshot():
with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response:
metrics = response.read().decode()
received_bytes = sum(
float(value)
for value in re.findall(
r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M
)
)
return {
"trace_ingest_proto_bytes": received_bytes,
**{name: prom(expression) for name, expression in COUNTERS.items()},
}
def workload(client, actor, chat, seconds, rate):
def operation(n):
if n % 45 == 0:
started = time.perf_counter()
client.attachment(actor, chat)
return (time.perf_counter() - started) * 1000
if n % 5 == 0:
response = client.message(actor, chat, "synthetic cost experiment")
else:
response = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=20",
actor=actor,
schema="MessagePage",
)
return response["elapsed"] * 1000
start = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
futures = []
for number in range(seconds * rate):
delay = start + number / rate - time.perf_counter()
if delay > 0:
time.sleep(delay)
futures.append(pool.submit(operation, number))
observations = sorted(future.result() for future in futures)
elapsed = time.perf_counter() - start
return {
"requests": len(observations),
"elapsed_s": elapsed,
"latency_ms": {
f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1]
for q in (0.50, 0.95, 0.99)
},
}
def stats():
services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo")
ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split()
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
)
result = []
for line in raw.splitlines():
row = json.loads(line)
result.append(
{
"container": row["Name"],
"cpu_percent": float(row["CPUPerc"].rstrip("%")),
"memory": row["MemUsage"].split(" / ")[0],
}
)
return result
def run_one(mode, seconds, rate):
label, enabled, ratio = mode
configure(enabled, ratio)
client = Client("http://127.0.0.1:8086")
actor, _, _, chat = client.bootstrap()
for _ in range(25):
client.message(actor, chat, "seed")
workload(client, actor, chat, 3, rate)
time.sleep(3)
initial = snapshot()
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
run = pool.submit(workload, client, actor, chat, seconds, rate)
samples = []
for _ in range(2):
time.sleep(2)
samples.append(stats())
receipt = run.result()
# Collector emits batches and Prometheus scrapes every two seconds.
time.sleep(4)
final = snapshot()
return {
"mode": label,
"sample_ratio": float(ratio),
"load": receipt,
"container_samples": samples,
"ingest": {k: final[k] - v for k, v in initial.items()},
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--seconds", type=int, default=10)
parser.add_argument("--rate", type=int, default=18)
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1:
parser.error("use a new output and a positive load for at least 8 s")
results = []
try:
for repeat in range(args.repeats):
for mode in MODES if repeat % 2 == 0 else MODES[::-1]:
row = run_one(mode, args.seconds, args.rate)
print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True)
results.append(row)
args.output.write_text(
json.dumps(
{
"seconds": args.seconds,
"offered_requests_per_s": args.rate,
"runs": results,
},
indent=2,
)
+ "\n"
)
finally:
configure("1", "1")
if __name__ == "__main__":
main()