185 lines
5.9 KiB
Python
185 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Fixed offered-load comparison. Stops only named services in the current lab6 project."""
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from urllib.request import urlopen
|
|
|
|
from observe import Client, compose, prom, until
|
|
|
|
MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25"))
|
|
COUNTERS = {
|
|
"accepted_spans": "sum(otelcol_receiver_accepted_spans_total)",
|
|
"sent_spans": "sum(otelcol_exporter_sent_spans_total)",
|
|
"failed_spans": "sum(otelcol_exporter_send_failed_spans_total)",
|
|
"accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)",
|
|
"accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)",
|
|
}
|
|
|
|
|
|
def configure(enabled, ratio):
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"--no-deps",
|
|
"--force-recreate",
|
|
"api",
|
|
"worker",
|
|
"relay",
|
|
env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio},
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
def ready():
|
|
try:
|
|
with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response:
|
|
return response.status == 200
|
|
except OSError:
|
|
return False
|
|
|
|
until(ready)
|
|
|
|
|
|
def snapshot():
|
|
with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response:
|
|
metrics = response.read().decode()
|
|
received_bytes = sum(
|
|
float(value)
|
|
for value in re.findall(
|
|
r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M
|
|
)
|
|
)
|
|
return {
|
|
"trace_ingest_proto_bytes": received_bytes,
|
|
**{name: prom(expression) for name, expression in COUNTERS.items()},
|
|
}
|
|
|
|
|
|
def workload(client, actor, chat, seconds, rate):
|
|
def operation(n):
|
|
if n % 45 == 0:
|
|
started = time.perf_counter()
|
|
client.attachment(actor, chat)
|
|
return (time.perf_counter() - started) * 1000
|
|
if n % 5 == 0:
|
|
response = client.message(actor, chat, "synthetic cost experiment")
|
|
else:
|
|
response = client.call(
|
|
"GET",
|
|
f"/api/v1/chats/{chat['id']}/messages?limit=20",
|
|
actor=actor,
|
|
schema="MessagePage",
|
|
)
|
|
return response["elapsed"] * 1000
|
|
|
|
start = time.perf_counter()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
|
futures = []
|
|
for number in range(seconds * rate):
|
|
delay = start + number / rate - time.perf_counter()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
futures.append(pool.submit(operation, number))
|
|
observations = sorted(future.result() for future in futures)
|
|
elapsed = time.perf_counter() - start
|
|
return {
|
|
"requests": len(observations),
|
|
"elapsed_s": elapsed,
|
|
"latency_ms": {
|
|
f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1]
|
|
for q in (0.50, 0.95, 0.99)
|
|
},
|
|
}
|
|
|
|
|
|
def stats():
|
|
services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo")
|
|
ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split()
|
|
raw = subprocess.check_output(
|
|
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
|
|
)
|
|
result = []
|
|
for line in raw.splitlines():
|
|
row = json.loads(line)
|
|
result.append(
|
|
{
|
|
"container": row["Name"],
|
|
"cpu_percent": float(row["CPUPerc"].rstrip("%")),
|
|
"memory": row["MemUsage"].split(" / ")[0],
|
|
}
|
|
)
|
|
return result
|
|
|
|
|
|
def run_one(mode, seconds, rate):
|
|
label, enabled, ratio = mode
|
|
configure(enabled, ratio)
|
|
client = Client("http://127.0.0.1:8086")
|
|
actor, _, _, chat = client.bootstrap()
|
|
for _ in range(25):
|
|
client.message(actor, chat, "seed")
|
|
workload(client, actor, chat, 3, rate)
|
|
time.sleep(3)
|
|
initial = snapshot()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
|
run = pool.submit(workload, client, actor, chat, seconds, rate)
|
|
samples = []
|
|
for _ in range(2):
|
|
time.sleep(2)
|
|
samples.append(stats())
|
|
receipt = run.result()
|
|
# Collector emits batches and Prometheus scrapes every two seconds.
|
|
time.sleep(4)
|
|
final = snapshot()
|
|
return {
|
|
"mode": label,
|
|
"sample_ratio": float(ratio),
|
|
"load": receipt,
|
|
"container_samples": samples,
|
|
"ingest": {k: final[k] - v for k, v in initial.items()},
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--seconds", type=int, default=10)
|
|
parser.add_argument("--rate", type=int, default=18)
|
|
parser.add_argument("--repeats", type=int, default=2)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1:
|
|
parser.error("use a new output and a positive load for at least 8 s")
|
|
results = []
|
|
try:
|
|
for repeat in range(args.repeats):
|
|
for mode in MODES if repeat % 2 == 0 else MODES[::-1]:
|
|
row = run_one(mode, args.seconds, args.rate)
|
|
print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True)
|
|
results.append(row)
|
|
args.output.write_text(
|
|
json.dumps(
|
|
{
|
|
"seconds": args.seconds,
|
|
"offered_requests_per_s": args.rate,
|
|
"runs": results,
|
|
},
|
|
indent=2,
|
|
)
|
|
+ "\n"
|
|
)
|
|
finally:
|
|
configure("1", "1")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|