366 lines
14 KiB
Python
366 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeat a paced synthetic message workload with telemetry off and sampled."""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
|
|
def command(args, capture=True):
|
|
result = subprocess.run(
|
|
args,
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.PIPE if capture else None,
|
|
)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def set_env_value(text, name, value):
|
|
lines = text.splitlines()
|
|
prefix = name + "="
|
|
for index, line in enumerate(lines):
|
|
if line.startswith(prefix):
|
|
lines[index] = prefix + value
|
|
break
|
|
else:
|
|
lines.append(prefix + value)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def parse_env(text):
|
|
values = {}
|
|
for line in text.splitlines():
|
|
if line and not line.startswith("#") and "=" in line:
|
|
name, value = line.split("=", 1)
|
|
values[name] = value
|
|
return values
|
|
|
|
|
|
def wait_ready(url, ca_file=None, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
context = ssl.create_default_context(cafile=ca_file)
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(url + "/health/ready", context=context, timeout=2) as response:
|
|
if response.status == 200:
|
|
return
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
pass
|
|
time.sleep(2)
|
|
raise RuntimeError("API did not become ready after the sampling mode change")
|
|
|
|
|
|
def parse_size(value):
|
|
match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value)
|
|
if not match:
|
|
return None
|
|
factors = {
|
|
"B": 1,
|
|
"kB": 1000,
|
|
"MB": 1000**2,
|
|
"GB": 1000**3,
|
|
"KiB": 1024,
|
|
"MiB": 1024**2,
|
|
"GiB": 1024**3,
|
|
}
|
|
return int(float(match.group(1)) * factors[match.group(2)])
|
|
|
|
|
|
def docker_resources():
|
|
rows = command(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{.Name}};{{.CPUPerc}};{{.MemUsage}}",
|
|
]
|
|
)
|
|
cpu_total = 0.0
|
|
memory_total = 0
|
|
found = False
|
|
for row in rows.splitlines():
|
|
fields = row.split(";")
|
|
if len(fields) != 3 or not any(
|
|
name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-")
|
|
):
|
|
continue
|
|
cpu = re.match(r"\s*([\d.]+)%", fields[1])
|
|
memory = parse_size(fields[2].split("/", 1)[0])
|
|
if cpu and memory is not None:
|
|
cpu_total += float(cpu.group(1))
|
|
memory_total += memory
|
|
found = True
|
|
return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None
|
|
|
|
|
|
def storage_sizes(compose):
|
|
paths = {
|
|
"prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]),
|
|
"loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]),
|
|
"tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]),
|
|
}
|
|
result = {}
|
|
for name, args in paths.items():
|
|
output = command(compose + args)
|
|
result[name] = int(output.split()[0])
|
|
return result
|
|
|
|
|
|
def tempo_message_traces(compose, start, end):
|
|
url = (
|
|
"http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000"
|
|
"&tags=service.name%%3Dmessenger-api" % (start, end)
|
|
)
|
|
output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url])
|
|
data = json.loads(output)
|
|
return [
|
|
item
|
|
for item in data.get("traces", [])
|
|
if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"
|
|
]
|
|
|
|
|
|
def mean_spans(compose, traces, sample_limit=20):
|
|
counts = []
|
|
for item in traces[:sample_limit]:
|
|
output = command(
|
|
compose
|
|
+ [
|
|
"exec",
|
|
"-T",
|
|
"tempo",
|
|
"wget",
|
|
"-qO-",
|
|
"http://127.0.0.1:3200/api/traces/" + item["traceID"],
|
|
]
|
|
)
|
|
data = json.loads(output)
|
|
count = sum(
|
|
len(scope.get("spans", []))
|
|
for batch in data.get("batches", [])
|
|
for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", []))
|
|
)
|
|
counts.append(count)
|
|
return round(sum(counts) / len(counts), 2) if counts else 0.0
|
|
|
|
|
|
def run_load(base_url, ca_file, duration, concurrency, target_rps, alice, chat):
|
|
from client import Client, Failure
|
|
|
|
barrier = threading.Barrier(concurrency + 1)
|
|
ticket_lock = threading.Lock()
|
|
ticket = 0
|
|
latencies = []
|
|
failures = 0
|
|
results_lock = threading.Lock()
|
|
start_clock = time.monotonic()
|
|
start_epoch = int(time.time())
|
|
deadline = start_clock + duration
|
|
|
|
def worker(worker_id):
|
|
nonlocal ticket, failures
|
|
client = Client(base_url, ca_file, timeout=5)
|
|
local_latencies = []
|
|
local_failures = 0
|
|
barrier.wait()
|
|
while True:
|
|
with ticket_lock:
|
|
current = ticket
|
|
ticket += 1
|
|
due = start_clock + current / target_rps
|
|
if due >= deadline:
|
|
break
|
|
delay = due - time.monotonic()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
started = time.monotonic()
|
|
try:
|
|
client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current))
|
|
local_latencies.append(time.monotonic() - started)
|
|
except Failure:
|
|
local_failures += 1
|
|
with results_lock:
|
|
latencies.extend(local_latencies)
|
|
failures += local_failures
|
|
|
|
samples = []
|
|
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
|
futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)]
|
|
barrier.wait()
|
|
while any(not future.done() for future in futures):
|
|
sample = docker_resources()
|
|
if sample is not None:
|
|
samples.append(sample)
|
|
time.sleep(2)
|
|
for future in futures:
|
|
future.result()
|
|
elapsed = time.monotonic() - start_clock
|
|
sorted_latencies = sorted(latencies)
|
|
|
|
def percentile(value):
|
|
if not sorted_latencies:
|
|
return None
|
|
index = max(0, math.ceil(value * len(sorted_latencies)) - 1)
|
|
return round(sorted_latencies[index] * 1000, 3)
|
|
|
|
total = len(sorted_latencies) + failures
|
|
report = {
|
|
"workload": "paced synthetic POST messages; no automatic retries",
|
|
"requested_duration_seconds": duration,
|
|
"target_rps": target_rps,
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"concurrency": concurrency,
|
|
"successful_requests": len(sorted_latencies),
|
|
"failed_requests": failures,
|
|
"error_ratio": round(failures / total, 6) if total else None,
|
|
"successful_rps": round(len(sorted_latencies) / elapsed, 3),
|
|
"successful_latency_ms": {
|
|
"p50": percentile(0.50),
|
|
"p95": percentile(0.95),
|
|
"p99": percentile(0.99),
|
|
},
|
|
"docker_resource_samples": len(samples),
|
|
}
|
|
if samples:
|
|
report["docker_resources"] = {
|
|
"mean_cpu_percent_sum": round(
|
|
sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2
|
|
),
|
|
"peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples),
|
|
"peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples),
|
|
"services": ["api", "worker", "publisher", "otel-collector"],
|
|
}
|
|
return report, start_epoch, int(time.time())
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--env-file", default=".env")
|
|
parser.add_argument("--base-url", default=None)
|
|
parser.add_argument("--ca-file", default=None)
|
|
parser.add_argument("--duration", type=int, default=15)
|
|
parser.add_argument("--concurrency", type=int, default=2)
|
|
parser.add_argument("--target-rps", type=int, default=20)
|
|
parser.add_argument("--warmup-seconds", type=int, default=5)
|
|
parser.add_argument("--output", default="evidence/track1-observability-cost.json")
|
|
args = parser.parse_args()
|
|
if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1:
|
|
parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive")
|
|
|
|
env_path = ROOT / args.env_file
|
|
if not env_path.is_file():
|
|
parser.error("create the local .env from .env.example and generate secrets first")
|
|
original = env_path.read_text(encoding="utf-8")
|
|
values = parse_env(original)
|
|
base_url = args.base_url or "https://localhost:" + values.get("APP_PORT", "8443")
|
|
compose = ["docker", "compose", "--env-file", str(env_path)]
|
|
output_path = ROOT / args.output
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
results = []
|
|
configurations = [
|
|
("off", "true", "1.0"),
|
|
("sample-100-percent", "false", "1.0"),
|
|
("sample-25-percent", "false", "0.25"),
|
|
("sample-10-percent", "false", "0.1"),
|
|
]
|
|
try:
|
|
for name, disabled, ratio in configurations:
|
|
changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled)
|
|
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio")
|
|
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio)
|
|
env_path.write_text(changed, encoding="utf-8")
|
|
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
|
|
wait_ready(base_url, args.ca_file)
|
|
time.sleep(args.warmup_seconds)
|
|
from client import Client
|
|
|
|
alice, _, _, chat = Client(base_url, args.ca_file, timeout=10).bootstrap()
|
|
time.sleep(2)
|
|
before = storage_sizes(compose)
|
|
load, start, end = run_load(
|
|
base_url, args.ca_file, args.duration, args.concurrency, args.target_rps, alice, chat
|
|
)
|
|
time.sleep(15)
|
|
after = storage_sizes(compose)
|
|
traces = tempo_message_traces(compose, start - 5, end + 20)
|
|
spans_per_trace = mean_spans(compose, traces)
|
|
results.append(
|
|
{
|
|
"configuration": name,
|
|
"sdk_disabled": disabled == "true",
|
|
"trace_sampling_ratio": float(ratio),
|
|
"load": load,
|
|
"window_unix_seconds": {"start": start, "end": end},
|
|
"sampled_message_trace_count": len(traces),
|
|
"estimated_spans_per_message_trace": spans_per_trace,
|
|
"persistent_storage_bytes_delta": {
|
|
signal: after[signal] - before[signal] for signal in before
|
|
},
|
|
}
|
|
)
|
|
|
|
baseline = next(
|
|
item["estimated_spans_per_message_trace"]
|
|
for item in results
|
|
if item["configuration"] == "sample-100-percent"
|
|
)
|
|
for item in results:
|
|
expected_spans = item["load"]["successful_requests"] * baseline
|
|
observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"]
|
|
item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2)
|
|
|
|
report = {
|
|
"experiment": "telemetry overhead under fixed-rate synthetic POST load",
|
|
"hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.",
|
|
"method": {
|
|
"duration_seconds": args.duration,
|
|
"concurrency": args.concurrency,
|
|
"target_rps": args.target_rps,
|
|
"warmup_seconds": args.warmup_seconds,
|
|
"telemetry_flush_wait_seconds": 15,
|
|
"hardware": {
|
|
"platform": platform.platform(),
|
|
"processor": platform.processor() or "unspecified",
|
|
"logical_cpus": os.cpu_count(),
|
|
},
|
|
"sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.",
|
|
"storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.",
|
|
},
|
|
"limitations": [
|
|
"The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.",
|
|
"Database history grows between modes; all modes use identical duration and concurrency.",
|
|
"Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.",
|
|
"Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.",
|
|
"CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.",
|
|
],
|
|
"results": results,
|
|
"balanced_configuration": "sample-25-percent",
|
|
}
|
|
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
finally:
|
|
env_path.write_text(original, encoding="utf-8")
|
|
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|