Files
mtusi2026lab7/student-tests/track1_benchmark.py
T
kaktuz20000 5a6fcff3ef
Skeleton contracts / contracts (pull_request) Canceled after 0s
выполение на 5
2026-11-04 19:22:35 +03:00

366 lines
14 KiB
Python

#!/usr/bin/env python3
"""Repeat a paced synthetic message workload with telemetry off and sampled."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import platform
import re
import ssl
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
def command(args, capture=True):
result = subprocess.run(
args,
cwd=ROOT,
check=True,
text=True,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None,
)
return result.stdout.strip() if capture else ""
def set_env_value(text, name, value):
lines = text.splitlines()
prefix = name + "="
for index, line in enumerate(lines):
if line.startswith(prefix):
lines[index] = prefix + value
break
else:
lines.append(prefix + value)
return "\n".join(lines) + "\n"
def parse_env(text):
values = {}
for line in text.splitlines():
if line and not line.startswith("#") and "=" in line:
name, value = line.split("=", 1)
values[name] = value
return values
def wait_ready(url, ca_file=None, timeout=90):
deadline = time.monotonic() + timeout
context = ssl.create_default_context(cafile=ca_file)
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url + "/health/ready", context=context, timeout=2) as response:
if response.status == 200:
return
except (urllib.error.URLError, TimeoutError, OSError):
pass
time.sleep(2)
raise RuntimeError("API did not become ready after the sampling mode change")
def parse_size(value):
match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value)
if not match:
return None
factors = {
"B": 1,
"kB": 1000,
"MB": 1000**2,
"GB": 1000**3,
"KiB": 1024,
"MiB": 1024**2,
"GiB": 1024**3,
}
return int(float(match.group(1)) * factors[match.group(2)])
def docker_resources():
rows = command(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{.Name}};{{.CPUPerc}};{{.MemUsage}}",
]
)
cpu_total = 0.0
memory_total = 0
found = False
for row in rows.splitlines():
fields = row.split(";")
if len(fields) != 3 or not any(
name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-")
):
continue
cpu = re.match(r"\s*([\d.]+)%", fields[1])
memory = parse_size(fields[2].split("/", 1)[0])
if cpu and memory is not None:
cpu_total += float(cpu.group(1))
memory_total += memory
found = True
return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None
def storage_sizes(compose):
paths = {
"prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]),
"loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]),
"tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]),
}
result = {}
for name, args in paths.items():
output = command(compose + args)
result[name] = int(output.split()[0])
return result
def tempo_message_traces(compose, start, end):
url = (
"http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000"
"&tags=service.name%%3Dmessenger-api" % (start, end)
)
output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url])
data = json.loads(output)
return [
item
for item in data.get("traces", [])
if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"
]
def mean_spans(compose, traces, sample_limit=20):
counts = []
for item in traces[:sample_limit]:
output = command(
compose
+ [
"exec",
"-T",
"tempo",
"wget",
"-qO-",
"http://127.0.0.1:3200/api/traces/" + item["traceID"],
]
)
data = json.loads(output)
count = sum(
len(scope.get("spans", []))
for batch in data.get("batches", [])
for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", []))
)
counts.append(count)
return round(sum(counts) / len(counts), 2) if counts else 0.0
def run_load(base_url, ca_file, duration, concurrency, target_rps, alice, chat):
from client import Client, Failure
barrier = threading.Barrier(concurrency + 1)
ticket_lock = threading.Lock()
ticket = 0
latencies = []
failures = 0
results_lock = threading.Lock()
start_clock = time.monotonic()
start_epoch = int(time.time())
deadline = start_clock + duration
def worker(worker_id):
nonlocal ticket, failures
client = Client(base_url, ca_file, timeout=5)
local_latencies = []
local_failures = 0
barrier.wait()
while True:
with ticket_lock:
current = ticket
ticket += 1
due = start_clock + current / target_rps
if due >= deadline:
break
delay = due - time.monotonic()
if delay > 0:
time.sleep(delay)
started = time.monotonic()
try:
client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current))
local_latencies.append(time.monotonic() - started)
except Failure:
local_failures += 1
with results_lock:
latencies.extend(local_latencies)
failures += local_failures
samples = []
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)]
barrier.wait()
while any(not future.done() for future in futures):
sample = docker_resources()
if sample is not None:
samples.append(sample)
time.sleep(2)
for future in futures:
future.result()
elapsed = time.monotonic() - start_clock
sorted_latencies = sorted(latencies)
def percentile(value):
if not sorted_latencies:
return None
index = max(0, math.ceil(value * len(sorted_latencies)) - 1)
return round(sorted_latencies[index] * 1000, 3)
total = len(sorted_latencies) + failures
report = {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": duration,
"target_rps": target_rps,
"elapsed_seconds": round(elapsed, 3),
"concurrency": concurrency,
"successful_requests": len(sorted_latencies),
"failed_requests": failures,
"error_ratio": round(failures / total, 6) if total else None,
"successful_rps": round(len(sorted_latencies) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(0.50),
"p95": percentile(0.95),
"p99": percentile(0.99),
},
"docker_resource_samples": len(samples),
}
if samples:
report["docker_resources"] = {
"mean_cpu_percent_sum": round(
sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2
),
"peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples),
"peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples),
"services": ["api", "worker", "publisher", "otel-collector"],
}
return report, start_epoch, int(time.time())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", default=".env")
parser.add_argument("--base-url", default=None)
parser.add_argument("--ca-file", default=None)
parser.add_argument("--duration", type=int, default=15)
parser.add_argument("--concurrency", type=int, default=2)
parser.add_argument("--target-rps", type=int, default=20)
parser.add_argument("--warmup-seconds", type=int, default=5)
parser.add_argument("--output", default="evidence/track1-observability-cost.json")
args = parser.parse_args()
if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1:
parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive")
env_path = ROOT / args.env_file
if not env_path.is_file():
parser.error("create the local .env from .env.example and generate secrets first")
original = env_path.read_text(encoding="utf-8")
values = parse_env(original)
base_url = args.base_url or "https://localhost:" + values.get("APP_PORT", "8443")
compose = ["docker", "compose", "--env-file", str(env_path)]
output_path = ROOT / args.output
output_path.parent.mkdir(parents=True, exist_ok=True)
results = []
configurations = [
("off", "true", "1.0"),
("sample-100-percent", "false", "1.0"),
("sample-25-percent", "false", "0.25"),
("sample-10-percent", "false", "0.1"),
]
try:
for name, disabled, ratio in configurations:
changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled)
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio")
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio)
env_path.write_text(changed, encoding="utf-8")
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
wait_ready(base_url, args.ca_file)
time.sleep(args.warmup_seconds)
from client import Client
alice, _, _, chat = Client(base_url, args.ca_file, timeout=10).bootstrap()
time.sleep(2)
before = storage_sizes(compose)
load, start, end = run_load(
base_url, args.ca_file, args.duration, args.concurrency, args.target_rps, alice, chat
)
time.sleep(15)
after = storage_sizes(compose)
traces = tempo_message_traces(compose, start - 5, end + 20)
spans_per_trace = mean_spans(compose, traces)
results.append(
{
"configuration": name,
"sdk_disabled": disabled == "true",
"trace_sampling_ratio": float(ratio),
"load": load,
"window_unix_seconds": {"start": start, "end": end},
"sampled_message_trace_count": len(traces),
"estimated_spans_per_message_trace": spans_per_trace,
"persistent_storage_bytes_delta": {
signal: after[signal] - before[signal] for signal in before
},
}
)
baseline = next(
item["estimated_spans_per_message_trace"]
for item in results
if item["configuration"] == "sample-100-percent"
)
for item in results:
expected_spans = item["load"]["successful_requests"] * baseline
observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"]
item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2)
report = {
"experiment": "telemetry overhead under fixed-rate synthetic POST load",
"hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.",
"method": {
"duration_seconds": args.duration,
"concurrency": args.concurrency,
"target_rps": args.target_rps,
"warmup_seconds": args.warmup_seconds,
"telemetry_flush_wait_seconds": 15,
"hardware": {
"platform": platform.platform(),
"processor": platform.processor() or "unspecified",
"logical_cpus": os.cpu_count(),
},
"sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.",
"storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.",
},
"limitations": [
"The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.",
"Database history grows between modes; all modes use identical duration and concurrency.",
"Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.",
"Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.",
"CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.",
],
"results": results,
"balanced_configuration": "sample-25-percent",
}
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
finally:
env_path.write_text(original, encoding="utf-8")
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
if __name__ == "__main__":
main()