#!/usr/bin/env python3 """Fixed offered-load comparison. Stops only named services in the current lab6 project.""" import argparse import concurrent.futures import json import math import os import re import subprocess import time from pathlib import Path from urllib.request import urlopen from observe import Client, compose, prom, until MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25")) COUNTERS = { "accepted_spans": "sum(otelcol_receiver_accepted_spans_total)", "sent_spans": "sum(otelcol_exporter_sent_spans_total)", "failed_spans": "sum(otelcol_exporter_send_failed_spans_total)", "accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)", "accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)", } def configure(enabled, ratio): compose( "up", "-d", "--no-build", "--no-deps", "--force-recreate", "api", "worker", "relay", env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio}, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) def ready(): try: with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response: return response.status == 200 except OSError: return False until(ready) def snapshot(): with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response: metrics = response.read().decode() received_bytes = sum( float(value) for value in re.findall( r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M ) ) return { "trace_ingest_proto_bytes": received_bytes, **{name: prom(expression) for name, expression in COUNTERS.items()}, } def workload(client, actor, chat, seconds, rate): def operation(n): if n % 45 == 0: started = time.perf_counter() client.attachment(actor, chat) return (time.perf_counter() - started) * 1000 if n % 5 == 0: response = client.message(actor, chat, "synthetic cost experiment") else: response = client.call( "GET", f"/api/v1/chats/{chat['id']}/messages?limit=20", actor=actor, schema="MessagePage", ) return response["elapsed"] * 1000 start = time.perf_counter() with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: futures = [] for number in range(seconds * rate): delay = start + number / rate - time.perf_counter() if delay > 0: time.sleep(delay) futures.append(pool.submit(operation, number)) observations = sorted(future.result() for future in futures) elapsed = time.perf_counter() - start return { "requests": len(observations), "elapsed_s": elapsed, "latency_ms": { f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1] for q in (0.50, 0.95, 0.99) }, } def stats(): services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo") ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split() raw = subprocess.check_output( ["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True ) result = [] for line in raw.splitlines(): row = json.loads(line) result.append( { "container": row["Name"], "cpu_percent": float(row["CPUPerc"].rstrip("%")), "memory": row["MemUsage"].split(" / ")[0], } ) return result def run_one(mode, seconds, rate): label, enabled, ratio = mode configure(enabled, ratio) client = Client("http://127.0.0.1:8086") actor, _, _, chat = client.bootstrap() for _ in range(25): client.message(actor, chat, "seed") workload(client, actor, chat, 3, rate) time.sleep(3) initial = snapshot() with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: run = pool.submit(workload, client, actor, chat, seconds, rate) samples = [] for _ in range(2): time.sleep(2) samples.append(stats()) receipt = run.result() # Collector emits batches and Prometheus scrapes every two seconds. time.sleep(4) final = snapshot() return { "mode": label, "sample_ratio": float(ratio), "load": receipt, "container_samples": samples, "ingest": {k: final[k] - v for k, v in initial.items()}, } def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--seconds", type=int, default=10) parser.add_argument("--rate", type=int, default=18) parser.add_argument("--repeats", type=int, default=2) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1: parser.error("use a new output and a positive load for at least 8 s") results = [] try: for repeat in range(args.repeats): for mode in MODES if repeat % 2 == 0 else MODES[::-1]: row = run_one(mode, args.seconds, args.rate) print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True) results.append(row) args.output.write_text( json.dumps( { "seconds": args.seconds, "offered_requests_per_s": args.rate, "runs": results, }, indent=2, ) + "\n" ) finally: configure("1", "1") if __name__ == "__main__": main()