import argparse import concurrent.futures import json import os import socket import subprocess import threading import time import uuid from contextlib import closing from http.client import HTTPConnection from pathlib import Path from obs_check import Api, http, now from obs_incidents import collector_metrics from obs_recovery import backends, compose MODES = { "off": ("true", "1", "collector.yaml", "10000"), "on": ("false", "1", "collector.yaml", "10000"), "head": ("false", "0.1", "collector.yaml", "10000"), "tail": ("false", "1", "collector-tail.yaml", "10000"), "balanced": ("false", "1", "collector-balanced.yaml", "30000"), } def configure(mode): disabled, ratio, config, interval = MODES[mode] os.environ.update( OTEL_SDK_DISABLED=disabled, OTEL_TRACE_RATIO=ratio, OTEL_COLLECTOR_CONFIG=config, OTEL_METRIC_EXPORT_INTERVAL=interval, COST_EXPERIMENT="true", ) compose( "up", "-d", "--no-deps", "--force-recreate", "--wait", "otel-collector", "api", "worker", "relay", ) subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True) class Engine(HTTPConnection): def __init__(self, path): super().__init__("localhost", timeout=10) self.path = path def connect(self): self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.settimeout(self.timeout) self.sock.connect(self.path) def engine_read(path, route): with closing(Engine(path)) as connection: connection.request("GET", route) response = connection.getresponse() if response.status != 200: raise RuntimeError("Docker stats unavailable") return json.load(response) class Resources: def __init__(self): host = subprocess.check_output( ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True ).strip() if not host.startswith("unix://"): raise RuntimeError("cost measurement requires a local Docker socket") self.path = host[7:] self.ids = compose("ps", "-q").splitlines() self.samples = [] self.stop = threading.Event() self.thread = threading.Thread(target=self.poll) def sample(self): rows = [] for identity in self.ids: data = engine_read( self.path, f"/containers/{identity}/stats?stream=false&one-shot=true" ) memory = data["memory_stats"] rows.append( { "id": identity[:12], "name": data["name"], "cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"], "working_set_bytes": memory["usage"] - memory.get("stats", {}).get("inactive_file", 0), "memory_limit": memory["limit"], } ) return {"at": time.time(), "containers": rows} def poll(self): while not self.stop.wait(2): self.samples.append(self.sample()) def begin(self): self.samples.append(self.sample()) self.thread.start() def finish(self): self.stop.set() self.thread.join() self.samples.append(self.sample()) return self.samples def counters(url, names): raw = http("GET", url + "/metrics", expect=(200,))[3].decode() result = {} for line in raw.splitlines(): if not line or line.startswith("#"): continue name = line.split("{", 1)[0].split(" ", 1)[0] if name in names: result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1]) return result def accounting(b, hosts): metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})') return { "collector": collector_metrics(), "sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0, "tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}), "loki": counters(b.loki, {"loki_distributor_bytes_received_total"}), "prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}), } def run(mode, repeat, duration, rate, destination): configure(mode) b = backends() api = Api(os.environ["BASE_URL"]) _, username, password, _ = api.register() auth, _ = api.login(username, password) for _ in range(20): api.call("GET", "/api/v1/users/me", headers=auth) time.sleep(35) hosts = "|".join( subprocess.check_output( ["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True ).strip() + "-.*" for cid in compose("ps", "-q", "api", "worker", "relay").splitlines() ) before = accounting(b, hosts) background = subprocess.check_output( ["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'], text=True, ).splitlines() resources = Resources() resources.begin() started = time.time() origin = time.perf_counter() prefix = "cost-" + uuid.uuid4().hex[:12] def request(index): category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal" rid = prefix + "-" + category + "-" + str(index) began = time.perf_counter() headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category}) try: status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0] except OSError: status = 0 return { "id": rid, "category": category, "status": status, "latency_ms": (time.perf_counter() - began) * 1000, "schedule_lag_ms": (began - origin - index / rate) * 1000, } with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool: futures = [] for index in range(int(duration * rate)): time.sleep(max(0, origin + index / rate - time.perf_counter())) futures.append(pool.submit(request, index)) requests = [future.result() for future in futures] finished = time.time() samples = resources.finish() time.sleep(35) after = accounting(b, hosts) retained = {} for category in ("normal", "error", "slow"): found = b.traceql( '{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }', started - 1, time.time() + 1, limit=5000, ) retained[category] = sorted({t["traceID"] for t in found}) api.logout(auth) unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)] report = { "utc": now(), "sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), "mode": mode, "repeat": repeat, "duration": duration, "rate": rate, "started": started, "finished": finished, "background": background, "requests": requests, "resources": samples, "before": before, "after": after, "retained_trace_ids": retained, "unexpected_responses": len(unexpected), "result": "PASS" if not unexpected else "FAIL", } destination.write_text(json.dumps(report, indent=2) + "\n") print( json.dumps( { "mode": mode, "repeat": repeat, "result": report["result"], "retained": {k: len(v) for k, v in retained.items()}, } ), flush=True, ) if unexpected: raise RuntimeError("unexpected benchmark responses; raw series saved") def main(): parser = argparse.ArgumentParser() parser.add_argument("--mode", choices=MODES) parser.add_argument("--repeat", type=int, default=1) parser.add_argument("--duration", type=int, default=40) parser.add_argument("--rate", type=int, default=20) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.exists(): raise RuntimeError("refusing to overwrite evidence") args.output.parent.mkdir(parents=True, exist_ok=True) try: run(args.mode, args.repeat, args.duration, args.rate, args.output) finally: os.environ.update( OTEL_SDK_DISABLED="false", OTEL_TRACE_RATIO="1", OTEL_COLLECTOR_CONFIG="collector.yaml", OTEL_METRIC_EXPORT_INTERVAL="10000", COST_EXPERIMENT="false", ) compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay") if __name__ == "__main__": main()