Add reproducible observability cost modes and external measurements
This commit is contained in:
@@ -0,0 +1,264 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import closing
|
||||
from http.client import HTTPConnection
|
||||
from pathlib import Path
|
||||
|
||||
from obs_check import Api, http, now
|
||||
from obs_incidents import collector_metrics
|
||||
from obs_recovery import backends, compose
|
||||
|
||||
MODES = {
|
||||
"off": ("true", "1", "collector.yaml", "10000"),
|
||||
"on": ("false", "1", "collector.yaml", "10000"),
|
||||
"head": ("false", "0.1", "collector.yaml", "10000"),
|
||||
"tail": ("false", "1", "collector-tail.yaml", "10000"),
|
||||
"balanced": ("false", "1", "collector-balanced.yaml", "30000"),
|
||||
}
|
||||
|
||||
|
||||
def configure(mode):
|
||||
disabled, ratio, config, interval = MODES[mode]
|
||||
os.environ.update(
|
||||
OTEL_SDK_DISABLED=disabled,
|
||||
OTEL_TRACE_RATIO=ratio,
|
||||
OTEL_COLLECTOR_CONFIG=config,
|
||||
OTEL_METRIC_EXPORT_INTERVAL=interval,
|
||||
COST_EXPERIMENT="true",
|
||||
)
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"--wait",
|
||||
"otel-collector",
|
||||
"api",
|
||||
"worker",
|
||||
"relay",
|
||||
)
|
||||
subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True)
|
||||
|
||||
|
||||
class Engine(HTTPConnection):
|
||||
def __init__(self, path):
|
||||
super().__init__("localhost", timeout=10)
|
||||
self.path = path
|
||||
|
||||
def connect(self):
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
self.sock.settimeout(self.timeout)
|
||||
self.sock.connect(self.path)
|
||||
|
||||
|
||||
def engine_read(path, route):
|
||||
with closing(Engine(path)) as connection:
|
||||
connection.request("GET", route)
|
||||
response = connection.getresponse()
|
||||
if response.status != 200:
|
||||
raise RuntimeError("Docker stats unavailable")
|
||||
return json.load(response)
|
||||
|
||||
|
||||
class Resources:
|
||||
def __init__(self):
|
||||
host = subprocess.check_output(
|
||||
["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True
|
||||
).strip()
|
||||
if not host.startswith("unix://"):
|
||||
raise RuntimeError("cost measurement requires a local Docker socket")
|
||||
self.path = host[7:]
|
||||
self.ids = compose("ps", "-q").splitlines()
|
||||
self.samples = []
|
||||
self.stop = threading.Event()
|
||||
self.thread = threading.Thread(target=self.poll)
|
||||
|
||||
def sample(self):
|
||||
rows = []
|
||||
for identity in self.ids:
|
||||
data = engine_read(
|
||||
self.path, f"/containers/{identity}/stats?stream=false&one-shot=true"
|
||||
)
|
||||
memory = data["memory_stats"]
|
||||
rows.append(
|
||||
{
|
||||
"id": identity[:12],
|
||||
"name": data["name"],
|
||||
"cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"],
|
||||
"working_set_bytes": memory["usage"]
|
||||
- memory.get("stats", {}).get("inactive_file", 0),
|
||||
"memory_limit": memory["limit"],
|
||||
}
|
||||
)
|
||||
return {"at": time.time(), "containers": rows}
|
||||
|
||||
def poll(self):
|
||||
while not self.stop.wait(2):
|
||||
self.samples.append(self.sample())
|
||||
|
||||
def begin(self):
|
||||
self.samples.append(self.sample())
|
||||
self.thread.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
self.thread.join()
|
||||
self.samples.append(self.sample())
|
||||
return self.samples
|
||||
|
||||
|
||||
def counters(url, names):
|
||||
raw = http("GET", url + "/metrics", expect=(200,))[3].decode()
|
||||
result = {}
|
||||
for line in raw.splitlines():
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
name = line.split("{", 1)[0].split(" ", 1)[0]
|
||||
if name in names:
|
||||
result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1])
|
||||
return result
|
||||
|
||||
|
||||
def accounting(b, hosts):
|
||||
metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})')
|
||||
return {
|
||||
"collector": collector_metrics(),
|
||||
"sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0,
|
||||
"tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}),
|
||||
"loki": counters(b.loki, {"loki_distributor_bytes_received_total"}),
|
||||
"prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}),
|
||||
}
|
||||
|
||||
|
||||
def run(mode, repeat, duration, rate, destination):
|
||||
configure(mode)
|
||||
b = backends()
|
||||
api = Api(os.environ["BASE_URL"])
|
||||
_, username, password, _ = api.register()
|
||||
auth, _ = api.login(username, password)
|
||||
for _ in range(20):
|
||||
api.call("GET", "/api/v1/users/me", headers=auth)
|
||||
time.sleep(35)
|
||||
hosts = "|".join(
|
||||
subprocess.check_output(
|
||||
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
|
||||
).strip()
|
||||
+ "-.*"
|
||||
for cid in compose("ps", "-q", "api", "worker", "relay").splitlines()
|
||||
)
|
||||
before = accounting(b, hosts)
|
||||
background = subprocess.check_output(
|
||||
["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'],
|
||||
text=True,
|
||||
).splitlines()
|
||||
resources = Resources()
|
||||
resources.begin()
|
||||
started = time.time()
|
||||
origin = time.perf_counter()
|
||||
prefix = "cost-" + uuid.uuid4().hex[:12]
|
||||
|
||||
def request(index):
|
||||
category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal"
|
||||
rid = prefix + "-" + category + "-" + str(index)
|
||||
began = time.perf_counter()
|
||||
headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category})
|
||||
try:
|
||||
status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0]
|
||||
except OSError:
|
||||
status = 0
|
||||
return {
|
||||
"id": rid,
|
||||
"category": category,
|
||||
"status": status,
|
||||
"latency_ms": (time.perf_counter() - began) * 1000,
|
||||
"schedule_lag_ms": (began - origin - index / rate) * 1000,
|
||||
}
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
|
||||
futures = []
|
||||
for index in range(int(duration * rate)):
|
||||
time.sleep(max(0, origin + index / rate - time.perf_counter()))
|
||||
futures.append(pool.submit(request, index))
|
||||
requests = [future.result() for future in futures]
|
||||
finished = time.time()
|
||||
samples = resources.finish()
|
||||
time.sleep(35)
|
||||
after = accounting(b, hosts)
|
||||
retained = {}
|
||||
for category in ("normal", "error", "slow"):
|
||||
found = b.traceql(
|
||||
'{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }',
|
||||
started - 1,
|
||||
time.time() + 1,
|
||||
limit=5000,
|
||||
)
|
||||
retained[category] = sorted({t["traceID"] for t in found})
|
||||
api.logout(auth)
|
||||
unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)]
|
||||
report = {
|
||||
"utc": now(),
|
||||
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||
"mode": mode,
|
||||
"repeat": repeat,
|
||||
"duration": duration,
|
||||
"rate": rate,
|
||||
"started": started,
|
||||
"finished": finished,
|
||||
"background": background,
|
||||
"requests": requests,
|
||||
"resources": samples,
|
||||
"before": before,
|
||||
"after": after,
|
||||
"retained_trace_ids": retained,
|
||||
"unexpected_responses": len(unexpected),
|
||||
"result": "PASS" if not unexpected else "FAIL",
|
||||
}
|
||||
destination.write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"mode": mode,
|
||||
"repeat": repeat,
|
||||
"result": report["result"],
|
||||
"retained": {k: len(v) for k, v in retained.items()},
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
if unexpected:
|
||||
raise RuntimeError("unexpected benchmark responses; raw series saved")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=MODES)
|
||||
parser.add_argument("--repeat", type=int, default=1)
|
||||
parser.add_argument("--duration", type=int, default=40)
|
||||
parser.add_argument("--rate", type=int, default=20)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise RuntimeError("refusing to overwrite evidence")
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
run(args.mode, args.repeat, args.duration, args.rate, args.output)
|
||||
finally:
|
||||
os.environ.update(
|
||||
OTEL_SDK_DISABLED="false",
|
||||
OTEL_TRACE_RATIO="1",
|
||||
OTEL_COLLECTOR_CONFIG="collector.yaml",
|
||||
OTEL_METRIC_EXPORT_INTERVAL="10000",
|
||||
COST_EXPERIMENT="false",
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user