Files
mtusi2026lab7/scripts/obs_cost.py

273 lines
8.9 KiB
Python

import argparse
import concurrent.futures
import json
import os
import socket
import subprocess
import threading
import time
import uuid
from contextlib import closing
from http.client import HTTPConnection
from pathlib import Path
from obs_check import Api, http, now
from obs_incidents import collector_metrics
from obs_recovery import backends, compose
MODES = {
"off": ("true", "1", "collector.yaml", "10000"),
"on": ("false", "1", "collector.yaml", "10000"),
"head": ("false", "0.1", "collector.yaml", "10000"),
"tail": ("false", "1", "collector-tail.yaml", "10000"),
"balanced": ("false", "1", "collector-balanced.yaml", "30000"),
}
def configure(mode):
disabled, ratio, config, interval = MODES[mode]
os.environ.update(
OTEL_SDK_DISABLED=disabled,
OTEL_TRACE_RATIO=ratio,
OTEL_COLLECTOR_CONFIG=config,
OTEL_METRIC_EXPORT_INTERVAL=interval,
COST_EXPERIMENT="true",
)
compose(
"up",
"-d",
"--no-deps",
"--force-recreate",
"--wait",
"otel-collector",
"api",
"worker",
"relay",
)
subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True)
class Engine(HTTPConnection):
def __init__(self, path):
super().__init__("localhost", timeout=10)
self.path = path
def connect(self):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.settimeout(self.timeout)
self.sock.connect(self.path)
def engine_read(path, route):
with closing(Engine(path)) as connection:
connection.request("GET", route)
response = connection.getresponse()
if response.status != 200:
raise RuntimeError("Docker stats unavailable")
return json.load(response)
class Resources:
def __init__(self):
host = subprocess.check_output(
["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True
).strip()
if not host.startswith("unix://"):
raise RuntimeError("cost measurement requires a local Docker socket")
self.path = host[7:]
self.ids = compose("ps", "-q").splitlines()
self.names = dict(
zip(
self.ids,
subprocess.check_output(
["docker", "inspect", "-f", "{{.Name}}", *self.ids], text=True
).splitlines(),
)
)
self.samples = []
self.stop = threading.Event()
self.thread = threading.Thread(target=self.poll)
def sample(self):
rows = []
for identity in self.ids:
data = engine_read(
self.path, f"/containers/{identity}/stats?stream=false&one-shot=true"
)
memory = data["memory_stats"]
rows.append(
{
"id": identity[:12],
"name": self.names[identity],
"cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"],
"working_set_bytes": memory["usage"]
- memory.get("stats", {}).get("inactive_file", 0),
"memory_limit": memory["limit"],
}
)
return {"at": time.time(), "containers": rows}
def poll(self):
while not self.stop.wait(2):
self.samples.append(self.sample())
def begin(self):
self.samples.append(self.sample())
self.thread.start()
def finish(self):
self.stop.set()
self.thread.join()
self.samples.append(self.sample())
return self.samples
def counters(url, names):
raw = http("GET", url + "/metrics", expect=(200,))[3].decode()
result = {}
for line in raw.splitlines():
if not line or line.startswith("#"):
continue
name = line.split("{", 1)[0].split(" ", 1)[0]
if name in names:
result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1])
return result
def accounting(b, hosts):
metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})')
return {
"collector": collector_metrics(),
"sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0,
"tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}),
"loki": counters(b.loki, {"loki_distributor_bytes_received_total"}),
"prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}),
}
def run(mode, repeat, duration, rate, destination):
configure(mode)
b = backends()
api = Api(os.environ["BASE_URL"])
_, username, password, _ = api.register()
auth, _ = api.login(username, password)
for _ in range(20):
api.call("GET", "/api/v1/users/me", headers=auth)
time.sleep(35)
hosts = "|".join(
subprocess.check_output(
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
).strip()
+ "-.*"
for cid in compose("ps", "-q", "api", "worker", "relay").splitlines()
)
before = accounting(b, hosts)
background = subprocess.check_output(
["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'],
text=True,
).splitlines()
resources = Resources()
resources.begin()
started = time.time()
origin = time.perf_counter()
prefix = "cost-" + uuid.uuid4().hex[:12]
def request(index):
category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal"
rid = prefix + "-" + category + "-" + str(index)
began = time.perf_counter()
headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category})
try:
status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0]
except OSError:
status = 0
return {
"id": rid,
"category": category,
"status": status,
"latency_ms": (time.perf_counter() - began) * 1000,
"schedule_lag_ms": (began - origin - index / rate) * 1000,
}
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
futures = []
for index in range(int(duration * rate)):
time.sleep(max(0, origin + index / rate - time.perf_counter()))
futures.append(pool.submit(request, index))
requests = [future.result() for future in futures]
finished = time.time()
samples = resources.finish()
time.sleep(35)
after = accounting(b, hosts)
retained = {}
for category in ("normal", "error", "slow"):
found = b.traceql(
'{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }',
started - 1,
time.time() + 1,
limit=5000,
)
retained[category] = sorted({t["traceID"] for t in found})
api.logout(auth)
unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)]
report = {
"utc": now(),
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
"mode": mode,
"repeat": repeat,
"duration": duration,
"rate": rate,
"started": started,
"finished": finished,
"background": background,
"requests": requests,
"resources": samples,
"before": before,
"after": after,
"retained_trace_ids": retained,
"unexpected_responses": len(unexpected),
"result": "PASS" if not unexpected else "FAIL",
}
destination.write_text(json.dumps(report, indent=2) + "\n")
print(
json.dumps(
{
"mode": mode,
"repeat": repeat,
"result": report["result"],
"retained": {k: len(v) for k, v in retained.items()},
}
),
flush=True,
)
if unexpected:
raise RuntimeError("unexpected benchmark responses; raw series saved")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", choices=MODES)
parser.add_argument("--repeat", type=int, default=1)
parser.add_argument("--duration", type=int, default=40)
parser.add_argument("--rate", type=int, default=20)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.output.exists():
raise RuntimeError("refusing to overwrite evidence")
args.output.parent.mkdir(parents=True, exist_ok=True)
try:
run(args.mode, args.repeat, args.duration, args.rate, args.output)
finally:
os.environ.update(
OTEL_SDK_DISABLED="false",
OTEL_TRACE_RATIO="1",
OTEL_COLLECTOR_CONFIG="collector.yaml",
OTEL_METRIC_EXPORT_INTERVAL="10000",
COST_EXPERIMENT="false",
)
compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay")
if __name__ == "__main__":
main()