forked from mtusi2026/mtusi2026lab6
265 lines
8.6 KiB
Python
265 lines
8.6 KiB
Python
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from contextlib import closing
|
|
from http.client import HTTPConnection
|
|
from pathlib import Path
|
|
|
|
from obs_check import Api, http, now
|
|
from obs_incidents import collector_metrics
|
|
from obs_recovery import backends, compose
|
|
|
|
MODES = {
|
|
"off": ("true", "1", "collector.yaml", "10000"),
|
|
"on": ("false", "1", "collector.yaml", "10000"),
|
|
"head": ("false", "0.1", "collector.yaml", "10000"),
|
|
"tail": ("false", "1", "collector-tail.yaml", "10000"),
|
|
"balanced": ("false", "1", "collector-balanced.yaml", "30000"),
|
|
}
|
|
|
|
|
|
def configure(mode):
|
|
disabled, ratio, config, interval = MODES[mode]
|
|
os.environ.update(
|
|
OTEL_SDK_DISABLED=disabled,
|
|
OTEL_TRACE_RATIO=ratio,
|
|
OTEL_COLLECTOR_CONFIG=config,
|
|
OTEL_METRIC_EXPORT_INTERVAL=interval,
|
|
COST_EXPERIMENT="true",
|
|
)
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--force-recreate",
|
|
"--wait",
|
|
"otel-collector",
|
|
"api",
|
|
"worker",
|
|
"relay",
|
|
)
|
|
subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True)
|
|
|
|
|
|
class Engine(HTTPConnection):
|
|
def __init__(self, path):
|
|
super().__init__("localhost", timeout=10)
|
|
self.path = path
|
|
|
|
def connect(self):
|
|
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
self.sock.settimeout(self.timeout)
|
|
self.sock.connect(self.path)
|
|
|
|
|
|
def engine_read(path, route):
|
|
with closing(Engine(path)) as connection:
|
|
connection.request("GET", route)
|
|
response = connection.getresponse()
|
|
if response.status != 200:
|
|
raise RuntimeError("Docker stats unavailable")
|
|
return json.load(response)
|
|
|
|
|
|
class Resources:
|
|
def __init__(self):
|
|
host = subprocess.check_output(
|
|
["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True
|
|
).strip()
|
|
if not host.startswith("unix://"):
|
|
raise RuntimeError("cost measurement requires a local Docker socket")
|
|
self.path = host[7:]
|
|
self.ids = compose("ps", "-q").splitlines()
|
|
self.samples = []
|
|
self.stop = threading.Event()
|
|
self.thread = threading.Thread(target=self.poll)
|
|
|
|
def sample(self):
|
|
rows = []
|
|
for identity in self.ids:
|
|
data = engine_read(
|
|
self.path, f"/containers/{identity}/stats?stream=false&one-shot=true"
|
|
)
|
|
memory = data["memory_stats"]
|
|
rows.append(
|
|
{
|
|
"id": identity[:12],
|
|
"name": data["name"],
|
|
"cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"],
|
|
"working_set_bytes": memory["usage"]
|
|
- memory.get("stats", {}).get("inactive_file", 0),
|
|
"memory_limit": memory["limit"],
|
|
}
|
|
)
|
|
return {"at": time.time(), "containers": rows}
|
|
|
|
def poll(self):
|
|
while not self.stop.wait(2):
|
|
self.samples.append(self.sample())
|
|
|
|
def begin(self):
|
|
self.samples.append(self.sample())
|
|
self.thread.start()
|
|
|
|
def finish(self):
|
|
self.stop.set()
|
|
self.thread.join()
|
|
self.samples.append(self.sample())
|
|
return self.samples
|
|
|
|
|
|
def counters(url, names):
|
|
raw = http("GET", url + "/metrics", expect=(200,))[3].decode()
|
|
result = {}
|
|
for line in raw.splitlines():
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
name = line.split("{", 1)[0].split(" ", 1)[0]
|
|
if name in names:
|
|
result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1])
|
|
return result
|
|
|
|
|
|
def accounting(b, hosts):
|
|
metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})')
|
|
return {
|
|
"collector": collector_metrics(),
|
|
"sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0,
|
|
"tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}),
|
|
"loki": counters(b.loki, {"loki_distributor_bytes_received_total"}),
|
|
"prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}),
|
|
}
|
|
|
|
|
|
def run(mode, repeat, duration, rate, destination):
|
|
configure(mode)
|
|
b = backends()
|
|
api = Api(os.environ["BASE_URL"])
|
|
_, username, password, _ = api.register()
|
|
auth, _ = api.login(username, password)
|
|
for _ in range(20):
|
|
api.call("GET", "/api/v1/users/me", headers=auth)
|
|
time.sleep(35)
|
|
hosts = "|".join(
|
|
subprocess.check_output(
|
|
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
|
|
).strip()
|
|
+ "-.*"
|
|
for cid in compose("ps", "-q", "api", "worker", "relay").splitlines()
|
|
)
|
|
before = accounting(b, hosts)
|
|
background = subprocess.check_output(
|
|
["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'],
|
|
text=True,
|
|
).splitlines()
|
|
resources = Resources()
|
|
resources.begin()
|
|
started = time.time()
|
|
origin = time.perf_counter()
|
|
prefix = "cost-" + uuid.uuid4().hex[:12]
|
|
|
|
def request(index):
|
|
category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal"
|
|
rid = prefix + "-" + category + "-" + str(index)
|
|
began = time.perf_counter()
|
|
headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category})
|
|
try:
|
|
status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0]
|
|
except OSError:
|
|
status = 0
|
|
return {
|
|
"id": rid,
|
|
"category": category,
|
|
"status": status,
|
|
"latency_ms": (time.perf_counter() - began) * 1000,
|
|
"schedule_lag_ms": (began - origin - index / rate) * 1000,
|
|
}
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
|
|
futures = []
|
|
for index in range(int(duration * rate)):
|
|
time.sleep(max(0, origin + index / rate - time.perf_counter()))
|
|
futures.append(pool.submit(request, index))
|
|
requests = [future.result() for future in futures]
|
|
finished = time.time()
|
|
samples = resources.finish()
|
|
time.sleep(35)
|
|
after = accounting(b, hosts)
|
|
retained = {}
|
|
for category in ("normal", "error", "slow"):
|
|
found = b.traceql(
|
|
'{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }',
|
|
started - 1,
|
|
time.time() + 1,
|
|
limit=5000,
|
|
)
|
|
retained[category] = sorted({t["traceID"] for t in found})
|
|
api.logout(auth)
|
|
unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)]
|
|
report = {
|
|
"utc": now(),
|
|
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
|
"mode": mode,
|
|
"repeat": repeat,
|
|
"duration": duration,
|
|
"rate": rate,
|
|
"started": started,
|
|
"finished": finished,
|
|
"background": background,
|
|
"requests": requests,
|
|
"resources": samples,
|
|
"before": before,
|
|
"after": after,
|
|
"retained_trace_ids": retained,
|
|
"unexpected_responses": len(unexpected),
|
|
"result": "PASS" if not unexpected else "FAIL",
|
|
}
|
|
destination.write_text(json.dumps(report, indent=2) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"mode": mode,
|
|
"repeat": repeat,
|
|
"result": report["result"],
|
|
"retained": {k: len(v) for k, v in retained.items()},
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
if unexpected:
|
|
raise RuntimeError("unexpected benchmark responses; raw series saved")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--mode", choices=MODES)
|
|
parser.add_argument("--repeat", type=int, default=1)
|
|
parser.add_argument("--duration", type=int, default=40)
|
|
parser.add_argument("--rate", type=int, default=20)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists():
|
|
raise RuntimeError("refusing to overwrite evidence")
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
run(args.mode, args.repeat, args.duration, args.rate, args.output)
|
|
finally:
|
|
os.environ.update(
|
|
OTEL_SDK_DISABLED="false",
|
|
OTEL_TRACE_RATIO="1",
|
|
OTEL_COLLECTOR_CONFIG="collector.yaml",
|
|
OTEL_METRIC_EXPORT_INTERVAL="10000",
|
|
COST_EXPERIMENT="false",
|
|
)
|
|
compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|