forked from mtusi2026/mtusi2026lab6
190 lines
6.7 KiB
Python
190 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare client timings with Prometheus histogram quantiles and buckets."""
|
|
|
|
import concurrent.futures
|
|
import json
|
|
import math
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from attachment_memory import cleanup
|
|
from client import Client
|
|
|
|
|
|
def json_get(base_url, path, params=None, timeout=10):
|
|
query = "?" + urllib.parse.urlencode(params, doseq=True) if params else ""
|
|
with urllib.request.urlopen(base_url.rstrip("/") + path + query, timeout=timeout) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def percentile(values, quantile):
|
|
ordered = sorted(values)
|
|
return ordered[max(0, math.ceil(quantile * len(ordered)) - 1)]
|
|
|
|
|
|
def prometheus_scalar(base_url, expression):
|
|
payload = json_get(base_url, "/api/v1/query", {"query": expression})
|
|
assert payload["status"] == "success", payload
|
|
results = payload["data"]["result"]
|
|
assert results, expression
|
|
return float(results[0]["value"][1])
|
|
|
|
|
|
def histogram_boundaries(base_url, metric_name):
|
|
payload = json_get(
|
|
base_url,
|
|
"/api/v1/series",
|
|
{"match[]": metric_name + "_bucket"},
|
|
)
|
|
assert payload["status"] == "success", payload
|
|
return {series["le"] for series in payload["data"]}
|
|
|
|
|
|
def comparable(observed, telemetry, lower=0.2, upper=5.0):
|
|
ratio = telemetry / observed
|
|
assert lower <= ratio <= upper, (observed, telemetry, ratio)
|
|
return ratio
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--prometheus-url", default="http://localhost:9090")
|
|
parser.add_argument("--timeout", type=int, default=120)
|
|
args = parser.parse_args()
|
|
|
|
client = Client(args.base_url, timeout=20)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
users = [alice["id"], bob["id"], eve["id"]]
|
|
content = open("tests/fixtures/sample.png", "rb").read()
|
|
try:
|
|
warmup_started = time.monotonic()
|
|
warmup = client.call(
|
|
"POST",
|
|
f'/api/v1/chats/{chat["id"]}/attachments?filename=calibration-warmup.png',
|
|
202,
|
|
actor=alice,
|
|
raw=content,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)["json"]
|
|
client.wait_ready(alice, warmup, args.timeout)
|
|
assert time.monotonic() > warmup_started
|
|
# Histograms are cumulative. Ensure Prometheus has scraped a baseline
|
|
# before the measured jobs, otherwise all jobs may land in the first
|
|
# sample and increase() correctly has no earlier value to subtract.
|
|
time.sleep(12)
|
|
|
|
attachment_started = {}
|
|
|
|
def upload(index):
|
|
local = Client(args.base_url, timeout=20)
|
|
started = time.monotonic()
|
|
attachment = local.call(
|
|
"POST",
|
|
f'/api/v1/chats/{chat["id"]}/attachments?filename=calibration-{index}.png',
|
|
202,
|
|
actor=alice,
|
|
raw=content,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)["json"]
|
|
attachment_started[attachment["id"]] = started
|
|
return attachment
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
|
|
attachments = list(pool.map(upload, range(12)))
|
|
ready_seconds = []
|
|
for attachment in attachments:
|
|
client.wait_ready(alice, attachment, args.timeout)
|
|
ready_seconds.append(
|
|
time.monotonic() - attachment_started[attachment["id"]]
|
|
)
|
|
|
|
http_seconds = []
|
|
for _ in range(400):
|
|
response = client.call("GET", "/api/v1/chats", actor=alice)
|
|
http_seconds.append(response["elapsed"])
|
|
|
|
time.sleep(12)
|
|
http_metric = prometheus_scalar(
|
|
args.prometheus_url,
|
|
"histogram_quantile(0.95, sum by (le) "
|
|
"(increase(messenger_http_server_duration_seconds_bucket[2m])))",
|
|
)
|
|
worker_metric = prometheus_scalar(
|
|
args.prometheus_url,
|
|
"histogram_quantile(0.95, sum by (le) "
|
|
"(increase(messenger_worker_job_duration_seconds_bucket[2m])))",
|
|
)
|
|
ready_metric = prometheus_scalar(
|
|
args.prometheus_url,
|
|
"histogram_quantile(0.95, sum by (le) "
|
|
"(increase(messenger_attachment_ready_duration_seconds_bucket[2m])))",
|
|
)
|
|
client_http_p95 = percentile(http_seconds, 0.95)
|
|
client_ready_p95 = percentile(ready_seconds, 0.95)
|
|
|
|
http_boundaries = histogram_boundaries(
|
|
args.prometheus_url, "messenger_http_server_duration_seconds"
|
|
)
|
|
worker_boundaries = histogram_boundaries(
|
|
args.prometheus_url, "messenger_worker_job_duration_seconds"
|
|
)
|
|
ready_boundaries = histogram_boundaries(
|
|
args.prometheus_url, "messenger_attachment_ready_duration_seconds"
|
|
)
|
|
assert {"0.005", "0.01", "0.05", "0.1", "1.0"}.issubset(
|
|
http_boundaries
|
|
), http_boundaries
|
|
assert {"0.05", "0.1", "0.25", "1.0", "10.0"}.issubset(
|
|
worker_boundaries
|
|
), worker_boundaries
|
|
assert {"0.1", "0.25", "1.0", "10.0", "120.0"}.issubset(
|
|
ready_boundaries
|
|
), ready_boundaries
|
|
assert 0 < http_metric < 1, http_metric
|
|
assert 0 < worker_metric < 10, worker_metric
|
|
assert 0 < ready_metric < 10, ready_metric
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "PASS",
|
|
"samples": {
|
|
"http": len(http_seconds),
|
|
"attachments": len(ready_seconds),
|
|
},
|
|
"http_p95_seconds": {
|
|
"client": client_http_p95,
|
|
"prometheus": http_metric,
|
|
"ratio": comparable(client_http_p95, http_metric),
|
|
},
|
|
"ready_p95_seconds": {
|
|
"client_polling_upper_bound": client_ready_p95,
|
|
"prometheus": ready_metric,
|
|
"ratio": comparable(
|
|
client_ready_p95, ready_metric, lower=0.1, upper=5.0
|
|
),
|
|
},
|
|
"worker_p95_seconds": worker_metric,
|
|
"boundaries": {
|
|
"http": sorted(http_boundaries),
|
|
"worker": sorted(worker_boundaries),
|
|
"ready": sorted(ready_boundaries),
|
|
},
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
finally:
|
|
cleanup(chat["id"], users)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|