Files
mtusi2026lab6/tests/histogram_calibration.py

190 lines
6.7 KiB
Python

#!/usr/bin/env python3
"""Compare client timings with Prometheus histogram quantiles and buckets."""
import concurrent.futures
import json
import math
import time
import urllib.parse
import urllib.request
from attachment_memory import cleanup
from client import Client
def json_get(base_url, path, params=None, timeout=10):
query = "?" + urllib.parse.urlencode(params, doseq=True) if params else ""
with urllib.request.urlopen(base_url.rstrip("/") + path + query, timeout=timeout) as response:
return json.load(response)
def percentile(values, quantile):
ordered = sorted(values)
return ordered[max(0, math.ceil(quantile * len(ordered)) - 1)]
def prometheus_scalar(base_url, expression):
payload = json_get(base_url, "/api/v1/query", {"query": expression})
assert payload["status"] == "success", payload
results = payload["data"]["result"]
assert results, expression
return float(results[0]["value"][1])
def histogram_boundaries(base_url, metric_name):
payload = json_get(
base_url,
"/api/v1/series",
{"match[]": metric_name + "_bucket"},
)
assert payload["status"] == "success", payload
return {series["le"] for series in payload["data"]}
def comparable(observed, telemetry, lower=0.2, upper=5.0):
ratio = telemetry / observed
assert lower <= ratio <= upper, (observed, telemetry, ratio)
return ratio
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--prometheus-url", default="http://localhost:9090")
parser.add_argument("--timeout", type=int, default=120)
args = parser.parse_args()
client = Client(args.base_url, timeout=20)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
content = open("tests/fixtures/sample.png", "rb").read()
try:
warmup_started = time.monotonic()
warmup = client.call(
"POST",
f'/api/v1/chats/{chat["id"]}/attachments?filename=calibration-warmup.png',
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
client.wait_ready(alice, warmup, args.timeout)
assert time.monotonic() > warmup_started
# Histograms are cumulative. Ensure Prometheus has scraped a baseline
# before the measured jobs, otherwise all jobs may land in the first
# sample and increase() correctly has no earlier value to subtract.
time.sleep(12)
attachment_started = {}
def upload(index):
local = Client(args.base_url, timeout=20)
started = time.monotonic()
attachment = local.call(
"POST",
f'/api/v1/chats/{chat["id"]}/attachments?filename=calibration-{index}.png',
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
attachment_started[attachment["id"]] = started
return attachment
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
attachments = list(pool.map(upload, range(12)))
ready_seconds = []
for attachment in attachments:
client.wait_ready(alice, attachment, args.timeout)
ready_seconds.append(
time.monotonic() - attachment_started[attachment["id"]]
)
http_seconds = []
for _ in range(400):
response = client.call("GET", "/api/v1/chats", actor=alice)
http_seconds.append(response["elapsed"])
time.sleep(12)
http_metric = prometheus_scalar(
args.prometheus_url,
"histogram_quantile(0.95, sum by (le) "
"(increase(messenger_http_server_duration_seconds_bucket[2m])))",
)
worker_metric = prometheus_scalar(
args.prometheus_url,
"histogram_quantile(0.95, sum by (le) "
"(increase(messenger_worker_job_duration_seconds_bucket[2m])))",
)
ready_metric = prometheus_scalar(
args.prometheus_url,
"histogram_quantile(0.95, sum by (le) "
"(increase(messenger_attachment_ready_duration_seconds_bucket[2m])))",
)
client_http_p95 = percentile(http_seconds, 0.95)
client_ready_p95 = percentile(ready_seconds, 0.95)
http_boundaries = histogram_boundaries(
args.prometheus_url, "messenger_http_server_duration_seconds"
)
worker_boundaries = histogram_boundaries(
args.prometheus_url, "messenger_worker_job_duration_seconds"
)
ready_boundaries = histogram_boundaries(
args.prometheus_url, "messenger_attachment_ready_duration_seconds"
)
assert {"0.005", "0.01", "0.05", "0.1", "1.0"}.issubset(
http_boundaries
), http_boundaries
assert {"0.05", "0.1", "0.25", "1.0", "10.0"}.issubset(
worker_boundaries
), worker_boundaries
assert {"0.1", "0.25", "1.0", "10.0", "120.0"}.issubset(
ready_boundaries
), ready_boundaries
assert 0 < http_metric < 1, http_metric
assert 0 < worker_metric < 10, worker_metric
assert 0 < ready_metric < 10, ready_metric
print(
json.dumps(
{
"status": "PASS",
"samples": {
"http": len(http_seconds),
"attachments": len(ready_seconds),
},
"http_p95_seconds": {
"client": client_http_p95,
"prometheus": http_metric,
"ratio": comparable(client_http_p95, http_metric),
},
"ready_p95_seconds": {
"client_polling_upper_bound": client_ready_p95,
"prometheus": ready_metric,
"ratio": comparable(
client_ready_p95, ready_metric, lower=0.1, upper=5.0
),
},
"worker_p95_seconds": worker_metric,
"boundaries": {
"http": sorted(http_boundaries),
"worker": sorted(worker_boundaries),
"ready": sorted(ready_boundaries),
},
},
ensure_ascii=False,
indent=2,
)
)
finally:
cleanup(chat["id"], users)
if __name__ == "__main__":
main()