Доработать инструменты экспериментов: чередовать повторы, останавливать лестницу на первой проваленной ступени, задавать maxconn и CPU базы в сериях, записывать замер скорости хоста и контроль шума, сократить комментарии
This commit is contained in:
@@ -67,17 +67,15 @@ logs:
|
||||
instances:
|
||||
sh scripts/instances.sh
|
||||
|
||||
# Replaces every API replica with one built from the current sources, without
|
||||
# downtime; restore a replica stopped by stop-one with `docker compose start api`.
|
||||
# Replaces every API replica without downtime; all of them must be running.
|
||||
rollout:
|
||||
sh scripts/rollout.sh
|
||||
|
||||
# Track 1, the cost of scaling: one series on fresh stands, then the charts.
|
||||
# RATES is required, e.g. make bench SERIES=scripts/bench/series/B.conf RATES=200,400,800
|
||||
# Track 1. Every run starts from empty volumes, so the stand's data is lost.
|
||||
# make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/B.conf" RATES=400,800,1200
|
||||
SERIES ?= scripts/bench/series/A.conf
|
||||
REPEATS ?= 3
|
||||
bench:
|
||||
sh scripts/bench/run-series.sh $(SERIES) $(REPEATS)
|
||||
sh scripts/bench/run-series.sh $(SERIES)
|
||||
|
||||
# Needs matplotlib from requirements-bench.txt: make plot PYTHON=<venv python>.
|
||||
plot:
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Lists the API containers of this compose project, one per line:
|
||||
# <container id> <name> <state> <alias> <image id>
|
||||
# The alias is what the process sends as X-Instance-Id. It is random per
|
||||
# process, so it is asked from the process itself, over the same wget the
|
||||
# Docker healthcheck uses, from inside the container. A container that is not
|
||||
# running has no alias and shows "-". The replicas are asked in parallel: the
|
||||
# failover hook that relies on this list must return within 10 seconds.
|
||||
# Lists the API containers: <id> <name> <state> <X-Instance-Id> <image>. The
|
||||
# alias is random per process, so each running replica is asked over
|
||||
# /health/live, in parallel: the failover hook must return within 10 seconds.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
|
||||
@@ -1,26 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Level 4: one API replica fails under a constant write load. The generator
|
||||
# sends messages at RATE for DURATION and retries uncertain sends with the
|
||||
# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM,
|
||||
# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started
|
||||
# again. Afterwards the generator reads the history back: every confirmed
|
||||
# message once.
|
||||
# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and
|
||||
# started again at RESTORE_AT, under a constant write load that retries
|
||||
# uncertain sends with the same key; then every confirmed message must be in
|
||||
# the history once.
|
||||
#
|
||||
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
|
||||
#
|
||||
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35),
|
||||
# OUT (default evidence/failover). Results in <OUT>/<kind>-<n>/:
|
||||
# timeline.json, seconds.csv (per-second outcomes, attributed by send time),
|
||||
# faults.json (when and what was stopped), compose.log (the balancer's
|
||||
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
|
||||
# Runs on the current stand (`make up`) and leaves every replica running.
|
||||
#
|
||||
# Every time used for the analysis comes from the Docker VM's clock: the log
|
||||
# timestamps, the container's stop and start times from `docker inspect`, and
|
||||
# the generator's own times. The host clock of Windows runs apart from it by
|
||||
# a fraction of a second, a large share of a 1.5 s detection window; host
|
||||
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
|
||||
# is the replica's own "shutdown started" line in compose.log.
|
||||
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT
|
||||
# (evidence/failover). Times come from the Docker VM clock; the Windows clock
|
||||
# differs by a fraction of a second and is kept for reference only.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
@@ -50,8 +38,6 @@ since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
||||
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
||||
generator=$!
|
||||
# The generator seeds two users first; its timeline starts after that. The
|
||||
# fault times below are absolute, as is started_at in timeline.json.
|
||||
sleep "$fault_at"
|
||||
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
docker "$kind" "$target" >/dev/null
|
||||
|
||||
+13
-8
@@ -3,14 +3,13 @@
|
||||
# The generator and the knobs the experiments vary live in compose.bench.yaml.
|
||||
export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml
|
||||
|
||||
# abspath prints the absolute path of an existing directory in the form the
|
||||
# Docker CLI accepts for a bind mount (a Windows path under Git Bash).
|
||||
# abspath prints a directory in the form the Docker CLI mounts (a Windows path
|
||||
# under Git Bash).
|
||||
abspath() {
|
||||
(cd "$1" && (pwd -W 2>/dev/null || pwd))
|
||||
}
|
||||
|
||||
# meta writes the header of one run: what ran, when, on which commit and
|
||||
# host. Arguments: file, then key=value pairs added to the record.
|
||||
# meta writes what ran, when, on which commit and host: file, then key=value.
|
||||
meta() {
|
||||
file=$1
|
||||
shift
|
||||
@@ -30,8 +29,13 @@ meta() {
|
||||
} >"$file"
|
||||
}
|
||||
|
||||
# sample_docker_stats appends the CPU and memory of every container of the
|
||||
# stand to a JSON-lines file, about every two seconds, until killed.
|
||||
# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container:
|
||||
# the same work every time, so a slow spell of the host shows in the results.
|
||||
host_probe() {
|
||||
docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }'
|
||||
}
|
||||
|
||||
# sample_docker_stats appends the CPU and memory of every container until killed.
|
||||
sample_docker_stats() {
|
||||
while :; do
|
||||
docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \
|
||||
@@ -39,8 +43,8 @@ sample_docker_stats() {
|
||||
done >>"$1"
|
||||
}
|
||||
|
||||
# sample_postgres appends, every second, the database's commit counters and
|
||||
# what its active sessions are waiting for (none: running on CPU).
|
||||
# sample_postgres appends, every second, the database counters and what the
|
||||
# active sessions wait for ("cpu": nothing).
|
||||
sample_postgres() {
|
||||
while :; do
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c "
|
||||
@@ -48,6 +52,7 @@ sample_postgres() {
|
||||
't', extract(epoch FROM now()),
|
||||
'xact_commit', xact_commit, 'xact_rollback', xact_rollback,
|
||||
'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit,
|
||||
'sessions', sessions, 'sessions_abandoned', sessions_abandoned,
|
||||
'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()),
|
||||
'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM (
|
||||
SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n
|
||||
|
||||
+48
-26
@@ -5,14 +5,16 @@
|
||||
|
||||
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
|
||||
output directory:
|
||||
summary.csv one row per step of every run: offered and successful rate,
|
||||
errors, latency percentiles, generator lag, mean CPU of each
|
||||
component and pool waits during the step;
|
||||
summary.csv one row per step of every run;
|
||||
capacity.csv the capacity of every run and the median per configuration;
|
||||
control.csv the noise checks of every run: the host speed probe and the
|
||||
loadgen CPU per request at the first step, flagged when more
|
||||
than 25 % off the median;
|
||||
<series>-throughput.png, <series>-p99.png, capacity.png,
|
||||
cpu-at-capacity.png, pool-waits.png.
|
||||
Capacity is the highest valid step (generator on schedule) with less than 1 %
|
||||
errors and a p99 of successful requests within 500 ms.
|
||||
Capacity is the highest step that passed the criterion loadgen applies (valid,
|
||||
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
|
||||
median of the repeats; a hollow marker means the generator fell behind.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
@@ -27,7 +29,7 @@ matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
SLO_P99_MS = 500.0
|
||||
MAX_ERROR_SHARE = 0.01
|
||||
CONTROL_TOLERANCE = 0.25
|
||||
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
|
||||
|
||||
|
||||
@@ -69,6 +71,7 @@ def load_runs(root):
|
||||
"steps": report["steps"],
|
||||
"docker": read_jsonl(d / "docker-stats.jsonl"),
|
||||
"metrics": read_jsonl(d / "metrics.jsonl"),
|
||||
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
|
||||
})
|
||||
return runs
|
||||
|
||||
@@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1):
|
||||
return out
|
||||
|
||||
|
||||
def pool_waits(run, t0, t1):
|
||||
"""Requests that waited for a pooled connection during [t0, t1), all
|
||||
replicas together, per second."""
|
||||
def pool_rate(run, t0, t1, counter):
|
||||
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
|
||||
first, last = {}, {}
|
||||
for row in run["metrics"]:
|
||||
t = run["start"] + row["t"]
|
||||
@@ -104,18 +106,14 @@ def pool_waits(run, t0, t1):
|
||||
for replica in row.get("replicas") or []:
|
||||
pool = replica.get("pgxpool") or {}
|
||||
key = replica.get("instance")
|
||||
if key and "empty_acquire_count" in pool:
|
||||
first.setdefault(key, pool["empty_acquire_count"])
|
||||
last[key] = pool["empty_acquire_count"]
|
||||
if key and counter in pool:
|
||||
first.setdefault(key, pool[counter])
|
||||
last[key] = pool[counter]
|
||||
if not last:
|
||||
return ""
|
||||
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
|
||||
|
||||
|
||||
def is_capacity(step):
|
||||
return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS
|
||||
|
||||
|
||||
def main(root, out):
|
||||
runs = load_runs(root)
|
||||
if not runs:
|
||||
@@ -129,17 +127,22 @@ def main(root, out):
|
||||
t0 = run["start"] + step["started_at_s"]
|
||||
t1 = t0 + step["seconds"]
|
||||
cpu = mean_cpu(run, t0, t1)
|
||||
classes = step["classes"]
|
||||
row = {
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
|
||||
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
|
||||
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
|
||||
"client_timeouts": classes.get("client_timeout", 0),
|
||||
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
|
||||
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
|
||||
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
|
||||
"pool_waits_per_s": pool_waits(run, t0, t1),
|
||||
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
|
||||
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
|
||||
}
|
||||
rows.append(row)
|
||||
if is_capacity(step):
|
||||
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
|
||||
best = row
|
||||
capacities.append({
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
@@ -164,13 +167,29 @@ def main(root, out):
|
||||
medians[key] = statistics.median(values)
|
||||
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"])
|
||||
|
||||
control = []
|
||||
for run in runs:
|
||||
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
|
||||
if first and first[0]["cpu_loadgen_pct"] != "":
|
||||
control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"],
|
||||
round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)])
|
||||
if control:
|
||||
median = statistics.median(c[5] for c in control)
|
||||
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"])
|
||||
for c in control:
|
||||
deviation = c[5] / median - 1
|
||||
w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE])
|
||||
|
||||
def median_by_offered(series, config, field):
|
||||
points = {}
|
||||
points, valid = {}, {}
|
||||
for r in rows:
|
||||
if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "":
|
||||
if r["series"] == series and r["config"] == config and r[field] != "":
|
||||
points.setdefault(r["offered_rps"], []).append(float(r[field]))
|
||||
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
|
||||
xs = sorted(points)
|
||||
return xs, [statistics.median(points[x]) for x in xs]
|
||||
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
|
||||
|
||||
for series in sorted({s for s, _ in configs}):
|
||||
names = [c for s, c in configs if s == series]
|
||||
@@ -181,8 +200,11 @@ def main(root, out):
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
top = 0
|
||||
for name in names:
|
||||
xs, ys = median_by_offered(series, name, field)
|
||||
ax.plot(xs, ys, marker="o", label=name)
|
||||
xs, ys, valid = median_by_offered(series, name, field)
|
||||
line, = ax.plot(xs, ys, label=name)
|
||||
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
|
||||
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
|
||||
facecolors="none", edgecolors=line.get_color())
|
||||
top = max([top, *xs])
|
||||
if field == "ok_rps" and top:
|
||||
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
|
||||
@@ -192,7 +214,7 @@ def main(root, out):
|
||||
ax.set_yscale("log")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
ax.set_ylabel(title)
|
||||
ax.set_title(f"Series {series}: median of repeats, valid steps only")
|
||||
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
@@ -233,7 +255,7 @@ def main(root, out):
|
||||
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
for series, name in configs:
|
||||
xs, ys = median_by_offered(series, name, "pool_waits_per_s")
|
||||
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
|
||||
if xs:
|
||||
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
|
||||
@@ -1,21 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Level 4: a rolling update under a constant write load. The generator sends
|
||||
# messages at RATE for DURATION, retrying uncertain sends with the same key;
|
||||
# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards
|
||||
# the generator reads the history back: every confirmed message once.
|
||||
# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a
|
||||
# constant write load that retries uncertain sends with the same key; then
|
||||
# every confirmed message must be in the history once.
|
||||
#
|
||||
# sh scripts/bench/rollout-under-load.sh
|
||||
#
|
||||
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default
|
||||
# evidence/rollout). Results in <OUT>/: timeline.json, seconds.csv,
|
||||
# rollout.log, update.json (when it ran, which containers it replaced),
|
||||
# compose.log, instances-*.txt.
|
||||
#
|
||||
# Times for the analysis come from the Docker VM's clock, as the generator's
|
||||
# do: log timestamps and the start times of the new containers. The host
|
||||
# clock of Windows runs apart from it and is kept for reference only. The
|
||||
# old replicas are removed by the update; their stop shows in compose.log as
|
||||
# their "shutdown started" lines.
|
||||
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout).
|
||||
# Times come from the Docker VM clock, as in failover.sh.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
@@ -29,8 +20,7 @@ export MSYS_NO_PATHCONV=1
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
docker compose up -d --wait >/dev/null 2>&1
|
||||
# Build first, as a delivery pipeline would: the update under load then
|
||||
# measures the replacement of the replicas, not the compiler.
|
||||
# Built beforehand, as by a delivery pipeline: the update replaces replicas.
|
||||
docker compose build api >/dev/null 2>&1
|
||||
meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at"
|
||||
sh scripts/instances.sh >"$dir/instances-before.txt"
|
||||
|
||||
+65
-44
@@ -1,61 +1,82 @@
|
||||
#!/bin/sh
|
||||
# Track 1, the cost of scaling: runs every configuration of one series file,
|
||||
# each repeated, each on a fresh stand (empty volumes, same seed), under the
|
||||
# same grid of offered rates.
|
||||
# Track 1: every configuration of the given series files, each on a fresh
|
||||
# stand, under one grid of offered rates. Repeats are interleaved (repeat 1
|
||||
# of every configuration, then repeat 2), so a slow spell of the host does not
|
||||
# fall on one configuration.
|
||||
#
|
||||
# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats]
|
||||
# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...]
|
||||
#
|
||||
# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root,
|
||||
# default evidence/track1/raw). Fixed for every configuration: HAProxy
|
||||
# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the
|
||||
# generator LOADGEN_CPUS (1.5). Results per run:
|
||||
# <OUT>/<series>/<config>/rep<k>/{meta.json,steps.json,seconds.csv,
|
||||
# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log}
|
||||
# The stand is removed at the end; `make up` brings the normal one back.
|
||||
# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s),
|
||||
# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT
|
||||
# (evidence/track1/raw). Results: <OUT>/<series>/<config>/rep<k>/.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
|
||||
series=${1:?usage: run-series.sh <series.conf> [repeats]}
|
||||
repeats=${2:-3}
|
||||
: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}"
|
||||
[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh <series.conf>..." >&2; exit 2; }
|
||||
: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}"
|
||||
repeats=${REPEATS:-3}
|
||||
step=${STEP:-30s}
|
||||
warmup=${WARMUP:-30s}
|
||||
root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf)
|
||||
out=${OUT:-evidence/track1/raw}
|
||||
stop=
|
||||
[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail
|
||||
|
||||
export APP_DEBUG_ADDR=:9090 LB_QUIET=1
|
||||
export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5}
|
||||
# Fixed for every configuration; the API, its pool, maxconn and the database
|
||||
# CPU come from the series line.
|
||||
export LB_QUIET=1 LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0
|
||||
export MSYS_NO_PATHCONV=1
|
||||
|
||||
grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do
|
||||
[ -n "$name" ] || continue
|
||||
for rep in $(seq 1 "$repeats"); do
|
||||
dir=$root/$name/rep$rep
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES"
|
||||
export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool
|
||||
# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus.
|
||||
run_config() {
|
||||
series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10}
|
||||
dir=$out/$series/$name/rep$rep
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU"
|
||||
export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \
|
||||
DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus
|
||||
|
||||
docker compose down -v --remove-orphans >/dev/null 2>&1
|
||||
docker compose up -d --build --wait >/dev/null 2>&1
|
||||
meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \
|
||||
"api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \
|
||||
"lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \
|
||||
"rates=$RATES" "step=$step" "warmup=$warmup"
|
||||
docker compose down -v --remove-orphans >/dev/null 2>&1
|
||||
if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then
|
||||
echo " the stand did not start, see $dir/up.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \
|
||||
"api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \
|
||||
"db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \
|
||||
"lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \
|
||||
"stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)"
|
||||
|
||||
sample_docker_stats "$dir/docker-stats.jsonl" &
|
||||
stats=$!
|
||||
sample_postgres "$dir/postgres.jsonl" &
|
||||
pg=$!
|
||||
# stdin is closed: `compose run` is interactive by default and would
|
||||
# otherwise read the rest of the series file this loop is reading.
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
|
||||
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 </dev/null ||
|
||||
echo " loadgen failed, see $dir/loadgen.log"
|
||||
kill "$stats" "$pg" 2>/dev/null || true
|
||||
wait "$stats" "$pg" 2>/dev/null || true
|
||||
cat "$dir/loadgen.log" | grep '^step' || true
|
||||
sample_docker_stats "$dir/docker-stats.jsonl" &
|
||||
stats=$!
|
||||
sample_postgres "$dir/postgres.jsonl" &
|
||||
pg=$!
|
||||
# stdin closed: `compose run` would otherwise read it.
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
|
||||
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 </dev/null ||
|
||||
echo " loadgen failed, see $dir/loadgen.log"
|
||||
kill "$stats" "$pg" 2>/dev/null || true
|
||||
wait "$stats" "$pg" 2>/dev/null || true
|
||||
docker compose logs --no-color --timestamps lb api >"$dir/compose.log" 2>&1
|
||||
grep '^step' "$dir/loadgen.log" || true
|
||||
}
|
||||
|
||||
docker compose --profile bench build loadgen >/dev/null
|
||||
default_ifs=$IFS
|
||||
newline='
|
||||
'
|
||||
for rep in $(seq 1 "$repeats"); do
|
||||
for conf in "$@"; do
|
||||
lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d')
|
||||
IFS=$newline
|
||||
for line in $lines; do
|
||||
IFS=$default_ifs
|
||||
# shellcheck disable=SC2086 # the line splits into the fields
|
||||
run_config "$(basename "$conf" .conf)" "$rep" $line
|
||||
done
|
||||
IFS=$default_ifs
|
||||
done
|
||||
done
|
||||
docker compose down -v --remove-orphans >/dev/null 2>&1
|
||||
echo "results in $root"
|
||||
echo "results in $out"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas.
|
||||
# The pool stays 10 per replica, so the total grows with the replicas.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
A1 1 2.0 1024M 920MiB 10
|
||||
A2 2 1.0 512M 460MiB 10
|
||||
A4 4 0.5 256M 230MiB 10
|
||||
# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas;
|
||||
# the pool stays 10 per replica, so the total grows with the replicas.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
A1 1 2.0 1024M 920MiB 10 0 1.0
|
||||
A2 2 1.0 512M 460MiB 10 0 1.0
|
||||
A4 4 0.5 256M 230MiB 10 0 1.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Series A': the budget of series A with the total pool fixed at 20
|
||||
# connections. A'2 equals A2 and is not run again.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
Ap1 1 2.0 1024M 920MiB 20
|
||||
Ap4 4 0.5 256M 230MiB 5
|
||||
# Series A': the budget of series A with the total pool fixed at 20 connections;
|
||||
# A'2 equals A2.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
Ap1 1 2.0 1024M 920MiB 20 0 1.0
|
||||
Ap4 4 0.5 256M 230MiB 5 0 1.0
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by
|
||||
# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run
|
||||
# again.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
B1 1 1.0 512M 460MiB 10
|
||||
B3 3 1.0 512M 460MiB 10
|
||||
B4 4 1.0 512M 460MiB 10
|
||||
# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
B1 1 1.0 512M 460MiB 10 0 1.0
|
||||
B3 3 1.0 512M 460MiB 10 0 1.0
|
||||
B4 4 1.0 512M 460MiB 10 0 1.0
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Improvement C: maxconn per replica on the balancer, before (0, no limit) and
|
||||
# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0);
|
||||
# 10 and 40 show how much the value matters.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
R1m0 1 1.0 512M 460MiB 10 0 1.0
|
||||
R1m20 1 1.0 512M 460MiB 10 20 1.0
|
||||
R4m0 4 1.0 512M 460MiB 10 0 1.0
|
||||
R4m20 4 1.0 512M 460MiB 10 20 1.0
|
||||
R1m10 1 1.0 512M 460MiB 10 10 1.0
|
||||
R1m40 1 1.0 512M 460MiB 10 40 1.0
|
||||
@@ -1,4 +0,0 @@
|
||||
# Calibration: the smallest API budget of the study (one replica, one CPU)
|
||||
# with a wide range of rates finds where the common rate grid must reach.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
cal 1 1.0 512M 460MiB 10
|
||||
@@ -0,0 +1,4 @@
|
||||
# Control: A2 with twice the database CPU. If the database is the bottleneck,
|
||||
# the capacity rises.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
A2pg2 2 1.0 512M 460MiB 10 0 2.0
|
||||
@@ -1,7 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Shows which container answers under which X-Instance-Id: the proof on the
|
||||
# defense that the header values belong to different running processes, and
|
||||
# which container a failover test actually stopped.
|
||||
# Which container answers under which X-Instance-Id.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
|
||||
+2
-5
@@ -1,9 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Restart hook of `make test-persistence` (tests/resilience.py): stops the
|
||||
# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the
|
||||
# same containers again (the one-shot migrate finds nothing to apply).
|
||||
# Volumes and data are kept and nothing is re-seeded. The test hides the
|
||||
# output, so failures surface through the exit code only.
|
||||
# Restart hook of `make test-persistence`: stops the whole stand gracefully and
|
||||
# starts the same containers again; volumes are kept.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop
|
||||
|
||||
+8
-18
@@ -1,21 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API without downtime, with docker rollout
|
||||
# (https://github.com/wowu/docker-rollout, MIT): it starts as many new
|
||||
# replicas as are running, waits until all of them are healthy, then stops
|
||||
# and removes the old ones. If a new replica does not become healthy it
|
||||
# removes the new ones and leaves the old ones serving, so the stand is never
|
||||
# left with a mix of versions.
|
||||
# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout,
|
||||
# MIT): new replicas start next to the old ones, and the old ones go once all
|
||||
# new ones are healthy; if one is not, the new ones are removed instead.
|
||||
#
|
||||
# Every replica must be running. With a stopped one (say, after stop-one),
|
||||
# docker rollout only replaces the stopped one and reports success, which
|
||||
# would leave old and new versions side by side; the script refuses instead.
|
||||
#
|
||||
# The migrations of the new version run first, once, while the old replicas
|
||||
# serve: they only add to the schema. docker rollout scales with `compose up`,
|
||||
# which also starts the dependencies of the API and would rerun the old,
|
||||
# exited migrate container. Recreating it beforehand from the new image,
|
||||
# without starting it, makes that `up` run the new migrations exactly once,
|
||||
# before any new replica starts.
|
||||
# With a stopped replica docker rollout would replace only that one and leave
|
||||
# two versions running, so the script refuses. The migrate container is
|
||||
# recreated from the new image without starting: the `up` of docker rollout
|
||||
# then applies the new migrations once, before any new replica starts.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
@@ -44,8 +35,7 @@ docker compose build api
|
||||
docker compose up --no-start --no-deps --force-recreate migrate
|
||||
# Run through sh: Windows cannot use the script as a docker CLI plugin.
|
||||
if ! sh "$tool" rollout --wait-after-healthy 4 api; then
|
||||
# A failed migration stops the scale-up after the new containers were
|
||||
# created but before they started; remove them for a clean next attempt.
|
||||
# A failed migration leaves the new containers created but not started.
|
||||
created=$(docker compose ps -aq --status created api)
|
||||
if [ -n "$created" ]; then
|
||||
docker rm $created >/dev/null
|
||||
|
||||
+3
-7
@@ -1,11 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Failover hook of `make test-failover` (tests/resilience.py): stops the one
|
||||
# API replica whose X-Instance-Id is $TARGET_INSTANCE_ID and leaves it
|
||||
# stopped. The stop is graceful: on SIGTERM the replica turns unready, keeps
|
||||
# serving for its drain delay while HAProxy takes it out of rotation, then
|
||||
# exits; `docker stop` returns once it has. Nothing else is touched, and an
|
||||
# alias that does not name exactly one running replica stops nothing.
|
||||
# Restore the replica with: docker compose start api
|
||||
# Failover hook of `make test-failover`: gracefully stops the one running API
|
||||
# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else.
|
||||
# Restore it with: docker compose start api
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
|
||||
@@ -52,9 +52,8 @@ func (c *client) postMessage(chatID, token, text string, out any) error {
|
||||
return c.do(req, token, http.StatusCreated, out)
|
||||
}
|
||||
|
||||
// sendWithKey sends one attempt of a message under the given key on a
|
||||
// connection of its own and reports the status, the message (on 201) and the
|
||||
// replica that answered.
|
||||
// sendWithKey sends one attempt on a connection of its own, so that the
|
||||
// attempts of one key reach different replicas.
|
||||
func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) {
|
||||
var msg chat.MessageResponse
|
||||
raw, err := json.Marshal(map[string]string{"text": text})
|
||||
|
||||
@@ -4,12 +4,9 @@
|
||||
// then the history is paged and must contain exactly N*M unique messages in
|
||||
// id order.
|
||||
//
|
||||
// Phase 2, shared idempotency: in each of K rounds one user sends the same
|
||||
// message with the same Idempotency-Key P times in parallel, each attempt on
|
||||
// its own connection, so the balancer spreads them over the replicas. Every
|
||||
// attempt must answer 201 with the same message, the attempts must have
|
||||
// reached at least two replicas, the history must hold exactly K messages,
|
||||
// and the key with another text must answer 409.
|
||||
// Phase 2: in each of K rounds one key is sent P times in parallel through the
|
||||
// balancer; every attempt must get 201 with one message, at least two replicas
|
||||
// must answer, the history must hold K messages, and another text must get 409.
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
+16
-25
@@ -1,21 +1,10 @@
|
||||
// Command loadgen is the open-loop load generator of the scaling study
|
||||
// (track 1) and of the failover and rolling-update experiments.
|
||||
// Command loadgen is the open-loop load generator of the experiments.
|
||||
//
|
||||
// loadgen bench — seeds users, chats and messages, warms up, then offers
|
||||
// each rate of -rates for -step: a mix of history reads
|
||||
// (-read-ratio) and message sends. Writes steps.json and
|
||||
// seconds.csv, and metrics.jsonl with -scrape.
|
||||
// loadgen timeline — one user sends messages into one chat at -rate for
|
||||
// -duration; a send with an uncertain outcome (no answer,
|
||||
// 5xx) is retried with the same Idempotency-Key. At the
|
||||
// end the whole history is read back: every confirmed
|
||||
// message must be there, and none twice. Writes
|
||||
// timeline.json and seconds.csv.
|
||||
//
|
||||
// Requests are sent on schedule whatever the response times (open loop), and
|
||||
// the generator records how late it sent them: a step it could not keep up
|
||||
// with is marked invalid rather than reported as the system's limit. Tokens
|
||||
// stay in memory and are never written out.
|
||||
// loadgen bench — seeds data, warms up, then offers each rate of -rates
|
||||
// for -step as a mix of history reads and message sends.
|
||||
// loadgen timeline — one user sends messages at -rate, retrying uncertain
|
||||
// sends with the same Idempotency-Key, then reads the
|
||||
// history back: every confirmed message once.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -52,6 +41,7 @@ type config struct {
|
||||
Timeout string `json:"client_timeout"`
|
||||
Seed uint64 `json:"seed"`
|
||||
Scrape bool `json:"scrape"`
|
||||
StopOnFail bool `json:"stop_on_fail"`
|
||||
StatsURL string `json:"stats_url"`
|
||||
APIHost string `json:"api_host"`
|
||||
DebugPort string `json:"debug_port"`
|
||||
@@ -98,6 +88,7 @@ func parse(mode string, args []string) (config, error) {
|
||||
fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration")
|
||||
fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)")
|
||||
fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix")
|
||||
fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step that fails the capacity criterion")
|
||||
fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second")
|
||||
fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV")
|
||||
fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas")
|
||||
@@ -138,8 +129,7 @@ func writeJSON(path string, v any) error {
|
||||
return os.WriteFile(path, append(raw, '\n'), 0o644)
|
||||
}
|
||||
|
||||
// startScrape samples the system in the background when asked to; the
|
||||
// returned function stops it and waits.
|
||||
// startScrape returns the function that stops the sampling and waits.
|
||||
func startScrape(ctx context.Context, cfg config, start time.Time) func() {
|
||||
if !cfg.Scrape {
|
||||
return func() {}
|
||||
@@ -177,8 +167,7 @@ func runBench(ctx context.Context, cfg config) error {
|
||||
stopScrape := startScrape(ctx, cfg, start)
|
||||
defer stopScrape()
|
||||
|
||||
// next picks the request on the scheduling goroutine, where the random
|
||||
// source may be used safely.
|
||||
// next runs on the scheduling goroutine, which owns the random source.
|
||||
counter := 0
|
||||
next := func(rec *recorder) func(time.Time) func() {
|
||||
return func(planned time.Time) func() {
|
||||
@@ -217,8 +206,11 @@ func runBench(ctx context.Context, cfg config) error {
|
||||
for _, x := range rec.all() {
|
||||
all.add(x)
|
||||
}
|
||||
fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v\n",
|
||||
rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid)
|
||||
fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n",
|
||||
rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed)
|
||||
if cfg.StopOnFail && !s.Passed {
|
||||
break
|
||||
}
|
||||
}
|
||||
stopScrape()
|
||||
if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil {
|
||||
@@ -247,8 +239,7 @@ type timelineReport struct {
|
||||
// retryDelay spaces the attempts of one message whose outcome was uncertain.
|
||||
const retryDelay = 200 * time.Millisecond
|
||||
|
||||
// retryGrace is how long after the schedule ends an uncertain message keeps
|
||||
// being retried before it counts as unresolved.
|
||||
// retryGrace bounds the retries of an uncertain message after the schedule ends.
|
||||
const retryGrace = 10 * time.Second
|
||||
|
||||
func runTimeline(ctx context.Context, cfg config) error {
|
||||
|
||||
@@ -6,14 +6,9 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// openLoop offers requests at a constant rate for d, whatever the response
|
||||
// times: request i is planned at start + i/rate and sent then, or at once if
|
||||
// the generator is already late. Waiting for responses before sending more
|
||||
// (a closed loop) would let a slow system lower its own load and hide its
|
||||
// latency: coordinated omission. next runs on the scheduling goroutine, so it
|
||||
// may use non-thread-safe state such as the random source; the job it
|
||||
// returns runs on a goroutine of its own. openLoop returns once every job
|
||||
// has finished.
|
||||
// openLoop sends request i at start + i/rate whatever the response times: a
|
||||
// closed loop would let a slow system lower its own load and hide its latency
|
||||
// (coordinated omission). It returns once every job has finished.
|
||||
func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) {
|
||||
n := int(rate * d.Seconds())
|
||||
interval := time.Duration(float64(time.Second) / rate)
|
||||
|
||||
@@ -12,11 +12,8 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// scraper samples, once a second, what the balancer and every replica report
|
||||
// about themselves, so a saturated step can be explained: the HAProxy
|
||||
// statistics per replica and the expvar of each API process (pool waits,
|
||||
// GOMAXPROCS, memory and GC). Replicas are found the way HAProxy finds them,
|
||||
// through Docker DNS, and asked directly on their internal debug port.
|
||||
// scraper samples the HAProxy statistics and every replica's expvar once a
|
||||
// second; replicas are found through Docker DNS, as HAProxy finds them.
|
||||
type scraper struct {
|
||||
statsURL string
|
||||
apiHost string
|
||||
|
||||
+15
-10
@@ -12,11 +12,17 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxValidLag is the generator's own tolerance: if requests left more than
|
||||
// this late at the 99th percentile, the generator, not the system, limited
|
||||
// the step, and the step is excluded from the conclusions.
|
||||
// maxValidLag: if requests left later than this at p99, the generator, not
|
||||
// the system, limited the step.
|
||||
const maxValidLag = 10 * time.Millisecond
|
||||
|
||||
// The capacity criterion of the study: a step passes when it is valid, has
|
||||
// fewer than 1 % errors and its successful requests have a p99 within 500 ms.
|
||||
const (
|
||||
maxErrorShare = 0.01
|
||||
sloP99 = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
// sample is one request as the generator saw it.
|
||||
type sample struct {
|
||||
planned time.Time
|
||||
@@ -27,11 +33,9 @@ type sample struct {
|
||||
write bool
|
||||
}
|
||||
|
||||
// classify names the outcome of a request. HTTP errors produced by the
|
||||
// balancer itself carry no X-Instance-Id and are kept apart from the API's
|
||||
// own: a 503 from HAProxy means no replica was available, a 503 from the API
|
||||
// means a dependency was. client_timeout is the generator giving up on a
|
||||
// response, not a timeout of the system.
|
||||
// classify names the outcome of a request. Errors of the balancer itself carry
|
||||
// no X-Instance-Id and get the suffix _lb; client_timeout is the generator
|
||||
// giving up, not a timeout of the system.
|
||||
func classify(r result) string {
|
||||
switch {
|
||||
case r.err != nil && isTimeout(r.err):
|
||||
@@ -78,12 +82,12 @@ type stepSummary struct {
|
||||
P99Ms float64 `json:"ok_p99_ms"`
|
||||
LagP99Ms float64 `json:"lag_p99_ms"`
|
||||
Valid bool `json:"valid"`
|
||||
Passed bool `json:"passed"`
|
||||
Instances map[string]int `json:"instances"`
|
||||
Writes int `json:"writes_ok"`
|
||||
}
|
||||
|
||||
// summarize reduces one step. Latency percentiles cover successful requests
|
||||
// only, as the course's report format asks; the errors are counted by class.
|
||||
// summarize reduces one step; latency percentiles cover successful requests.
|
||||
func summarize(rate float64, d time.Duration, samples []sample) stepSummary {
|
||||
s := stepSummary{
|
||||
OfferedRPS: rate,
|
||||
@@ -117,6 +121,7 @@ func summarize(rate float64, d time.Duration, samples []sample) stepSummary {
|
||||
lagP99 := percentile(lags, 99)
|
||||
s.LagP99Ms = ms(lagP99)
|
||||
s.Valid = lagP99 <= maxValidLag
|
||||
s.Passed = s.Valid && s.ErrorShare < maxErrorShare && percentile(ok, 99) <= sloP99
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
@@ -104,3 +104,25 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) {
|
||||
t.Fatalf("open loop took %v, want about 1.2 s", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepPassesOnlyWithinTheObjective(t *testing.T) {
|
||||
ok := func(latency, lag time.Duration) sample {
|
||||
return sample{class: "ok", latency: latency, lag: lag}
|
||||
}
|
||||
fast := []sample{ok(10*time.Millisecond, time.Millisecond), ok(20*time.Millisecond, time.Millisecond)}
|
||||
if s := summarize(2, time.Second, fast); !s.Passed {
|
||||
t.Fatalf("fast clean step must pass: %+v", s)
|
||||
}
|
||||
slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)}
|
||||
if s := summarize(2, time.Second, slow); s.Passed {
|
||||
t.Fatalf("p99 above 500 ms must fail: %+v", s)
|
||||
}
|
||||
failing := append(fast, sample{class: "http_503_lb", lag: time.Millisecond})
|
||||
if s := summarize(3, time.Second, failing); s.Passed {
|
||||
t.Fatalf("errors above 1 %% must fail: %+v", s)
|
||||
}
|
||||
late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)}
|
||||
if s := summarize(2, time.Second, late); s.Passed {
|
||||
t.Fatalf("an invalid step must fail: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user