Доработать инструменты экспериментов: чередовать повторы, останавливать лестницу на первой проваленной ступени, задавать maxconn и CPU базы в сериях, записывать замер скорости хоста и контроль шума, сократить комментарии

This commit is contained in:
2026-11-04 12:39:04 +03:00
parent ce47c138e4
commit baa5995938
24 changed files with 253 additions and 240 deletions
+4 -6
View File
@@ -67,17 +67,15 @@ logs:
instances:
sh scripts/instances.sh
# Replaces every API replica with one built from the current sources, without
# downtime; restore a replica stopped by stop-one with `docker compose start api`.
# Replaces every API replica without downtime; all of them must be running.
rollout:
sh scripts/rollout.sh
# Track 1, the cost of scaling: one series on fresh stands, then the charts.
# RATES is required, e.g. make bench SERIES=scripts/bench/series/B.conf RATES=200,400,800
# Track 1. Every run starts from empty volumes, so the stand's data is lost.
# make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/B.conf" RATES=400,800,1200
SERIES ?= scripts/bench/series/A.conf
REPEATS ?= 3
bench:
sh scripts/bench/run-series.sh $(SERIES) $(REPEATS)
sh scripts/bench/run-series.sh $(SERIES)
# Needs matplotlib from requirements-bench.txt: make plot PYTHON=<venv python>.
plot:
+3 -7
View File
@@ -1,11 +1,7 @@
#!/bin/sh
# Lists the API containers of this compose project, one per line:
# <container id> <name> <state> <alias> <image id>
# The alias is what the process sends as X-Instance-Id. It is random per
# process, so it is asked from the process itself, over the same wget the
# Docker healthcheck uses, from inside the container. A container that is not
# running has no alias and shows "-". The replicas are asked in parallel: the
# failover hook that relies on this list must return within 10 seconds.
# Lists the API containers: <id> <name> <state> <X-Instance-Id> <image>. The
# alias is random per process, so each running replica is asked over
# /health/live, in parallel: the failover hook must return within 10 seconds.
set -eu
cd "$(dirname "$0")/.."
+7 -21
View File
@@ -1,26 +1,14 @@
#!/bin/sh
# Level 4: one API replica fails under a constant write load. The generator
# sends messages at RATE for DURATION and retries uncertain sends with the
# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM,
# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started
# again. Afterwards the generator reads the history back: every confirmed
# message once.
# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and
# started again at RESTORE_AT, under a constant write load that retries
# uncertain sends with the same key; then every confirmed message must be in
# the history once.
#
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
#
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35),
# OUT (default evidence/failover). Results in <OUT>/<kind>-<n>/:
# timeline.json, seconds.csv (per-second outcomes, attributed by send time),
# faults.json (when and what was stopped), compose.log (the balancer's
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
# Runs on the current stand (`make up`) and leaves every replica running.
#
# Every time used for the analysis comes from the Docker VM's clock: the log
# timestamps, the container's stop and start times from `docker inspect`, and
# the generator's own times. The host clock of Windows runs apart from it by
# a fraction of a second, a large share of a 1.5 s detection window; host
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
# is the replica's own "shutdown started" line in compose.log.
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT
# (evidence/failover). Times come from the Docker VM clock; the Windows clock
# differs by a fraction of a second and is kept for reference only.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
@@ -50,8 +38,6 @@ since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
generator=$!
# The generator seeds two users first; its timeline starts after that. The
# fault times below are absolute, as is started_at in timeline.json.
sleep "$fault_at"
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker "$kind" "$target" >/dev/null
+13 -8
View File
@@ -3,14 +3,13 @@
# The generator and the knobs the experiments vary live in compose.bench.yaml.
export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml
# abspath prints the absolute path of an existing directory in the form the
# Docker CLI accepts for a bind mount (a Windows path under Git Bash).
# abspath prints a directory in the form the Docker CLI mounts (a Windows path
# under Git Bash).
abspath() {
(cd "$1" && (pwd -W 2>/dev/null || pwd))
}
# meta writes the header of one run: what ran, when, on which commit and
# host. Arguments: file, then key=value pairs added to the record.
# meta writes what ran, when, on which commit and host: file, then key=value.
meta() {
file=$1
shift
@@ -30,8 +29,13 @@ meta() {
} >"$file"
}
# sample_docker_stats appends the CPU and memory of every container of the
# stand to a JSON-lines file, about every two seconds, until killed.
# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container:
# the same work every time, so a slow spell of the host shows in the results.
host_probe() {
docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }'
}
# sample_docker_stats appends the CPU and memory of every container until killed.
sample_docker_stats() {
while :; do
docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \
@@ -39,8 +43,8 @@ sample_docker_stats() {
done >>"$1"
}
# sample_postgres appends, every second, the database's commit counters and
# what its active sessions are waiting for (none: running on CPU).
# sample_postgres appends, every second, the database counters and what the
# active sessions wait for ("cpu": nothing).
sample_postgres() {
while :; do
docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c "
@@ -48,6 +52,7 @@ sample_postgres() {
't', extract(epoch FROM now()),
'xact_commit', xact_commit, 'xact_rollback', xact_rollback,
'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit,
'sessions', sessions, 'sessions_abandoned', sessions_abandoned,
'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()),
'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM (
SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n
+48 -26
View File
@@ -5,14 +5,16 @@
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
output directory:
summary.csv one row per step of every run: offered and successful rate,
errors, latency percentiles, generator lag, mean CPU of each
component and pool waits during the step;
summary.csv one row per step of every run;
capacity.csv the capacity of every run and the median per configuration;
control.csv the noise checks of every run: the host speed probe and the
loadgen CPU per request at the first step, flagged when more
than 25 % off the median;
<series>-throughput.png, <series>-p99.png, capacity.png,
cpu-at-capacity.png, pool-waits.png.
Capacity is the highest valid step (generator on schedule) with less than 1 %
errors and a p99 of successful requests within 500 ms.
Capacity is the highest step that passed the criterion loadgen applies (valid,
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
median of the repeats; a hollow marker means the generator fell behind.
"""
import csv
import json
@@ -27,7 +29,7 @@ matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
SLO_P99_MS = 500.0
MAX_ERROR_SHARE = 0.01
CONTROL_TOLERANCE = 0.25
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
@@ -69,6 +71,7 @@ def load_runs(root):
"steps": report["steps"],
"docker": read_jsonl(d / "docker-stats.jsonl"),
"metrics": read_jsonl(d / "metrics.jsonl"),
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
})
return runs
@@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1):
return out
def pool_waits(run, t0, t1):
"""Requests that waited for a pooled connection during [t0, t1), all
replicas together, per second."""
def pool_rate(run, t0, t1, counter):
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
first, last = {}, {}
for row in run["metrics"]:
t = run["start"] + row["t"]
@@ -104,18 +106,14 @@ def pool_waits(run, t0, t1):
for replica in row.get("replicas") or []:
pool = replica.get("pgxpool") or {}
key = replica.get("instance")
if key and "empty_acquire_count" in pool:
first.setdefault(key, pool["empty_acquire_count"])
last[key] = pool["empty_acquire_count"]
if key and counter in pool:
first.setdefault(key, pool[counter])
last[key] = pool[counter]
if not last:
return ""
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
def is_capacity(step):
return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS
def main(root, out):
runs = load_runs(root)
if not runs:
@@ -129,17 +127,22 @@ def main(root, out):
t0 = run["start"] + step["started_at_s"]
t1 = t0 + step["seconds"]
cpu = mean_cpu(run, t0, t1)
classes = step["classes"]
row = {
"series": run["series"], "config": run["config"], "rep": run["rep"],
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"],
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
"client_timeouts": classes.get("client_timeout", 0),
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
"pool_waits_per_s": pool_waits(run, t0, t1),
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
}
rows.append(row)
if is_capacity(step):
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
best = row
capacities.append({
"series": run["series"], "config": run["config"], "rep": run["rep"],
@@ -164,13 +167,29 @@ def main(root, out):
medians[key] = statistics.median(values)
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"])
control = []
for run in runs:
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
if first and first[0]["cpu_loadgen_pct"] != "":
control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"],
round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)])
if control:
median = statistics.median(c[5] for c in control)
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"])
for c in control:
deviation = c[5] / median - 1
w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE])
def median_by_offered(series, config, field):
points = {}
points, valid = {}, {}
for r in rows:
if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "":
if r["series"] == series and r["config"] == config and r[field] != "":
points.setdefault(r["offered_rps"], []).append(float(r[field]))
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
xs = sorted(points)
return xs, [statistics.median(points[x]) for x in xs]
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
for series in sorted({s for s, _ in configs}):
names = [c for s, c in configs if s == series]
@@ -181,8 +200,11 @@ def main(root, out):
fig, ax = plt.subplots(figsize=(7, 4.5))
top = 0
for name in names:
xs, ys = median_by_offered(series, name, field)
ax.plot(xs, ys, marker="o", label=name)
xs, ys, valid = median_by_offered(series, name, field)
line, = ax.plot(xs, ys, label=name)
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
facecolors="none", edgecolors=line.get_color())
top = max([top, *xs])
if field == "ok_rps" and top:
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
@@ -192,7 +214,7 @@ def main(root, out):
ax.set_yscale("log")
ax.set_xlabel("Offered requests per second")
ax.set_ylabel(title)
ax.set_title(f"Series {series}: median of repeats, valid steps only")
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
ax.grid(alpha=0.3)
ax.legend()
fig.tight_layout()
@@ -233,7 +255,7 @@ def main(root, out):
fig, ax = plt.subplots(figsize=(7, 4.5))
for series, name in configs:
xs, ys = median_by_offered(series, name, "pool_waits_per_s")
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
if xs:
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
ax.set_xlabel("Offered requests per second")
+6 -16
View File
@@ -1,21 +1,12 @@
#!/bin/sh
# Level 4: a rolling update under a constant write load. The generator sends
# messages at RATE for DURATION, retrying uncertain sends with the same key;
# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards
# the generator reads the history back: every confirmed message once.
# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a
# constant write load that retries uncertain sends with the same key; then
# every confirmed message must be in the history once.
#
# sh scripts/bench/rollout-under-load.sh
#
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default
# evidence/rollout). Results in <OUT>/: timeline.json, seconds.csv,
# rollout.log, update.json (when it ran, which containers it replaced),
# compose.log, instances-*.txt.
#
# Times for the analysis come from the Docker VM's clock, as the generator's
# do: log timestamps and the start times of the new containers. The host
# clock of Windows runs apart from it and is kept for reference only. The
# old replicas are removed by the update; their stop shows in compose.log as
# their "shutdown started" lines.
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout).
# Times come from the Docker VM clock, as in failover.sh.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
@@ -29,8 +20,7 @@ export MSYS_NO_PATHCONV=1
rm -rf "$dir"
mkdir -p "$dir"
docker compose up -d --wait >/dev/null 2>&1
# Build first, as a delivery pipeline would: the update under load then
# measures the replacement of the replicas, not the compiler.
# Built beforehand, as by a delivery pipeline: the update replaces replicas.
docker compose build api >/dev/null 2>&1
meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at"
sh scripts/instances.sh >"$dir/instances-before.txt"
+65 -44
View File
@@ -1,61 +1,82 @@
#!/bin/sh
# Track 1, the cost of scaling: runs every configuration of one series file,
# each repeated, each on a fresh stand (empty volumes, same seed), under the
# same grid of offered rates.
# Track 1: every configuration of the given series files, each on a fresh
# stand, under one grid of offered rates. Repeats are interleaved (repeat 1
# of every configuration, then repeat 2), so a slow spell of the host does not
# fall on one configuration.
#
# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats]
# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...]
#
# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root,
# default evidence/track1/raw). Fixed for every configuration: HAProxy
# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the
# generator LOADGEN_CPUS (1.5). Results per run:
# <OUT>/<series>/<config>/rep<k>/{meta.json,steps.json,seconds.csv,
# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log}
# The stand is removed at the end; `make up` brings the normal one back.
# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s),
# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT
# (evidence/track1/raw). Results: <OUT>/<series>/<config>/rep<k>/.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
series=${1:?usage: run-series.sh <series.conf> [repeats]}
repeats=${2:-3}
: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}"
[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh <series.conf>..." >&2; exit 2; }
: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}"
repeats=${REPEATS:-3}
step=${STEP:-30s}
warmup=${WARMUP:-30s}
root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf)
out=${OUT:-evidence/track1/raw}
stop=
[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail
export APP_DEBUG_ADDR=:9090 LB_QUIET=1
export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5}
# Fixed for every configuration; the API, its pool, maxconn and the database
# CPU come from the series line.
export LB_QUIET=1 LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0
export MSYS_NO_PATHCONV=1
grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do
[ -n "$name" ] || continue
for rep in $(seq 1 "$repeats"); do
dir=$root/$name/rep$rep
rm -rf "$dir"
mkdir -p "$dir"
echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES"
export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool
# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus.
run_config() {
series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10}
dir=$out/$series/$name/rep$rep
rm -rf "$dir"
mkdir -p "$dir"
echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU"
export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \
DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus
docker compose down -v --remove-orphans >/dev/null 2>&1
docker compose up -d --build --wait >/dev/null 2>&1
meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \
"api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \
"lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \
"rates=$RATES" "step=$step" "warmup=$warmup"
docker compose down -v --remove-orphans >/dev/null 2>&1
if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then
echo " the stand did not start, see $dir/up.log" >&2
exit 1
fi
meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \
"api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \
"db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \
"lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \
"stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)"
sample_docker_stats "$dir/docker-stats.jsonl" &
stats=$!
sample_postgres "$dir/postgres.jsonl" &
pg=$!
# stdin is closed: `compose run` is interactive by default and would
# otherwise read the rest of the series file this loop is reading.
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 </dev/null ||
echo " loadgen failed, see $dir/loadgen.log"
kill "$stats" "$pg" 2>/dev/null || true
wait "$stats" "$pg" 2>/dev/null || true
cat "$dir/loadgen.log" | grep '^step' || true
sample_docker_stats "$dir/docker-stats.jsonl" &
stats=$!
sample_postgres "$dir/postgres.jsonl" &
pg=$!
# stdin closed: `compose run` would otherwise read it.
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 </dev/null ||
echo " loadgen failed, see $dir/loadgen.log"
kill "$stats" "$pg" 2>/dev/null || true
wait "$stats" "$pg" 2>/dev/null || true
docker compose logs --no-color --timestamps lb api >"$dir/compose.log" 2>&1
grep '^step' "$dir/loadgen.log" || true
}
docker compose --profile bench build loadgen >/dev/null
default_ifs=$IFS
newline='
'
for rep in $(seq 1 "$repeats"); do
for conf in "$@"; do
lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d')
IFS=$newline
for line in $lines; do
IFS=$default_ifs
# shellcheck disable=SC2086 # the line splits into the fields
run_config "$(basename "$conf" .conf)" "$rep" $line
done
IFS=$default_ifs
done
done
docker compose down -v --remove-orphans >/dev/null 2>&1
echo "results in $root"
echo "results in $out"
+6 -6
View File
@@ -1,6 +1,6 @@
# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas.
# The pool stays 10 per replica, so the total grows with the replicas.
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
A1 1 2.0 1024M 920MiB 10
A2 2 1.0 512M 460MiB 10
A4 4 0.5 256M 230MiB 10
# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas;
# the pool stays 10 per replica, so the total grows with the replicas.
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
A1 1 2.0 1024M 920MiB 10 0 1.0
A2 2 1.0 512M 460MiB 10 0 1.0
A4 4 0.5 256M 230MiB 10 0 1.0
+5 -5
View File
@@ -1,5 +1,5 @@
# Series A': the budget of series A with the total pool fixed at 20
# connections. A'2 equals A2 and is not run again.
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
Ap1 1 2.0 1024M 920MiB 20
Ap4 4 0.5 256M 230MiB 5
# Series A': the budget of series A with the total pool fixed at 20 connections;
# A'2 equals A2.
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
Ap1 1 2.0 1024M 920MiB 20 0 1.0
Ap4 4 0.5 256M 230MiB 5 0 1.0
+5 -7
View File
@@ -1,7 +1,5 @@
# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by
# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run
# again.
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
B1 1 1.0 512M 460MiB 10
B3 3 1.0 512M 460MiB 10
B4 4 1.0 512M 460MiB 10
# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2.
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
B1 1 1.0 512M 460MiB 10 0 1.0
B3 3 1.0 512M 460MiB 10 0 1.0
B4 4 1.0 512M 460MiB 10 0 1.0
+10
View File
@@ -0,0 +1,10 @@
# Improvement C: maxconn per replica on the balancer, before (0, no limit) and
# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0);
# 10 and 40 show how much the value matters.
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
R1m0 1 1.0 512M 460MiB 10 0 1.0
R1m20 1 1.0 512M 460MiB 10 20 1.0
R4m0 4 1.0 512M 460MiB 10 0 1.0
R4m20 4 1.0 512M 460MiB 10 20 1.0
R1m10 1 1.0 512M 460MiB 10 10 1.0
R1m40 1 1.0 512M 460MiB 10 40 1.0
-4
View File
@@ -1,4 +0,0 @@
# Calibration: the smallest API budget of the study (one replica, one CPU)
# with a wide range of rates finds where the common rate grid must reach.
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
cal 1 1.0 512M 460MiB 10
+4
View File
@@ -0,0 +1,4 @@
# Control: A2 with twice the database CPU. If the database is the bottleneck,
# the capacity rises.
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
A2pg2 2 1.0 512M 460MiB 10 0 2.0
+1 -3
View File
@@ -1,7 +1,5 @@
#!/bin/sh
# Shows which container answers under which X-Instance-Id: the proof on the
# defense that the header values belong to different running processes, and
# which container a failover test actually stopped.
# Which container answers under which X-Instance-Id.
set -eu
cd "$(dirname "$0")/.."
+2 -5
View File
@@ -1,9 +1,6 @@
#!/bin/sh
# Restart hook of `make test-persistence` (tests/resilience.py): stops the
# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the
# same containers again (the one-shot migrate finds nothing to apply).
# Volumes and data are kept and nothing is re-seeded. The test hides the
# output, so failures surface through the exit code only.
# Restart hook of `make test-persistence`: stops the whole stand gracefully and
# starts the same containers again; volumes are kept.
set -eu
cd "$(dirname "$0")/.."
docker compose stop
+8 -18
View File
@@ -1,21 +1,12 @@
#!/bin/sh
# Rolling update of the API without downtime, with docker rollout
# (https://github.com/wowu/docker-rollout, MIT): it starts as many new
# replicas as are running, waits until all of them are healthy, then stops
# and removes the old ones. If a new replica does not become healthy it
# removes the new ones and leaves the old ones serving, so the stand is never
# left with a mix of versions.
# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout,
# MIT): new replicas start next to the old ones, and the old ones go once all
# new ones are healthy; if one is not, the new ones are removed instead.
#
# Every replica must be running. With a stopped one (say, after stop-one),
# docker rollout only replaces the stopped one and reports success, which
# would leave old and new versions side by side; the script refuses instead.
#
# The migrations of the new version run first, once, while the old replicas
# serve: they only add to the schema. docker rollout scales with `compose up`,
# which also starts the dependencies of the API and would rerun the old,
# exited migrate container. Recreating it beforehand from the new image,
# without starting it, makes that `up` run the new migrations exactly once,
# before any new replica starts.
# With a stopped replica docker rollout would replace only that one and leave
# two versions running, so the script refuses. The migrate container is
# recreated from the new image without starting: the `up` of docker rollout
# then applies the new migrations once, before any new replica starts.
set -eu
cd "$(dirname "$0")/.."
@@ -44,8 +35,7 @@ docker compose build api
docker compose up --no-start --no-deps --force-recreate migrate
# Run through sh: Windows cannot use the script as a docker CLI plugin.
if ! sh "$tool" rollout --wait-after-healthy 4 api; then
# A failed migration stops the scale-up after the new containers were
# created but before they started; remove them for a clean next attempt.
# A failed migration leaves the new containers created but not started.
created=$(docker compose ps -aq --status created api)
if [ -n "$created" ]; then
docker rm $created >/dev/null
+3 -7
View File
@@ -1,11 +1,7 @@
#!/bin/sh
# Failover hook of `make test-failover` (tests/resilience.py): stops the one
# API replica whose X-Instance-Id is $TARGET_INSTANCE_ID and leaves it
# stopped. The stop is graceful: on SIGTERM the replica turns unready, keeps
# serving for its drain delay while HAProxy takes it out of rotation, then
# exits; `docker stop` returns once it has. Nothing else is touched, and an
# alias that does not name exactly one running replica stops nothing.
# Restore the replica with: docker compose start api
# Failover hook of `make test-failover`: gracefully stops the one running API
# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else.
# Restore it with: docker compose start api
set -eu
cd "$(dirname "$0")/.."
+2 -3
View File
@@ -52,9 +52,8 @@ func (c *client) postMessage(chatID, token, text string, out any) error {
return c.do(req, token, http.StatusCreated, out)
}
// sendWithKey sends one attempt of a message under the given key on a
// connection of its own and reports the status, the message (on 201) and the
// replica that answered.
// sendWithKey sends one attempt on a connection of its own, so that the
// attempts of one key reach different replicas.
func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) {
var msg chat.MessageResponse
raw, err := json.Marshal(map[string]string{"text": text})
+3 -6
View File
@@ -4,12 +4,9 @@
// then the history is paged and must contain exactly N*M unique messages in
// id order.
//
// Phase 2, shared idempotency: in each of K rounds one user sends the same
// message with the same Idempotency-Key P times in parallel, each attempt on
// its own connection, so the balancer spreads them over the replicas. Every
// attempt must answer 201 with the same message, the attempts must have
// reached at least two replicas, the history must hold exactly K messages,
// and the key with another text must answer 409.
// Phase 2: in each of K rounds one key is sent P times in parallel through the
// balancer; every attempt must get 201 with one message, at least two replicas
// must answer, the history must hold K messages, and another text must get 409.
package main
import (
+16 -25
View File
@@ -1,21 +1,10 @@
// Command loadgen is the open-loop load generator of the scaling study
// (track 1) and of the failover and rolling-update experiments.
// Command loadgen is the open-loop load generator of the experiments.
//
// loadgen bench — seeds users, chats and messages, warms up, then offers
// each rate of -rates for -step: a mix of history reads
// (-read-ratio) and message sends. Writes steps.json and
// seconds.csv, and metrics.jsonl with -scrape.
// loadgen timeline — one user sends messages into one chat at -rate for
// -duration; a send with an uncertain outcome (no answer,
// 5xx) is retried with the same Idempotency-Key. At the
// end the whole history is read back: every confirmed
// message must be there, and none twice. Writes
// timeline.json and seconds.csv.
//
// Requests are sent on schedule whatever the response times (open loop), and
// the generator records how late it sent them: a step it could not keep up
// with is marked invalid rather than reported as the system's limit. Tokens
// stay in memory and are never written out.
// loadgen bench — seeds data, warms up, then offers each rate of -rates
// for -step as a mix of history reads and message sends.
// loadgen timeline — one user sends messages at -rate, retrying uncertain
// sends with the same Idempotency-Key, then reads the
// history back: every confirmed message once.
package main
import (
@@ -52,6 +41,7 @@ type config struct {
Timeout string `json:"client_timeout"`
Seed uint64 `json:"seed"`
Scrape bool `json:"scrape"`
StopOnFail bool `json:"stop_on_fail"`
StatsURL string `json:"stats_url"`
APIHost string `json:"api_host"`
DebugPort string `json:"debug_port"`
@@ -98,6 +88,7 @@ func parse(mode string, args []string) (config, error) {
fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration")
fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)")
fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix")
fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step that fails the capacity criterion")
fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second")
fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV")
fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas")
@@ -138,8 +129,7 @@ func writeJSON(path string, v any) error {
return os.WriteFile(path, append(raw, '\n'), 0o644)
}
// startScrape samples the system in the background when asked to; the
// returned function stops it and waits.
// startScrape returns the function that stops the sampling and waits.
func startScrape(ctx context.Context, cfg config, start time.Time) func() {
if !cfg.Scrape {
return func() {}
@@ -177,8 +167,7 @@ func runBench(ctx context.Context, cfg config) error {
stopScrape := startScrape(ctx, cfg, start)
defer stopScrape()
// next picks the request on the scheduling goroutine, where the random
// source may be used safely.
// next runs on the scheduling goroutine, which owns the random source.
counter := 0
next := func(rec *recorder) func(time.Time) func() {
return func(planned time.Time) func() {
@@ -217,8 +206,11 @@ func runBench(ctx context.Context, cfg config) error {
for _, x := range rec.all() {
all.add(x)
}
fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v\n",
rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid)
fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n",
rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed)
if cfg.StopOnFail && !s.Passed {
break
}
}
stopScrape()
if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil {
@@ -247,8 +239,7 @@ type timelineReport struct {
// retryDelay spaces the attempts of one message whose outcome was uncertain.
const retryDelay = 200 * time.Millisecond
// retryGrace is how long after the schedule ends an uncertain message keeps
// being retried before it counts as unresolved.
// retryGrace bounds the retries of an uncertain message after the schedule ends.
const retryGrace = 10 * time.Second
func runTimeline(ctx context.Context, cfg config) error {
+3 -8
View File
@@ -6,14 +6,9 @@ import (
"time"
)
// openLoop offers requests at a constant rate for d, whatever the response
// times: request i is planned at start + i/rate and sent then, or at once if
// the generator is already late. Waiting for responses before sending more
// (a closed loop) would let a slow system lower its own load and hide its
// latency: coordinated omission. next runs on the scheduling goroutine, so it
// may use non-thread-safe state such as the random source; the job it
// returns runs on a goroutine of its own. openLoop returns once every job
// has finished.
// openLoop sends request i at start + i/rate whatever the response times: a
// closed loop would let a slow system lower its own load and hide its latency
// (coordinated omission). It returns once every job has finished.
func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) {
n := int(rate * d.Seconds())
interval := time.Duration(float64(time.Second) / rate)
+2 -5
View File
@@ -12,11 +12,8 @@ import (
"time"
)
// scraper samples, once a second, what the balancer and every replica report
// about themselves, so a saturated step can be explained: the HAProxy
// statistics per replica and the expvar of each API process (pool waits,
// GOMAXPROCS, memory and GC). Replicas are found the way HAProxy finds them,
// through Docker DNS, and asked directly on their internal debug port.
// scraper samples the HAProxy statistics and every replica's expvar once a
// second; replicas are found through Docker DNS, as HAProxy finds them.
type scraper struct {
statsURL string
apiHost string
+15 -10
View File
@@ -12,11 +12,17 @@ import (
"time"
)
// maxValidLag is the generator's own tolerance: if requests left more than
// this late at the 99th percentile, the generator, not the system, limited
// the step, and the step is excluded from the conclusions.
// maxValidLag: if requests left later than this at p99, the generator, not
// the system, limited the step.
const maxValidLag = 10 * time.Millisecond
// The capacity criterion of the study: a step passes when it is valid, has
// fewer than 1 % errors and its successful requests have a p99 within 500 ms.
const (
maxErrorShare = 0.01
sloP99 = 500 * time.Millisecond
)
// sample is one request as the generator saw it.
type sample struct {
planned time.Time
@@ -27,11 +33,9 @@ type sample struct {
write bool
}
// classify names the outcome of a request. HTTP errors produced by the
// balancer itself carry no X-Instance-Id and are kept apart from the API's
// own: a 503 from HAProxy means no replica was available, a 503 from the API
// means a dependency was. client_timeout is the generator giving up on a
// response, not a timeout of the system.
// classify names the outcome of a request. Errors of the balancer itself carry
// no X-Instance-Id and get the suffix _lb; client_timeout is the generator
// giving up, not a timeout of the system.
func classify(r result) string {
switch {
case r.err != nil && isTimeout(r.err):
@@ -78,12 +82,12 @@ type stepSummary struct {
P99Ms float64 `json:"ok_p99_ms"`
LagP99Ms float64 `json:"lag_p99_ms"`
Valid bool `json:"valid"`
Passed bool `json:"passed"`
Instances map[string]int `json:"instances"`
Writes int `json:"writes_ok"`
}
// summarize reduces one step. Latency percentiles cover successful requests
// only, as the course's report format asks; the errors are counted by class.
// summarize reduces one step; latency percentiles cover successful requests.
func summarize(rate float64, d time.Duration, samples []sample) stepSummary {
s := stepSummary{
OfferedRPS: rate,
@@ -117,6 +121,7 @@ func summarize(rate float64, d time.Duration, samples []sample) stepSummary {
lagP99 := percentile(lags, 99)
s.LagP99Ms = ms(lagP99)
s.Valid = lagP99 <= maxValidLag
s.Passed = s.Valid && s.ErrorShare < maxErrorShare && percentile(ok, 99) <= sloP99
return s
}
+22
View File
@@ -104,3 +104,25 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) {
t.Fatalf("open loop took %v, want about 1.2 s", took)
}
}
func TestStepPassesOnlyWithinTheObjective(t *testing.T) {
ok := func(latency, lag time.Duration) sample {
return sample{class: "ok", latency: latency, lag: lag}
}
fast := []sample{ok(10*time.Millisecond, time.Millisecond), ok(20*time.Millisecond, time.Millisecond)}
if s := summarize(2, time.Second, fast); !s.Passed {
t.Fatalf("fast clean step must pass: %+v", s)
}
slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)}
if s := summarize(2, time.Second, slow); s.Passed {
t.Fatalf("p99 above 500 ms must fail: %+v", s)
}
failing := append(fast, sample{class: "http_503_lb", lag: time.Millisecond})
if s := summarize(3, time.Second, failing); s.Passed {
t.Fatalf("errors above 1 %% must fail: %+v", s)
}
late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)}
if s := summarize(2, time.Second, late); s.Passed {
t.Fatalf("an invalid step must fail: %+v", s)
}
}