diff --git a/Makefile b/Makefile index 39794250..21b7346d 100644 --- a/Makefile +++ b/Makefile @@ -67,17 +67,15 @@ logs: instances: sh scripts/instances.sh -# Replaces every API replica with one built from the current sources, without -# downtime; restore a replica stopped by stop-one with `docker compose start api`. +# Replaces every API replica without downtime; all of them must be running. rollout: sh scripts/rollout.sh -# Track 1, the cost of scaling: one series on fresh stands, then the charts. -# RATES is required, e.g. make bench SERIES=scripts/bench/series/B.conf RATES=200,400,800 +# Track 1. Every run starts from empty volumes, so the stand's data is lost. +# make bench SERIES="scripts/bench/series/A.conf scripts/bench/series/B.conf" RATES=400,800,1200 SERIES ?= scripts/bench/series/A.conf -REPEATS ?= 3 bench: - sh scripts/bench/run-series.sh $(SERIES) $(REPEATS) + sh scripts/bench/run-series.sh $(SERIES) # Needs matplotlib from requirements-bench.txt: make plot PYTHON=. plot: diff --git a/scripts/api-instances.sh b/scripts/api-instances.sh index 4a90d4f1..be6ce307 100755 --- a/scripts/api-instances.sh +++ b/scripts/api-instances.sh @@ -1,11 +1,7 @@ #!/bin/sh -# Lists the API containers of this compose project, one per line: -# -# The alias is what the process sends as X-Instance-Id. It is random per -# process, so it is asked from the process itself, over the same wget the -# Docker healthcheck uses, from inside the container. A container that is not -# running has no alias and shows "-". The replicas are asked in parallel: the -# failover hook that relies on this list must return within 10 seconds. +# Lists the API containers: . The +# alias is random per process, so each running replica is asked over +# /health/live, in parallel: the failover hook must return within 10 seconds. set -eu cd "$(dirname "$0")/.." diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh index d223fc21..afff54d9 100755 --- a/scripts/bench/failover.sh +++ b/scripts/bench/failover.sh @@ -1,26 +1,14 @@ #!/bin/sh -# Level 4: one API replica fails under a constant write load. The generator -# sends messages at RATE for DURATION and retries uncertain sends with the -# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM, -# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started -# again. Afterwards the generator reads the history back: every confirmed -# message once. +# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and +# started again at RESTORE_AT, under a constant write load that retries +# uncertain sends with the same key; then every confirmed message must be in +# the history once. # # sh scripts/bench/failover.sh stop|kill # -# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), -# OUT (default evidence/failover). Results in /-/: -# timeline.json, seconds.csv (per-second outcomes, attributed by send time), -# faults.json (when and what was stopped), compose.log (the balancer's -# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. -# Runs on the current stand (`make up`) and leaves every replica running. -# -# Every time used for the analysis comes from the Docker VM's clock: the log -# timestamps, the container's stop and start times from `docker inspect`, and -# the generator's own times. The host clock of Windows runs apart from it by -# a fraction of a second, a large share of a 1.5 s detection window; host -# times are kept for reference only. With `stop`, the moment SIGTERM arrived -# is the replica's own "shutdown started" line in compose.log. +# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT +# (evidence/failover). Times come from the Docker VM clock; the Windows clock +# differs by a fraction of a second and is kept for reference only. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -50,8 +38,6 @@ since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & generator=$! -# The generator seeds two users first; its timeline starts after that. The -# fault times below are absolute, as is started_at in timeline.json. sleep "$fault_at" fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker "$kind" "$target" >/dev/null diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index 2b8c3115..24232ee8 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -3,14 +3,13 @@ # The generator and the knobs the experiments vary live in compose.bench.yaml. export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml -# abspath prints the absolute path of an existing directory in the form the -# Docker CLI accepts for a bind mount (a Windows path under Git Bash). +# abspath prints a directory in the form the Docker CLI mounts (a Windows path +# under Git Bash). abspath() { (cd "$1" && (pwd -W 2>/dev/null || pwd)) } -# meta writes the header of one run: what ran, when, on which commit and -# host. Arguments: file, then key=value pairs added to the record. +# meta writes what ran, when, on which commit and host: file, then key=value. meta() { file=$1 shift @@ -30,8 +29,13 @@ meta() { } >"$file" } -# sample_docker_stats appends the CPU and memory of every container of the -# stand to a JSON-lines file, about every two seconds, until killed. +# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container: +# the same work every time, so a slow spell of the host shows in the results. +host_probe() { + docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' +} + +# sample_docker_stats appends the CPU and memory of every container until killed. sample_docker_stats() { while :; do docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \ @@ -39,8 +43,8 @@ sample_docker_stats() { done >>"$1" } -# sample_postgres appends, every second, the database's commit counters and -# what its active sessions are waiting for (none: running on CPU). +# sample_postgres appends, every second, the database counters and what the +# active sessions wait for ("cpu": nothing). sample_postgres() { while :; do docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c " @@ -48,6 +52,7 @@ sample_postgres() { 't', extract(epoch FROM now()), 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, 'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit, + 'sessions', sessions, 'sessions_abandoned', sessions_abandoned, 'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()), 'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM ( SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py index 20493b2d..33305612 100755 --- a/scripts/bench/plot.py +++ b/scripts/bench/plot.py @@ -5,14 +5,16 @@ The raw root holds //rep/ directories. Written to the output directory: - summary.csv one row per step of every run: offered and successful rate, - errors, latency percentiles, generator lag, mean CPU of each - component and pool waits during the step; + summary.csv one row per step of every run; capacity.csv the capacity of every run and the median per configuration; + control.csv the noise checks of every run: the host speed probe and the + loadgen CPU per request at the first step, flagged when more + than 25 % off the median; -throughput.png, -p99.png, capacity.png, cpu-at-capacity.png, pool-waits.png. -Capacity is the highest valid step (generator on schedule) with less than 1 % -errors and a p99 of successful requests within 500 ms. +Capacity is the highest step that passed the criterion loadgen applies (valid, +under 1 % errors, p99 of successful requests within 500 ms). Charts show the +median of the repeats; a hollow marker means the generator fell behind. """ import csv import json @@ -27,7 +29,7 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 SLO_P99_MS = 500.0 -MAX_ERROR_SHARE = 0.01 +CONTROL_TOLERANCE = 0.25 COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") @@ -69,6 +71,7 @@ def load_runs(root): "steps": report["steps"], "docker": read_jsonl(d / "docker-stats.jsonl"), "metrics": read_jsonl(d / "metrics.jsonl"), + "meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {}, }) return runs @@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1): return out -def pool_waits(run, t0, t1): - """Requests that waited for a pooled connection during [t0, t1), all - replicas together, per second.""" +def pool_rate(run, t0, t1, counter): + """Growth of a pgxpool counter during [t0, t1), all replicas, per second.""" first, last = {}, {} for row in run["metrics"]: t = run["start"] + row["t"] @@ -104,18 +106,14 @@ def pool_waits(run, t0, t1): for replica in row.get("replicas") or []: pool = replica.get("pgxpool") or {} key = replica.get("instance") - if key and "empty_acquire_count" in pool: - first.setdefault(key, pool["empty_acquire_count"]) - last[key] = pool["empty_acquire_count"] + if key and counter in pool: + first.setdefault(key, pool[counter]) + last[key] = pool[counter] if not last: return "" return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) -def is_capacity(step): - return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS - - def main(root, out): runs = load_runs(root) if not runs: @@ -129,17 +127,22 @@ def main(root, out): t0 = run["start"] + step["started_at_s"] t1 = t0 + step["seconds"] cpu = mean_cpu(run, t0, t1) + classes = step["classes"] row = { "series": run["series"], "config": run["config"], "rep": run["rep"], "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], - "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], + "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"], + "client_timeouts": classes.get("client_timeout", 0), + "lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")), + "api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")), **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, - "pool_waits_per_s": pool_waits(run, t0, t1), + "pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"), + "new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"), } rows.append(row) - if is_capacity(step): + if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]): best = row capacities.append({ "series": run["series"], "config": run["config"], "rep": run["rep"], @@ -164,13 +167,29 @@ def main(root, out): medians[key] = statistics.median(values) w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"]) + control = [] + for run in runs: + first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] + if first and first[0]["cpu_loadgen_pct"] != "": + control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"], + round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)]) + if control: + median = statistics.median(c[5] for c in control) + with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"]) + for c in control: + deviation = c[5] / median - 1 + w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE]) + def median_by_offered(series, config, field): - points = {} + points, valid = {}, {} for r in rows: - if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "": + if r["series"] == series and r["config"] == config and r[field] != "": points.setdefault(r["offered_rps"], []).append(float(r[field])) + valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"] xs = sorted(points) - return xs, [statistics.median(points[x]) for x in xs] + return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs] for series in sorted({s for s, _ in configs}): names = [c for s, c in configs if s == series] @@ -181,8 +200,11 @@ def main(root, out): fig, ax = plt.subplots(figsize=(7, 4.5)) top = 0 for name in names: - xs, ys = median_by_offered(series, name, field) - ax.plot(xs, ys, marker="o", label=name) + xs, ys, valid = median_by_offered(series, name, field) + line, = ax.plot(xs, ys, label=name) + ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color()) + ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], + facecolors="none", edgecolors=line.get_color()) top = max([top, *xs]) if field == "ok_rps" and top: ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") @@ -192,7 +214,7 @@ def main(root, out): ax.set_yscale("log") ax.set_xlabel("Offered requests per second") ax.set_ylabel(title) - ax.set_title(f"Series {series}: median of repeats, valid steps only") + ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)") ax.grid(alpha=0.3) ax.legend() fig.tight_layout() @@ -233,7 +255,7 @@ def main(root, out): fig, ax = plt.subplots(figsize=(7, 4.5)) for series, name in configs: - xs, ys = median_by_offered(series, name, "pool_waits_per_s") + xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s") if xs: ax.plot(xs, ys, marker="o", label=f"{series}/{name}") ax.set_xlabel("Offered requests per second") diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh index 2390155a..44cfb40b 100755 --- a/scripts/bench/rollout-under-load.sh +++ b/scripts/bench/rollout-under-load.sh @@ -1,21 +1,12 @@ #!/bin/sh -# Level 4: a rolling update under a constant write load. The generator sends -# messages at RATE for DURATION, retrying uncertain sends with the same key; -# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards -# the generator reads the history back: every confirmed message once. +# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a +# constant write load that retries uncertain sends with the same key; then +# every confirmed message must be in the history once. # # sh scripts/bench/rollout-under-load.sh # -# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default -# evidence/rollout). Results in /: timeline.json, seconds.csv, -# rollout.log, update.json (when it ran, which containers it replaced), -# compose.log, instances-*.txt. -# -# Times for the analysis come from the Docker VM's clock, as the generator's -# do: log timestamps and the start times of the new containers. The host -# clock of Windows runs apart from it and is kept for reference only. The -# old replicas are removed by the update; their stop shows in compose.log as -# their "shutdown started" lines. +# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout). +# Times come from the Docker VM clock, as in failover.sh. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -29,8 +20,7 @@ export MSYS_NO_PATHCONV=1 rm -rf "$dir" mkdir -p "$dir" docker compose up -d --wait >/dev/null 2>&1 -# Build first, as a delivery pipeline would: the update under load then -# measures the replacement of the replicas, not the compiler. +# Built beforehand, as by a delivery pipeline: the update replaces replicas. docker compose build api >/dev/null 2>&1 meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" sh scripts/instances.sh >"$dir/instances-before.txt" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index 11d98aa8..b25d7389 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -1,61 +1,82 @@ #!/bin/sh -# Track 1, the cost of scaling: runs every configuration of one series file, -# each repeated, each on a fresh stand (empty volumes, same seed), under the -# same grid of offered rates. +# Track 1: every configuration of the given series files, each on a fresh +# stand, under one grid of offered rates. Repeats are interleaved (repeat 1 +# of every configuration, then repeat 2), so a slow spell of the host does not +# fall on one configuration. # -# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats] +# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...] # -# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root, -# default evidence/track1/raw). Fixed for every configuration: HAProxy -# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the -# generator LOADGEN_CPUS (1.5). Results per run: -# ///rep/{meta.json,steps.json,seconds.csv, -# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log} -# The stand is removed at the end; `make up` brings the normal one back. +# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s), +# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT +# (evidence/track1/raw). Results: ///rep/. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh -series=${1:?usage: run-series.sh [repeats]} -repeats=${2:-3} -: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}" +[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh ..." >&2; exit 2; } +: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}" +repeats=${REPEATS:-3} step=${STEP:-30s} warmup=${WARMUP:-30s} -root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf) +out=${OUT:-evidence/track1/raw} +stop= +[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail -export APP_DEBUG_ADDR=:9090 LB_QUIET=1 -export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5} +# Fixed for every configuration; the API, its pool, maxconn and the database +# CPU come from the series line. +export LB_QUIET=1 LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0 export MSYS_NO_PATHCONV=1 -grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do - [ -n "$name" ] || continue - for rep in $(seq 1 "$repeats"); do - dir=$root/$name/rep$rep - rm -rf "$dir" - mkdir -p "$dir" - echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES" - export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool +# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus. +run_config() { + series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10} + dir=$out/$series/$name/rep$rep + rm -rf "$dir" + mkdir -p "$dir" + echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU" + export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \ + DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus - docker compose down -v --remove-orphans >/dev/null 2>&1 - docker compose up -d --build --wait >/dev/null 2>&1 - meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \ - "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \ - "lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \ - "rates=$RATES" "step=$step" "warmup=$warmup" + docker compose down -v --remove-orphans >/dev/null 2>&1 + if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then + echo " the stand did not start, see $dir/up.log" >&2 + exit 1 + fi + meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \ + "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \ + "db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \ + "lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \ + "stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)" - sample_docker_stats "$dir/docker-stats.jsonl" & - stats=$! - sample_postgres "$dir/postgres.jsonl" & - pg=$! - # stdin is closed: `compose run` is interactive by default and would - # otherwise read the rest of the series file this loop is reading. - BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ - -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 /dev/null || true - wait "$stats" "$pg" 2>/dev/null || true - cat "$dir/loadgen.log" | grep '^step' || true + sample_docker_stats "$dir/docker-stats.jsonl" & + stats=$! + sample_postgres "$dir/postgres.jsonl" & + pg=$! + # stdin closed: `compose run` would otherwise read it. + BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 /dev/null || true + wait "$stats" "$pg" 2>/dev/null || true + docker compose logs --no-color --timestamps lb api >"$dir/compose.log" 2>&1 + grep '^step' "$dir/loadgen.log" || true +} + +docker compose --profile bench build loadgen >/dev/null +default_ifs=$IFS +newline=' +' +for rep in $(seq 1 "$repeats"); do + for conf in "$@"; do + lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d') + IFS=$newline + for line in $lines; do + IFS=$default_ifs + # shellcheck disable=SC2086 # the line splits into the fields + run_config "$(basename "$conf" .conf)" "$rep" $line + done + IFS=$default_ifs done done docker compose down -v --remove-orphans >/dev/null 2>&1 -echo "results in $root" +echo "results in $out" diff --git a/scripts/bench/series/A.conf b/scripts/bench/series/A.conf index 1a305380..a461f2b3 100644 --- a/scripts/bench/series/A.conf +++ b/scripts/bench/series/A.conf @@ -1,6 +1,6 @@ -# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas. -# The pool stays 10 per replica, so the total grows with the replicas. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -A1 1 2.0 1024M 920MiB 10 -A2 2 1.0 512M 460MiB 10 -A4 4 0.5 256M 230MiB 10 +# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas; +# the pool stays 10 per replica, so the total grows with the replicas. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A1 1 2.0 1024M 920MiB 10 0 1.0 +A2 2 1.0 512M 460MiB 10 0 1.0 +A4 4 0.5 256M 230MiB 10 0 1.0 diff --git a/scripts/bench/series/Aprime.conf b/scripts/bench/series/Aprime.conf index d0952eff..6ebe2902 100644 --- a/scripts/bench/series/Aprime.conf +++ b/scripts/bench/series/Aprime.conf @@ -1,5 +1,5 @@ -# Series A': the budget of series A with the total pool fixed at 20 -# connections. A'2 equals A2 and is not run again. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -Ap1 1 2.0 1024M 920MiB 20 -Ap4 4 0.5 256M 230MiB 5 +# Series A': the budget of series A with the total pool fixed at 20 connections; +# A'2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +Ap1 1 2.0 1024M 920MiB 20 0 1.0 +Ap4 4 0.5 256M 230MiB 5 0 1.0 diff --git a/scripts/bench/series/B.conf b/scripts/bench/series/B.conf index b882d44e..9f4fc11c 100644 --- a/scripts/bench/series/B.conf +++ b/scripts/bench/series/B.conf @@ -1,7 +1,5 @@ -# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by -# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run -# again. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -B1 1 1.0 512M 460MiB 10 -B3 3 1.0 512M 460MiB 10 -B4 4 1.0 512M 460MiB 10 +# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +B1 1 1.0 512M 460MiB 10 0 1.0 +B3 3 1.0 512M 460MiB 10 0 1.0 +B4 4 1.0 512M 460MiB 10 0 1.0 diff --git a/scripts/bench/series/C.conf b/scripts/bench/series/C.conf new file mode 100644 index 00000000..20d012f3 --- /dev/null +++ b/scripts/bench/series/C.conf @@ -0,0 +1,10 @@ +# Improvement C: maxconn per replica on the balancer, before (0, no limit) and +# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0); +# 10 and 40 show how much the value matters. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +R1m0 1 1.0 512M 460MiB 10 0 1.0 +R1m20 1 1.0 512M 460MiB 10 20 1.0 +R4m0 4 1.0 512M 460MiB 10 0 1.0 +R4m20 4 1.0 512M 460MiB 10 20 1.0 +R1m10 1 1.0 512M 460MiB 10 10 1.0 +R1m40 1 1.0 512M 460MiB 10 40 1.0 diff --git a/scripts/bench/series/calibrate.conf b/scripts/bench/series/calibrate.conf deleted file mode 100644 index 0e3c976e..00000000 --- a/scripts/bench/series/calibrate.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Calibration: the smallest API budget of the study (one replica, one CPU) -# with a wide range of rates finds where the common rate grid must reach. -# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica -cal 1 1.0 512M 460MiB 10 diff --git a/scripts/bench/series/control.conf b/scripts/bench/series/control.conf new file mode 100644 index 00000000..d00c775c --- /dev/null +++ b/scripts/bench/series/control.conf @@ -0,0 +1,4 @@ +# Control: A2 with twice the database CPU. If the database is the bottleneck, +# the capacity rises. +# name replicas cpus memory gomemlimit pool maxconn pg_cpus +A2pg2 2 1.0 512M 460MiB 10 0 2.0 diff --git a/scripts/instances.sh b/scripts/instances.sh index 52150a3f..02fbbefd 100755 --- a/scripts/instances.sh +++ b/scripts/instances.sh @@ -1,7 +1,5 @@ #!/bin/sh -# Shows which container answers under which X-Instance-Id: the proof on the -# defense that the header values belong to different running processes, and -# which container a failover test actually stopped. +# Which container answers under which X-Instance-Id. set -eu cd "$(dirname "$0")/.." diff --git a/scripts/restart.sh b/scripts/restart.sh index 29aeb757..7eda3b6a 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,9 +1,6 @@ #!/bin/sh -# Restart hook of `make test-persistence` (tests/resilience.py): stops the -# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the -# same containers again (the one-shot migrate finds nothing to apply). -# Volumes and data are kept and nothing is re-seeded. The test hides the -# output, so failures surface through the exit code only. +# Restart hook of `make test-persistence`: stops the whole stand gracefully and +# starts the same containers again; volumes are kept. set -eu cd "$(dirname "$0")/.." docker compose stop diff --git a/scripts/rollout.sh b/scripts/rollout.sh index c46e52c4..4710e64c 100755 --- a/scripts/rollout.sh +++ b/scripts/rollout.sh @@ -1,21 +1,12 @@ #!/bin/sh -# Rolling update of the API without downtime, with docker rollout -# (https://github.com/wowu/docker-rollout, MIT): it starts as many new -# replicas as are running, waits until all of them are healthy, then stops -# and removes the old ones. If a new replica does not become healthy it -# removes the new ones and leaves the old ones serving, so the stand is never -# left with a mix of versions. +# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout, +# MIT): new replicas start next to the old ones, and the old ones go once all +# new ones are healthy; if one is not, the new ones are removed instead. # -# Every replica must be running. With a stopped one (say, after stop-one), -# docker rollout only replaces the stopped one and reports success, which -# would leave old and new versions side by side; the script refuses instead. -# -# The migrations of the new version run first, once, while the old replicas -# serve: they only add to the schema. docker rollout scales with `compose up`, -# which also starts the dependencies of the API and would rerun the old, -# exited migrate container. Recreating it beforehand from the new image, -# without starting it, makes that `up` run the new migrations exactly once, -# before any new replica starts. +# With a stopped replica docker rollout would replace only that one and leave +# two versions running, so the script refuses. The migrate container is +# recreated from the new image without starting: the `up` of docker rollout +# then applies the new migrations once, before any new replica starts. set -eu cd "$(dirname "$0")/.." @@ -44,8 +35,7 @@ docker compose build api docker compose up --no-start --no-deps --force-recreate migrate # Run through sh: Windows cannot use the script as a docker CLI plugin. if ! sh "$tool" rollout --wait-after-healthy 4 api; then - # A failed migration stops the scale-up after the new containers were - # created but before they started; remove them for a clean next attempt. + # A failed migration leaves the new containers created but not started. created=$(docker compose ps -aq --status created api) if [ -n "$created" ]; then docker rm $created >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index d2730e94..6971e1b4 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -1,11 +1,7 @@ #!/bin/sh -# Failover hook of `make test-failover` (tests/resilience.py): stops the one -# API replica whose X-Instance-Id is $TARGET_INSTANCE_ID and leaves it -# stopped. The stop is graceful: on SIGTERM the replica turns unready, keeps -# serving for its drain delay while HAProxy takes it out of rotation, then -# exits; `docker stop` returns once it has. Nothing else is touched, and an -# alias that does not name exactly one running replica stops nothing. -# Restore the replica with: docker compose start api +# Failover hook of `make test-failover`: gracefully stops the one running API +# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else. +# Restore it with: docker compose start api set -eu cd "$(dirname "$0")/.." diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go index b56eb137..9b583e0d 100644 --- a/tools/concurrency/client.go +++ b/tools/concurrency/client.go @@ -52,9 +52,8 @@ func (c *client) postMessage(chatID, token, text string, out any) error { return c.do(req, token, http.StatusCreated, out) } -// sendWithKey sends one attempt of a message under the given key on a -// connection of its own and reports the status, the message (on 201) and the -// replica that answered. +// sendWithKey sends one attempt on a connection of its own, so that the +// attempts of one key reach different replicas. func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) { var msg chat.MessageResponse raw, err := json.Marshal(map[string]string{"text": text}) diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go index 7011c071..a1ac25e6 100644 --- a/tools/concurrency/main.go +++ b/tools/concurrency/main.go @@ -4,12 +4,9 @@ // then the history is paged and must contain exactly N*M unique messages in // id order. // -// Phase 2, shared idempotency: in each of K rounds one user sends the same -// message with the same Idempotency-Key P times in parallel, each attempt on -// its own connection, so the balancer spreads them over the replicas. Every -// attempt must answer 201 with the same message, the attempts must have -// reached at least two replicas, the history must hold exactly K messages, -// and the key with another text must answer 409. +// Phase 2: in each of K rounds one key is sent P times in parallel through the +// balancer; every attempt must get 201 with one message, at least two replicas +// must answer, the history must hold K messages, and another text must get 409. package main import ( diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go index fabeddd7..e9dd9fc0 100644 --- a/tools/loadgen/main.go +++ b/tools/loadgen/main.go @@ -1,21 +1,10 @@ -// Command loadgen is the open-loop load generator of the scaling study -// (track 1) and of the failover and rolling-update experiments. +// Command loadgen is the open-loop load generator of the experiments. // -// loadgen bench — seeds users, chats and messages, warms up, then offers -// each rate of -rates for -step: a mix of history reads -// (-read-ratio) and message sends. Writes steps.json and -// seconds.csv, and metrics.jsonl with -scrape. -// loadgen timeline — one user sends messages into one chat at -rate for -// -duration; a send with an uncertain outcome (no answer, -// 5xx) is retried with the same Idempotency-Key. At the -// end the whole history is read back: every confirmed -// message must be there, and none twice. Writes -// timeline.json and seconds.csv. -// -// Requests are sent on schedule whatever the response times (open loop), and -// the generator records how late it sent them: a step it could not keep up -// with is marked invalid rather than reported as the system's limit. Tokens -// stay in memory and are never written out. +// loadgen bench — seeds data, warms up, then offers each rate of -rates +// for -step as a mix of history reads and message sends. +// loadgen timeline — one user sends messages at -rate, retrying uncertain +// sends with the same Idempotency-Key, then reads the +// history back: every confirmed message once. package main import ( @@ -52,6 +41,7 @@ type config struct { Timeout string `json:"client_timeout"` Seed uint64 `json:"seed"` Scrape bool `json:"scrape"` + StopOnFail bool `json:"stop_on_fail"` StatsURL string `json:"stats_url"` APIHost string `json:"api_host"` DebugPort string `json:"debug_port"` @@ -98,6 +88,7 @@ func parse(mode string, args []string) (config, error) { fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") + fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step that fails the capacity criterion") fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") @@ -138,8 +129,7 @@ func writeJSON(path string, v any) error { return os.WriteFile(path, append(raw, '\n'), 0o644) } -// startScrape samples the system in the background when asked to; the -// returned function stops it and waits. +// startScrape returns the function that stops the sampling and waits. func startScrape(ctx context.Context, cfg config, start time.Time) func() { if !cfg.Scrape { return func() {} @@ -177,8 +167,7 @@ func runBench(ctx context.Context, cfg config) error { stopScrape := startScrape(ctx, cfg, start) defer stopScrape() - // next picks the request on the scheduling goroutine, where the random - // source may be used safely. + // next runs on the scheduling goroutine, which owns the random source. counter := 0 next := func(rec *recorder) func(time.Time) func() { return func(planned time.Time) func() { @@ -217,8 +206,11 @@ func runBench(ctx context.Context, cfg config) error { for _, x := range rec.all() { all.add(x) } - fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v\n", - rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid) + fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", + rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) + if cfg.StopOnFail && !s.Passed { + break + } } stopScrape() if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { @@ -247,8 +239,7 @@ type timelineReport struct { // retryDelay spaces the attempts of one message whose outcome was uncertain. const retryDelay = 200 * time.Millisecond -// retryGrace is how long after the schedule ends an uncertain message keeps -// being retried before it counts as unresolved. +// retryGrace bounds the retries of an uncertain message after the schedule ends. const retryGrace = 10 * time.Second func runTimeline(ctx context.Context, cfg config) error { diff --git a/tools/loadgen/run.go b/tools/loadgen/run.go index 21019fd3..7a84f6a2 100644 --- a/tools/loadgen/run.go +++ b/tools/loadgen/run.go @@ -6,14 +6,9 @@ import ( "time" ) -// openLoop offers requests at a constant rate for d, whatever the response -// times: request i is planned at start + i/rate and sent then, or at once if -// the generator is already late. Waiting for responses before sending more -// (a closed loop) would let a slow system lower its own load and hide its -// latency: coordinated omission. next runs on the scheduling goroutine, so it -// may use non-thread-safe state such as the random source; the job it -// returns runs on a goroutine of its own. openLoop returns once every job -// has finished. +// openLoop sends request i at start + i/rate whatever the response times: a +// closed loop would let a slow system lower its own load and hide its latency +// (coordinated omission). It returns once every job has finished. func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) { n := int(rate * d.Seconds()) interval := time.Duration(float64(time.Second) / rate) diff --git a/tools/loadgen/scrape.go b/tools/loadgen/scrape.go index 69790f23..6b95db75 100644 --- a/tools/loadgen/scrape.go +++ b/tools/loadgen/scrape.go @@ -12,11 +12,8 @@ import ( "time" ) -// scraper samples, once a second, what the balancer and every replica report -// about themselves, so a saturated step can be explained: the HAProxy -// statistics per replica and the expvar of each API process (pool waits, -// GOMAXPROCS, memory and GC). Replicas are found the way HAProxy finds them, -// through Docker DNS, and asked directly on their internal debug port. +// scraper samples the HAProxy statistics and every replica's expvar once a +// second; replicas are found through Docker DNS, as HAProxy finds them. type scraper struct { statsURL string apiHost string diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go index f74336e9..541299f0 100644 --- a/tools/loadgen/stats.go +++ b/tools/loadgen/stats.go @@ -12,11 +12,17 @@ import ( "time" ) -// maxValidLag is the generator's own tolerance: if requests left more than -// this late at the 99th percentile, the generator, not the system, limited -// the step, and the step is excluded from the conclusions. +// maxValidLag: if requests left later than this at p99, the generator, not +// the system, limited the step. const maxValidLag = 10 * time.Millisecond +// The capacity criterion of the study: a step passes when it is valid, has +// fewer than 1 % errors and its successful requests have a p99 within 500 ms. +const ( + maxErrorShare = 0.01 + sloP99 = 500 * time.Millisecond +) + // sample is one request as the generator saw it. type sample struct { planned time.Time @@ -27,11 +33,9 @@ type sample struct { write bool } -// classify names the outcome of a request. HTTP errors produced by the -// balancer itself carry no X-Instance-Id and are kept apart from the API's -// own: a 503 from HAProxy means no replica was available, a 503 from the API -// means a dependency was. client_timeout is the generator giving up on a -// response, not a timeout of the system. +// classify names the outcome of a request. Errors of the balancer itself carry +// no X-Instance-Id and get the suffix _lb; client_timeout is the generator +// giving up, not a timeout of the system. func classify(r result) string { switch { case r.err != nil && isTimeout(r.err): @@ -78,12 +82,12 @@ type stepSummary struct { P99Ms float64 `json:"ok_p99_ms"` LagP99Ms float64 `json:"lag_p99_ms"` Valid bool `json:"valid"` + Passed bool `json:"passed"` Instances map[string]int `json:"instances"` Writes int `json:"writes_ok"` } -// summarize reduces one step. Latency percentiles cover successful requests -// only, as the course's report format asks; the errors are counted by class. +// summarize reduces one step; latency percentiles cover successful requests. func summarize(rate float64, d time.Duration, samples []sample) stepSummary { s := stepSummary{ OfferedRPS: rate, @@ -117,6 +121,7 @@ func summarize(rate float64, d time.Duration, samples []sample) stepSummary { lagP99 := percentile(lags, 99) s.LagP99Ms = ms(lagP99) s.Valid = lagP99 <= maxValidLag + s.Passed = s.Valid && s.ErrorShare < maxErrorShare && percentile(ok, 99) <= sloP99 return s } diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go index 1e590016..df7b6fea 100644 --- a/tools/loadgen/stats_test.go +++ b/tools/loadgen/stats_test.go @@ -104,3 +104,25 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { t.Fatalf("open loop took %v, want about 1.2 s", took) } } + +func TestStepPassesOnlyWithinTheObjective(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + fast := []sample{ok(10*time.Millisecond, time.Millisecond), ok(20*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, fast); !s.Passed { + t.Fatalf("fast clean step must pass: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); s.Passed { + t.Fatalf("p99 above 500 ms must fail: %+v", s) + } + failing := append(fast, sample{class: "http_503_lb", lag: time.Millisecond}) + if s := summarize(3, time.Second, failing); s.Passed { + t.Fatalf("errors above 1 %% must fail: %+v", s) + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.Passed { + t.Fatalf("an invalid step must fail: %+v", s) + } +}