Доработать инструменты экспериментов: чередовать повторы, останавливать лестницу на первой проваленной ступени, задавать maxconn и CPU базы в сериях, записывать замер скорости хоста и контроль шума, сократить комментарии
This commit is contained in:
@@ -1,11 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Lists the API containers of this compose project, one per line:
|
||||
# <container id> <name> <state> <alias> <image id>
|
||||
# The alias is what the process sends as X-Instance-Id. It is random per
|
||||
# process, so it is asked from the process itself, over the same wget the
|
||||
# Docker healthcheck uses, from inside the container. A container that is not
|
||||
# running has no alias and shows "-". The replicas are asked in parallel: the
|
||||
# failover hook that relies on this list must return within 10 seconds.
|
||||
# Lists the API containers: <id> <name> <state> <X-Instance-Id> <image>. The
|
||||
# alias is random per process, so each running replica is asked over
|
||||
# /health/live, in parallel: the failover hook must return within 10 seconds.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
|
||||
@@ -1,26 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Level 4: one API replica fails under a constant write load. The generator
|
||||
# sends messages at RATE for DURATION and retries uncertain sends with the
|
||||
# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM,
|
||||
# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started
|
||||
# again. Afterwards the generator reads the history back: every confirmed
|
||||
# message once.
|
||||
# Level 4: one replica stopped (SIGTERM) or killed (SIGKILL) at FAULT_AT and
|
||||
# started again at RESTORE_AT, under a constant write load that retries
|
||||
# uncertain sends with the same key; then every confirmed message must be in
|
||||
# the history once.
|
||||
#
|
||||
# sh scripts/bench/failover.sh stop|kill <replica number, 1-based>
|
||||
#
|
||||
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35),
|
||||
# OUT (default evidence/failover). Results in <OUT>/<kind>-<n>/:
|
||||
# timeline.json, seconds.csv (per-second outcomes, attributed by send time),
|
||||
# faults.json (when and what was stopped), compose.log (the balancer's
|
||||
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
|
||||
# Runs on the current stand (`make up`) and leaves every replica running.
|
||||
#
|
||||
# Every time used for the analysis comes from the Docker VM's clock: the log
|
||||
# timestamps, the container's stop and start times from `docker inspect`, and
|
||||
# the generator's own times. The host clock of Windows runs apart from it by
|
||||
# a fraction of a second, a large share of a 1.5 s detection window; host
|
||||
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
|
||||
# is the replica's own "shutdown started" line in compose.log.
|
||||
# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), OUT
|
||||
# (evidence/failover). Times come from the Docker VM clock; the Windows clock
|
||||
# differs by a fraction of a second and is kept for reference only.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
@@ -50,8 +38,6 @@ since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
||||
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
||||
generator=$!
|
||||
# The generator seeds two users first; its timeline starts after that. The
|
||||
# fault times below are absolute, as is started_at in timeline.json.
|
||||
sleep "$fault_at"
|
||||
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
docker "$kind" "$target" >/dev/null
|
||||
|
||||
+13
-8
@@ -3,14 +3,13 @@
|
||||
# The generator and the knobs the experiments vary live in compose.bench.yaml.
|
||||
export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml
|
||||
|
||||
# abspath prints the absolute path of an existing directory in the form the
|
||||
# Docker CLI accepts for a bind mount (a Windows path under Git Bash).
|
||||
# abspath prints a directory in the form the Docker CLI mounts (a Windows path
|
||||
# under Git Bash).
|
||||
abspath() {
|
||||
(cd "$1" && (pwd -W 2>/dev/null || pwd))
|
||||
}
|
||||
|
||||
# meta writes the header of one run: what ran, when, on which commit and
|
||||
# host. Arguments: file, then key=value pairs added to the record.
|
||||
# meta writes what ran, when, on which commit and host: file, then key=value.
|
||||
meta() {
|
||||
file=$1
|
||||
shift
|
||||
@@ -30,8 +29,13 @@ meta() {
|
||||
} >"$file"
|
||||
}
|
||||
|
||||
# sample_docker_stats appends the CPU and memory of every container of the
|
||||
# stand to a JSON-lines file, about every two seconds, until killed.
|
||||
# host_probe prints the seconds a fixed CPU loop takes in a one-CPU container:
|
||||
# the same work every time, so a slow spell of the host shows in the results.
|
||||
host_probe() {
|
||||
docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }'
|
||||
}
|
||||
|
||||
# sample_docker_stats appends the CPU and memory of every container until killed.
|
||||
sample_docker_stats() {
|
||||
while :; do
|
||||
docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \
|
||||
@@ -39,8 +43,8 @@ sample_docker_stats() {
|
||||
done >>"$1"
|
||||
}
|
||||
|
||||
# sample_postgres appends, every second, the database's commit counters and
|
||||
# what its active sessions are waiting for (none: running on CPU).
|
||||
# sample_postgres appends, every second, the database counters and what the
|
||||
# active sessions wait for ("cpu": nothing).
|
||||
sample_postgres() {
|
||||
while :; do
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c "
|
||||
@@ -48,6 +52,7 @@ sample_postgres() {
|
||||
't', extract(epoch FROM now()),
|
||||
'xact_commit', xact_commit, 'xact_rollback', xact_rollback,
|
||||
'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit,
|
||||
'sessions', sessions, 'sessions_abandoned', sessions_abandoned,
|
||||
'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()),
|
||||
'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM (
|
||||
SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n
|
||||
|
||||
+48
-26
@@ -5,14 +5,16 @@
|
||||
|
||||
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
|
||||
output directory:
|
||||
summary.csv one row per step of every run: offered and successful rate,
|
||||
errors, latency percentiles, generator lag, mean CPU of each
|
||||
component and pool waits during the step;
|
||||
summary.csv one row per step of every run;
|
||||
capacity.csv the capacity of every run and the median per configuration;
|
||||
control.csv the noise checks of every run: the host speed probe and the
|
||||
loadgen CPU per request at the first step, flagged when more
|
||||
than 25 % off the median;
|
||||
<series>-throughput.png, <series>-p99.png, capacity.png,
|
||||
cpu-at-capacity.png, pool-waits.png.
|
||||
Capacity is the highest valid step (generator on schedule) with less than 1 %
|
||||
errors and a p99 of successful requests within 500 ms.
|
||||
Capacity is the highest step that passed the criterion loadgen applies (valid,
|
||||
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
|
||||
median of the repeats; a hollow marker means the generator fell behind.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
@@ -27,7 +29,7 @@ matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
SLO_P99_MS = 500.0
|
||||
MAX_ERROR_SHARE = 0.01
|
||||
CONTROL_TOLERANCE = 0.25
|
||||
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
|
||||
|
||||
|
||||
@@ -69,6 +71,7 @@ def load_runs(root):
|
||||
"steps": report["steps"],
|
||||
"docker": read_jsonl(d / "docker-stats.jsonl"),
|
||||
"metrics": read_jsonl(d / "metrics.jsonl"),
|
||||
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
|
||||
})
|
||||
return runs
|
||||
|
||||
@@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1):
|
||||
return out
|
||||
|
||||
|
||||
def pool_waits(run, t0, t1):
|
||||
"""Requests that waited for a pooled connection during [t0, t1), all
|
||||
replicas together, per second."""
|
||||
def pool_rate(run, t0, t1, counter):
|
||||
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
|
||||
first, last = {}, {}
|
||||
for row in run["metrics"]:
|
||||
t = run["start"] + row["t"]
|
||||
@@ -104,18 +106,14 @@ def pool_waits(run, t0, t1):
|
||||
for replica in row.get("replicas") or []:
|
||||
pool = replica.get("pgxpool") or {}
|
||||
key = replica.get("instance")
|
||||
if key and "empty_acquire_count" in pool:
|
||||
first.setdefault(key, pool["empty_acquire_count"])
|
||||
last[key] = pool["empty_acquire_count"]
|
||||
if key and counter in pool:
|
||||
first.setdefault(key, pool[counter])
|
||||
last[key] = pool[counter]
|
||||
if not last:
|
||||
return ""
|
||||
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
|
||||
|
||||
|
||||
def is_capacity(step):
|
||||
return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS
|
||||
|
||||
|
||||
def main(root, out):
|
||||
runs = load_runs(root)
|
||||
if not runs:
|
||||
@@ -129,17 +127,22 @@ def main(root, out):
|
||||
t0 = run["start"] + step["started_at_s"]
|
||||
t1 = t0 + step["seconds"]
|
||||
cpu = mean_cpu(run, t0, t1)
|
||||
classes = step["classes"]
|
||||
row = {
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
|
||||
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
|
||||
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
|
||||
"client_timeouts": classes.get("client_timeout", 0),
|
||||
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
|
||||
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
|
||||
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
|
||||
"pool_waits_per_s": pool_waits(run, t0, t1),
|
||||
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
|
||||
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
|
||||
}
|
||||
rows.append(row)
|
||||
if is_capacity(step):
|
||||
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
|
||||
best = row
|
||||
capacities.append({
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
@@ -164,13 +167,29 @@ def main(root, out):
|
||||
medians[key] = statistics.median(values)
|
||||
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"])
|
||||
|
||||
control = []
|
||||
for run in runs:
|
||||
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
|
||||
if first and first[0]["cpu_loadgen_pct"] != "":
|
||||
control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"],
|
||||
round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)])
|
||||
if control:
|
||||
median = statistics.median(c[5] for c in control)
|
||||
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"])
|
||||
for c in control:
|
||||
deviation = c[5] / median - 1
|
||||
w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE])
|
||||
|
||||
def median_by_offered(series, config, field):
|
||||
points = {}
|
||||
points, valid = {}, {}
|
||||
for r in rows:
|
||||
if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "":
|
||||
if r["series"] == series and r["config"] == config and r[field] != "":
|
||||
points.setdefault(r["offered_rps"], []).append(float(r[field]))
|
||||
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
|
||||
xs = sorted(points)
|
||||
return xs, [statistics.median(points[x]) for x in xs]
|
||||
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
|
||||
|
||||
for series in sorted({s for s, _ in configs}):
|
||||
names = [c for s, c in configs if s == series]
|
||||
@@ -181,8 +200,11 @@ def main(root, out):
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
top = 0
|
||||
for name in names:
|
||||
xs, ys = median_by_offered(series, name, field)
|
||||
ax.plot(xs, ys, marker="o", label=name)
|
||||
xs, ys, valid = median_by_offered(series, name, field)
|
||||
line, = ax.plot(xs, ys, label=name)
|
||||
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
|
||||
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
|
||||
facecolors="none", edgecolors=line.get_color())
|
||||
top = max([top, *xs])
|
||||
if field == "ok_rps" and top:
|
||||
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
|
||||
@@ -192,7 +214,7 @@ def main(root, out):
|
||||
ax.set_yscale("log")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
ax.set_ylabel(title)
|
||||
ax.set_title(f"Series {series}: median of repeats, valid steps only")
|
||||
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
@@ -233,7 +255,7 @@ def main(root, out):
|
||||
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
for series, name in configs:
|
||||
xs, ys = median_by_offered(series, name, "pool_waits_per_s")
|
||||
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
|
||||
if xs:
|
||||
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
|
||||
@@ -1,21 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Level 4: a rolling update under a constant write load. The generator sends
|
||||
# messages at RATE for DURATION, retrying uncertain sends with the same key;
|
||||
# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards
|
||||
# the generator reads the history back: every confirmed message once.
|
||||
# Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a
|
||||
# constant write load that retries uncertain sends with the same key; then
|
||||
# every confirmed message must be in the history once.
|
||||
#
|
||||
# sh scripts/bench/rollout-under-load.sh
|
||||
#
|
||||
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default
|
||||
# evidence/rollout). Results in <OUT>/: timeline.json, seconds.csv,
|
||||
# rollout.log, update.json (when it ran, which containers it replaced),
|
||||
# compose.log, instances-*.txt.
|
||||
#
|
||||
# Times for the analysis come from the Docker VM's clock, as the generator's
|
||||
# do: log timestamps and the start times of the new containers. The host
|
||||
# clock of Windows runs apart from it and is kept for reference only. The
|
||||
# old replicas are removed by the update; their stop shows in compose.log as
|
||||
# their "shutdown started" lines.
|
||||
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout).
|
||||
# Times come from the Docker VM clock, as in failover.sh.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
@@ -29,8 +20,7 @@ export MSYS_NO_PATHCONV=1
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
docker compose up -d --wait >/dev/null 2>&1
|
||||
# Build first, as a delivery pipeline would: the update under load then
|
||||
# measures the replacement of the replicas, not the compiler.
|
||||
# Built beforehand, as by a delivery pipeline: the update replaces replicas.
|
||||
docker compose build api >/dev/null 2>&1
|
||||
meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at"
|
||||
sh scripts/instances.sh >"$dir/instances-before.txt"
|
||||
|
||||
+65
-44
@@ -1,61 +1,82 @@
|
||||
#!/bin/sh
|
||||
# Track 1, the cost of scaling: runs every configuration of one series file,
|
||||
# each repeated, each on a fresh stand (empty volumes, same seed), under the
|
||||
# same grid of offered rates.
|
||||
# Track 1: every configuration of the given series files, each on a fresh
|
||||
# stand, under one grid of offered rates. Repeats are interleaved (repeat 1
|
||||
# of every configuration, then repeat 2), so a slow spell of the host does not
|
||||
# fall on one configuration.
|
||||
#
|
||||
# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats]
|
||||
# RATES=400,800,1200 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [...]
|
||||
#
|
||||
# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root,
|
||||
# default evidence/track1/raw). Fixed for every configuration: HAProxy
|
||||
# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the
|
||||
# generator LOADGEN_CPUS (1.5). Results per run:
|
||||
# <OUT>/<series>/<config>/rep<k>/{meta.json,steps.json,seconds.csv,
|
||||
# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log}
|
||||
# The stand is removed at the end; `make up` brings the normal one back.
|
||||
# Environment: RATES (required), REPEATS (3), STEP (30s), WARMUP (30s),
|
||||
# STOP_ON_FAIL (1: end a ladder at its first failing step), OUT
|
||||
# (evidence/track1/raw). Results: <OUT>/<series>/<config>/rep<k>/.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
|
||||
series=${1:?usage: run-series.sh <series.conf> [repeats]}
|
||||
repeats=${2:-3}
|
||||
: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}"
|
||||
[ $# -gt 0 ] || { echo "usage: RATES=... run-series.sh <series.conf>..." >&2; exit 2; }
|
||||
: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 400,800,1200}"
|
||||
repeats=${REPEATS:-3}
|
||||
step=${STEP:-30s}
|
||||
warmup=${WARMUP:-30s}
|
||||
root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf)
|
||||
out=${OUT:-evidence/track1/raw}
|
||||
stop=
|
||||
[ "${STOP_ON_FAIL:-1}" = 1 ] && stop=-stop-on-fail
|
||||
|
||||
export APP_DEBUG_ADDR=:9090 LB_QUIET=1
|
||||
export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5}
|
||||
# Fixed for every configuration; the API, its pool, maxconn and the database
|
||||
# CPU come from the series line.
|
||||
export LB_QUIET=1 LB_CPUS=1.0 PG_MEMORY=1G LOADGEN_CPUS=2.0
|
||||
export MSYS_NO_PATHCONV=1
|
||||
|
||||
grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do
|
||||
[ -n "$name" ] || continue
|
||||
for rep in $(seq 1 "$repeats"); do
|
||||
dir=$root/$name/rep$rep
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES"
|
||||
export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool
|
||||
# One series line: name replicas cpus memory gomemlimit pool maxconn pg_cpus.
|
||||
run_config() {
|
||||
series=$1 rep=$2 name=$3 replicas=$4 cpus=$5 memory=$6 gomemlimit=$7 pool=$8 maxconn=$9 pg_cpus=${10}
|
||||
dir=$out/$series/$name/rep$rep
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
echo "== $series/$name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, maxconn $maxconn, PostgreSQL $pg_cpus CPU"
|
||||
export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit \
|
||||
DB_MAX_CONNS=$pool API_MAXCONN=$maxconn PG_CPUS=$pg_cpus
|
||||
|
||||
docker compose down -v --remove-orphans >/dev/null 2>&1
|
||||
docker compose up -d --build --wait >/dev/null 2>&1
|
||||
meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \
|
||||
"api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \
|
||||
"lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \
|
||||
"rates=$RATES" "step=$step" "warmup=$warmup"
|
||||
docker compose down -v --remove-orphans >/dev/null 2>&1
|
||||
if ! docker compose up -d --build --wait >"$dir/up.log" 2>&1; then
|
||||
echo " the stand did not start, see $dir/up.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
meta "$dir/meta.json" "series=$series" "config=$name" "repeat=$rep" \
|
||||
"api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" \
|
||||
"db_max_conns=$pool" "api_maxconn=$maxconn" "pg_cpus=$pg_cpus" "pg_memory=$PG_MEMORY" \
|
||||
"lb_cpus=$LB_CPUS" "loadgen_cpus=$LOADGEN_CPUS" "rates=$RATES" "step=$step" "warmup=$warmup" \
|
||||
"stop_on_fail=${STOP_ON_FAIL:-1}" "host_probe_s=$(host_probe)"
|
||||
|
||||
sample_docker_stats "$dir/docker-stats.jsonl" &
|
||||
stats=$!
|
||||
sample_postgres "$dir/postgres.jsonl" &
|
||||
pg=$!
|
||||
# stdin is closed: `compose run` is interactive by default and would
|
||||
# otherwise read the rest of the series file this loop is reading.
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
|
||||
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 </dev/null ||
|
||||
echo " loadgen failed, see $dir/loadgen.log"
|
||||
kill "$stats" "$pg" 2>/dev/null || true
|
||||
wait "$stats" "$pg" 2>/dev/null || true
|
||||
cat "$dir/loadgen.log" | grep '^step' || true
|
||||
sample_docker_stats "$dir/docker-stats.jsonl" &
|
||||
stats=$!
|
||||
sample_postgres "$dir/postgres.jsonl" &
|
||||
pg=$!
|
||||
# stdin closed: `compose run` would otherwise read it.
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
|
||||
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape $stop >"$dir/loadgen.log" 2>&1 </dev/null ||
|
||||
echo " loadgen failed, see $dir/loadgen.log"
|
||||
kill "$stats" "$pg" 2>/dev/null || true
|
||||
wait "$stats" "$pg" 2>/dev/null || true
|
||||
docker compose logs --no-color --timestamps lb api >"$dir/compose.log" 2>&1
|
||||
grep '^step' "$dir/loadgen.log" || true
|
||||
}
|
||||
|
||||
docker compose --profile bench build loadgen >/dev/null
|
||||
default_ifs=$IFS
|
||||
newline='
|
||||
'
|
||||
for rep in $(seq 1 "$repeats"); do
|
||||
for conf in "$@"; do
|
||||
lines=$(grep -v '^#' "$conf" | sed '/^[[:space:]]*$/d')
|
||||
IFS=$newline
|
||||
for line in $lines; do
|
||||
IFS=$default_ifs
|
||||
# shellcheck disable=SC2086 # the line splits into the fields
|
||||
run_config "$(basename "$conf" .conf)" "$rep" $line
|
||||
done
|
||||
IFS=$default_ifs
|
||||
done
|
||||
done
|
||||
docker compose down -v --remove-orphans >/dev/null 2>&1
|
||||
echo "results in $root"
|
||||
echo "results in $out"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas.
|
||||
# The pool stays 10 per replica, so the total grows with the replicas.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
A1 1 2.0 1024M 920MiB 10
|
||||
A2 2 1.0 512M 460MiB 10
|
||||
A4 4 0.5 256M 230MiB 10
|
||||
# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas;
|
||||
# the pool stays 10 per replica, so the total grows with the replicas.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
A1 1 2.0 1024M 920MiB 10 0 1.0
|
||||
A2 2 1.0 512M 460MiB 10 0 1.0
|
||||
A4 4 0.5 256M 230MiB 10 0 1.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Series A': the budget of series A with the total pool fixed at 20
|
||||
# connections. A'2 equals A2 and is not run again.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
Ap1 1 2.0 1024M 920MiB 20
|
||||
Ap4 4 0.5 256M 230MiB 5
|
||||
# Series A': the budget of series A with the total pool fixed at 20 connections;
|
||||
# A'2 equals A2.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
Ap1 1 2.0 1024M 920MiB 20 0 1.0
|
||||
Ap4 4 0.5 256M 230MiB 5 0 1.0
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by
|
||||
# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run
|
||||
# again.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
B1 1 1.0 512M 460MiB 10
|
||||
B3 3 1.0 512M 460MiB 10
|
||||
B4 4 1.0 512M 460MiB 10
|
||||
# Series B: replicas of 1 CPU, 512 MiB and pool 10 added one by one; B2 equals A2.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
B1 1 1.0 512M 460MiB 10 0 1.0
|
||||
B3 3 1.0 512M 460MiB 10 0 1.0
|
||||
B4 4 1.0 512M 460MiB 10 0 1.0
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Improvement C: maxconn per replica on the balancer, before (0, no limit) and
|
||||
# after (20) on 1 and 4 replicas, under an overload ladder (STOP_ON_FAIL=0);
|
||||
# 10 and 40 show how much the value matters.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
R1m0 1 1.0 512M 460MiB 10 0 1.0
|
||||
R1m20 1 1.0 512M 460MiB 10 20 1.0
|
||||
R4m0 4 1.0 512M 460MiB 10 0 1.0
|
||||
R4m20 4 1.0 512M 460MiB 10 20 1.0
|
||||
R1m10 1 1.0 512M 460MiB 10 10 1.0
|
||||
R1m40 1 1.0 512M 460MiB 10 40 1.0
|
||||
@@ -1,4 +0,0 @@
|
||||
# Calibration: the smallest API budget of the study (one replica, one CPU)
|
||||
# with a wide range of rates finds where the common rate grid must reach.
|
||||
# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica
|
||||
cal 1 1.0 512M 460MiB 10
|
||||
@@ -0,0 +1,4 @@
|
||||
# Control: A2 with twice the database CPU. If the database is the bottleneck,
|
||||
# the capacity rises.
|
||||
# name replicas cpus memory gomemlimit pool maxconn pg_cpus
|
||||
A2pg2 2 1.0 512M 460MiB 10 0 2.0
|
||||
@@ -1,7 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Shows which container answers under which X-Instance-Id: the proof on the
|
||||
# defense that the header values belong to different running processes, and
|
||||
# which container a failover test actually stopped.
|
||||
# Which container answers under which X-Instance-Id.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
|
||||
+2
-5
@@ -1,9 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Restart hook of `make test-persistence` (tests/resilience.py): stops the
|
||||
# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the
|
||||
# same containers again (the one-shot migrate finds nothing to apply).
|
||||
# Volumes and data are kept and nothing is re-seeded. The test hides the
|
||||
# output, so failures surface through the exit code only.
|
||||
# Restart hook of `make test-persistence`: stops the whole stand gracefully and
|
||||
# starts the same containers again; volumes are kept.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop
|
||||
|
||||
+8
-18
@@ -1,21 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API without downtime, with docker rollout
|
||||
# (https://github.com/wowu/docker-rollout, MIT): it starts as many new
|
||||
# replicas as are running, waits until all of them are healthy, then stops
|
||||
# and removes the old ones. If a new replica does not become healthy it
|
||||
# removes the new ones and leaves the old ones serving, so the stand is never
|
||||
# left with a mix of versions.
|
||||
# Rolling update of the API with docker rollout (github.com/wowu/docker-rollout,
|
||||
# MIT): new replicas start next to the old ones, and the old ones go once all
|
||||
# new ones are healthy; if one is not, the new ones are removed instead.
|
||||
#
|
||||
# Every replica must be running. With a stopped one (say, after stop-one),
|
||||
# docker rollout only replaces the stopped one and reports success, which
|
||||
# would leave old and new versions side by side; the script refuses instead.
|
||||
#
|
||||
# The migrations of the new version run first, once, while the old replicas
|
||||
# serve: they only add to the schema. docker rollout scales with `compose up`,
|
||||
# which also starts the dependencies of the API and would rerun the old,
|
||||
# exited migrate container. Recreating it beforehand from the new image,
|
||||
# without starting it, makes that `up` run the new migrations exactly once,
|
||||
# before any new replica starts.
|
||||
# With a stopped replica docker rollout would replace only that one and leave
|
||||
# two versions running, so the script refuses. The migrate container is
|
||||
# recreated from the new image without starting: the `up` of docker rollout
|
||||
# then applies the new migrations once, before any new replica starts.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
@@ -44,8 +35,7 @@ docker compose build api
|
||||
docker compose up --no-start --no-deps --force-recreate migrate
|
||||
# Run through sh: Windows cannot use the script as a docker CLI plugin.
|
||||
if ! sh "$tool" rollout --wait-after-healthy 4 api; then
|
||||
# A failed migration stops the scale-up after the new containers were
|
||||
# created but before they started; remove them for a clean next attempt.
|
||||
# A failed migration leaves the new containers created but not started.
|
||||
created=$(docker compose ps -aq --status created api)
|
||||
if [ -n "$created" ]; then
|
||||
docker rm $created >/dev/null
|
||||
|
||||
+3
-7
@@ -1,11 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Failover hook of `make test-failover` (tests/resilience.py): stops the one
|
||||
# API replica whose X-Instance-Id is $TARGET_INSTANCE_ID and leaves it
|
||||
# stopped. The stop is graceful: on SIGTERM the replica turns unready, keeps
|
||||
# serving for its drain delay while HAProxy takes it out of rotation, then
|
||||
# exits; `docker stop` returns once it has. Nothing else is touched, and an
|
||||
# alias that does not name exactly one running replica stops nothing.
|
||||
# Restore the replica with: docker compose start api
|
||||
# Failover hook of `make test-failover`: gracefully stops the one running API
|
||||
# replica whose X-Instance-Id is $TARGET_INSTANCE_ID, and nothing else.
|
||||
# Restore it with: docker compose start api
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
|
||||
Reference in New Issue
Block a user