diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index a27b1206..936f2ec4 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -36,7 +36,12 @@ jobs: run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt - name: Persistence across a restart of the API and both stores run: make test-persistence PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/restart.sh | tee evidence-ci/persistence.txt - - name: Concurrency check + - name: Failover of one replica behind the balancer + run: | + make test-failover PYTHON=python BASE_URL="$BASE_URL" ACTION_SCRIPT=scripts/stop-one.sh | tee evidence-ci/failover.txt + docker compose start api + docker compose up -d --wait + - name: Concurrency and shared idempotency across replicas run: go run ./tools/concurrency --base-url "$BASE_URL" --output evidence-ci/concurrency.json - name: Container logs if: always() diff --git a/.gitignore b/.gitignore index b5a71896..94c5da0a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +# Raw output of local benchmark runs; curated results go to evidence/. +bench-out/ diff --git a/Dockerfile b/Dockerfile index 571f6bb7..f32eecf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,34 @@ # Build stage: compile static binaries with the exact toolchain pinned in go.mod. +# Modules live in their own layer, rebuilt only when go.mod or go.sum change. +# Only the Go sources are copied after it, and the compiler's cache persists +# between builds: a change elsewhere in the repository does not trigger a full +# recompilation, which matters when a rolling update rebuilds the image. FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download && go mod verify -COPY . . -RUN CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate +COPY cmd ./cmd +COPY internal ./internal +COPY migrations ./migrations +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate -# Runtime stage: minimal image, unprivileged user, the server is PID 1. The -# same image runs the one-shot migrate service with its own entrypoint. +# Load generator of the experiments, an image of its own: the API image does +# not carry it. Built only for `docker compose --profile bench`. +FROM build AS loadgen-build +COPY tools ./tools +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen + +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen +RUN adduser -D -H -u 10001 app +COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen +USER app +ENTRYPOINT ["/usr/local/bin/loadgen"] + +# Runtime stage, the last one so that `build: .` builds it: minimal image, +# unprivileged user, the server is PID 1. The same image runs the one-shot +# migrate service with its own entrypoint. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app COPY --from=build /out/server /out/migrate /usr/local/bin/ diff --git a/Makefile b/Makefile index 4c74f9de..39794250 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ test-failover: # --------------------------------------------------------------------------- EVIDENCE_DIR ?= evidence -.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout +.PHONY: build vet unit secrets up down destroy logs concurrency explain instances rollout bench plot failover-load rollout-load build: go build -o bin/server ./cmd/server @@ -72,6 +72,27 @@ instances: rollout: sh scripts/rollout.sh +# Track 1, the cost of scaling: one series on fresh stands, then the charts. +# RATES is required, e.g. make bench SERIES=scripts/bench/series/B.conf RATES=200,400,800 +SERIES ?= scripts/bench/series/A.conf +REPEATS ?= 3 +bench: + sh scripts/bench/run-series.sh $(SERIES) $(REPEATS) + +# Needs matplotlib from requirements-bench.txt: make plot PYTHON=. +plot: + $(PYTHON) scripts/bench/plot.py evidence/track1/raw evidence/track1 + +# Level 4 under load: one replica stopped (KIND=stop) or killed (KIND=kill), +# and the rolling update. +KIND ?= stop +REPLICA ?= 1 +failover-load: + sh scripts/bench/failover.sh $(KIND) $(REPLICA) + +rollout-load: + sh scripts/bench/rollout-under-load.sh + concurrency: go run ./tools/concurrency --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/concurrency.json" diff --git a/cmd/server/main.go b/cmd/server/main.go index f3cbafcf..72e2440f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,7 @@ package main import ( "context" + "expvar" "fmt" "log/slog" "net" @@ -13,6 +14,8 @@ import ( "syscall" "time" + "github.com/jackc/pgx/v5/pgxpool" + "messenger/internal/app" "messenger/internal/chat" "messenger/internal/platform/config" @@ -80,10 +83,45 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) var background sync.WaitGroup defer background.Wait() // before the deferred pool.Close background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if cfg.DebugAddr != "" { + publishDebugVars(alias, pool) + debugLn, err := net.Listen("tcp", cfg.DebugAddr) + if err != nil { + return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err) + } + background.Go(func() { + if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { + log.Error("debug listener stopped", "error", err.Error()) + } + }) + } return server.Run(ctx, log, ln, a.Engine, a.Health, cfg.DrainDelay, cfg.ShutdownTimeout) } +// publishDebugVars exposes, next to the runtime's memstats, what the scaling +// study needs from a replica: who it is, how many CPUs Go uses inside the +// container's limit, and how often requests had to wait for a pooled +// database connection. +func publishDebugVars(alias string, pool *pgxpool.Pool) { + expvar.NewString("instance").Set(alias) + expvar.Publish("gomaxprocs", expvar.Func(func() any { return runtime.GOMAXPROCS(0) })) + expvar.Publish("pgxpool", expvar.Func(func() any { + s := pool.Stat() + return map[string]any{ + "acquire_count": s.AcquireCount(), + "acquire_duration_ms": s.AcquireDuration().Milliseconds(), + "acquired_conns": s.AcquiredConns(), + "idle_conns": s.IdleConns(), + "total_conns": s.TotalConns(), + "max_conns": s.MaxConns(), + "empty_acquire_count": s.EmptyAcquireCount(), + "empty_acquire_wait_ms": s.EmptyAcquireWaitTime().Milliseconds(), + "canceled_acquire_count": s.CanceledAcquireCount(), + } + })) +} + func newLogger(cfg config.Config) *slog.Logger { opts := &slog.HandlerOptions{Level: cfg.LogLevel} if cfg.Env == "dev" { diff --git a/compose.yaml b/compose.yaml index ccaa6ab9..5e1421eb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -165,6 +165,25 @@ services: # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M + # Load generator of the experiments (tools/loadgen). Only + # `docker compose --profile bench run --rm loadgen ...` starts it; it talks to + # the balancer and, for metrics, to the replicas' internal debug port. + loadgen: + profiles: ["bench"] + build: + context: . + target: loadgen + image: mtusi-messenger-loadgen:lab3 + volumes: + - ${BENCH_OUT:-./bench-out}:/out + networks: + - edge + deploy: + resources: + limits: + cpus: "${LOADGEN_CPUS:-1.5}" + memory: 512M + volumes: pgdata: valkeydata: diff --git a/internal/server/server.go b/internal/server/server.go index 718c66df..c33fc2b1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -67,3 +67,24 @@ func Run(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler, log.Info("shutdown completed") return nil } + +// RunDebug serves h, the internal metrics of the process, on ln until ctx is +// cancelled. Nothing outside the stand's network reaches it and no request +// depends on it, so it closes at once instead of draining. +func RunDebug(ctx context.Context, log *slog.Logger, ln net.Listener, h http.Handler) error { + srv := &http.Server{ + Handler: h, + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelWarn), + } + go func() { + <-ctx.Done() + _ = srv.Close() + }() + log.Info("debug listening", "addr", ln.Addr().String()) + if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 4f447f85..8a2caa39 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -143,3 +143,27 @@ func waitFor(t *testing.T, url string, status int) { } t.Fatalf("%s did not return %d in time", url, status) } + +// The debug listener has no drain: it serves the benchmark's metrics scraper +// only and stops with the process. +func TestRunDebugServesUntilCancelled(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTeapot) }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- server.RunDebug(ctx, log, ln, h) }() + waitFor(t, "http://"+ln.Addr().String()+"/debug/vars", http.StatusTeapot) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("RunDebug returned %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RunDebug did not stop after cancel") + } +} diff --git a/requirements-bench.txt b/requirements-bench.txt new file mode 100644 index 00000000..b1ec8530 --- /dev/null +++ b/requirements-bench.txt @@ -0,0 +1,2 @@ +# Plotting of the track 1 results (scripts/bench/plot.py); install into a venv. +matplotlib==3.10.3 diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh new file mode 100755 index 00000000..624ede52 --- /dev/null +++ b/scripts/bench/failover.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# Level 4: one API replica fails under a constant write load. The generator +# sends messages at RATE for DURATION and retries uncertain sends with the +# same key; at FAULT_AT seconds the chosen replica is stopped (stop: SIGTERM, +# graceful) or killed (kill: SIGKILL, a crash), at RESTORE_AT it is started +# again. Afterwards the generator reads the history back: every confirmed +# message once. +# +# sh scripts/bench/failover.sh stop|kill +# +# Environment: RATE (50), DURATION (60s), FAULT_AT (15), RESTORE_AT (35), +# OUT (default evidence/failover). Results in /-/: +# timeline.json, seconds.csv (per-second outcomes, attributed by send time), +# faults.json (when and what was stopped), compose.log (the balancer's +# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. +# Runs on the current stand (`make up`) and leaves every replica running. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +kind=${1:?usage: failover.sh stop|kill } +n=${2:?usage: failover.sh stop|kill } +case "$kind" in stop | kill) ;; *) echo "kind must be stop or kill" >&2; exit 2 ;; esac +rate=${RATE:-50} +duration=${DURATION:-60s} +fault_at=${FAULT_AT:-15} +restore_at=${RESTORE_AT:-35} +dir=${OUT:-evidence/failover}/$kind-$n +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +target=$(docker compose ps -q api | sed -n "${n}p") +[ -n "$target" ] || { echo "no running replica number $n" >&2; exit 2; } +alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -c1-12)" '$1 == id { print $4 }') +meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ + "fault_at_s=$fault_at" "restore_at_s=$restore_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +# The generator seeds two users first; its timeline starts after that. The +# fault times below are absolute, as is started_at in timeline.json. +sleep "$fault_at" +fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker "$kind" "$target" >/dev/null +fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +sleep $((restore_at - fault_at)) +restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +docker compose start api >/dev/null 2>&1 +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \ + "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json" +docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh new file mode 100644 index 00000000..d7b9d48c --- /dev/null +++ b/scripts/bench/lib.sh @@ -0,0 +1,57 @@ +# Shared helpers of the experiment scripts; sourced, not run. + +# abspath prints the absolute path of an existing directory in the form the +# Docker CLI accepts for a bind mount (a Windows path under Git Bash). +abspath() { + (cd "$1" && (pwd -W 2>/dev/null || pwd)) +} + +# meta writes the header of one run: what ran, when, on which commit and +# host. Arguments: file, then key=value pairs added to the record. +meta() { + file=$1 + shift + { + printf '{\n' + printf ' "date_utc": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf ' "commit": "%s",\n' "$(git rev-parse --short HEAD)" + printf ' "branch": "%s",\n' "$(git branch --show-current)" + printf ' "docker": "%s",\n' "$(docker version --format '{{.Server.Version}}')" + printf ' "host_cpus": "%s",\n' "$(docker info --format '{{.NCPU}}')" + printf ' "host_memory_bytes": "%s",\n' "$(docker info --format '{{.MemTotal}}')" + for pair in "$@"; do + printf ' "%s": "%s",\n' "${pair%%=*}" "${pair#*=}" + done + printf ' "go": "%s"\n' "$(go env GOVERSION 2>/dev/null || echo unknown)" + printf '}\n' + } >"$file" +} + +# sample_docker_stats appends the CPU and memory of every container of the +# stand to a JSON-lines file, about every two seconds, until killed. +sample_docker_stats() { + while :; do + docker stats --no-stream --format "{\"t\":$(date +%s),\"name\":\"{{.Name}}\",\"cpu\":\"{{.CPUPerc}}\",\"mem\":\"{{.MemUsage}}\"}" \ + $(docker ps -q --filter label=com.docker.compose.project=lab3) 2>/dev/null || true + done >>"$1" +} + +# sample_postgres appends, every second, the database's commit counters and +# what its active sessions are waiting for (none: running on CPU). +sample_postgres() { + while :; do + docker compose exec -T postgres psql -U messenger -d messenger -XAtq -c " + SELECT json_build_object( + 't', extract(epoch FROM now()), + 'xact_commit', xact_commit, 'xact_rollback', xact_rollback, + 'tup_inserted', tup_inserted, 'blks_read', blks_read, 'blks_hit', blks_hit, + 'active', (SELECT count(*) FROM pg_stat_activity WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid()), + 'waits', (SELECT coalesce(json_object_agg(w, n), '{}') FROM ( + SELECT coalesce(wait_event_type || ':' || wait_event, 'cpu') AS w, count(*) AS n + FROM pg_stat_activity + WHERE state = 'active' AND backend_type = 'client backend' AND pid <> pg_backend_pid() + GROUP BY 1) AS x)) + FROM pg_stat_database WHERE datname = 'messenger'" 2>/dev/null || true + sleep 1 + done >>"$1" +} diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py new file mode 100755 index 00000000..20493b2d --- /dev/null +++ b/scripts/bench/plot.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Tables and charts of track 1 from the raw results of run-series.sh. + + python scripts/bench/plot.py + +The raw root holds //rep/ directories. Written to the +output directory: + summary.csv one row per step of every run: offered and successful rate, + errors, latency percentiles, generator lag, mean CPU of each + component and pool waits during the step; + capacity.csv the capacity of every run and the median per configuration; + -throughput.png, -p99.png, capacity.png, + cpu-at-capacity.png, pool-waits.png. +Capacity is the highest valid step (generator on schedule) with less than 1 % +errors and a p99 of successful requests within 500 ms. +""" +import csv +import json +import pathlib +import statistics +import sys +from datetime import datetime + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +SLO_P99_MS = 500.0 +MAX_ERROR_SHARE = 0.01 +COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") + + +def read_jsonl(path): + if not path.exists(): + return [] + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + pass + return rows + + +def component(name): + for c in COMPONENTS: + if f"-{c}-" in name: + return c + return None + + +def parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + + +def load_runs(root): + runs = [] + for steps_path in sorted(root.glob("*/*/rep*/steps.json")): + d = steps_path.parent + report = json.loads(steps_path.read_text(encoding="utf-8")) + runs.append({ + "series": d.parent.parent.name, + "config": d.parent.name, + "rep": d.name, + "start": parse_time(report["started_at"]), + "steps": report["steps"], + "docker": read_jsonl(d / "docker-stats.jsonl"), + "metrics": read_jsonl(d / "metrics.jsonl"), + }) + return runs + + +def mean_cpu(run, t0, t1): + """Mean CPU per component in [t0, t1); replicas of the API are summed.""" + per_time = {} + for row in run["docker"]: + c = component(row.get("name", "")) + if c is None or not t0 <= row["t"] < t1: + continue + try: + cpu = float(row["cpu"].rstrip("%")) + except (ValueError, KeyError): + continue + per_time.setdefault(row["t"], {}).setdefault(c, 0.0) + per_time[row["t"]][c] += cpu + out = {} + for c in COMPONENTS: + values = [v[c] for v in per_time.values() if c in v] + out[c] = round(statistics.mean(values), 1) if values else "" + return out + + +def pool_waits(run, t0, t1): + """Requests that waited for a pooled connection during [t0, t1), all + replicas together, per second.""" + first, last = {}, {} + for row in run["metrics"]: + t = run["start"] + row["t"] + if not t0 <= t < t1: + continue + for replica in row.get("replicas") or []: + pool = replica.get("pgxpool") or {} + key = replica.get("instance") + if key and "empty_acquire_count" in pool: + first.setdefault(key, pool["empty_acquire_count"]) + last[key] = pool["empty_acquire_count"] + if not last: + return "" + return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) + + +def is_capacity(step): + return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS + + +def main(root, out): + runs = load_runs(root) + if not runs: + sys.exit(f"no runs under {root}") + out.mkdir(parents=True, exist_ok=True) + + rows, capacities = [], [] + for run in runs: + best = None + for step in run["steps"]: + t0 = run["start"] + step["started_at_s"] + t1 = t0 + step["seconds"] + cpu = mean_cpu(run, t0, t1) + row = { + "series": run["series"], "config": run["config"], "rep": run["rep"], + "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), + "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], + "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], + "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], + **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, + "pool_waits_per_s": pool_waits(run, t0, t1), + } + rows.append(row) + if is_capacity(step): + best = row + capacities.append({ + "series": run["series"], "config": run["config"], "rep": run["rep"], + "capacity_rps": best["offered_rps"] if best else 0, "at": best, + }) + + with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + + configs = [] + for c in capacities: + if (c["series"], c["config"]) not in configs: + configs.append((c["series"], c["config"])) + with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median"]) + medians = {} + for key in configs: + values = [c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == key] + medians[key] = statistics.median(values) + w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"]) + + def median_by_offered(series, config, field): + points = {} + for r in rows: + if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "": + points.setdefault(r["offered_rps"], []).append(float(r[field])) + xs = sorted(points) + return xs, [statistics.median(points[x]) for x in xs] + + for series in sorted({s for s, _ in configs}): + names = [c for s, c in configs if s == series] + for field, title, filename, log in ( + ("ok_rps", "Successful requests per second", "throughput", False), + ("ok_p99_ms", "p99 of successful requests, ms", "p99", True), + ): + fig, ax = plt.subplots(figsize=(7, 4.5)) + top = 0 + for name in names: + xs, ys = median_by_offered(series, name, field) + ax.plot(xs, ys, marker="o", label=name) + top = max([top, *xs]) + if field == "ok_rps" and top: + ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") + if field == "ok_p99_ms": + ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms") + if log: + ax.set_yscale("log") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel(title) + ax.set_title(f"Series {series}: median of repeats, valid steps only") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / f"{series}-{filename}.png", dpi=120) + plt.close(fig) + + labels = [f"{s}/{c}" for s, c in configs] + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5)) + meds = [medians[k] for k in configs] + lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs] + highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs] + ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8") + ax.set_ylabel("Capacity, requests per second") + ax.set_title("Capacity per configuration (median, range of repeats)") + ax.tick_params(axis="x", rotation=30) + ax.grid(axis="y", alpha=0.3) + fig.tight_layout() + fig.savefig(out / "capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5)) + width = 0.8 / len(COMPONENTS) + for i, comp in enumerate(COMPONENTS): + values = [] + for key in configs: + at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]] + cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""] + values.append(statistics.median(cpu) if cpu else 0) + ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp) + ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30) + ax.set_ylabel("Mean CPU at capacity, % of one core") + ax.set_title("Which component is busy at capacity") + ax.grid(axis="y", alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "cpu-at-capacity.png", dpi=120) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(7, 4.5)) + for series, name in configs: + xs, ys = median_by_offered(series, name, "pool_waits_per_s") + if xs: + ax.plot(xs, ys, marker="o", label=f"{series}/{name}") + ax.set_xlabel("Offered requests per second") + ax.set_ylabel("Waits for a pooled connection per second") + ax.set_title("Connection pool waits, all replicas") + ax.grid(alpha=0.3) + ax.legend() + fig.tight_layout() + fig.savefig(out / "pool-waits.png", dpi=120) + plt.close(fig) + print(f"{len(runs)} runs, {len(rows)} steps -> {out}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])) diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh new file mode 100755 index 00000000..1bab9a47 --- /dev/null +++ b/scripts/bench/rollout-under-load.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Level 4: a rolling update under a constant write load. The generator sends +# messages at RATE for DURATION, retrying uncertain sends with the same key; +# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards +# the generator reads the history back: every confirmed message once. +# +# sh scripts/bench/rollout-under-load.sh +# +# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default +# evidence/rollout). Results in /: timeline.json, seconds.csv, +# rollout.log, update.json (when it ran), compose.log, instances-*.txt. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +rate=${RATE:-50} +duration=${DURATION:-60s} +update_at=${UPDATE_AT:-15} +dir=${OUT:-evidence/rollout} +export MSYS_NO_PATHCONV=1 + +rm -rf "$dir" +mkdir -p "$dir" +docker compose up -d --wait >/dev/null 2>&1 +# Build first, as a delivery pipeline would: the update under load then +# measures the replacement of the replicas, not the compiler. +docker compose build api >/dev/null 2>&1 +meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" +sh scripts/instances.sh >"$dir/instances-before.txt" +since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ + -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & +generator=$! +sleep "$update_at" +started=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +status=0 +sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? +finished=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" + +printf '{\n "update_started_utc": "%s",\n "update_finished_utc": "%s",\n "rollout_exit_code": %s\n}\n' \ + "$started" "$finished" "$status" >"$dir/update.json" +docker compose logs --no-color --since "$since" lb api migrate >"$dir/compose.log" 2>&1 +sh scripts/instances.sh >"$dir/instances-after.txt" +cat "$dir/loadgen.log" | tail -n 1 +echo "results in $dir" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh new file mode 100755 index 00000000..df62c9e7 --- /dev/null +++ b/scripts/bench/run-series.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# Track 1, the cost of scaling: runs every configuration of one series file, +# each repeated, each on a fresh stand (empty volumes, same seed), under the +# same grid of offered rates. +# +# RATES=200,400,800 sh scripts/bench/run-series.sh scripts/bench/series/A.conf [repeats] +# +# Environment: RATES (required), STEP (30s), WARMUP (30s), OUT (results root, +# default evidence/track1/raw). Fixed for every configuration: HAProxy +# LB_CPUS (1.0), PostgreSQL PG_CPUS/PG_MEMORY (1.0/1G), Valkey 0.5 CPU and the +# generator LOADGEN_CPUS (1.5). Results per run: +# ///rep/{meta.json,steps.json,seconds.csv, +# metrics.jsonl,docker-stats.jsonl,postgres.jsonl,loadgen.log} +# The stand is removed at the end; `make up` brings the normal one back. +set -eu +cd "$(dirname "$0")/../.." +. scripts/bench/lib.sh + +series=${1:?usage: run-series.sh [repeats]} +repeats=${2:-3} +: "${RATES:?set RATES, the offered requests per second of the steps, e.g. 200,400,800}" +step=${STEP:-30s} +warmup=${WARMUP:-30s} +root=${OUT:-evidence/track1/raw}/$(basename "$series" .conf) + +export APP_DEBUG_ADDR=:9090 LB_QUIET=1 +export LB_CPUS=${LB_CPUS:-1.0} PG_CPUS=${PG_CPUS:-1.0} PG_MEMORY=${PG_MEMORY:-1G} LOADGEN_CPUS=${LOADGEN_CPUS:-1.5} +export MSYS_NO_PATHCONV=1 + +grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool; do + [ -n "$name" ] || continue + for rep in $(seq 1 "$repeats"); do + dir=$root/$name/rep$rep + rm -rf "$dir" + mkdir -p "$dir" + echo "== $name rep $rep: $replicas x $cpus CPU / $memory, pool $pool, rates $RATES" + export API_REPLICAS=$replicas API_CPUS=$cpus API_MEMORY=$memory API_GOMEMLIMIT=$gomemlimit DB_MAX_CONNS=$pool + + docker compose down -v --remove-orphans >/dev/null 2>&1 + docker compose up -d --build --wait >/dev/null 2>&1 + meta "$dir/meta.json" "series=$(basename "$series" .conf)" "config=$name" "repeat=$rep" \ + "api_replicas=$replicas" "api_cpus=$cpus" "api_memory=$memory" "gomemlimit=$gomemlimit" "db_max_conns=$pool" \ + "lb_cpus=$LB_CPUS" "pg_cpus=$PG_CPUS" "pg_memory=$PG_MEMORY" "loadgen_cpus=$LOADGEN_CPUS" \ + "rates=$RATES" "step=$step" "warmup=$warmup" + + sample_docker_stats "$dir/docker-stats.jsonl" & + stats=$! + sample_postgres "$dir/postgres.jsonl" & + pg=$! + BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 || echo " loadgen failed, see $dir/loadgen.log" + kill "$stats" "$pg" 2>/dev/null || true + wait "$stats" "$pg" 2>/dev/null || true + cat "$dir/loadgen.log" | grep '^step' || true + done +done +docker compose down -v --remove-orphans >/dev/null 2>&1 +echo "results in $root" diff --git a/scripts/bench/series/A.conf b/scripts/bench/series/A.conf new file mode 100644 index 00000000..1a305380 --- /dev/null +++ b/scripts/bench/series/A.conf @@ -0,0 +1,6 @@ +# Series A: the same API budget, 2 CPU and 1 GiB, cut into 1, 2 or 4 replicas. +# The pool stays 10 per replica, so the total grows with the replicas. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +A1 1 2.0 1024M 920MiB 10 +A2 2 1.0 512M 460MiB 10 +A4 4 0.5 256M 230MiB 10 diff --git a/scripts/bench/series/Aprime.conf b/scripts/bench/series/Aprime.conf new file mode 100644 index 00000000..d0952eff --- /dev/null +++ b/scripts/bench/series/Aprime.conf @@ -0,0 +1,5 @@ +# Series A': the budget of series A with the total pool fixed at 20 +# connections. A'2 equals A2 and is not run again. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +Ap1 1 2.0 1024M 920MiB 20 +Ap4 4 0.5 256M 230MiB 5 diff --git a/scripts/bench/series/B.conf b/scripts/bench/series/B.conf new file mode 100644 index 00000000..b882d44e --- /dev/null +++ b/scripts/bench/series/B.conf @@ -0,0 +1,7 @@ +# Series B: replicas of one fixed size (1 CPU, 512 MiB, pool 10) added one by +# one, so the API budget grows from 1 to 4 CPU. B2 equals A2 and is not run +# again. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +B1 1 1.0 512M 460MiB 10 +B3 3 1.0 512M 460MiB 10 +B4 4 1.0 512M 460MiB 10 diff --git a/scripts/bench/series/calibrate.conf b/scripts/bench/series/calibrate.conf new file mode 100644 index 00000000..0e3c976e --- /dev/null +++ b/scripts/bench/series/calibrate.conf @@ -0,0 +1,4 @@ +# Calibration: the smallest API budget of the study (one replica, one CPU) +# with a wide range of rates finds where the common rate grid must reach. +# name replicas cpus_per_replica memory_per_replica gomemlimit pool_per_replica +cal 1 1.0 512M 460MiB 10 diff --git a/scripts/restart.sh b/scripts/restart.sh index 11591f9c..29aeb757 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,6 +1,7 @@ #!/bin/sh # Restart hook of `make test-persistence` (tests/resilience.py): stops the -# API, PostgreSQL and Valkey gracefully and starts the same containers again. +# balancer, every API replica, PostgreSQL and Valkey gracefully and starts the +# same containers again (the one-shot migrate finds nothing to apply). # Volumes and data are kept and nothing is re-seeded. The test hides the # output, so failures surface through the exit code only. set -eu diff --git a/tools/concurrency/client.go b/tools/concurrency/client.go index 62bd2d5e..b56eb137 100644 --- a/tools/concurrency/client.go +++ b/tools/concurrency/client.go @@ -10,6 +10,8 @@ import ( "time" "github.com/google/uuid" + + "messenger/internal/chat" ) // client is a minimal typed HTTP client for the messenger API. @@ -50,6 +52,40 @@ func (c *client) postMessage(chatID, token, text string, out any) error { return c.do(req, token, http.StatusCreated, out) } +// sendWithKey sends one attempt of a message under the given key on a +// connection of its own and reports the status, the message (on 201) and the +// replica that answered. +func (c *client) sendWithKey(chatID, token, key, text string) (int, chat.MessageResponse, string, error) { + var msg chat.MessageResponse + raw, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return 0, msg, "", err + } + req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(raw)) + if err != nil { + return 0, msg, "", err + } + req.Close = true + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Idempotency-Key", key) + resp, err := c.http.Do(req) + if err != nil { + return 0, msg, "", err + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return 0, msg, "", err + } + if resp.StatusCode == http.StatusCreated { + if err := json.Unmarshal(payload, &msg); err != nil { + return 0, msg, "", err + } + } + return resp.StatusCode, msg, resp.Header.Get("X-Instance-Id"), nil +} + // login opens a session with Basic credentials and returns its bearer token. func (c *client) login(username, password string) (string, error) { req, err := http.NewRequest(http.MethodPost, c.base+"/api/v1/auth/sessions", nil) diff --git a/tools/concurrency/main.go b/tools/concurrency/main.go index eea5e108..7011c071 100644 --- a/tools/concurrency/main.go +++ b/tools/concurrency/main.go @@ -1,12 +1,22 @@ -// Command concurrency is the black-box race check from the grading rubric: -// N clients send M messages each into one chat at the same time, then the -// history is paged and must contain exactly N*M unique messages in id order. +// Command concurrency is the black-box race check from the grading rubric. +// +// Phase 1: N clients send M messages each into one chat at the same time, +// then the history is paged and must contain exactly N*M unique messages in +// id order. +// +// Phase 2, shared idempotency: in each of K rounds one user sends the same +// message with the same Idempotency-Key P times in parallel, each attempt on +// its own connection, so the balancer spreads them over the replicas. Every +// attempt must answer 201 with the same message, the attempts must have +// reached at least two replicas, the history must hold exactly K messages, +// and the key with another text must answer 409. package main import ( "encoding/json" "flag" "fmt" + "net/http" "os" "path/filepath" "sort" @@ -35,17 +45,37 @@ type Report struct { Ordered bool `json:"history_ordered_by_id"` Pages int `json:"pages_read"` ElapsedSeconds float64 `json:"elapsed_seconds"` - Passed bool `json:"passed"` + // Idempotency is phase 2. + Idempotency IdempotencyReport `json:"idempotency"` + Passed bool `json:"passed"` +} + +type IdempotencyReport struct { + Rounds int `json:"rounds"` + Parallel int `json:"parallel"` + // SameMessageRounds counts rounds whose attempts all answered 201 with + // one and the same message. + SameMessageRounds int `json:"same_message_rounds"` + // InstancesSeen is the number of distinct X-Instance-Id values among all + // attempts. + InstancesSeen int `json:"instances_seen"` + RequireReplicas bool `json:"require_two_replicas"` + Conflict409 bool `json:"different_body_409"` + History int `json:"history_messages"` + Passed bool `json:"passed"` } func main() { baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") clients := flag.Int("clients", 10, "number of concurrent clients") perClient := flag.Int("per-client", 10, "messages sent by each client") + rounds := flag.Int("rounds", 20, "idempotency rounds, one key each") + parallel := flag.Int("parallel", 8, "parallel attempts with the same key per round") + requireReplicas := flag.Bool("require-replicas", true, "fail unless the attempts reached at least two replicas") output := flag.String("output", "", "optional path for the JSON report") flag.Parse() - report, err := run(*baseURL, *clients, *perClient) + report, err := run(*baseURL, *clients, *perClient, *rounds, *parallel, *requireReplicas) if err != nil { fail(err) } @@ -61,6 +91,8 @@ func main() { } fmt.Printf("PASS: %d clients x %d messages produced exactly %d unique ordered messages\n", report.Clients, report.PerClient, report.Unique) + fmt.Printf("PASS: %d rounds x %d parallel attempts with one key produced %d messages across %d instances\n", + report.Idempotency.Rounds, report.Idempotency.Parallel, report.Idempotency.History, report.Idempotency.InstancesSeen) } func fail(err error) { @@ -75,9 +107,9 @@ func writeReport(path string, text []byte) error { return os.WriteFile(path, append(text, '\n'), 0o644) } -func run(baseURL string, clients, perClient int) (Report, error) { - if clients < 1 || perClient < 1 { - return Report{}, fmt.Errorf("clients and per-client must be positive") +func run(baseURL string, clients, perClient, rounds, parallel int, requireReplicas bool) (Report, error) { + if clients < 2 || perClient < 1 || rounds < 1 || parallel < 1 { + return Report{}, fmt.Errorf("clients must be at least 2; per-client, rounds and parallel positive") } report := Report{ BaseURL: baseURL, @@ -112,15 +144,82 @@ func run(baseURL string, clients, perClient int) (Report, error) { report.Listed = len(listed) report.Unique, report.Missing = compare(created, listed) report.Ordered = sort.StringsAreSorted(listed) + report.Idempotency, err = checkIdempotency(api, baseURL, runID, members[:2], tokens[0], rounds, parallel, requireReplicas) + if err != nil { + return report, err + } report.Passed = report.Failed == 0 && report.Created == report.Expected && report.Listed == report.Expected && report.Unique == report.Expected && report.Missing == 0 && - report.Ordered + report.Ordered && + report.Idempotency.Passed return report, nil } +// checkIdempotency runs phase 2 in a chat of its own. +func checkIdempotency(api *client, baseURL, runID string, members []string, token string, rounds, parallel int, requireReplicas bool) (IdempotencyReport, error) { + r := IdempotencyReport{Rounds: rounds, Parallel: parallel, RequireReplicas: requireReplicas} + room, err := createChat(api, runID+"-idem", members, token) + if err != nil { + return r, err + } + instances := map[string]bool{} + var key, text string + for round := range rounds { + key, text = uuid.NewString(), fmt.Sprintf("idempotency round %d", round) + type attempt struct { + status int + id string + instance string + } + attempts := make([]attempt, parallel) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range parallel { + wg.Go(func() { + own := newClient(baseURL) + <-start + status, msg, instance, err := own.sendWithKey(room.ID, token, key, text) + if err == nil { + attempts[i] = attempt{status, msg.ID, instance} + } + }) + } + close(start) + wg.Wait() + same := true + for _, a := range attempts { + if a.instance != "" { + instances[a.instance] = true + } + if a.status != http.StatusCreated || a.id == "" || a.id != attempts[0].id { + same = false + } + } + if same { + r.SameMessageRounds++ + } + } + r.InstancesSeen = len(instances) + status, _, _, err := newClient(baseURL).sendWithKey(room.ID, token, key, text+" changed") + if err != nil { + return r, err + } + r.Conflict409 = status == http.StatusConflict + listed, _, err := readHistory(api, room.ID, token) + if err != nil { + return r, err + } + r.History = len(listed) + r.Passed = r.SameMessageRounds == rounds && + r.History == rounds && + r.Conflict409 && + (!requireReplicas || r.InstancesSeen >= 2) + return r, nil +} + // clientPassword is shared by the synthetic clients of one run. const clientPassword = "concurrency-check-password" diff --git a/tools/concurrency/main_test.go b/tools/concurrency/main_test.go index 7c9158b4..f0e1b045 100644 --- a/tools/concurrency/main_test.go +++ b/tools/concurrency/main_test.go @@ -43,7 +43,7 @@ func TestRunAgainstLiveServer(t *testing.T) { srv := httptest.NewServer(a.Engine) defer srv.Close() - report, err := run(srv.URL, 10, 10) + report, err := run(srv.URL, 10, 10, 5, 4, false) if err != nil { t.Fatalf("run error: %v", err) } @@ -56,10 +56,34 @@ func TestRunAgainstLiveServer(t *testing.T) { if !report.Ordered || !report.Passed { t.Fatalf("report = %+v", report) } + idem := report.Idempotency + if idem.Rounds != 5 || idem.Parallel != 4 || idem.SameMessageRounds != 5 || idem.History != 5 || !idem.Conflict409 || !idem.Passed { + t.Fatalf("idempotency = %+v", idem) + } + if idem.InstancesSeen != 1 { + t.Fatalf("one in-process server must be seen as one instance: %+v", idem) + } +} + +// Through the balancer the same key must reach at least two replicas; one +// replica alone does not prove the idempotency is shared. +func TestIdempotencyRequiresTwoReplicasByDefault(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := app.New(app.Deps{Log: log, DB: db.Pool, Sessions: kv.Client, SessionTTL: time.Hour, MaxBodyBytes: 64 * 1024, InstanceID: "concurrency-test"}) + srv := httptest.NewServer(a.Engine) + defer srv.Close() + + report, err := run(srv.URL, 2, 1, 2, 2, true) + if err != nil { + t.Fatalf("run error: %v", err) + } + if report.Idempotency.Passed || report.Passed { + t.Fatalf("a single instance passed the shared idempotency check: %+v", report.Idempotency) + } } func TestRunFailsOnUnreachableServer(t *testing.T) { - if _, err := run("http://127.0.0.1:1", 2, 2); err == nil { + if _, err := run("http://127.0.0.1:1", 2, 2, 1, 1, false); err == nil { t.Fatal("expected an error for an unreachable server") } } diff --git a/tools/loadgen/client.go b/tools/loadgen/client.go new file mode 100644 index 00000000..fce2c380 --- /dev/null +++ b/tools/loadgen/client.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// result is one HTTP exchange; instance is the X-Instance-Id of the answer, +// empty when the balancer answered itself or nothing answered. +type result struct { + status int + instance string + body []byte + latency time.Duration + err error +} + +type client struct { + base string + http *http.Client + timeout time.Duration +} + +// newClient keeps many connections open to the balancer, as many concurrent +// users would; HAProxy balances every request, not every connection. +func newClient(base string, timeout time.Duration) *client { + transport := &http.Transport{ + MaxIdleConns: 10000, + MaxIdleConnsPerHost: 10000, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + } + return &client{base: base, http: &http.Client{Transport: transport}, timeout: timeout} +} + +func (c *client) do(ctx context.Context, method, path, token, key string, body []byte, basic string) result { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader) + if err != nil { + return result{err: err} + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + switch { + case basic != "": + req.Header.Set("Authorization", "Basic "+basic) + case token != "": + req.Header.Set("Authorization", "Bearer "+token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + start := time.Now() + resp, err := c.http.Do(req) + if err != nil { + return result{err: err, latency: time.Since(start)} + } + defer resp.Body.Close() + payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + r := result{status: resp.StatusCode, instance: resp.Header.Get("X-Instance-Id"), body: payload, latency: time.Since(start), err: err} + return r +} + +func (c *client) postJSON(ctx context.Context, path, token, key string, body any) result { + raw, err := json.Marshal(body) + if err != nil { + return result{err: err} + } + return c.do(ctx, http.MethodPost, path, token, key, raw, "") +} + +// expect decodes the body of a result with the wanted status. +func expect(r result, want int, out any) error { + if r.err != nil { + return r.err + } + if r.status != want { + return fmt.Errorf("status %d, want %d: %s", r.status, want, r.body) + } + if out == nil { + return nil + } + return json.Unmarshal(r.body, out) +} + +func (c *client) register(ctx context.Context, username, password string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]string{"username": username, "display_name": username, "password": password} + err := expect(c.postJSON(ctx, "/api/v1/users", "", "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) login(ctx context.Context, username, password string) (string, error) { + var session struct { + Token string `json:"token"` + } + basic := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + err := expect(c.do(ctx, http.MethodPost, "/api/v1/auth/sessions", "", "", nil, basic), http.StatusCreated, &session) + return session.Token, err +} + +func (c *client) createChat(ctx context.Context, token, title string, members []string) (string, error) { + var created struct { + ID string `json:"id"` + } + body := map[string]any{"title": title, "member_ids": members} + err := expect(c.postJSON(ctx, "/api/v1/chats", token, "", body), http.StatusCreated, &created) + return created.ID, err +} + +func (c *client) send(ctx context.Context, chatID, token, key, text string) result { + return c.postJSON(ctx, "/api/v1/chats/"+chatID+"/messages", token, key, map[string]string{"text": text}) +} + +// firstPage reads the oldest page of a history: the read of the profile. +func (c *client) firstPage(ctx context.Context, chatID, token string) result { + return c.do(ctx, http.MethodGet, "/api/v1/chats/"+chatID+"/messages?limit=50", token, "", nil, "") +} + +type message struct { + ID string `json:"id"` + Text string `json:"text"` +} + +// history reads a whole chat, page by page. +func (c *client) history(ctx context.Context, chatID, token string) ([]message, error) { + var all []message + path := "/api/v1/chats/" + chatID + "/messages?limit=100" + for range 100000 { + var page struct { + Items []message `json:"items"` + NextCursor *string `json:"next_cursor"` + } + if err := expect(c.do(ctx, http.MethodGet, path, token, "", nil, ""), http.StatusOK, &page); err != nil { + return nil, err + } + all = append(all, page.Items...) + if page.NextCursor == nil { + return all, nil + } + path = "/api/v1/chats/" + chatID + "/messages?limit=100&cursor=" + url.QueryEscape(*page.NextCursor) + } + return nil, fmt.Errorf("history of %s did not end", chatID) +} diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go new file mode 100644 index 00000000..fabeddd7 --- /dev/null +++ b/tools/loadgen/main.go @@ -0,0 +1,352 @@ +// Command loadgen is the open-loop load generator of the scaling study +// (track 1) and of the failover and rolling-update experiments. +// +// loadgen bench — seeds users, chats and messages, warms up, then offers +// each rate of -rates for -step: a mix of history reads +// (-read-ratio) and message sends. Writes steps.json and +// seconds.csv, and metrics.jsonl with -scrape. +// loadgen timeline — one user sends messages into one chat at -rate for +// -duration; a send with an uncertain outcome (no answer, +// 5xx) is retried with the same Idempotency-Key. At the +// end the whole history is read back: every confirmed +// message must be there, and none twice. Writes +// timeline.json and seconds.csv. +// +// Requests are sent on schedule whatever the response times (open loop), and +// the generator records how late it sent them: a step it could not keep up +// with is marked invalid rather than reported as the system's limit. Tokens +// stay in memory and are never written out. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + mrand "math/rand/v2" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" +) + +type config struct { + Mode string `json:"mode"` + BaseURL string `json:"base_url"` + Users int `json:"users"` + Chats int `json:"chats"` + SeedMessages int `json:"seed_messages_per_chat"` + ReadRatio float64 `json:"read_ratio"` + Rates []float64 `json:"rates"` + Step string `json:"step"` + Warmup string `json:"warmup"` + Rate float64 `json:"rate"` + Duration string `json:"duration"` + Timeout string `json:"client_timeout"` + Seed uint64 `json:"seed"` + Scrape bool `json:"scrape"` + StatsURL string `json:"stats_url"` + APIHost string `json:"api_host"` + DebugPort string `json:"debug_port"` + Out string `json:"-"` + + step, warmup, duration, timeout time.Duration +} + +func main() { + if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") { + fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags") + os.Exit(2) + } + cfg, err := parse(os.Args[1], os.Args[2:]) + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if cfg.Mode == "bench" { + err = runBench(ctx, cfg) + } else { + err = runTimeline(ctx, cfg) + } + if err != nil { + fmt.Fprintln(os.Stderr, "loadgen:", err) + os.Exit(1) + } +} + +func parse(mode string, args []string) (config, error) { + cfg := config{Mode: mode} + fs := flag.NewFlagSet(mode, flag.ContinueOnError) + rates := fs.String("rates", "200,400", "bench: offered requests per second of each step") + fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API") + fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users") + fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members") + fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring") + fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends") + fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step") + fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate") + fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second") + fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") + fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") + fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") + fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") + fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") + fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") + fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)") + fs.StringVar(&cfg.Out, "out", "/out", "directory for the results") + if err := fs.Parse(args); err != nil { + return cfg, err + } + for _, part := range strings.Split(*rates, ",") { + r, err := strconv.ParseFloat(strings.TrimSpace(part), 64) + if err != nil || r <= 0 { + return cfg, fmt.Errorf("rates: %q is not a positive number", part) + } + cfg.Rates = append(cfg.Rates, r) + } + for _, d := range []struct { + name string + raw string + dst *time.Duration + }{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} { + v, err := time.ParseDuration(d.raw) + if err != nil || v < 0 { + return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw) + } + *d.dst = v + } + if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 { + return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive") + } + return cfg, os.MkdirAll(cfg.Out, 0o755) +} + +func writeJSON(path string, v any) error { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0o644) +} + +// startScrape samples the system in the background when asked to; the +// returned function stops it and waits. +func startScrape(ctx context.Context, cfg config, start time.Time) func() { + if !cfg.Scrape { + return func() {} + } + ctx, cancel := context.WithCancel(ctx) + s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}} + var wg sync.WaitGroup + wg.Go(func() { + if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil { + fmt.Fprintln(os.Stderr, "loadgen: scrape:", err) + } + }) + return func() { cancel(); wg.Wait() } +} + +type benchReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Users int `json:"users"` + Chats int `json:"chats"` + Seeded int `json:"seeded_messages"` + Steps []stepSummary `json:"steps"` + WritesOK int `json:"messages_written_during_steps"` +} + +func runBench(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164)) + ds, err := seed(ctx, c, rng, cfg.Users, cfg.Chats, cfg.SeedMessages) + if err != nil { + return err + } + report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages} + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + + // next picks the request on the scheduling goroutine, where the random + // source may be used safely. + counter := 0 + next := func(rec *recorder) func(time.Time) func() { + return func(planned time.Time) func() { + r := ds.rooms[rng.IntN(len(ds.rooms))] + a := ds.actors[r.members[rng.IntN(len(r.members))]] + write := rng.Float64() >= cfg.ReadRatio + counter++ + text := fmt.Sprintf("load %d", counter) + return func() { + sent := time.Now() + var res result + if write { + res = c.send(ctx, r.id, a.token, uuid.NewString(), text) + } else { + res = c.firstPage(ctx, r.id, a.token) + } + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write}) + } + } + } + if cfg.warmup > 0 { + openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{})) + } + all := &recorder{} + for _, rate := range cfg.Rates { + if ctx.Err() != nil { + break + } + stepStart := time.Now() + rec := &recorder{} + openLoop(ctx, rate, cfg.step, next(rec)) + s := summarize(rate, cfg.step, rec.all()) + s.StartedAt = stepStart.Sub(start).Seconds() + report.Steps = append(report.Steps, s) + report.WritesOK += s.Writes + for _, x := range rec.all() { + all.add(x) + } + fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v\n", + rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid) + } + stopScrape() + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { + return err + } + return writeJSON(filepath.Join(cfg.Out, "steps.json"), report) +} + +type timelineReport struct { + StartedAt time.Time `json:"started_at"` + Config config `json:"config"` + Jobs int `json:"messages_attempted"` + Attempts int `json:"requests"` + Confirmed int `json:"confirmed"` + RetriedJobs int `json:"messages_retried"` + Unresolved int `json:"unresolved"` + Rejected int `json:"rejected_4xx"` + Classes map[string]int `json:"classes"` + Instances map[string]int `json:"instances"` + History int `json:"history_messages"` + Duplicates int `json:"duplicates"` + Missing int `json:"missing_confirmed"` + Passed bool `json:"passed"` +} + +// retryDelay spaces the attempts of one message whose outcome was uncertain. +const retryDelay = 200 * time.Millisecond + +// retryGrace is how long after the schedule ends an uncertain message keeps +// being retried before it counts as unresolved. +const retryGrace = 10 * time.Second + +func runTimeline(ctx context.Context, cfg config) error { + c := newClient(cfg.BaseURL, cfg.timeout) + rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65)) + ds, err := seed(ctx, c, rng, 2, 1, 0) + if err != nil { + return err + } + r := ds.rooms[0] + sender := ds.actors[r.members[0]] + report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}} + runID := randomHex(3) + start := time.Now() + stopScrape := startScrape(ctx, cfg, start) + defer stopScrape() + deadline := start.Add(cfg.duration + retryGrace) + + var mu sync.Mutex + confirmed := map[string]string{} // text -> message id + rec := &recorder{} + jobs := 0 + openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() { + jobs++ + key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs) + return func() { + for attempt := 0; ; attempt++ { + sent := time.Now() + res := c.send(ctx, r.id, sender.token, key, text) + cls := classify(res) + rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true}) + mu.Lock() + switch { + case cls == "ok": + var m message + if json.Unmarshal(res.body, &m) == nil { + confirmed[text] = m.ID + } + case res.err == nil && res.status >= 400 && res.status < 500: + report.Rejected++ + case time.Now().After(deadline) || ctx.Err() != nil: + report.Unresolved++ + default: + if attempt == 0 { + report.RetriedJobs++ + } + mu.Unlock() + time.Sleep(retryDelay) + planned = time.Now() + continue + } + mu.Unlock() + return + } + } + }) + stopScrape() + samples := rec.all() + report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed) + for _, x := range samples { + report.Classes[x.class]++ + if x.instance != "" { + report.Instances[x.instance]++ + } + } + + history, err := c.history(ctx, r.id, sender.token) + if err != nil { + return fmt.Errorf("read history back: %w", err) + } + report.History = len(history) + seen := map[string]int{} + byID := map[string]string{} + for _, m := range history { + seen[m.Text]++ + byID[m.ID] = m.Text + } + for _, n := range seen { + if n > 1 { + report.Duplicates += n - 1 + } + } + for text, id := range confirmed { + if byID[id] != text { + report.Missing++ + } + } + report.Passed = report.Duplicates == 0 && report.Missing == 0 + if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil { + return err + } + if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil { + return err + } + fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n", + report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing) + if !report.Passed { + return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing) + } + return nil +} diff --git a/tools/loadgen/main_test.go b/tools/loadgen/main_test.go new file mode 100644 index 00000000..78dba77c --- /dev/null +++ b/tools/loadgen/main_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "messenger/internal/app" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +func liveServer(t *testing.T) string { + t.Helper() + a := app.New(app.Deps{ + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + DB: db.Pool, + Sessions: kv.Client, + SessionTTL: time.Hour, + MaxBodyBytes: 64 * 1024, + InstanceID: "loadgen-test", + }) + srv := httptest.NewServer(a.Engine) + t.Cleanup(srv.Close) + return srv.URL +} + +func TestBenchAgainstLiveServer(t *testing.T) { + out := t.TempDir() + cfg, err := parse("bench", []string{"-base-url", liveServer(t), "-users", "4", "-chats", "2", "-seed-messages", "3", + "-rates", "20,40", "-step", "1s", "-warmup", "0s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runBench(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report benchReport + raw, err := os.ReadFile(filepath.Join(out, "steps.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("steps.json: %v %s", err, raw) + } + if len(report.Steps) != 2 || report.Seeded != 6 || report.Users != 4 || report.Chats != 2 { + t.Fatalf("report = %+v", report) + } + for _, s := range report.Steps { + if s.Sent != int(s.OfferedRPS) || s.OK != s.Sent || s.ErrorShare != 0 || s.Instances["loadgen-test"] != s.Sent { + t.Fatalf("step = %+v", s) + } + } + csv, err := os.ReadFile(filepath.Join(out, "seconds.csv")) + if err != nil || !strings.HasPrefix(string(csv), "second,sent,ok,") || strings.Count(string(csv), "\n") < 3 { + t.Fatalf("seconds.csv: %v\n%s", err, csv) + } +} + +func TestTimelineVerifiesHistory(t *testing.T) { + out := t.TempDir() + cfg, err := parse("timeline", []string{"-base-url", liveServer(t), "-rate", "20", "-duration", "1s", "-out", out}) + if err != nil { + t.Fatal(err) + } + if err := runTimeline(context.Background(), cfg); err != nil { + t.Fatal(err) + } + var report timelineReport + raw, err := os.ReadFile(filepath.Join(out, "timeline.json")) + if err != nil || json.Unmarshal(raw, &report) != nil { + t.Fatalf("timeline.json: %v %s", err, raw) + } + if report.Jobs != 20 || report.Confirmed != 20 || report.History != 20 || report.Duplicates != 0 || report.Missing != 0 || !report.Passed { + t.Fatalf("report = %+v", report) + } +} diff --git a/tools/loadgen/run.go b/tools/loadgen/run.go new file mode 100644 index 00000000..21019fd3 --- /dev/null +++ b/tools/loadgen/run.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "sync" + "time" +) + +// openLoop offers requests at a constant rate for d, whatever the response +// times: request i is planned at start + i/rate and sent then, or at once if +// the generator is already late. Waiting for responses before sending more +// (a closed loop) would let a slow system lower its own load and hide its +// latency: coordinated omission. next runs on the scheduling goroutine, so it +// may use non-thread-safe state such as the random source; the job it +// returns runs on a goroutine of its own. openLoop returns once every job +// has finished. +func openLoop(ctx context.Context, rate float64, d time.Duration, next func(planned time.Time) func()) { + n := int(rate * d.Seconds()) + interval := time.Duration(float64(time.Second) / rate) + start := time.Now() + var wg sync.WaitGroup + for i := range n { + planned := start.Add(time.Duration(i) * interval) + if wait := time.Until(planned); wait > 0 { + select { + case <-ctx.Done(): + case <-time.After(wait): + } + } + if ctx.Err() != nil { + break + } + wg.Go(next(planned)) + } + wg.Wait() +} + +// recorder collects samples from concurrent jobs. +type recorder struct { + mu sync.Mutex + samples []sample +} + +func (r *recorder) add(s sample) { + r.mu.Lock() + r.samples = append(r.samples, s) + r.mu.Unlock() +} + +func (r *recorder) all() []sample { + r.mu.Lock() + defer r.mu.Unlock() + return append([]sample(nil), r.samples...) +} diff --git a/tools/loadgen/scrape.go b/tools/loadgen/scrape.go new file mode 100644 index 00000000..69790f23 --- /dev/null +++ b/tools/loadgen/scrape.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "encoding/csv" + "encoding/json" + "io" + "net" + "net/http" + "os" + "strings" + "time" +) + +// scraper samples, once a second, what the balancer and every replica report +// about themselves, so a saturated step can be explained: the HAProxy +// statistics per replica and the expvar of each API process (pool waits, +// GOMAXPROCS, memory and GC). Replicas are found the way HAProxy finds them, +// through Docker DNS, and asked directly on their internal debug port. +type scraper struct { + statsURL string + apiHost string + debugPort string + http *http.Client +} + +type scrapeLine struct { + T float64 `json:"t"` + HAProxy []map[string]string `json:"haproxy,omitempty"` + Replicas []json.RawMessage `json:"replicas,omitempty"` +} + +// haproxyFields are the columns of the statistics CSV that are kept. +var haproxyFields = []string{"svname", "status", "scur", "qcur", "stot", "rate", "rtime", "ttime", "hrsp_5xx", "chkfail", "econ", "eresp", "wretr", "wredis"} + +func (s scraper) run(ctx context.Context, start time.Time, path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + line := scrapeLine{T: time.Since(start).Seconds(), HAProxy: s.haproxy(ctx), Replicas: s.replicas(ctx)} + if err := enc.Encode(line); err != nil { + return err + } + } +} + +func (s scraper) get(ctx context.Context, url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := s.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) +} + +// haproxy keeps the rows of the replicas backend. +func (s scraper) haproxy(ctx context.Context) []map[string]string { + raw, err := s.get(ctx, s.statsURL) + if err != nil { + return nil + } + rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(string(raw), "# "))).ReadAll() + if err != nil || len(rows) < 2 { + return nil + } + header := rows[0] + col := map[string]int{} + for i, name := range header { + col[name] = i + } + var out []map[string]string + for _, row := range rows[1:] { + if len(row) != len(header) || row[col["pxname"]] != "replicas" || strings.HasPrefix(row[col["status"]], "MAINT") { + continue + } + kept := map[string]string{} + for _, name := range haproxyFields { + if i, ok := col[name]; ok { + kept[name] = row[i] + } + } + out = append(out, kept) + } + return out +} + +// replicas returns the expvar documents of the running replicas, trimmed to +// the fields the study uses. +func (s scraper) replicas(ctx context.Context) []json.RawMessage { + addrs, err := net.DefaultResolver.LookupHost(ctx, s.apiHost) + if err != nil { + return nil + } + var out []json.RawMessage + for _, addr := range addrs { + raw, err := s.get(ctx, "http://"+net.JoinHostPort(addr, s.debugPort)+"/debug/vars") + if err != nil { + continue + } + var vars struct { + Instance string `json:"instance"` + GOMAXPROCS int `json:"gomaxprocs"` + Pool json.RawMessage `json:"pgxpool"` + Mem struct { + HeapAlloc uint64 `json:"HeapAlloc"` + Sys uint64 `json:"Sys"` + NumGC uint32 `json:"NumGC"` + PauseTotalNs uint64 `json:"PauseTotalNs"` + GCCPUFraction float64 `json:"GCCPUFraction"` + } `json:"memstats"` + } + if json.Unmarshal(raw, &vars) != nil { + continue + } + kept, err := json.Marshal(map[string]any{ + "addr": addr, "instance": vars.Instance, "gomaxprocs": vars.GOMAXPROCS, + "pgxpool": vars.Pool, "heap_alloc": vars.Mem.HeapAlloc, "sys": vars.Mem.Sys, + "num_gc": vars.Mem.NumGC, "gc_pause_total_ns": vars.Mem.PauseTotalNs, "gc_cpu_fraction": vars.Mem.GCCPUFraction, + }) + if err == nil { + out = append(out, kept) + } + } + return out +} diff --git a/tools/loadgen/seed.go b/tools/loadgen/seed.go new file mode 100644 index 00000000..748b0c18 --- /dev/null +++ b/tools/loadgen/seed.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + mrand "math/rand/v2" + "sync" + + "github.com/google/uuid" +) + +// seedPassword is shared by the synthetic users of one run. +const seedPassword = "loadgen-synthetic-password" + +// seedWorkers bounds the parallel registrations: each one hashes a password +// with Argon2id, and the setup must not be what the study measures. +const seedWorkers = 8 + +type actor struct { + id, token string +} + +type room struct { + id string + members []int // indexes into dataset.actors +} + +type dataset struct { + actors []actor + rooms []room + // messages is the number of messages written while seeding. + messages int +} + +// seed registers users and logs each in once, creates chats of two to five +// random members and writes perChat messages into each. Logins happen here, +// so the password hashing never shows up in the measured steps. +func seed(ctx context.Context, c *client, rng *mrand.Rand, users, chats, perChat int) (dataset, error) { + if users < 2 || chats < 1 { + return dataset{}, fmt.Errorf("need at least 2 users and 1 chat") + } + runID := randomHex(3) + ds := dataset{actors: make([]actor, users)} + if err := parallel(users, func(i int) error { + name := fmt.Sprintf("lg_%s_%d", runID, i) + id, err := c.register(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("register %s: %w", name, err) + } + token, err := c.login(ctx, name, seedPassword) + if err != nil { + return fmt.Errorf("log in %s: %w", name, err) + } + ds.actors[i] = actor{id: id, token: token} + return nil + }); err != nil { + return dataset{}, err + } + + for j := range chats { + size := 2 + rng.IntN(4) + members := rng.Perm(users)[:min(size, users)] + ids := make([]string, len(members)) + for k, m := range members { + ids[k] = ds.actors[m].id + } + id, err := c.createChat(ctx, ds.actors[members[0]].token, fmt.Sprintf("load %s %d", runID, j), ids) + if err != nil { + return dataset{}, fmt.Errorf("create chat %d: %w", j, err) + } + ds.rooms = append(ds.rooms, room{id: id, members: members}) + } + + if err := parallel(len(ds.rooms), func(j int) error { + r := ds.rooms[j] + for k := range perChat { + sender := ds.actors[r.members[k%len(r.members)]] + res := c.send(ctx, r.id, sender.token, uuid.NewString(), fmt.Sprintf("seed %d", k)) + if err := expect(res, 201, nil); err != nil { + return fmt.Errorf("seed message into chat %d: %w", j, err) + } + } + return nil + }); err != nil { + return dataset{}, err + } + ds.messages = len(ds.rooms) * perChat + return ds, nil +} + +// parallel runs fn(0..n-1) on seedWorkers goroutines and returns the first +// error. +func parallel(n int, fn func(i int) error) error { + var ( + wg sync.WaitGroup + mu sync.Mutex + first error + ) + next := make(chan int) + for range seedWorkers { + wg.Go(func() { + for i := range next { + if err := fn(i); err != nil { + mu.Lock() + if first == nil { + first = err + } + mu.Unlock() + } + } + }) + } + for i := range n { + next <- i + } + close(next) + wg.Wait() + return first +} + +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go new file mode 100644 index 00000000..f74336e9 --- /dev/null +++ b/tools/loadgen/stats.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "errors" + "fmt" + "math" + "net" + "slices" + "sort" + "strings" + "time" +) + +// maxValidLag is the generator's own tolerance: if requests left more than +// this late at the 99th percentile, the generator, not the system, limited +// the step, and the step is excluded from the conclusions. +const maxValidLag = 10 * time.Millisecond + +// sample is one request as the generator saw it. +type sample struct { + planned time.Time + lag time.Duration // actual send time minus planned send time + latency time.Duration + class string + instance string + write bool +} + +// classify names the outcome of a request. HTTP errors produced by the +// balancer itself carry no X-Instance-Id and are kept apart from the API's +// own: a 503 from HAProxy means no replica was available, a 503 from the API +// means a dependency was. client_timeout is the generator giving up on a +// response, not a timeout of the system. +func classify(r result) string { + switch { + case r.err != nil && isTimeout(r.err): + return "client_timeout" + case r.err != nil: + return "conn_error" + case r.status >= 200 && r.status < 300: + return "ok" + case r.instance == "": + return fmt.Sprintf("http_%d_lb", r.status) + default: + return fmt.Sprintf("http_%d", r.status) + } +} + +func isTimeout(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// percentile is the nearest-rank percentile of an ascending slice. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + rank := int(math.Ceil(p / 100 * float64(len(sorted)))) + return sorted[min(max(rank, 1), len(sorted))-1] +} + +type stepSummary struct { + OfferedRPS float64 `json:"offered_rps"` + Seconds float64 `json:"seconds"` + StartedAt float64 `json:"started_at_s"` + Sent int `json:"sent"` + OK int `json:"ok"` + OKRPS float64 `json:"ok_rps"` + ErrorShare float64 `json:"error_share"` + Classes map[string]int `json:"classes"` + P50Ms float64 `json:"ok_p50_ms"` + P95Ms float64 `json:"ok_p95_ms"` + P99Ms float64 `json:"ok_p99_ms"` + LagP99Ms float64 `json:"lag_p99_ms"` + Valid bool `json:"valid"` + Instances map[string]int `json:"instances"` + Writes int `json:"writes_ok"` +} + +// summarize reduces one step. Latency percentiles cover successful requests +// only, as the course's report format asks; the errors are counted by class. +func summarize(rate float64, d time.Duration, samples []sample) stepSummary { + s := stepSummary{ + OfferedRPS: rate, + Seconds: d.Seconds(), + Sent: len(samples), + Classes: map[string]int{}, + Instances: map[string]int{}, + } + var ok, lags []time.Duration + for _, x := range samples { + s.Classes[x.class]++ + lags = append(lags, x.lag) + if x.instance != "" { + s.Instances[x.instance]++ + } + if x.class == "ok" { + ok = append(ok, x.latency) + if x.write { + s.Writes++ + } + } + } + slices.Sort(ok) + slices.Sort(lags) + s.OK = len(ok) + s.OKRPS = float64(s.OK) / d.Seconds() + if s.Sent > 0 { + s.ErrorShare = float64(s.Sent-s.OK) / float64(s.Sent) + } + s.P50Ms, s.P95Ms, s.P99Ms = ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)) + lagP99 := percentile(lags, 99) + s.LagP99Ms = ms(lagP99) + s.Valid = lagP99 <= maxValidLag + return s +} + +func ms(d time.Duration) float64 { + return float64(d.Microseconds()) / 1000 +} + +// errorColumns are the fixed error columns of the per-second CSV; any other +// class lands in err_other. +var errorColumns = []string{"client_timeout", "conn_error"} + +// secondsCSV renders the per-second timeline, attributed by planned send +// time so it lines up with the moment a fault was injected. +func secondsCSV(start time.Time, samples []sample) string { + type bucket struct { + samples []sample + } + buckets := map[int]*bucket{} + last := 0 + for _, x := range samples { + sec := int(x.planned.Sub(start) / time.Second) + if buckets[sec] == nil { + buckets[sec] = &bucket{} + } + buckets[sec].samples = append(buckets[sec].samples, x) + last = max(last, sec) + } + var b strings.Builder + b.WriteString("second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances\n") + for sec := 0; sec <= last; sec++ { + bk := buckets[sec] + if bk == nil { + fmt.Fprintf(&b, "%d,0,0,0,0,0,0,0,0,0,0,0,0,\n", sec) + continue + } + var ok, lags []time.Duration + counts := map[string]int{} + instances := map[string]int{} + for _, x := range bk.samples { + lags = append(lags, x.lag) + if x.instance != "" { + instances[x.instance]++ + } + switch { + case x.class == "ok": + ok = append(ok, x.latency) + case slices.Contains(errorColumns, x.class): + counts[x.class]++ + case strings.HasPrefix(x.class, "http_5") && strings.HasSuffix(x.class, "_lb"): + counts["http_5xx_lb"]++ + case strings.HasPrefix(x.class, "http_5"): + counts["http_5xx_api"]++ + case strings.HasPrefix(x.class, "http_4"): + counts["http_4xx"]++ + default: + counts["err_other"]++ + } + } + slices.Sort(ok) + slices.Sort(lags) + names := make([]string, 0, len(instances)) + for name := range instances { + names = append(names, name) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, name := range names { + parts = append(parts, fmt.Sprintf("%s=%d", name, instances[name])) + } + fmt.Fprintf(&b, "%d,%d,%d,%d,%d,%d,%d,%d,%d,%.1f,%.1f,%.1f,%.1f,%s\n", + sec, len(bk.samples), len(ok), + counts["client_timeout"], counts["conn_error"], counts["http_5xx_api"], counts["http_5xx_lb"], counts["http_4xx"], counts["err_other"], + ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)), ms(percentile(lags, 99)), + strings.Join(parts, "|")) + } + return b.String() +} diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go new file mode 100644 index 00000000..d9b32e1d --- /dev/null +++ b/tools/loadgen/stats_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "sync/atomic" + "testing" + "time" +) + +type timeoutErr struct{} + +func (timeoutErr) Error() string { return "i/o timeout" } +func (timeoutErr) Timeout() bool { return true } +func (timeoutErr) Temporary() bool { return true } + +var _ net.Error = timeoutErr{} + +func TestClassify(t *testing.T) { + for _, c := range []struct { + res result + want string + }{ + {result{status: 201, instance: "a"}, "ok"}, + {result{status: 200, instance: "a"}, "ok"}, + {result{status: 409, instance: "a"}, "http_409"}, + {result{status: 503, instance: "a"}, "http_503"}, + {result{status: 503}, "http_503_lb"}, + {result{status: 502}, "http_502_lb"}, + {result{err: context.DeadlineExceeded}, "client_timeout"}, + {result{err: fmt.Errorf("post: %w", timeoutErr{})}, "client_timeout"}, + {result{err: errors.New("connection refused")}, "conn_error"}, + } { + if got := classify(c.res); got != c.want { + t.Errorf("classify(%+v) = %q, want %q", c.res, got, c.want) + } + } +} + +func TestPercentileNearestRank(t *testing.T) { + ms := func(v ...int) []time.Duration { + out := make([]time.Duration, len(v)) + for i, x := range v { + out[i] = time.Duration(x) * time.Millisecond + } + return out + } + sorted := ms(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + for p, want := range map[float64]time.Duration{50: 5 * time.Millisecond, 95: 10 * time.Millisecond, 99: 10 * time.Millisecond, 10: time.Millisecond} { + if got := percentile(sorted, p); got != want { + t.Errorf("p%v = %v, want %v", p, got, want) + } + } + if got := percentile(nil, 99); got != 0 { + t.Errorf("empty p99 = %v, want 0", got) + } +} + +// A step is invalid when the generator itself fell behind its schedule: the +// system was then offered less than the nominal rate. +func TestStepSummaryValidityAndErrorShare(t *testing.T) { + samples := []sample{ + {class: "ok", latency: 10 * time.Millisecond, lag: time.Millisecond}, + {class: "ok", latency: 20 * time.Millisecond, lag: time.Millisecond}, + {class: "client_timeout", latency: 2 * time.Second, lag: time.Millisecond}, + {class: "http_503_lb", latency: time.Millisecond, lag: time.Millisecond}, + } + s := summarize(4, time.Second, samples) + if s.Sent != 4 || s.OK != 2 || s.OKRPS != 2 || s.ErrorShare != 0.5 || !s.Valid { + t.Fatalf("summary = %+v", s) + } + if s.Classes["client_timeout"] != 1 || s.Classes["http_503_lb"] != 1 { + t.Fatalf("classes = %v", s.Classes) + } + if s.P99Ms != 20 { + t.Fatalf("p99 of successful requests = %v ms, want 20", s.P99Ms) + } + samples[0].lag = 50 * time.Millisecond + samples[1].lag = 50 * time.Millisecond + if s := summarize(4, time.Second, samples); s.Valid { + t.Fatalf("lagging generator marked valid: %+v", s) + } +} + +// Open loop: requests leave on schedule even when responses are slow. A +// closed loop with one worker would send only a handful here. +func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { + var fired atomic.Int32 + start := time.Now() + openLoop(context.Background(), 50, time.Second, func(time.Time) func() { + return func() { + fired.Add(1) + time.Sleep(200 * time.Millisecond) + } + }) + if n := fired.Load(); n < 45 || n > 50 { + t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n) + } + if took := time.Since(start); took > 1500*time.Millisecond { + t.Fatalf("open loop took %v, want about 1.2 s", took) + } +}