From c9d0f732212e4011134a1d8eeea614c73d153c1a Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 2 Nov 2026 18:23:26 +0300 Subject: [PATCH] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E:=20=D0=B7=D0=B0=D1=89?= =?UTF-8?q?=D0=B8=D1=82=D0=B0=20rollout=20=D0=BE=D1=82=20=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=BD=D1=8B?= =?UTF-8?q?=D1=85=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA,=20=D1=87=D1=82?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D1=81=D0=B5=D1=80=D0=B8=D0=B9,=20?= =?UTF-8?q?=D0=B2=D1=80=D0=B5=D0=BC=D1=8F=20=D1=8D=D0=BA=D1=81=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=BE=D0=B2=20=D0=BF=D0=BE?= =?UTF-8?q?=20=D1=87=D0=B0=D1=81=D0=B0=D0=BC=20Docker,=20=D0=B2=D1=8B?= =?UTF-8?q?=D1=85=D0=BE=D0=B4=20=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80=D0=B0?= =?UTF-8?q?=20=D0=BF=D1=80=D0=B8=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D0=B5=20?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D1=80=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/server/main.go | 19 +++++--- cmd/server/main_test.go | 72 +++++++++++++++++++++++++++++ internal/auth/service_test.go | 18 ++++++-- scripts/bench/failover.sh | 23 ++++++--- scripts/bench/rollout-under-load.sh | 27 ++++++++--- scripts/bench/run-series.sh | 5 +- scripts/rollout.sh | 10 ++++ tools/loadgen/stats_test.go | 4 +- 8 files changed, 153 insertions(+), 25 deletions(-) create mode 100644 cmd/server/main_test.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 72e2440f..6bf7d66b 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -80,15 +80,22 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) InstanceID: alias, IdempotencyTTL: cfg.IdempotencyTTL, }) - var background sync.WaitGroup - defer background.Wait() // before the deferred pool.Close - background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + var debugLn net.Listener if cfg.DebugAddr != "" { - publishDebugVars(alias, pool) - debugLn, err := net.Listen("tcp", cfg.DebugAddr) - if err != nil { + if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil { return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err) } + publishDebugVars(alias, pool) + } + + // The background work ends with the server: on a signal, and also when + // the server fails, so the deferred Wait cannot block the exit. + var background sync.WaitGroup + defer background.Wait() // before the deferred pool.Close + ctx, cancel := context.WithCancel(ctx) + defer cancel() // before Wait + background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) + if debugLn != nil { background.Go(func() { if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil { log.Error("debug listener stopped", "error", err.Error()) diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go new file mode 100644 index 00000000..3885495a --- /dev/null +++ b/cmd/server/main_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "os" + "testing" + "time" + + "messenger/internal/platform/config" + "messenger/internal/testenv" +) + +var ( + db *testenv.DB + kv *testenv.KV +) + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + if db, err = testenv.StartPostgres(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if kv, err = testenv.StartValkey(ctx); err != nil { + db.Close() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + code := m.Run() + kv.Close() + db.Close() + os.Exit(code) +} + +// A start-up failure after the background work began must still end the +// process with an error, not leave it waiting for a signal. +func TestRunFailsFastWhenTheDebugListenerCannotStart(t *testing.T) { + busy, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer busy.Close() + cfg := config.Config{ + Host: "127.0.0.1", + Port: 0, + ShutdownTimeout: time.Second, + MaxBodyBytes: 1024, + DatabaseURL: db.DSN, + SessionStoreURL: kv.URL, + SessionTTL: time.Hour, + IdempotencyTTL: time.Hour, + DBMaxConns: 2, + DebugAddr: busy.Addr().String(), + } + done := make(chan error, 1) + go func() { + done <- run(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), cfg, "alias") + }() + select { + case err := <-done: + if err == nil { + t.Fatal("run succeeded on a busy debug address") + } + case <-time.After(10 * time.Second): + t.Fatal("run hung instead of returning the listen error") + } +} diff --git a/internal/auth/service_test.go b/internal/auth/service_test.go index fb5d767b..b5c8173f 100644 --- a/internal/auth/service_test.go +++ b/internal/auth/service_test.go @@ -129,7 +129,11 @@ func TestLogoutRevokesOnlyThatSession(t *testing.T) { } } +// ExpiresAt is written by this process's clock and enforced by Valkey's. In +// the tests Valkey runs in the Docker VM, whose clock may lag the Windows host +// by a fraction of a second, so the expiry is awaited with a tolerance. func TestSessionExpiresAfterTTL(t *testing.T) { + const clockTolerance = 3 * time.Second f := newFixture(t, time.Second) issued, err := f.svc.Login(f.ctx, "alice", pw) if err != nil { @@ -138,8 +142,16 @@ func TestSessionExpiresAfterTTL(t *testing.T) { if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil { t.Fatalf("fresh session: %v", err) } - time.Sleep(time.Until(issued.ExpiresAt) + 100*time.Millisecond) - if _, err := f.svc.Authenticate(f.ctx, issued.Token); !errors.Is(err, auth.ErrUnauthenticated) { - t.Fatalf("expired session: err = %v, want ErrUnauthenticated", err) + time.Sleep(time.Until(issued.ExpiresAt)) + deadline := time.Now().Add(clockTolerance) + for { + _, err := f.svc.Authenticate(f.ctx, issued.Token) + if errors.Is(err, auth.ErrUnauthenticated) { + return + } + if time.Now().After(deadline) { + t.Fatalf("expired session: err = %v %v after ExpiresAt, want ErrUnauthenticated", err, clockTolerance) + } + time.Sleep(100 * time.Millisecond) } } diff --git a/scripts/bench/failover.sh b/scripts/bench/failover.sh index 624ede52..d223fc21 100755 --- a/scripts/bench/failover.sh +++ b/scripts/bench/failover.sh @@ -14,6 +14,13 @@ # faults.json (when and what was stopped), compose.log (the balancer's # DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt. # Runs on the current stand (`make up`) and leaves every replica running. +# +# Every time used for the analysis comes from the Docker VM's clock: the log +# timestamps, the container's stop and start times from `docker inspect`, and +# the generator's own times. The host clock of Windows runs apart from it by +# a fraction of a second, a large share of a 1.5 s detection window; host +# times are kept for reference only. With `stop`, the moment SIGTERM arrived +# is the replica's own "shutdown started" line in compose.log. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -37,7 +44,8 @@ alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut - meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \ "fault_at_s=$fault_at" "restore_at_s=$restore_at" sh scripts/instances.sh >"$dir/instances-before.txt" -since=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & @@ -45,17 +53,18 @@ generator=$! # The generator seeds two users first; its timeline starts after that. The # fault times below are absolute, as is started_at in timeline.json. sleep "$fault_at" -fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker "$kind" "$target" >/dev/null -fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target") sleep $((restore_at - fault_at)) -restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) docker compose start api >/dev/null 2>&1 +started=$(docker inspect --format '{{.State.StartedAt}}' "$target") wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" -printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \ - "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json" -docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1 +printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \ + "$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json" +docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1 sh scripts/instances.sh >"$dir/instances-after.txt" cat "$dir/loadgen.log" | tail -n 1 echo "results in $dir" diff --git a/scripts/bench/rollout-under-load.sh b/scripts/bench/rollout-under-load.sh index 1bab9a47..2390155a 100755 --- a/scripts/bench/rollout-under-load.sh +++ b/scripts/bench/rollout-under-load.sh @@ -8,7 +8,14 @@ # # Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default # evidence/rollout). Results in /: timeline.json, seconds.csv, -# rollout.log, update.json (when it ran), compose.log, instances-*.txt. +# rollout.log, update.json (when it ran, which containers it replaced), +# compose.log, instances-*.txt. +# +# Times for the analysis come from the Docker VM's clock, as the generator's +# do: log timestamps and the start times of the new containers. The host +# clock of Windows runs apart from it and is kept for reference only. The +# old replicas are removed by the update; their stop shows in compose.log as +# their "shutdown started" lines. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh @@ -27,21 +34,27 @@ docker compose up -d --wait >/dev/null 2>&1 docker compose build api >/dev/null 2>&1 meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" sh scripts/instances.sh >"$dir/instances-before.txt" -since=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# A margin for the host clock, which may run ahead of the VM's. +since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) +old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ') BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & generator=$! sleep "$update_at" -started=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) status=0 sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? -finished=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) +finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" -printf '{\n "update_started_utc": "%s",\n "update_finished_utc": "%s",\n "rollout_exit_code": %s\n}\n' \ - "$started" "$finished" "$status" >"$dir/update.json" -docker compose logs --no-color --since "$since" lb api migrate >"$dir/compose.log" 2>&1 +new="" +for id in $(docker compose ps -q api); do + new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, " +done +printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \ + "$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json" +docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1 sh scripts/instances.sh >"$dir/instances-after.txt" cat "$dir/loadgen.log" | tail -n 1 echo "results in $dir" diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index df62c9e7..11d98aa8 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -47,8 +47,11 @@ grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool stats=$! sample_postgres "$dir/postgres.jsonl" & pg=$! + # stdin is closed: `compose run` is interactive by default and would + # otherwise read the rest of the series file this loop is reading. BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \ - -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 || echo " loadgen failed, see $dir/loadgen.log" + -rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 /dev/null || true wait "$stats" "$pg" 2>/dev/null || true cat "$dir/loadgen.log" | grep '^step' || true diff --git a/scripts/rollout.sh b/scripts/rollout.sh index 1a223a83..c46e52c4 100755 --- a/scripts/rollout.sh +++ b/scripts/rollout.sh @@ -6,6 +6,10 @@ # removes the new ones and leaves the old ones serving, so the stand is never # left with a mix of versions. # +# Every replica must be running. With a stopped one (say, after stop-one), +# docker rollout only replaces the stopped one and reports success, which +# would leave old and new versions side by side; the script refuses instead. +# # The migrations of the new version run first, once, while the old replicas # serve: they only add to the schema. docker rollout scales with `compose up`, # which also starts the dependencies of the API and would rerun the old, @@ -15,6 +19,12 @@ set -eu cd "$(dirname "$0")/.." +idle=$(docker compose ps -aq --status exited --status created --status paused --status dead api) +if [ -n "$idle" ]; then + echo "some API replicas are not running; start them first: docker compose start api" >&2 + exit 2 +fi + version=v0.14 checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b tool=bin/docker-rollout diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go index d9b32e1d..1e590016 100644 --- a/tools/loadgen/stats_test.go +++ b/tools/loadgen/stats_test.go @@ -98,7 +98,9 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) { if n := fired.Load(); n < 45 || n > 50 { t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n) } - if took := time.Since(start); took > 1500*time.Millisecond { + // About 1.2 s; the bound leaves room for -race on a shared CI runner and + // still fails a closed loop, which would need 50 x 200 ms = 10 s. + if took := time.Since(start); took > 2500*time.Millisecond { t.Fatalf("open loop took %v, want about 1.2 s", took) } }