Исправить замечания ревью: защита rollout от остановленных реплик, чтение серий, время экспериментов по часам Docker, выход сервера при ошибке старта

This commit is contained in:
2026-11-02 18:23:26 +03:00
parent b2973405f3
commit c9d0f73221
8 changed files with 153 additions and 25 deletions
+13 -6
View File
@@ -80,15 +80,22 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
InstanceID: alias,
IdempotencyTTL: cfg.IdempotencyTTL,
})
var background sync.WaitGroup
defer background.Wait() // before the deferred pool.Close
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
var debugLn net.Listener
if cfg.DebugAddr != "" {
publishDebugVars(alias, pool)
debugLn, err := net.Listen("tcp", cfg.DebugAddr)
if err != nil {
if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil {
return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err)
}
publishDebugVars(alias, pool)
}
// The background work ends with the server: on a signal, and also when
// the server fails, so the deferred Wait cannot block the exit.
var background sync.WaitGroup
defer background.Wait() // before the deferred pool.Close
ctx, cancel := context.WithCancel(ctx)
defer cancel() // before Wait
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
if debugLn != nil {
background.Go(func() {
if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil {
log.Error("debug listener stopped", "error", err.Error())
+72
View File
@@ -0,0 +1,72 @@
package main
import (
"context"
"fmt"
"io"
"log/slog"
"net"
"os"
"testing"
"time"
"messenger/internal/platform/config"
"messenger/internal/testenv"
)
var (
db *testenv.DB
kv *testenv.KV
)
func TestMain(m *testing.M) {
ctx := context.Background()
var err error
if db, err = testenv.StartPostgres(ctx); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if kv, err = testenv.StartValkey(ctx); err != nil {
db.Close()
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
code := m.Run()
kv.Close()
db.Close()
os.Exit(code)
}
// A start-up failure after the background work began must still end the
// process with an error, not leave it waiting for a signal.
func TestRunFailsFastWhenTheDebugListenerCannotStart(t *testing.T) {
busy, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer busy.Close()
cfg := config.Config{
Host: "127.0.0.1",
Port: 0,
ShutdownTimeout: time.Second,
MaxBodyBytes: 1024,
DatabaseURL: db.DSN,
SessionStoreURL: kv.URL,
SessionTTL: time.Hour,
IdempotencyTTL: time.Hour,
DBMaxConns: 2,
DebugAddr: busy.Addr().String(),
}
done := make(chan error, 1)
go func() {
done <- run(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), cfg, "alias")
}()
select {
case err := <-done:
if err == nil {
t.Fatal("run succeeded on a busy debug address")
}
case <-time.After(10 * time.Second):
t.Fatal("run hung instead of returning the listen error")
}
}
+15 -3
View File
@@ -129,7 +129,11 @@ func TestLogoutRevokesOnlyThatSession(t *testing.T) {
}
}
// ExpiresAt is written by this process's clock and enforced by Valkey's. In
// the tests Valkey runs in the Docker VM, whose clock may lag the Windows host
// by a fraction of a second, so the expiry is awaited with a tolerance.
func TestSessionExpiresAfterTTL(t *testing.T) {
const clockTolerance = 3 * time.Second
f := newFixture(t, time.Second)
issued, err := f.svc.Login(f.ctx, "alice", pw)
if err != nil {
@@ -138,8 +142,16 @@ func TestSessionExpiresAfterTTL(t *testing.T) {
if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil {
t.Fatalf("fresh session: %v", err)
}
time.Sleep(time.Until(issued.ExpiresAt) + 100*time.Millisecond)
if _, err := f.svc.Authenticate(f.ctx, issued.Token); !errors.Is(err, auth.ErrUnauthenticated) {
t.Fatalf("expired session: err = %v, want ErrUnauthenticated", err)
time.Sleep(time.Until(issued.ExpiresAt))
deadline := time.Now().Add(clockTolerance)
for {
_, err := f.svc.Authenticate(f.ctx, issued.Token)
if errors.Is(err, auth.ErrUnauthenticated) {
return
}
if time.Now().After(deadline) {
t.Fatalf("expired session: err = %v %v after ExpiresAt, want ErrUnauthenticated", err, clockTolerance)
}
time.Sleep(100 * time.Millisecond)
}
}
+16 -7
View File
@@ -14,6 +14,13 @@
# faults.json (when and what was stopped), compose.log (the balancer's
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
# Runs on the current stand (`make up`) and leaves every replica running.
#
# Every time used for the analysis comes from the Docker VM's clock: the log
# timestamps, the container's stop and start times from `docker inspect`, and
# the generator's own times. The host clock of Windows runs apart from it by
# a fraction of a second, a large share of a 1.5 s detection window; host
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
# is the replica's own "shutdown started" line in compose.log.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
@@ -37,7 +44,8 @@ alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -
meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \
"fault_at_s=$fault_at" "restore_at_s=$restore_at"
sh scripts/instances.sh >"$dir/instances-before.txt"
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
# A margin for the host clock, which may run ahead of the VM's.
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
@@ -45,17 +53,18 @@ generator=$!
# The generator seeds two users first; its timeline starts after that. The
# fault times below are absolute, as is started_at in timeline.json.
sleep "$fault_at"
fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker "$kind" "$target" >/dev/null
fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target")
sleep $((restore_at - fault_at))
restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
docker compose start api >/dev/null 2>&1
started=$(docker inspect --format '{{.State.StartedAt}}' "$target")
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json"
docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json"
docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1
sh scripts/instances.sh >"$dir/instances-after.txt"
cat "$dir/loadgen.log" | tail -n 1
echo "results in $dir"
+20 -7
View File
@@ -8,7 +8,14 @@
#
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default
# evidence/rollout). Results in <OUT>/: timeline.json, seconds.csv,
# rollout.log, update.json (when it ran), compose.log, instances-*.txt.
# rollout.log, update.json (when it ran, which containers it replaced),
# compose.log, instances-*.txt.
#
# Times for the analysis come from the Docker VM's clock, as the generator's
# do: log timestamps and the start times of the new containers. The host
# clock of Windows runs apart from it and is kept for reference only. The
# old replicas are removed by the update; their stop shows in compose.log as
# their "shutdown started" lines.
set -eu
cd "$(dirname "$0")/../.."
. scripts/bench/lib.sh
@@ -27,21 +34,27 @@ docker compose up -d --wait >/dev/null 2>&1
docker compose build api >/dev/null 2>&1
meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at"
sh scripts/instances.sh >"$dir/instances-before.txt"
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
# A margin for the host clock, which may run ahead of the VM's.
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ')
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
generator=$!
sleep "$update_at"
started=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
status=0
sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$?
finished=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
printf '{\n "update_started_utc": "%s",\n "update_finished_utc": "%s",\n "rollout_exit_code": %s\n}\n' \
"$started" "$finished" "$status" >"$dir/update.json"
docker compose logs --no-color --since "$since" lb api migrate >"$dir/compose.log" 2>&1
new=""
for id in $(docker compose ps -q api); do
new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, "
done
printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \
"$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json"
docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1
sh scripts/instances.sh >"$dir/instances-after.txt"
cat "$dir/loadgen.log" | tail -n 1
echo "results in $dir"
+4 -1
View File
@@ -47,8 +47,11 @@ grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool
stats=$!
sample_postgres "$dir/postgres.jsonl" &
pg=$!
# stdin is closed: `compose run` is interactive by default and would
# otherwise read the rest of the series file this loop is reading.
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 || echo " loadgen failed, see $dir/loadgen.log"
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 </dev/null ||
echo " loadgen failed, see $dir/loadgen.log"
kill "$stats" "$pg" 2>/dev/null || true
wait "$stats" "$pg" 2>/dev/null || true
cat "$dir/loadgen.log" | grep '^step' || true
+10
View File
@@ -6,6 +6,10 @@
# removes the new ones and leaves the old ones serving, so the stand is never
# left with a mix of versions.
#
# Every replica must be running. With a stopped one (say, after stop-one),
# docker rollout only replaces the stopped one and reports success, which
# would leave old and new versions side by side; the script refuses instead.
#
# The migrations of the new version run first, once, while the old replicas
# serve: they only add to the schema. docker rollout scales with `compose up`,
# which also starts the dependencies of the API and would rerun the old,
@@ -15,6 +19,12 @@
set -eu
cd "$(dirname "$0")/.."
idle=$(docker compose ps -aq --status exited --status created --status paused --status dead api)
if [ -n "$idle" ]; then
echo "some API replicas are not running; start them first: docker compose start api" >&2
exit 2
fi
version=v0.14
checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b
tool=bin/docker-rollout
+3 -1
View File
@@ -98,7 +98,9 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) {
if n := fired.Load(); n < 45 || n > 50 {
t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n)
}
if took := time.Since(start); took > 1500*time.Millisecond {
// About 1.2 s; the bound leaves room for -race on a shared CI runner and
// still fails a closed loop, which would need 50 x 200 ms = 10 s.
if took := time.Since(start); took > 2500*time.Millisecond {
t.Fatalf("open loop took %v, want about 1.2 s", took)
}
}