Исправить замечания ревью: защита rollout от остановленных реплик, чтение серий, время экспериментов по часам Docker, выход сервера при ошибке старта
This commit is contained in:
+13
-6
@@ -80,15 +80,22 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
InstanceID: alias,
|
||||
IdempotencyTTL: cfg.IdempotencyTTL,
|
||||
})
|
||||
var background sync.WaitGroup
|
||||
defer background.Wait() // before the deferred pool.Close
|
||||
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
|
||||
var debugLn net.Listener
|
||||
if cfg.DebugAddr != "" {
|
||||
publishDebugVars(alias, pool)
|
||||
debugLn, err := net.Listen("tcp", cfg.DebugAddr)
|
||||
if err != nil {
|
||||
if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil {
|
||||
return fmt.Errorf("listen on %s: %w", cfg.DebugAddr, err)
|
||||
}
|
||||
publishDebugVars(alias, pool)
|
||||
}
|
||||
|
||||
// The background work ends with the server: on a signal, and also when
|
||||
// the server fails, so the deferred Wait cannot block the exit.
|
||||
var background sync.WaitGroup
|
||||
defer background.Wait() // before the deferred pool.Close
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel() // before Wait
|
||||
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
|
||||
if debugLn != nil {
|
||||
background.Go(func() {
|
||||
if err := server.RunDebug(ctx, log, debugLn, expvar.Handler()); err != nil {
|
||||
log.Error("debug listener stopped", "error", err.Error())
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
var (
|
||||
db *testenv.DB
|
||||
kv *testenv.KV
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
var err error
|
||||
if db, err = testenv.StartPostgres(ctx); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if kv, err = testenv.StartValkey(ctx); err != nil {
|
||||
db.Close()
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
code := m.Run()
|
||||
kv.Close()
|
||||
db.Close()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// A start-up failure after the background work began must still end the
|
||||
// process with an error, not leave it waiting for a signal.
|
||||
func TestRunFailsFastWhenTheDebugListenerCannotStart(t *testing.T) {
|
||||
busy, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer busy.Close()
|
||||
cfg := config.Config{
|
||||
Host: "127.0.0.1",
|
||||
Port: 0,
|
||||
ShutdownTimeout: time.Second,
|
||||
MaxBodyBytes: 1024,
|
||||
DatabaseURL: db.DSN,
|
||||
SessionStoreURL: kv.URL,
|
||||
SessionTTL: time.Hour,
|
||||
IdempotencyTTL: time.Hour,
|
||||
DBMaxConns: 2,
|
||||
DebugAddr: busy.Addr().String(),
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- run(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), cfg, "alias")
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("run succeeded on a busy debug address")
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("run hung instead of returning the listen error")
|
||||
}
|
||||
}
|
||||
@@ -129,7 +129,11 @@ func TestLogoutRevokesOnlyThatSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ExpiresAt is written by this process's clock and enforced by Valkey's. In
|
||||
// the tests Valkey runs in the Docker VM, whose clock may lag the Windows host
|
||||
// by a fraction of a second, so the expiry is awaited with a tolerance.
|
||||
func TestSessionExpiresAfterTTL(t *testing.T) {
|
||||
const clockTolerance = 3 * time.Second
|
||||
f := newFixture(t, time.Second)
|
||||
issued, err := f.svc.Login(f.ctx, "alice", pw)
|
||||
if err != nil {
|
||||
@@ -138,8 +142,16 @@ func TestSessionExpiresAfterTTL(t *testing.T) {
|
||||
if _, err := f.svc.Authenticate(f.ctx, issued.Token); err != nil {
|
||||
t.Fatalf("fresh session: %v", err)
|
||||
}
|
||||
time.Sleep(time.Until(issued.ExpiresAt) + 100*time.Millisecond)
|
||||
if _, err := f.svc.Authenticate(f.ctx, issued.Token); !errors.Is(err, auth.ErrUnauthenticated) {
|
||||
t.Fatalf("expired session: err = %v, want ErrUnauthenticated", err)
|
||||
time.Sleep(time.Until(issued.ExpiresAt))
|
||||
deadline := time.Now().Add(clockTolerance)
|
||||
for {
|
||||
_, err := f.svc.Authenticate(f.ctx, issued.Token)
|
||||
if errors.Is(err, auth.ErrUnauthenticated) {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("expired session: err = %v %v after ExpiresAt, want ErrUnauthenticated", err, clockTolerance)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,13 @@
|
||||
# faults.json (when and what was stopped), compose.log (the balancer's
|
||||
# DOWN/UP lines and the replicas' shutdown/start lines), instances-*.txt.
|
||||
# Runs on the current stand (`make up`) and leaves every replica running.
|
||||
#
|
||||
# Every time used for the analysis comes from the Docker VM's clock: the log
|
||||
# timestamps, the container's stop and start times from `docker inspect`, and
|
||||
# the generator's own times. The host clock of Windows runs apart from it by
|
||||
# a fraction of a second, a large share of a 1.5 s detection window; host
|
||||
# times are kept for reference only. With `stop`, the moment SIGTERM arrived
|
||||
# is the replica's own "shutdown started" line in compose.log.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
@@ -37,7 +44,8 @@ alias=$(sh scripts/api-instances.sh | awk -v id="$(printf '%s' "$target" | cut -
|
||||
meta "$dir/meta.json" "experiment=failover" "kind=$kind" "replica=$n" "rate=$rate" "duration=$duration" \
|
||||
"fault_at_s=$fault_at" "restore_at_s=$restore_at"
|
||||
sh scripts/instances.sh >"$dir/instances-before.txt"
|
||||
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
# A margin for the host clock, which may run ahead of the VM's.
|
||||
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
||||
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
||||
@@ -45,17 +53,18 @@ generator=$!
|
||||
# The generator seeds two users first; its timeline starts after that. The
|
||||
# fault times below are absolute, as is started_at in timeline.json.
|
||||
sleep "$fault_at"
|
||||
fault=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
fault_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
docker "$kind" "$target" >/dev/null
|
||||
fault_done=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
exited=$(docker inspect --format '{{.State.FinishedAt}}' "$target")
|
||||
sleep $((restore_at - fault_at))
|
||||
restore=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
restore_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
docker compose start api >/dev/null 2>&1
|
||||
started=$(docker inspect --format '{{.State.StartedAt}}' "$target")
|
||||
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
|
||||
|
||||
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "fault_utc": "%s",\n "fault_command_returned_utc": "%s",\n "restore_utc": "%s"\n}\n' \
|
||||
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$fault" "$fault_done" "$restore" >"$dir/faults.json"
|
||||
docker compose logs --no-color --since "$since" lb api >"$dir/compose.log" 2>&1
|
||||
printf '{\n "kind": "%s",\n "container": "%s",\n "x_instance_id": "%s",\n "process_exited_vm_utc": "%s",\n "restarted_vm_utc": "%s",\n "fault_command_host_utc": "%s",\n "restore_command_host_utc": "%s"\n}\n' \
|
||||
"$kind" "$(printf '%s' "$target" | cut -c1-12)" "$alias" "$exited" "$started" "$fault_host" "$restore_host" >"$dir/faults.json"
|
||||
docker compose logs --no-color --timestamps --since "$since" lb api >"$dir/compose.log" 2>&1
|
||||
sh scripts/instances.sh >"$dir/instances-after.txt"
|
||||
cat "$dir/loadgen.log" | tail -n 1
|
||||
echo "results in $dir"
|
||||
|
||||
@@ -8,7 +8,14 @@
|
||||
#
|
||||
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default
|
||||
# evidence/rollout). Results in <OUT>/: timeline.json, seconds.csv,
|
||||
# rollout.log, update.json (when it ran), compose.log, instances-*.txt.
|
||||
# rollout.log, update.json (when it ran, which containers it replaced),
|
||||
# compose.log, instances-*.txt.
|
||||
#
|
||||
# Times for the analysis come from the Docker VM's clock, as the generator's
|
||||
# do: log timestamps and the start times of the new containers. The host
|
||||
# clock of Windows runs apart from it and is kept for reference only. The
|
||||
# old replicas are removed by the update; their stop shows in compose.log as
|
||||
# their "shutdown started" lines.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
. scripts/bench/lib.sh
|
||||
@@ -27,21 +34,27 @@ docker compose up -d --wait >/dev/null 2>&1
|
||||
docker compose build api >/dev/null 2>&1
|
||||
meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at"
|
||||
sh scripts/instances.sh >"$dir/instances-before.txt"
|
||||
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
# A margin for the host clock, which may run ahead of the VM's.
|
||||
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
|
||||
old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ')
|
||||
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
||||
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
||||
generator=$!
|
||||
sleep "$update_at"
|
||||
started=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
status=0
|
||||
sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$?
|
||||
finished=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
||||
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
|
||||
|
||||
printf '{\n "update_started_utc": "%s",\n "update_finished_utc": "%s",\n "rollout_exit_code": %s\n}\n' \
|
||||
"$started" "$finished" "$status" >"$dir/update.json"
|
||||
docker compose logs --no-color --since "$since" lb api migrate >"$dir/compose.log" 2>&1
|
||||
new=""
|
||||
for id in $(docker compose ps -q api); do
|
||||
new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, "
|
||||
done
|
||||
printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \
|
||||
"$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json"
|
||||
docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1
|
||||
sh scripts/instances.sh >"$dir/instances-after.txt"
|
||||
cat "$dir/loadgen.log" | tail -n 1
|
||||
echo "results in $dir"
|
||||
|
||||
@@ -47,8 +47,11 @@ grep -v '^#' "$series" | while read -r name replicas cpus memory gomemlimit pool
|
||||
stats=$!
|
||||
sample_postgres "$dir/postgres.jsonl" &
|
||||
pg=$!
|
||||
# stdin is closed: `compose run` is interactive by default and would
|
||||
# otherwise read the rest of the series file this loop is reading.
|
||||
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen bench \
|
||||
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 || echo " loadgen failed, see $dir/loadgen.log"
|
||||
-rates "$RATES" -step "$step" -warmup "$warmup" -scrape >"$dir/loadgen.log" 2>&1 </dev/null ||
|
||||
echo " loadgen failed, see $dir/loadgen.log"
|
||||
kill "$stats" "$pg" 2>/dev/null || true
|
||||
wait "$stats" "$pg" 2>/dev/null || true
|
||||
cat "$dir/loadgen.log" | grep '^step' || true
|
||||
|
||||
@@ -6,6 +6,10 @@
|
||||
# removes the new ones and leaves the old ones serving, so the stand is never
|
||||
# left with a mix of versions.
|
||||
#
|
||||
# Every replica must be running. With a stopped one (say, after stop-one),
|
||||
# docker rollout only replaces the stopped one and reports success, which
|
||||
# would leave old and new versions side by side; the script refuses instead.
|
||||
#
|
||||
# The migrations of the new version run first, once, while the old replicas
|
||||
# serve: they only add to the schema. docker rollout scales with `compose up`,
|
||||
# which also starts the dependencies of the API and would rerun the old,
|
||||
@@ -15,6 +19,12 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
idle=$(docker compose ps -aq --status exited --status created --status paused --status dead api)
|
||||
if [ -n "$idle" ]; then
|
||||
echo "some API replicas are not running; start them first: docker compose start api" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
version=v0.14
|
||||
checksum=cdeaba6ae9eee3b0b606286e585bbda6787283d801a6ad6d9b9d2bc347fda05b
|
||||
tool=bin/docker-rollout
|
||||
|
||||
@@ -98,7 +98,9 @@ func TestOpenLoopDoesNotWaitForResponses(t *testing.T) {
|
||||
if n := fired.Load(); n < 45 || n > 50 {
|
||||
t.Fatalf("fired %d requests in 1 s at 50/s, want about 50", n)
|
||||
}
|
||||
if took := time.Since(start); took > 1500*time.Millisecond {
|
||||
// About 1.2 s; the bound leaves room for -race on a shared CI runner and
|
||||
// still fails a closed loop, which would need 50 x 200 ms = 10 s.
|
||||
if took := time.Since(start); took > 2500*time.Millisecond {
|
||||
t.Fatalf("open loop took %v, want about 1.2 s", took)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user