feat: process attachments through durable versioned jobs

This commit is contained in:
Vladislav Agapov
2026-10-29 16:56:06 +03:00
parent 276dbdad8a
commit e35c8f0b75
235 changed files with 17028 additions and 44 deletions
+35
View File
@@ -0,0 +1,35 @@
# Aggregate covered statements once per merged block; keep files with no statements as N/A.
FNR == NR {
split($0, fields, "\t")
paths[fields[1]] = fields[2]
hashes[fields[1]] = fields[3]
executable[fields[1]] = fields[4]
next
}
/^mode:/ { next }
{
file = $1
sub(/:[^:]*$/, "", file)
if (!(file in paths)) {
print "unknown coverage source: " file > "/dev/stderr"
invalid = 1
}
total[file] += $2
if ($3 > 0) covered[file] += $2
}
END {
print "{\"threshold_per_file\":90,\"files\":[" > summary
separator = ""
for (file in paths) {
percent = total[file] ? sprintf("%.8f", covered[file] * 100 / total[file]) : "null"
printf "%s{\"file\":\"%s\",\"statements\":%d,\"covered\":%d,\"percent\":%s,\"sha256\":\"%s\"}", separator, paths[file], total[file], covered[file], percent, hashes[file] > summary
separator = ",\n"
printf "%8s %s\n", (total[file] ? sprintf("%.2f%%", covered[file] * 100 / total[file]) : "N/A"), paths[file]
if (covered[file] * 100 < total[file] * 90 || (executable[file] && !total[file])) invalid = 1
}
printf "\n],\"passed\":%s}\n", (invalid ? "false" : "true") > summary
if (invalid) {
print "FAIL: per-file coverage must reach 90%" > "/dev/stderr"
exit 1
}
}
+91
View File
@@ -0,0 +1,91 @@
#!/bin/sh
# Measure unit and real-process coverage, enforcing 90% separately for each Go source file.
set -eu
cd "$(dirname "$0")/.."
GO=${GO:-go}
: "${DATABASE_URL:?Use an isolated test database}" "${VALKEY_SHARDS:?Use test shards}"
: "${TEST_DATABASE_URL:?Use an isolated test database}" "${TEST_VALKEY_ADDRESS:?Use a test shard}"
export APP_PORT=${COVERAGE_PORT:-18080}
export LOG_LEVEL=error
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
worker_pid=
server_pid=
watchdog_pid=
mkdir -p coverage
run_dir=$(mktemp -d "$PWD/coverage/run.XXXXXX")
cleanup() {
if [ -n "$worker_pid" ]; then kill "$worker_pid" 2>/dev/null || true; wait "$worker_pid" 2>/dev/null || true; fi
if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; fi
if [ -n "$watchdog_pid" ]; then kill "$watchdog_pid" 2>/dev/null || true; wait "$watchdog_pid" 2>/dev/null || true; fi
rm -rf "$run_dir"
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
mkdir "$run_dir/process"
"$GO" build -race -cover -covermode=atomic -coverpkg=./... -o "$run_dir/server" ./cmd/server
"$run_dir/server" migrate
"$GO" test -count=1 -race -coverpkg=./... -coverprofile="$run_dir/unit.out" ./...
export GOCOVERDIR="$run_dir/process"
if "$run_dir/server" unknown >"$run_dir/server-invalid.log" 2>&1; then
echo 'server accepted an unknown command' >&2
exit 1
else
test "$?" -eq 1
fi
"$run_dir/server" worker >"$run_dir/worker.log" 2>&1 &
worker_pid=$!
"$run_dir/server" >"$run_dir/server.log" 2>&1 &
server_pid=$!
ready=false
for attempt in $(seq 1 60); do
kill -0 "$server_pid" 2>/dev/null || { cat "$run_dir/server.log"; exit 1; }
if curl --fail --silent --max-time 1 "http://localhost:$APP_PORT/health/ready" >/dev/null; then ready=true; break; fi
sleep .25
done
$ready || { cat "$run_dir/server.log"; exit 1; }
"$run_dir/server" healthcheck
"$run_dir/server" migrate
"${PYTHON:-python3}" tests/smoke.py --base-url "http://localhost:$APP_PORT"
kill -TERM "$server_pid"
# Bound the real-process shutdown check; a forced kill makes wait fail.
(sleep 12; kill -KILL "$server_pid" 2>/dev/null || true) &
watchdog_pid=$!
wait "$server_pid"
server_pid=
kill "$watchdog_pid" 2>/dev/null || true
wait "$watchdog_pid" 2>/dev/null || true
watchdog_pid=
kill -TERM "$worker_pid"
wait "$worker_pid"
worker_pid=
unset GOCOVERDIR
"$GO" tool covdata textfmt -i="$run_dir/process" -o="$run_dir/process.out"
awk '
/^mode:/ { if ($0 != "mode: atomic") exit 1; next }
{
if (($1 in statements) && statements[$1] != $2) exit 1;
statements[$1] = $2; hits[$1] += $3
}
END {
print "mode: atomic";
for (block in statements) print block, statements[block], hits[block]
}' "$run_dir/unit.out" "$run_dir/process.out" > coverage/coverage.out
"$GO" list -f '{{.ImportPath}} {{.Dir}} {{join .GoFiles ","}}' ./... > "$run_dir/packages.txt"
: > "$run_dir/inventory.tsv"
while read -r import_path directory names; do
old_ifs=$IFS
IFS=,
for name in $names; do
checksum=$(shasum -a 256 "$directory/$name")
checksum=${checksum%% *}
relative=${directory#"$PWD/"}/$name
executable=$(awk '/^func[ (]/ {found=1} END {print found+0}' "$directory/$name")
printf '%s\t%s\t%s\t%s
' "$import_path/$name" "$relative" "$checksum" "$executable" >> "$run_dir/inventory.tsv"
done
IFS=$old_ifs
done < "$run_dir/packages.txt"
awk -v summary=coverage/summary.json -f scripts/coverage.awk "$run_dir/inventory.tsv" coverage/coverage.out
"$GO" version > coverage/go-version.txt
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
# Create lab credentials once, outside Helm values and release history.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
: "${REGISTRY:=asisti.registry.twcstorage.ru}" "${REGISTRY_USER:=token}"
: "${REGISTRY_TOKEN_FILE:=$HOME/.secrets/twc-registry-token}"
test -s "$REGISTRY_TOKEN_FILE" || { echo 'Registry token file is missing or empty' >&2; exit 1; }
umask 077
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
kubectl --context "$KUBE_CONTEXT" create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" apply -f -
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets >/dev/null 2>&1; then
openssl rand -hex 32 | tr -d '\n' > "$work/postgres-password"
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$(cat "$work/postgres-password")" > "$work/database-url"
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url" --from-file="$work/cursor-key"
fi
# Add the shared cursor key to an existing lab2 secret without replacing database credentials.
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets -o json | jq -e '.data["cursor-key"] | length > 0' >/dev/null; then
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
jq -n --rawfile key "$work/cursor-key" '{stringData:{"cursor-key":$key}}' > "$work/patch.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" patch secret messenger-secrets --type merge --patch-file "$work/patch.json"
fi
# Build a portable Docker config; macOS credential-helper references cannot be used by kubelet.
jq -n --arg registry "$REGISTRY" --arg user "$REGISTRY_USER" --rawfile token "$REGISTRY_TOKEN_FILE" \
'{auths:{($registry):{auth:(($user+":"+($token | sub("[\\r\\n]+$"; "")))|@base64)}}}' > "$work/config.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic registry-pull \
--type=kubernetes.io/dockerconfigjson --from-file=".dockerconfigjson=$work/config.json" \
--dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-s3 >/dev/null 2>&1; then
printf 'operator' > "$work/root-user"
printf 'messenger' > "$work/access-key"
openssl rand -hex 32 | tr -d '\n' > "$work/root-password"
openssl rand -hex 32 | tr -d '\n' > "$work/secret-key"
jq -n --rawfile admin "$work/root-user" --rawfile adminSecret "$work/root-password" --rawfile app "$work/access-key" --rawfile appSecret "$work/secret-key" '{identities:[{name:"operator",credentials:[{accessKey:$admin,secretKey:$adminSecret}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:$app,secretKey:$appSecret}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$work/identities.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-s3 --from-file="$work/root-user" --from-file="$work/root-password" --from-file="$work/access-key" --from-file="$work/secret-key" --from-file="$work/identities.json"
fi
+59
View File
@@ -0,0 +1,59 @@
#!/bin/sh
# Operate only on the isolated processes and containers recorded by test-local.sh.
set -eu
exec 2>> evidence/local/actions-errors.txt
: "${LOCAL_TEST_WORK:?Run through scripts/test-local.sh}"
. "$LOCAL_TEST_WORK/environment"
stop_api() {
label=$1
pid=$(cat "$LOCAL_TEST_WORK/$label.pid")
if kill -0 "$pid" 2>/dev/null; then
state=$(ps -p "$pid" -o stat=)
case "$state" in Z*) return ;; esac
command=$(ps -p "$pid" -o command=)
case "$command" in "$LOCAL_TEST_WORK/server"|"$LOCAL_TEST_WORK/server worker") ;; *) echo 'API PID ownership mismatch' >&2; exit 1 ;; esac
kill -TERM "$pid"
for attempt in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || break; sleep .1; done
fi
}
start_api() {
label=$1
port=$2
INSTANCE_ID="$label" APP_PORT="$port" nohup "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 < /dev/null &
echo "$!" > "$LOCAL_TEST_WORK/$label.pid"
}
if [ "${LOCAL_ACTION:-restart}" = failover ]; then
case "${TARGET_INSTANCE_ID:?Missing observed replica}" in local-a|local-b) ;; *) echo 'Unknown isolated replica' >&2; exit 1 ;; esac
printf "%s target=%s action=stop\n" "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/local/actions.txt
stop_api "$TARGET_INSTANCE_ID"
exit 0
fi
if [ "${LOCAL_ACTION:-restart}" = pause-worker ]; then
stop_api worker
printf "%s action=stop-worker\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt
exit 0
fi
stop_api local-a
stop_api local-b
stop_api worker
printf "%s action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt
docker restart "$LOCAL_TEST_PG" "$LOCAL_TEST_VK" "$LOCAL_TEST_S3" "$LOCAL_TEST_BROKER" >/dev/null
# Docker may assign new host ports when restarting containers with ephemeral bindings.
export DATABASE_URL="postgres://postgres@$(docker port "$LOCAL_TEST_PG" 5432/tcp)/messenger?sslmode=disable"
export VALKEY_SHARDS="shard0=$(docker port "$LOCAL_TEST_VK" 6379/tcp)"
export S3_ENDPOINT="$(docker port "$LOCAL_TEST_S3" 9000/tcp)"
export BROKER_ADDRESS="$(docker port "$LOCAL_TEST_BROKER" 6379/tcp)"
for attempt in $(seq 1 60); do
if docker exec "$LOCAL_TEST_PG" pg_isready -U postgres >/dev/null 2>&1 && "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi
sleep 1
done
nohup "$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/worker.log" 2>&1 < /dev/null &
echo "$!" > "$LOCAL_TEST_WORK/worker.pid"
start_api local-a "$LOCAL_TEST_PORT_A"
start_api local-b "$LOCAL_TEST_PORT_B"
for attempt in $(seq 1 60); do
if curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_A/health/ready" >/dev/null && curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_B/health/ready" >/dev/null; then exit 0; fi
sleep 1
done
echo 'Isolated restart readiness timed out' >&2
exit 1
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Reconnect the loopback tunnel after the API pod is replaced during restart tests.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
LOCAL_PORT=${LOCAL_PORT:-8083}
child=
cleanup() {
if [ -n "$child" ]; then
kill "$child" 2>/dev/null || true
wait "$child" 2>/dev/null || true
fi
exit 0
}
trap cleanup INT TERM
while :; do
kubectl --context "$KUBE_CONTEXT" -n traefik port-forward service/traefik "$LOCAL_PORT:80" --address 127.0.0.1 &
child=$!
wait "$child" || true
child=
sleep 1
done
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Restart only this lab's API and persistent stores, retaining every PVC.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
# HPA must be disabled explicitly for this destructive test window.
if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get hpa api >/dev/null 2>&1; then
echo 'Disable autoscaling in the lab release before the restart test, then restore it.' >&2
exit 1
fi
replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.replicas}')
mkdir -p evidence
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,postgres,valkey,s3)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-before.txt
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas=0
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l app=api --timeout=60s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout restart statefulset/postgres statefulset/valkey statefulset/s3
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/postgres --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/valkey --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/s3 --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas="$replicas"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,postgres,valkey,s3)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-after.txt
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Terminate the exact observed API pod; its replacement receives a different instance ID.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${TARGET_INSTANCE_ID:?Expected API pod name}"
case "$TARGET_INSTANCE_ID" in api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac
label=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$TARGET_INSTANCE_ID" -o jsonpath='{.metadata.labels.app}')
test "$label" = api
mkdir -p evidence
printf '%s target=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$TARGET_INSTANCE_ID" >> evidence/failover-targets.txt
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$TARGET_INSTANCE_ID" --wait=false
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
# Demonstrate accepted upload survival with worker stopped and all dependencies restarted.
set -eu
: "${LOCAL_TEST_WORK:?Run through test-local.sh}" "${BASE_URL:?}"
umask 077
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
LOCAL_ACTION=pause-worker scripts/local-action.sh
username="durable_$(openssl rand -hex 6)"
password=$(openssl rand -hex 16)
jq -n --arg username "$username" --arg password "$password" '{username:$username,display_name:"Durability test",password:$password}' > "$work/user.json"
curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$work/user.json" > "$work/user-response.json"
# Temporary curl config avoids placing credentials or bearer tokens in process arguments.
printf 'user = "%s:%s"\n' "$username" "$password" > "$work/basic.conf"
curl -fsS --config "$work/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$work/session.json"
printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$work/session.json")" > "$work/auth.conf"
jq -n --arg id "$(jq -r .id "$work/user-response.json")" '{title:"Durability",member_ids:[$id]}' > "$work/chat.json"
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$work/chat.json" > "$work/chat-response.json"
chat=$(jq -r .id "$work/chat-response.json")
status=$(curl -sS --config "$work/auth.conf" -o "$work/attachment.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
test "$status" = 202
jq -e '.status=="queued"' "$work/attachment.json" >/dev/null
id=$(jq -r .id "$work/attachment.json")
jq -n --arg id "$id" '{text:"",attachment_ids:[$id]}' > "$work/message.json"
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats/$chat/messages" -H 'Content-Type: application/json' -H 'Idempotency-Key: durable-upload' --data-binary @"$work/message.json" > /dev/null
status=$(curl -sS --config "$work/auth.conf" -o /dev/null -w '%{http_code}' "$BASE_URL/api/v1/attachments/$id/content")
test "$status" = 409
printf 'worker_stopped=true upload_status=202 snapshot=queued message_created=true content_status=409\n'
LOCAL_ACTION=restart scripts/local-action.sh
ready=false
for attempt in $(seq 1 60); do
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$work/status.json"
if jq -e '.status=="ready" and (.variants|length)==3' "$work/status.json" >/dev/null; then ready=true; break; fi
sleep 1
done
$ready
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id/content" > "$work/original.png"
cmp tests/fixtures/sample.png "$work/original.png"
printf 'PASS: accepted intent survived API, worker, PostgreSQL, S3 and AOF broker restart; ready=1 variants=3 original_unchanged=true\n'
+110
View File
@@ -0,0 +1,110 @@
#!/bin/sh
# Disposable integration harness; Helm remains the deployment source of truth.
set -eu
cd "$(dirname "$0")/.."
umask 077
export LOCAL_TEST_WORK=$(mktemp -d)
containers=
cleanup() {
for label in local-a local-b worker; do
if [ -f "$LOCAL_TEST_WORK/$label.pid" ]; then kill -TERM "$(cat "$LOCAL_TEST_WORK/$label.pid")" 2>/dev/null || true; fi
done
for container in $containers; do docker rm -f "$container" >/dev/null; done
for label in local-a local-b worker; do if [ -f "$LOCAL_TEST_WORK/$label.log" ]; then cp "$LOCAL_TEST_WORK/$label.log" "evidence/local/$label.txt"; fi; done
rm -rf "$LOCAL_TEST_WORK"
}
trap cleanup EXIT
trap 'exit 130' INT TERM
mkdir -p evidence/local
pg=$(docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=messenger -p 127.0.0.1::5432 postgres:17.6-alpine)
containers="$pg"
vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine)
containers="$containers $vk"
broker=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine valkey-server --appendonly yes --appendfsync always --maxmemory 128mb --maxmemory-policy noeviction)
containers="$containers $broker"
admin=$(openssl rand -hex 24)
app=$(openssl rand -hex 24)
jq -n --arg admin "$admin" --arg app "$app" '{identities:[{name:"operator",credentials:[{accessKey:"operator",secretKey:$admin}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:"messenger",secretKey:$app}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$LOCAL_TEST_WORK/identities.json"
s3=$(docker run -d -p 127.0.0.1::9000 -v "$LOCAL_TEST_WORK/identities.json:/identity.json:ro" chrislusf/seaweedfs:3.85 server -dir=/data -s3 -s3.port=9000 -s3.config=/identity.json -master.volumeSizeLimitMB=100 -volume.max=5)
containers="$containers $s3"
port_a=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close')
port_b=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close')
cat > "$LOCAL_TEST_WORK/environment" <<ENV
export DATABASE_URL='postgres://postgres@$(docker port "$pg" 5432/tcp)/messenger?sslmode=disable'
export VALKEY_SHARDS='shard0=$(docker port "$vk" 6379/tcp)'
export S3_ENDPOINT='$(docker port "$s3" 9000/tcp)'
export S3_TLS=false
export S3_BUCKET=attachments
export S3_ACCESS_KEY=messenger
export S3_SECRET_KEY='$app'
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
export REQUEST_TIMEOUT_SECONDS=15
export LOG_LEVEL=error
export BROKER_ADDRESS='$(docker port "$broker" 6379/tcp)'
export LOCAL_TEST_BROKER='$broker'
export LOCAL_TEST_PG='$pg'
export LOCAL_TEST_VK='$vk'
export LOCAL_TEST_S3='$s3'
export LOCAL_TEST_PORT_A='$port_a'
export LOCAL_TEST_PORT_B='$port_b'
ENV
. "$LOCAL_TEST_WORK/environment"
GOTOOLCHAIN=go1.26.1 go build -o "$LOCAL_TEST_WORK/server" ./cmd/server
for attempt in $(seq 1 60); do
if S3_ACCESS_KEY=operator S3_SECRET_KEY="$admin" "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi
sleep 1
done
"$LOCAL_TEST_WORK/server" migrate
if [ "${LOCAL_COVERAGE:-false}" = true ]; then
test_vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine)
containers="$containers $test_vk"
export TEST_DATABASE_URL="$DATABASE_URL"
export TEST_VALKEY_ADDRESS=$(docker port "$test_vk" 6379/tcp)
export S3_ADMIN_SECRET="$admin"
GOTOOLCHAIN=go1.26.1 scripts/coverage.sh
exit 0
fi
for label in local-a local-b worker; do
port=$port_a
if [ "$label" = local-b ]; then port=$port_b; fi
if [ "$label" = worker ]; then
"$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/$label.log" 2>&1 &
else
INSTANCE_ID="$label" APP_PORT="$port" "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 &
fi
echo "$!" > "$LOCAL_TEST_WORK/$label.pid"
done
cat > "$LOCAL_TEST_WORK/nginx.conf" <<CONF
events {}
http {
upstream api {
server host.docker.internal:$port_a max_fails=1 fail_timeout=1s;
server host.docker.internal:$port_b max_fails=1 fail_timeout=1s;
}
server {
listen 8080;
client_max_body_size 12m;
location / {
proxy_pass http://api;
proxy_connect_timeout 1s;
proxy_read_timeout 15s;
proxy_next_upstream error timeout http_502 http_503;
}
}
}
CONF
lb=$(docker run -d --add-host=host.docker.internal:host-gateway -p 127.0.0.1::8080 -v "$LOCAL_TEST_WORK/nginx.conf:/etc/nginx/nginx.conf:ro" nginx:1.27-alpine)
containers="$containers $lb"
export BASE_URL="http://$(docker port "$lb" 8080/tcp)"
for attempt in $(seq 1 60); do
if curl --fail --silent --max-time 1 "$BASE_URL/health/ready" >/dev/null; then break; fi
sleep 1
done
make test > evidence/local/smoke.txt 2>&1
LOCAL_ACTION=failover make test-failover ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/failover.txt 2>&1
make test-persistence ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/persistence.txt 2>&1
make test > evidence/local/smoke-after.txt 2>&1
scripts/test-durable.sh > evidence/local/durable.txt 2>&1
GOTOOLCHAIN=go1.26.1 go version > evidence/local/go-version.txt
shasum -a 256 "$LOCAL_TEST_WORK/server" > evidence/local/binary.sha256
echo 'PASS: isolated smoke, attachment failover and persistence'