Forward the lab 4 foundation and migrate the lab 5 pipeline to SQLAlchemy
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SETTINGS = (
|
||||
"COMPOSE_PROJECT_NAME",
|
||||
"COMPOSE_FILE",
|
||||
"LB_HTTP_PORT",
|
||||
"LB_STATS_PORT",
|
||||
"BASE_URL",
|
||||
"EVIDENCE_DIR",
|
||||
"RECONCILE_EVIDENCE_DIR",
|
||||
"NO_PROXY",
|
||||
"S3_CPUS",
|
||||
"S3_MEMORY",
|
||||
"UPLOAD_GRACE_SECONDS",
|
||||
"API_IMAGE",
|
||||
"API_REPLICAS",
|
||||
"WEB_CONCURRENCY",
|
||||
"MALLOC_MMAP_THRESHOLD_",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Capture a command without overwriting evidence")
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
command = args.command[1:] if args.command[:1] == ["--"] else args.command
|
||||
if not command:
|
||||
parser.error("command is required")
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
started = time.monotonic()
|
||||
with args.output.open("x") as output:
|
||||
|
||||
def write(line):
|
||||
line = line.replace(str(ROOT), "<repo>")
|
||||
output.write(line)
|
||||
output.flush()
|
||||
sys.stdout.write(line)
|
||||
sys.stdout.flush()
|
||||
|
||||
write(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit: {commit}\n")
|
||||
write(
|
||||
"environment: "
|
||||
+ json.dumps({k: os.environ[k] for k in SETTINGS if k in os.environ})
|
||||
+ "\n"
|
||||
)
|
||||
write("$ " + shlex.join(command) + "\n")
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
errors="replace",
|
||||
) as process:
|
||||
for line in process.stdout:
|
||||
write(line)
|
||||
code = process.wait()
|
||||
write(f"exit_code: {code} elapsed_seconds: {time.monotonic() - started:.3f}\n")
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
repo=$PWD
|
||||
project="hlpipe-lab5-clean-$(date -u +%Y%m%d%H%M%S)"
|
||||
dir="$repo/.verification/$project"
|
||||
out="$repo/${EVIDENCE_DIR:-evidence}/clean-clone.txt"
|
||||
sha=$(git rev-parse HEAD)
|
||||
[ ! -e "$out" ] || { echo "Choose a new EVIDENCE_DIR" >&2; exit 2; }
|
||||
mkdir -p "$(dirname "$dir")" "$(dirname "$out")"
|
||||
exec 3>&1
|
||||
exec > "$out" 2>&1
|
||||
cleanup() {
|
||||
status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ -d "$dir/.git" ]; then
|
||||
cd "$dir"
|
||||
docker compose --profile bench --profile tools --profile reconcile down || status=1
|
||||
docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
fi
|
||||
echo "exit_code: $status; volumes retained"
|
||||
sed "s#$repo#<repo>#g" "$out" >&3
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' HUP INT TERM
|
||||
step() { echo "\$ $*"; "$@"; }
|
||||
export COMPOSE_PROJECT_NAME="$project" COMPOSE_FILE=compose.yaml
|
||||
export LB_HTTP_PORT=18085 LB_STATS_PORT=25085 BROKER_MANAGEMENT_PORT=25086 BASE_URL=http://127.0.0.1:18085
|
||||
export NO_PROXY=localhost,127.0.0.1,::1
|
||||
unset API_IMAGE
|
||||
echo "date: $(date -u +%FT%TZ) commit: $sha project: $project"
|
||||
echo "HTTP=$LB_HTTP_PORT stats=$LB_STATS_PORT"
|
||||
step git clone --quiet --no-hardlinks "$repo" "$dir"
|
||||
cd "$dir"
|
||||
step git checkout --quiet --detach "$sha"
|
||||
step make secrets
|
||||
step docker compose --env-file .env config --quiet
|
||||
step make up
|
||||
step docker compose ps -a --format '{{.Service}} {{.State}} {{.Status}} {{.Ports}}'
|
||||
step make test
|
||||
step make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
step make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
step docker compose ps -a --format '{{.Service}} {{.Name}} {{.State}}' api
|
||||
step make restore
|
||||
step make scenario SCENARIO=failover-upload
|
||||
step make restore
|
||||
step make scenario SCENARIO=s3-outage
|
||||
step sh scripts/s3-check.sh
|
||||
step make reconcile
|
||||
step make migrate
|
||||
step make down
|
||||
step docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
@@ -8,7 +8,7 @@ replica() { docker compose ps -q "${API_SERVICE:-api}" | head -1; }
|
||||
detect() { python3 scripts/probe.py detect "$@"; }
|
||||
failed=0
|
||||
detect --max 1.5 --within-action --restore scripts/restore.sh --out "$out/sigterm.json" \
|
||||
-- docker stop -t "${STOP_TIMEOUT:-6}" "$(replica)" || failed=1
|
||||
-- docker stop -t "${STOP_TIMEOUT:-7}" "$(replica)" || failed=1
|
||||
detect --max 1.5 --restore scripts/restore.sh --out "$out/kill.json" \
|
||||
-- docker kill "$(replica)" || failed=1
|
||||
hung=$(replica)
|
||||
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: docker-stats.sh <csv>}
|
||||
vm=$(dirname "$out")/vm_stat.csv
|
||||
cd "$(dirname "$0")/.."
|
||||
project=$(docker compose config | sed -n 's/^name: //p')
|
||||
echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out"
|
||||
echo 'ts,busy,total,cpus' > "$vm"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true
|
||||
ids=$(docker ps -q --filter "label=com.docker.compose.project=$project")
|
||||
[ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
@@ -1,12 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -X -c "$1"; }
|
||||
value() { docker compose exec -T postgres psql -U messenger -d messenger -X -tA -c "$1"; }
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
bucket=${S3_BUCKET:-messenger-attachments}
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
|
||||
echo "\$ docker compose ps --format '{{.Service}} {{.Status}} {{.Ports}}'"
|
||||
docker compose ps -a --format '{{.Service}} {{.Status}} {{.Ports}}'
|
||||
for table in attachments message_attachments; do
|
||||
@@ -36,5 +38,5 @@ bucket=${S3_BUCKET:-messenger-attachments}
|
||||
docker compose config --format json | python3 -c 'import json,sys
|
||||
for name, service in sorted(json.load(sys.stdin)["services"].items()):
|
||||
print(name, sorted(s["source"] for s in service.get("secrets", [])))'
|
||||
} > evidence/storage-layout.txt
|
||||
cat evidence/storage-layout.txt
|
||||
} > "$out"/storage-layout.txt
|
||||
cat "$out"/storage-layout.txt
|
||||
|
||||
@@ -11,14 +11,25 @@ cli "info persistence" | grep -E '^aof_enabled|^aof_last_write_status'
|
||||
echo "\$ valkey-cli dbsize; info stats | grep keyspace"
|
||||
cli dbsize
|
||||
cli "info stats" | grep -E '^keyspace_(hits|misses)'
|
||||
key=$(cli "--scan --pattern session:* --count 1000" | head -1)
|
||||
hash=${key#session:}
|
||||
echo "\$ сравнение срока ключа кэша и срока сессии для одного ключа (значение и токен не печатаются)"
|
||||
echo "valkey pexpiretime, ms: $(cli "pexpiretime $key")"
|
||||
echo "postgres expires_at, ms: $(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('$hash', 'hex')")"
|
||||
echo "\$ количество отметок отзыва и закэшированных сессий"
|
||||
revoked=0; cached=0
|
||||
echo "\$ valkey-cli --scan --pattern 'session:*' и вид значения (хеш сокращён, значения не печатаются)"
|
||||
revoked=0; cached=0; key=""; stored=""
|
||||
for k in $(cli "--scan --pattern session:* --count 1000"); do
|
||||
if [ "$(cli "get $k")" = "-" ]; then revoked=$((revoked + 1)); else cached=$((cached + 1)); fi
|
||||
if [ "$(cli "get $k")" = "-" ]; then
|
||||
revoked=$((revoked + 1)); printf '%.16s… отметка отзыва "-"\n' "$k"; continue
|
||||
fi
|
||||
cached=$((cached + 1)); printf '%.16s… сессия\n' "$k"
|
||||
if [ -z "$key" ]; then
|
||||
stored=$(psql "SELECT (extract(epoch FROM expires_at) * 1000)::bigint FROM sessions WHERE token_hash = decode('${k#session:}', 'hex')")
|
||||
if [ -n "$stored" ]; then key=$k; fi
|
||||
fi
|
||||
done
|
||||
echo "tombstones: $revoked, cached sessions: $cached"
|
||||
if [ "$revoked" -eq 0 ] || [ -z "$key" ]; then
|
||||
echo "нужны закэшированная сессия из БД и отметка отзыва: сначала make test" >&2
|
||||
exit 1
|
||||
fi
|
||||
expires=$(cli "pexpiretime $key")
|
||||
echo "\$ срок ключа кэша против срока сессии для $(printf '%.16s' "$key")…"
|
||||
echo "valkey pexpiretime, ms: $expires"
|
||||
echo "postgres expires_at, ms: $stored"
|
||||
if [ "$expires" -le "$stored" ]; then echo "ключ не переживает сессию: да"; else echo "ключ не переживает сессию: НЕТ"; exit 1; fi
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
base=${1:-http://$(docker compose port lb 8080)}
|
||||
stats=http://$(docker compose port lb 8404)/stats\;csv
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
|
||||
{
|
||||
header
|
||||
echo "\$ docker compose --env-file .env config --quiet && echo valid"
|
||||
@@ -25,7 +27,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
print(r["svname"], r["status"], r["check_status"], r["check_code"])'
|
||||
echo "\$ for i in 1..6: curl -D - $base/health/live | X-Instance-Id"
|
||||
for _ in 1 2 3 4 5 6; do curl -fsS -D - -o /dev/null "$base/health/live" | tr -d '\r' | grep -i '^x-instance-id'; done
|
||||
} > evidence/stack.txt
|
||||
} > "$out"/stack.txt
|
||||
{
|
||||
header
|
||||
echo "host: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m), $(sysctl -n hw.ncpu 2>/dev/null) CPU, $(($(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824)) GiB RAM, $(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)"
|
||||
@@ -38,7 +40,7 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
|
||||
echo "rabbitmq: $(docker compose exec -T broker rabbitmqctl -q version 2>/dev/null | head -1)"
|
||||
echo "packages (uv.lock):"
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
|
||||
for name in fastapi uvicorn uvicorn-worker gunicorn sqlalchemy alembic asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow \
|
||||
faststream aio-pika aiormq dishka-faststream; do
|
||||
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
|
||||
done
|
||||
@@ -47,5 +49,5 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
|
||||
service=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$id")
|
||||
docker top "$id" -eo pid,uid,comm | awk -v s="$service" 'NR > 1 {print " " s, $2, $3}'
|
||||
done | sort -u
|
||||
} > evidence/environment.txt
|
||||
cat evidence/stack.txt evidence/environment.txt
|
||||
} > "$out"/environment.txt
|
||||
cat "$out"/stack.txt "$out"/environment.txt
|
||||
|
||||
+16
-13
@@ -1,20 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
psql() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; }
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM schema_migrations) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, count(*) AS users, min(length(password_hash)) AS hash_length FROM users GROUP BY 1"
|
||||
header() { echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"; }
|
||||
counts="SELECT (SELECT count(*) FROM users) users, (SELECT count(*) FROM sessions) sessions, (SELECT count(*) FROM chats) chats, (SELECT count(*) FROM chat_members) members, (SELECT count(*) FROM messages) messages, (SELECT count(*) FROM idempotency_keys) keys, (SELECT count(*) FROM attachments) attachments, (SELECT count(*) FROM message_attachments) links, (SELECT count(*) FROM alembic_version) versions"
|
||||
kdf="SELECT substring(password_hash from '^[$]argon2id[$]v=[0-9]+[$]m=[0-9]+,t=[0-9]+,p=[0-9]+[$]') AS kdf_prefix, username = 'seed_author' AS seed_stub, count(*) AS users, min(length(password_hash)) AS min_length, max(length(password_hash)) AS max_length FROM users GROUP BY 1, 2 ORDER BY 2"
|
||||
tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FROM sessions GROUP BY 1"
|
||||
{
|
||||
header
|
||||
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments schema_migrations; do
|
||||
for table in users sessions chats chat_members messages idempotency_keys attachments message_attachments alembic_version; do
|
||||
echo "\$ psql -c '\\d+ $table'"
|
||||
psql "\\d+ $table"
|
||||
done
|
||||
echo "\$ psql -c 'SELECT version, left(checksum, 12), applied_at FROM schema_migrations'"
|
||||
psql "SELECT version, left(checksum, 12) AS checksum, applied_at FROM schema_migrations"
|
||||
} > evidence/schema.txt
|
||||
echo "\$ psql -c 'SELECT version_num FROM alembic_version'"
|
||||
psql "SELECT version_num FROM alembic_version"
|
||||
} > "$out"/schema.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$counts\""
|
||||
@@ -23,14 +25,14 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
|
||||
docker compose run --rm migrate 2>/dev/null
|
||||
echo "\$ psql -c \"$counts\""
|
||||
psql "$counts"
|
||||
} > evidence/migrations.txt
|
||||
} > "$out"/migrations.txt
|
||||
{
|
||||
header
|
||||
echo "\$ psql -c \"$kdf\""
|
||||
psql "$kdf"
|
||||
echo "\$ psql -c \"$tokens\""
|
||||
psql "$tokens"
|
||||
} > evidence/kdf.txt
|
||||
} > "$out"/kdf.txt
|
||||
{
|
||||
header
|
||||
echo "\$ git ls-files secrets .env backups volumes"
|
||||
@@ -38,7 +40,8 @@ tokens="SELECT octet_length(token_hash) AS stored_bytes, count(*) AS sessions FR
|
||||
echo "(пусто: секретов, .env, дампов и volumes в Git нет)"
|
||||
echo "\$ git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/"
|
||||
git check-ignore -v secrets/database_url secrets/postgres_password secrets/s3_root_password secrets/s3_secret_access_key .env backups/
|
||||
echo "\$ docker volume ls --filter name=mtusi2026lab5"
|
||||
docker volume ls --filter name=mtusi2026lab5
|
||||
} > evidence/secrets.txt
|
||||
echo "evidence/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
project=$(docker compose config --format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"])')
|
||||
echo "\$ docker volume ls --filter label=com.docker.compose.project=$project"
|
||||
docker volume ls --filter "label=com.docker.compose.project=$project"
|
||||
} > "$out"/secrets.txt
|
||||
echo ""$out"/{schema,migrations,kdf,secrets}.txt обновлены"
|
||||
|
||||
@@ -10,7 +10,7 @@ failed=0
|
||||
set --
|
||||
for host in $hosts; do
|
||||
TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \
|
||||
-- sh -c "scripts/stop-one.sh && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1
|
||||
-- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1
|
||||
scripts/restore.sh
|
||||
set -- "$@" "$out/$host/result.json"
|
||||
done
|
||||
|
||||
@@ -1,28 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
replicas() {
|
||||
docker inspect -f '{{.Config.Hostname}} {{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' $(docker compose ps -aq "${API_SERVICE:-api}")
|
||||
}
|
||||
tmp=evidence/failover.tmp
|
||||
url=${1:-http://localhost:8080}
|
||||
shift=0
|
||||
while [ "$shift" -lt "${2:-0}" ]; do curl -fsS -o /dev/null "$url/health/live"; shift=$((shift + 1)); done
|
||||
tmp="$out"/failover.tmp
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
|
||||
echo "== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt"
|
||||
replicas
|
||||
echo "\$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
make --no-print-directory test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$url"
|
||||
echo "== after: api replicas"
|
||||
replicas
|
||||
} > "$tmp" 2>&1
|
||||
stopped=$(replicas | awk '$2 == "exited" {print $1}')
|
||||
{
|
||||
echo "== stopped replica: $stopped; shutdown: signal, drain window with readiness 503, close, exit"
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|Draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
docker logs --since "$since" "$stopped" 2>&1 | grep -E 'Handling signal|service.draining|"status":503|Shutting down|Waiting for conn|Finished server|service.stopped|SIGTERM' | tail -16
|
||||
echo "\$ scripts/restore.sh"
|
||||
scripts/restore.sh
|
||||
echo "== after restore"
|
||||
replicas
|
||||
} >> "$tmp" 2>&1
|
||||
mv "$tmp" "evidence/failover-$stopped.txt"
|
||||
cat "evidence/failover-$stopped.txt"
|
||||
mv "$tmp" "$out/failover-$stopped.txt"
|
||||
cat "$out/failover-$stopped.txt"
|
||||
|
||||
@@ -23,6 +23,19 @@ def compose(*args, stdin=None):
|
||||
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def containers(service):
|
||||
ids = compose("ps", "-aq", service).stdout.split()
|
||||
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
|
||||
template = (
|
||||
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
|
||||
" {{.State.StartedAt}} {{.State.FinishedAt}}"
|
||||
)
|
||||
out = subprocess.run(
|
||||
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
|
||||
)
|
||||
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
|
||||
|
||||
|
||||
def upload(c, actor, chat, expected=202):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
||||
schema = "Attachment" if expected == 202 else None
|
||||
@@ -90,6 +103,10 @@ def failover_upload(c, args):
|
||||
recovered = time.monotonic() - started
|
||||
served = got["headers"].get("X-Instance-Id")
|
||||
check(served != target, "download served by the stopped replica")
|
||||
during = containers(args.service)
|
||||
states = {item["hostname"]: item["status"] for item in during}
|
||||
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
|
||||
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
|
||||
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
||||
check(
|
||||
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
||||
@@ -104,7 +121,9 @@ def failover_upload(c, args):
|
||||
"sha256_equal": True,
|
||||
"hook_seconds": round(hook_seconds, 3),
|
||||
"download_ok_after_seconds": round(recovered, 3),
|
||||
"replicas_at_download": during,
|
||||
"restored": args.restore,
|
||||
"replicas_after_restore": containers(args.service) if args.restore else None,
|
||||
}
|
||||
|
||||
|
||||
@@ -124,6 +143,7 @@ def s3_outage(c, args):
|
||||
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
||||
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
||||
}
|
||||
stopped = containers(args.s3_service)
|
||||
finally:
|
||||
started = time.monotonic()
|
||||
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
||||
@@ -131,8 +151,11 @@ def s3_outage(c, args):
|
||||
got = wait(lambda: content(c, bob, att), 30)
|
||||
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
||||
expected = {"upload": 503, "download": 503, "text_message": 201}
|
||||
check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}")
|
||||
check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded")
|
||||
return {
|
||||
"scenario": "s3-outage",
|
||||
"s3_container_during_outage": stopped,
|
||||
"during_outage": during,
|
||||
"expected": expected,
|
||||
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
||||
@@ -180,6 +203,8 @@ def abort_upload(c, args):
|
||||
)
|
||||
rows = out.stdout.split()
|
||||
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
||||
check(out.returncode == 0, "cannot inspect aborted upload")
|
||||
check(counts.get("ready", 0) == 0, "aborted upload became ready")
|
||||
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
||||
return {
|
||||
"scenario": "abort-upload",
|
||||
@@ -216,10 +241,17 @@ def main():
|
||||
"s3-outage": s3_outage,
|
||||
"abort-upload": abort_upload,
|
||||
}[args.scenario]
|
||||
commit = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False
|
||||
)
|
||||
stamp = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"commit": commit.stdout.decode().strip(),
|
||||
}
|
||||
try:
|
||||
report = {"ok": True, **run(c, args)}
|
||||
report = {"ok": True, **stamp, **run(c, args)}
|
||||
except Failure as exc:
|
||||
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
|
||||
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n")
|
||||
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: lb-stats.sh <csv>}
|
||||
cd "$(dirname "$0")/.."
|
||||
url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv}
|
||||
curl -fsS "$url" | sed -n '1s/^# /ts,/p' > "$out"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
curl -fsS "$url" | grep -E '^(public|api),' | sed "s/^/$ts,/" >> "$out" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
+29
-5
@@ -9,12 +9,20 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3_CLIENT_SECRETS = [
|
||||
"s3_access_key_id",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
"s3_worker_access_key_id",
|
||||
"s3_worker_secret_access_key",
|
||||
"s3_root_password",
|
||||
"s3_root_user",
|
||||
"s3_secret_access_key",
|
||||
"s3_worker_access_key_id",
|
||||
"s3_worker_secret_access_key",
|
||||
]
|
||||
RECONCILER_SECRETS = [
|
||||
"database_url",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
]
|
||||
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
|
||||
POLICY = {
|
||||
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
|
||||
"api": {
|
||||
@@ -29,6 +37,11 @@ POLICY = {
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
|
||||
"reconciler": {
|
||||
"secrets": RECONCILER_SECRETS,
|
||||
"volumes": [],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
|
||||
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
|
||||
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
|
||||
@@ -94,6 +107,9 @@ def declared(name, service, networks, problems):
|
||||
problems.append(f"{name}: publishes {ports} beyond loopback")
|
||||
if name == "api" and not service.get("read_only"):
|
||||
problems.append("api: root filesystem is writable")
|
||||
user = str(service.get("user", ""))
|
||||
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
|
||||
problems.append(f"{name}: runs as root")
|
||||
internal = [n for n in nets if networks.get(n, {}).get("internal")]
|
||||
return [
|
||||
name,
|
||||
@@ -103,6 +119,7 @@ def declared(name, service, networks, problems):
|
||||
",".join(volumes) or "-",
|
||||
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
|
||||
str(service.get("read_only", False)).lower(),
|
||||
user or "image",
|
||||
]
|
||||
|
||||
|
||||
@@ -149,6 +166,8 @@ def main():
|
||||
"bench",
|
||||
"--profile",
|
||||
"tools",
|
||||
"--profile",
|
||||
"reconcile",
|
||||
"config",
|
||||
"--format",
|
||||
"json",
|
||||
@@ -162,9 +181,14 @@ def main():
|
||||
if not any(row[0] == "api" for row in containers):
|
||||
problems.append("no running api replicas to inspect")
|
||||
lines = [
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
|
||||
"$ docker compose --profile bench --profile tools config --format json (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} "
|
||||
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
|
||||
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
|
||||
" (declared)",
|
||||
*table(
|
||||
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
|
||||
services,
|
||||
),
|
||||
"",
|
||||
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
|
||||
*table(["service", "hostname", "cpus", "memory", "published"], containers),
|
||||
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
runs="${RUNS:-3}"
|
||||
pause="${PAUSE:-15}"
|
||||
project=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$(docker compose ps -q lb)")
|
||||
network="${project}_edge"
|
||||
image=$(sed -n 's/^FROM \(python:[^ ]*\) AS runtime$/\1/p' Dockerfile)
|
||||
name="$project-load-client"
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out/load"
|
||||
[ -z "$(ls -A "$out/load")" ] || { echo "Choose an empty EVIDENCE_DIR/load" >&2; exit 2; }
|
||||
client() {
|
||||
docker run --rm --name "$name" --network "$network" --cpus 2 --memory 512m \
|
||||
--sysctl net.ipv4.tcp_tw_reuse=1 -v "$PWD/tests:/w/tests:ro" -v "$PWD/contracts:/w/contracts:ro" \
|
||||
-v "$PWD/lab.json:/w/lab.json:ro" -v "$PWD/$out/load:/out" "$image" python /w/tests/load.py "$@"
|
||||
}
|
||||
handled() {
|
||||
docker compose logs --no-log-prefix --since "$1" api | python3 -c 'import collections, json, sys
|
||||
seen = collections.Counter()
|
||||
for line in sys.stdin:
|
||||
if line.startswith("{"):
|
||||
event = json.loads(line)
|
||||
if event.get("event") == "request.handled":
|
||||
seen[(event["status"], event["method"], event["route"])] += 1
|
||||
for (status, method, route), count in sorted(seen.items()):
|
||||
print(status, method, route, count)'
|
||||
}
|
||||
{
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) network: $network"
|
||||
echo "# make load-evidence (scripts/load-evidence.sh): закрытый контур tests/load.py, 64 потока, 8 с, все записи"
|
||||
echo "# прогона в один новый чат, $runs повтора с паузой $pause с; клиент — одноразовый контейнер в сети <проект>_edge"
|
||||
echo "# стенда (2 CPU, 512 MiB, базовый образ API по digest, tests/, contracts/ и lab.json только для чтения,"
|
||||
echo "# tcp_tw_reuse=1), адрес http://lb:8080. На 4-й секунде — docker stats; после прогона — request.handled реплик."
|
||||
echo "\$ docker ps --format '{{.Names}}' | grep -v '^$project-' # чужие контейнеры на VM во время серии"
|
||||
docker ps --format '{{.Names}}' | grep -v "^$project-" || true
|
||||
echo "\$ client() { docker run --rm --name $name --network $network --cpus 2 --memory 512m --sysctl net.ipv4.tcp_tw_reuse=1 -v \"\$PWD/tests:/w/tests:ro\" -v \"\$PWD/contracts:/w/contracts:ro\" -v \"\$PWD/lab.json:/w/lab.json:ro\" -v \"\$PWD/$out/load:/out\" $image python /w/tests/load.py \"\$@\"; }"
|
||||
run=1
|
||||
while [ "$run" -le "$runs" ]; do
|
||||
since=$(date -u +%FT%TZ)
|
||||
echo "\$ client --base-url http://lb:8080 --duration 8 --concurrency 64 --output /out/run$run.json"
|
||||
client --base-url http://lb:8080 --duration 8 --concurrency 64 --output "/out/run$run.json" > "$work/out" 2>&1 &
|
||||
pid=$!
|
||||
sleep 4
|
||||
stats=$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' $(docker compose ps -q) "$name" | sed "s/^$project-//" | sort)
|
||||
if wait "$pid"; then code=0; else code=$?; fi
|
||||
echo "exit $code"
|
||||
cat "$work/out"
|
||||
echo "\$ docker stats --no-stream # 4-я секунда прогона: контейнер CPU% память"
|
||||
echo "$stats"
|
||||
echo "\$ docker compose logs --since $since api | request.handled: статус метод маршрут число"
|
||||
handled "$since"
|
||||
[ "$code" -eq 0 ] || { exit "$code"; }
|
||||
[ "$run" -lt "$runs" ] && sleep "$pause"
|
||||
run=$((run + 1))
|
||||
done
|
||||
echo "\$ docker compose exec -T postgres psql -c 'SELECT count(*) FROM messages'"
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -qAt -c 'SELECT count(*) FROM messages'
|
||||
} > "$out/load.txt"
|
||||
cat "$out/load.txt"
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
limit="${WAIT_SECONDS:-1500}"
|
||||
out=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$out"
|
||||
sql() { docker compose exec -T postgres psql -U messenger -d messenger -X "$@"; }
|
||||
pending() { sql -tAc "SELECT count(*) FROM attachments WHERE status = 'pending'"; }
|
||||
statuses="SELECT status, count(*), min(created_at), max(created_at) FROM attachments GROUP BY status ORDER BY status"
|
||||
since=$(date -u +%FT%TZ)
|
||||
{
|
||||
echo "date: $since commit: $(git rev-parse HEAD)"
|
||||
echo "# make overload-leftovers (scripts/overload-leftovers.sh) после серии памяти: невидимые намерения от 503 по"
|
||||
echo "# таймауту видны сверке свежими, уборка API удаляет их после UPLOAD_GRACE_SECONDS; стенд тот же"
|
||||
echo "\$ make reconcile # dry-run"
|
||||
docker compose run --rm --no-deps -T reconciler 2>/dev/null
|
||||
echo "\$ psql -c \"$statuses\""
|
||||
sql -c "$statuses"
|
||||
echo "\$ until [ \"\$(psql -tAc \"SELECT count(*) FROM attachments WHERE status = 'pending'\")\" = 0 ]; do sleep 10; done"
|
||||
started=$(date +%s)
|
||||
until [ "$(pending)" = 0 ] || [ $(( $(date +%s) - started )) -gt "$limit" ]; do sleep 10; done
|
||||
echo "pending = $(pending) в $(date -u +%FT%TZ)"
|
||||
[ "$(pending)" = 0 ] || { echo "FAIL: pending cleanup timed out"; exit 1; }
|
||||
echo "\$ docker compose logs --since $since api | audit.attachment_discarded и attachments.swept"
|
||||
docker compose logs --no-log-prefix --since "$since" api | python3 -c 'import collections, json, sys
|
||||
reasons, swept, times = collections.Counter(), 0, []
|
||||
for line in sys.stdin:
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
event = json.loads(line)
|
||||
if event.get("event") == "audit.attachment_discarded":
|
||||
reasons[event["reason"]] += 1
|
||||
times.append(event["ts"][:19] + "Z")
|
||||
elif event.get("event") == "attachments.swept":
|
||||
swept += event["count"]
|
||||
print(dict(reasons), "swept", swept, "first", min(times, default="-"), "last", max(times, default="-"))'
|
||||
echo "\$ psql -c \"$statuses\""
|
||||
sql -c "$statuses"
|
||||
echo "\$ make reconcile # dry-run после уборки"
|
||||
docker compose run --rm --no-deps -T reconciler 2>/dev/null
|
||||
} > "$out/overload-leftovers.txt"
|
||||
cat "$out/overload-leftovers.txt"
|
||||
@@ -6,7 +6,7 @@ started() {
|
||||
docker inspect --format '{{index .Config.Labels "com.docker.compose.service"}} {{.Config.Hostname}} {{.State.StartedAt}} {{.State.Status}}' "$id"
|
||||
done | sort
|
||||
}
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD)"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD)"
|
||||
echo "== before: service hostname StartedAt status"
|
||||
started
|
||||
make --no-print-directory test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL="${1:-http://localhost:8080}"
|
||||
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mode=${1:?usage: pg-stats.sh activity <csv> | statements <csv> | size | reset}
|
||||
cd "$(dirname "$0")/.."
|
||||
psql() {
|
||||
docker compose exec -T postgres sh -c 'exec psql -X -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "${POSTGRES_DB:-$POSTGRES_USER}" "$@"' psql "$@"
|
||||
}
|
||||
case $mode in
|
||||
reset)
|
||||
psql -c 'set client_min_messages = warning' -c 'create extension if not exists pg_stat_statements' -c 'select pg_stat_statements_reset()' >/dev/null
|
||||
;;
|
||||
size)
|
||||
psql -At -F, -c "select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
|
||||
;;
|
||||
statements)
|
||||
psql -c "copy (select calls, round(total_exec_time::numeric, 3) as total_ms, round(mean_exec_time::numeric, 3) as mean_ms, rows, shared_blks_hit, shared_blks_read, left(regexp_replace(query, '\s+', ' ', 'g'), 200) as query from pg_stat_statements where dbid = (select oid from pg_database where datname = current_database()) and query not like '%pg_stat%' order by total_exec_time desc limit ${TOP:-20}) to stdout with csv header" > "${2:?csv}"
|
||||
;;
|
||||
activity)
|
||||
echo 'ts,state,wait_event_type,wait_event,backends,max_connections' > "${2:?csv}"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
psql -At -F, -c "select coalesce(state, ''), coalesce(wait_event_type, ''), coalesce(wait_event, ''), count(*), current_setting('max_connections') from pg_stat_activity where backend_type = 'client backend' and pid <> pg_backend_pid() group by 1, 2, 3" | sed "s/^/$ts,/" >> "$2" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "unknown mode $mode" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
+163
-7
@@ -10,12 +10,16 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx")
|
||||
PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"]
|
||||
PSQL += ["-X", "-q", "-At"]
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -207,16 +211,112 @@ def check(args):
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def output(*command):
|
||||
done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
return done.stdout + done.stderr
|
||||
|
||||
|
||||
def replicas():
|
||||
ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split()
|
||||
form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"
|
||||
return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()]
|
||||
|
||||
|
||||
def answered(container, request_id):
|
||||
for line in output("docker", "logs", container).splitlines():
|
||||
if request_id in line and "request.handled" in line:
|
||||
return json.loads(line).get("status")
|
||||
return None
|
||||
|
||||
|
||||
def lost(alice, bob, chat, kill, args):
|
||||
key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex
|
||||
request_id = "lost-" + key[:12]
|
||||
lock = subprocess.Popen(
|
||||
PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
lock.stdin.write(
|
||||
f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n"
|
||||
f"SELECT pg_sleep({args.hold});\nCOMMIT;\n"
|
||||
)
|
||||
lock.stdin.flush()
|
||||
lock.stdout.readline()
|
||||
sent = time.time()
|
||||
headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id})
|
||||
path = f"/api/v1/chats/{chat}/messages"
|
||||
first = call("POST", path, {"text": text}, headers, args.timeout)[0]
|
||||
sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"
|
||||
waiting = set(output(*PSQL, "-c", sql).split())
|
||||
holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None])
|
||||
if kill and holder[0]:
|
||||
subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL)
|
||||
lock.stdin.close()
|
||||
lock.wait()
|
||||
retried = time.time()
|
||||
attempts, (status, instance, body) = 1, send(alice, chat, text, key)
|
||||
while status != 201 and attempts < 20:
|
||||
time.sleep(0.2)
|
||||
attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key)
|
||||
page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []}
|
||||
copies = [m for m in page["items"] if m["text"] == text]
|
||||
stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None
|
||||
report = {
|
||||
"variant": "kill" if kill else "timeout",
|
||||
"lost_attempt": {
|
||||
"status": first,
|
||||
"client_timeout_s": args.timeout,
|
||||
"chat_lock_held_s": args.hold,
|
||||
"replica": holder[1],
|
||||
"replica_killed": bool(kill and holder[0]),
|
||||
"replica_answered": answered(holder[0], request_id) if holder[0] else None,
|
||||
},
|
||||
"retry": {
|
||||
"after_s": round(retried - sent, 3),
|
||||
"status": status,
|
||||
"attempts": attempts,
|
||||
"replica": instance,
|
||||
},
|
||||
"copies_in_history": len(copies),
|
||||
"same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"),
|
||||
"written_by": None
|
||||
if stamp is None
|
||||
else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"),
|
||||
}
|
||||
if kill and holder[0]:
|
||||
subprocess.call(
|
||||
[str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
report["pass"] = (
|
||||
first == 0
|
||||
and holder[0] is not None
|
||||
and status == 201
|
||||
and report["copies_in_history"] == 1
|
||||
and report["same_message"]
|
||||
and (kill or report["lost_attempt"]["replica_answered"] == 201)
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def retry(args):
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
chat = chat_of(alice, bob)
|
||||
variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)]
|
||||
passed = all(v["pass"] for v in variants)
|
||||
emit({"variants": variants, "pass": passed}, args.out)
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
probes, trace, confirmed = [], [], []
|
||||
|
||||
def prober():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.time()
|
||||
status, instance, _ = send(alice, chat, "probe " + key, key)
|
||||
status, instance, body = send(alice, chat, "probe " + key, key)
|
||||
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
||||
if status == 201:
|
||||
confirmed.append(body)
|
||||
key = uuid.uuid4().hex
|
||||
time.sleep(0.1)
|
||||
|
||||
@@ -228,7 +328,38 @@ def record(stop, alice, chat):
|
||||
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
return probes, trace, threads
|
||||
return probes, trace, threads, confirmed
|
||||
|
||||
|
||||
def history_check(reader, chat, confirmed):
|
||||
items, cursors = [], set()
|
||||
cursor = None
|
||||
for _ in range(len(confirmed) // 100 + 2):
|
||||
path = f"/api/v1/chats/{chat}/messages?limit=100"
|
||||
if cursor:
|
||||
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
|
||||
status, _, page = call("GET", path, headers=reader["auth"])
|
||||
if status != 200 or not isinstance(page, dict):
|
||||
break
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
|
||||
return {
|
||||
"confirmed": len(confirmed),
|
||||
"stored": len(items),
|
||||
"exact_match": bool(confirmed) and same,
|
||||
"unique": len({m["id"] for m in items}) == len(items),
|
||||
}
|
||||
if cursor in cursors:
|
||||
break
|
||||
cursors.add(cursor)
|
||||
return {
|
||||
"confirmed": len(confirmed),
|
||||
"stored": len(items),
|
||||
"exact_match": False,
|
||||
"unique": False,
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path, header, rows):
|
||||
@@ -241,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
|
||||
t0, end = action["start"], action["end"]
|
||||
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
||||
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
||||
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
|
||||
recovery = (
|
||||
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
|
||||
)
|
||||
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
||||
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
||||
outage = [t for t in down if not back or t < back[0]] or [end]
|
||||
@@ -256,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
|
||||
"action_finished_before_load_end": end < action["load_end"],
|
||||
"lb_up_before": expected,
|
||||
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
||||
"lb_samples_missing": sum(up is None for _, up in trace),
|
||||
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
||||
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
||||
"probe": {
|
||||
"attempts": len(probes),
|
||||
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
||||
"first_failure_s": round(failures[0], 3) if failures else None,
|
||||
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
||||
"first_success_s": round(successes[0][1], 3) if successes else None,
|
||||
"recovery_s": round(recovery, 3) if recovery is not None else None,
|
||||
"instances_before": dict(before),
|
||||
"instances_during_outage": dict(during),
|
||||
"instances_after": dict(after),
|
||||
@@ -275,7 +412,13 @@ def verdict(args, result):
|
||||
checks = {
|
||||
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
||||
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
||||
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
||||
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
|
||||
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
|
||||
"write_recovery_ok": all(
|
||||
probe[k] is not None and probe[k] <= args.max_recovery
|
||||
for k in ("first_success_s", "recovery_s")
|
||||
),
|
||||
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
|
||||
}
|
||||
if args.expect == "dip":
|
||||
checks["detected"] = result["detection_seconds"] is not None
|
||||
@@ -296,7 +439,8 @@ def load(args):
|
||||
expected = lb_up() or 0
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
stop, action = threading.Event(), {}
|
||||
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
||||
chat = chat_of(alice, bob)
|
||||
probes, trace, threads, confirmed = record(stop, alice, chat)
|
||||
|
||||
def act():
|
||||
time.sleep(args.warmup + args.lead)
|
||||
@@ -327,6 +471,7 @@ def load(args):
|
||||
)
|
||||
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
||||
result = summarize(args, probes, trace, action, expected)
|
||||
result["history"] = history_check(bob, chat, confirmed)
|
||||
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
||||
result["k6"] = dict(
|
||||
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
||||
@@ -344,10 +489,13 @@ def detect(args):
|
||||
started = time.time()
|
||||
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
||||
detected = acted = None
|
||||
missing = 0
|
||||
while time.time() - started < args.timeout and (detected is None or acted is None):
|
||||
if acted is None and process.poll() is not None:
|
||||
acted = time.time() - started
|
||||
if detected is None and (lb_up() or 0) <= until:
|
||||
up = lb_up()
|
||||
missing += up is None
|
||||
if detected is None and up is not None and up <= until:
|
||||
detected = time.time() - started
|
||||
time.sleep(0.05)
|
||||
code = process.wait()
|
||||
@@ -362,6 +510,7 @@ def detect(args):
|
||||
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
||||
checks = {
|
||||
"action_ok": code == 0,
|
||||
"lb_observed": before >= 2 and missing == 0,
|
||||
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
||||
"lb_answer_ok": status == args.lb_status,
|
||||
}
|
||||
@@ -374,6 +523,7 @@ def detect(args):
|
||||
"action_exit": code,
|
||||
"action_seconds": round(acted, 3),
|
||||
"lb_up_before": before,
|
||||
"lb_samples_missing": missing,
|
||||
"lb_up_counted_as_detected": until,
|
||||
"detection_seconds": None if detected is None else round(detected, 3),
|
||||
"declared_max_seconds": args.max,
|
||||
@@ -413,6 +563,10 @@ def main():
|
||||
under.add_argument("--max-failed-ratio", type=float, default=0.01)
|
||||
under.add_argument("--max-recovery", type=float, default=10.0)
|
||||
under.add_argument("command", nargs=argparse.REMAINDER)
|
||||
again = sub.add_parser("retry", help="retry a write whose answer was lost, same key")
|
||||
again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)")
|
||||
again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s")
|
||||
again.add_argument("--out", help="JSON file, written only after a complete report")
|
||||
watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation")
|
||||
watch.add_argument("--max", type=float, required=True, help="declared worst detection, s")
|
||||
watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s")
|
||||
@@ -426,6 +580,8 @@ def main():
|
||||
args = parser.parse_args()
|
||||
if args.mode == "check":
|
||||
return check(args)
|
||||
if args.mode == "retry":
|
||||
return retry(args)
|
||||
args.command = [c for c in args.command if c != "--"]
|
||||
if not args.command:
|
||||
parser.error(f"{args.mode} needs an action command after --")
|
||||
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
|
||||
count="${2:-1}"
|
||||
age="${INJECT_AGE_SECONDS:-7200}"
|
||||
prefix="${S3_PREFIX:-attachments/}"
|
||||
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
|
||||
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
|
||||
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
|
||||
pending() {
|
||||
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
|
||||
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
|
||||
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
|
||||
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
|
||||
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
|
||||
RETURNING id, object_key"
|
||||
}
|
||||
fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; }
|
||||
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
|
||||
case "$kind" in
|
||||
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
|
||||
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
|
||||
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
|
||||
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
|
||||
stale-pending) pending | sed "s/^/stale-pending /" ;;
|
||||
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
|
||||
multipart)
|
||||
fresh | while read -r _ key; do
|
||||
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
|
||||
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
|
||||
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
|
||||
echo "multipart - $key"
|
||||
done
|
||||
;;
|
||||
*) echo "unknown kind: $kind" >&2; exit 2 ;;
|
||||
esac
|
||||
Executable
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SQL = (
|
||||
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256,"
|
||||
" floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint,"
|
||||
" EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id),"
|
||||
" (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL"
|
||||
" FROM attachments a ORDER BY a.object_key"
|
||||
)
|
||||
HASH = (
|
||||
'while read -r k; do printf "%s %s\\n" "$k" '
|
||||
'"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done'
|
||||
)
|
||||
CLASSES = (
|
||||
"orphan_stale",
|
||||
"orphan_fresh",
|
||||
"missing_referenced",
|
||||
"missing_unreferenced",
|
||||
"size_mismatch",
|
||||
"sha_mismatch",
|
||||
"pending_stale_with_object",
|
||||
"pending_stale_without_object",
|
||||
"pending_fresh",
|
||||
"quarantined",
|
||||
"incomplete_stale",
|
||||
"incomplete_fresh",
|
||||
"lost_ready",
|
||||
)
|
||||
TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"}
|
||||
BROKEN = (
|
||||
"missing_referenced",
|
||||
"missing_unreferenced",
|
||||
"size_mismatch",
|
||||
"sha_mismatch",
|
||||
"quarantined",
|
||||
)
|
||||
|
||||
|
||||
def timed(cmd, stdin=None):
|
||||
started = time.monotonic()
|
||||
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
||||
if out.returncode:
|
||||
raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}")
|
||||
return out.stdout, round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
def s3(*args, stdin=None):
|
||||
return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin)
|
||||
|
||||
|
||||
def age(stamp, now):
|
||||
return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp()
|
||||
|
||||
|
||||
def load(bucket, prefix, deep):
|
||||
listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}")
|
||||
uploads, uploads_seconds = s3(
|
||||
"object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}"
|
||||
)
|
||||
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
||||
psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"]
|
||||
rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL])
|
||||
objects = {}
|
||||
for item in json.loads(listing).get("items", []):
|
||||
key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"]
|
||||
objects[key] = item
|
||||
rows = {}
|
||||
for line in rows_raw.splitlines():
|
||||
rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t")
|
||||
rows[key] = {
|
||||
"id": rid,
|
||||
"status": status,
|
||||
"size": int(size),
|
||||
"sha256": sha,
|
||||
"age": int(row_age),
|
||||
"referenced": referenced == "t",
|
||||
"quarantined": quarantined == "t",
|
||||
}
|
||||
hashes, deep_seconds = {}, 0.0
|
||||
if deep:
|
||||
present = "".join(
|
||||
f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects
|
||||
)
|
||||
out, deep_seconds = s3("sh", "-c", HASH, stdin=present)
|
||||
hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line)
|
||||
seconds = {
|
||||
"s3_list": list_seconds,
|
||||
"s3_incomplete": uploads_seconds,
|
||||
"pg_query": pg_seconds,
|
||||
"deep_hash": deep_seconds,
|
||||
}
|
||||
return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds
|
||||
|
||||
|
||||
def row_class(key, row, objects, hashes, grace):
|
||||
if row["quarantined"]:
|
||||
return "quarantined"
|
||||
if row["status"] == "pending":
|
||||
stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object"
|
||||
return "pending_fresh" if row["age"] <= grace else stale
|
||||
if key not in objects:
|
||||
return "missing_referenced" if row["referenced"] else "missing_unreferenced"
|
||||
if objects[key]["size_bytes"] != row["size"]:
|
||||
return "size_mismatch"
|
||||
return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None
|
||||
|
||||
|
||||
def classify(objects, rows, hashes, uploads, grace):
|
||||
now = time.time()
|
||||
classes = {name: [] for name in CLASSES}
|
||||
for key, item in objects.items():
|
||||
if key not in rows:
|
||||
classes[
|
||||
"orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh"
|
||||
].append(key)
|
||||
for key, row in rows.items():
|
||||
name = row_class(key, row, objects, hashes, grace)
|
||||
if name:
|
||||
classes[name].append(key if name.startswith("orphan") else row["id"])
|
||||
for item in uploads:
|
||||
classes[
|
||||
"incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh"
|
||||
].append(item["key"])
|
||||
return classes
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(
|
||||
description="Independent PG <-> S3 invariant check for lab4 attachments"
|
||||
)
|
||||
p.add_argument(
|
||||
"--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600"))
|
||||
)
|
||||
p.add_argument(
|
||||
"--deep", action="store_true", help="stream every present object and compare sha256"
|
||||
)
|
||||
p.add_argument(
|
||||
"--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments"
|
||||
)
|
||||
p.add_argument(
|
||||
"--expect-ready",
|
||||
metavar="FILE",
|
||||
help="fail if any attachment from FILE is no longer healthy",
|
||||
)
|
||||
p.add_argument("--limit", type=int, default=20, help="max ids per class in the report")
|
||||
p.add_argument("--output", metavar="FILE")
|
||||
args = p.parse_args()
|
||||
started = time.monotonic()
|
||||
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
||||
objects, rows, hashes, uploads, seconds = load(
|
||||
os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep
|
||||
)
|
||||
classes = classify(objects, rows, hashes, uploads, args.grace)
|
||||
bad = {i for c in BROKEN for i in classes[c]}
|
||||
healthy = {
|
||||
r["id"]: r["sha256"]
|
||||
for k, r in rows.items()
|
||||
if r["status"] == "ready" and k in objects and r["id"] not in bad
|
||||
}
|
||||
if args.expect_ready:
|
||||
expected = json.loads(Path(args.expect_ready).read_text())
|
||||
classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha)
|
||||
if args.save_ready:
|
||||
Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True))
|
||||
violations = [c for c in CLASSES if classes[c] and c not in TOLERATED]
|
||||
seconds["total"] = round(time.monotonic() - started, 3)
|
||||
report = {
|
||||
"checked_at": dt.datetime.now(dt.timezone.utc)
|
||||
.isoformat(timespec="seconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"grace_seconds": args.grace,
|
||||
"deep": args.deep,
|
||||
"objects": len(objects),
|
||||
"rows": len(rows),
|
||||
"ready_healthy": len(healthy),
|
||||
"counts": {c: len(v) for c, v in classes.items()},
|
||||
"samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v},
|
||||
"seconds": seconds,
|
||||
"violations": violations,
|
||||
"ok": not violations,
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n")
|
||||
print(text)
|
||||
sys.exit(0 if report["ok"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,823 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
client = __import__("client")
|
||||
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
GRACE = 60
|
||||
FLAGS = ("--grace", str(GRACE), "--skew", "0")
|
||||
INTENT_AGE = 300
|
||||
LOCK_KEY = 20260004
|
||||
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
|
||||
USER_HZ = 100
|
||||
BUSY = (0, 1, 2, 5, 6, 7)
|
||||
|
||||
|
||||
def run(cmd, env=None, stdin=None):
|
||||
started = time.monotonic()
|
||||
done = subprocess.run(
|
||||
cmd,
|
||||
cwd=ROOT,
|
||||
input=stdin,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env=dict(os.environ, **(env or {})),
|
||||
)
|
||||
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
|
||||
return done.returncode, out, err, round(time.monotonic() - started, 3)
|
||||
|
||||
|
||||
def must(cmd, env=None, stdin=None):
|
||||
code, out, err, _ = run(cmd, env, stdin)
|
||||
if code:
|
||||
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
|
||||
return out
|
||||
|
||||
|
||||
def compose(*args, env=None, stdin=None):
|
||||
return must(["docker", "compose", *args], env, stdin)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-qAt",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-c",
|
||||
sql,
|
||||
).strip()
|
||||
|
||||
|
||||
def reconcile(*flags, env=None):
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
|
||||
code, out, err, wall = run(cmd, env)
|
||||
if code not in (0, 1) or not out.strip():
|
||||
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
|
||||
report = json.loads(out)
|
||||
audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line]
|
||||
report.update(exit_code=code, wall_seconds=wall, audit=audit)
|
||||
return report
|
||||
|
||||
|
||||
def oracle(*flags):
|
||||
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
|
||||
if not out.strip():
|
||||
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def inject(kind, count, ids=(), age=None):
|
||||
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
|
||||
if age is not None:
|
||||
env["INJECT_AGE_SECONDS"] = str(age)
|
||||
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
|
||||
return [line.split()[1] for line in out.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def head():
|
||||
return must(["git", "rev-parse", "HEAD"]).strip()
|
||||
|
||||
|
||||
def evidence_root():
|
||||
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
||||
|
||||
|
||||
def save(name, data, *, stamp=True):
|
||||
path = evidence_root() / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
|
||||
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
class Checks:
|
||||
def __init__(self):
|
||||
self.items = []
|
||||
|
||||
def __call__(self, name, ok, **detail):
|
||||
self.items.append(dict(name=name, ok=bool(ok), **detail))
|
||||
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
return all(item["ok"] for item in self.items)
|
||||
|
||||
|
||||
def chunk(tag, data):
|
||||
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
|
||||
|
||||
|
||||
def tiny_png():
|
||||
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
|
||||
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(raw))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def pick(found, names):
|
||||
return {name: found.get(name, 0) for name in names}
|
||||
|
||||
|
||||
def faults(c, check, pre):
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
|
||||
linked = ids[:6]
|
||||
for part in (linked[:3], linked[3:]):
|
||||
c.message(alice, chat, "", attachments=part)
|
||||
targets = {
|
||||
"missing": [linked[0], linked[1], ids[6]],
|
||||
"corrupt": [linked[2], ids[7], ids[8]],
|
||||
"bitflip": [linked[3], ids[9], ids[10]],
|
||||
}
|
||||
damaged = {i for group in targets.values() for i in group}
|
||||
ready = evidence_root() / "ready-before.json"
|
||||
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
|
||||
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
|
||||
save("ready-untouched.json", untouched, stamp=False)
|
||||
started = time.monotonic()
|
||||
injected = {
|
||||
"orphan_stale": inject("orphan", 3),
|
||||
"missing": inject("missing", 3, targets["missing"]),
|
||||
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
|
||||
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
|
||||
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
|
||||
+ inject("pending-object", 3, age=INTENT_AGE),
|
||||
}
|
||||
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
|
||||
injected["orphan_fresh"] = inject("orphan", 2)
|
||||
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
|
||||
expected = {name: len(keys) for name, keys in injected.items()}
|
||||
expected.update(quarantined=pre["counts"]["quarantined"])
|
||||
before = oracle(
|
||||
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
|
||||
)
|
||||
dry = reconcile("--dry-run", "--deep", *FLAGS)
|
||||
save("dry-run.json", dry)
|
||||
after_dry = oracle("--grace", str(GRACE), "--deep")
|
||||
named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected}
|
||||
check(
|
||||
"dry-run находит каждый внедрённый класс, и найдено ровно внедрённое",
|
||||
pick(dry["found"], expected) == expected
|
||||
and named == {name: sorted(ids) for name, ids in injected.items()},
|
||||
expected=expected,
|
||||
found=dry["found"],
|
||||
)
|
||||
check(
|
||||
"dry-run ничего не меняет",
|
||||
not any(dry["fixed"].values())
|
||||
and after_dry["counts"] == before["counts"]
|
||||
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
|
||||
objects=[before["objects"], after_dry["objects"]],
|
||||
rows=[before["rows"], after_dry["rows"]],
|
||||
)
|
||||
first = reconcile("--apply", "--deep", *FLAGS)
|
||||
save("apply-1.json", first)
|
||||
discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile")
|
||||
check(
|
||||
"apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита",
|
||||
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE)
|
||||
and not first["errors"]
|
||||
and discarded == sorted(injected["pending_stale"]),
|
||||
dry_run=pick(dry["found"], FIXABLE),
|
||||
fixed=pick(first["fixed"], FIXABLE),
|
||||
audit_discarded=len(discarded),
|
||||
)
|
||||
second = reconcile("--apply", "--deep", *FLAGS)
|
||||
save("apply-2.json", second)
|
||||
check(
|
||||
"повторный apply — 0 действий",
|
||||
not any(second["fixed"].values()),
|
||||
fixed=second["fixed"],
|
||||
found=second["found"],
|
||||
)
|
||||
after = oracle(
|
||||
"--grace",
|
||||
str(GRACE),
|
||||
"--deep",
|
||||
"--expect-ready",
|
||||
str(evidence_root() / "ready-untouched.json"),
|
||||
"--output",
|
||||
str(evidence_root() / "check-after.json"),
|
||||
)
|
||||
tolerated = ("orphan_fresh", "pending_fresh")
|
||||
check(
|
||||
"независимая проверка после apply: нарушений нет, готовые не потеряны",
|
||||
after["ok"]
|
||||
and not after["counts"]["lost_ready"]
|
||||
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
|
||||
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
|
||||
counts=after["counts"],
|
||||
)
|
||||
marks = dict(
|
||||
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
|
||||
)
|
||||
links = psql("SELECT attachment_id FROM message_attachments").split()
|
||||
kept = {i: marks.get(i) for i in sorted(damaged)}
|
||||
check(
|
||||
"строки с расхождениями и ссылки сообщений сохранены",
|
||||
all(kept.values()) and sum(i in linked for i in links) == len(linked),
|
||||
quarantine=kept,
|
||||
message_links=sum(i in linked for i in links),
|
||||
)
|
||||
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
|
||||
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
|
||||
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
|
||||
check(
|
||||
"карантин: метаданные 200, /content 404; нетронутое скачивается",
|
||||
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
|
||||
)
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))):
|
||||
compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"s3-cli",
|
||||
"pipe",
|
||||
f"root/{bucket}/attachments/{attachment}",
|
||||
stdin=data,
|
||||
)
|
||||
healed = reconcile("--apply", *FLAGS)
|
||||
save("healed.json", healed)
|
||||
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
|
||||
foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob)
|
||||
check(
|
||||
"вернули те же байты — карантин снят; другие байты той же длины — остался",
|
||||
healed["fixed"]["healed"] == 1
|
||||
and healed["samples"].get("healed") == [linked[0]]
|
||||
and linked[1] in healed["samples"].get("quarantined", [])
|
||||
and restored["bytes"] == FIXTURE
|
||||
and foreign["status"] == 404,
|
||||
fixed=healed["fixed"],
|
||||
s3_get=healed["cost"]["requests"]["s3_get"],
|
||||
)
|
||||
return {"injected": injected, "targets": targets, "linked": linked}
|
||||
|
||||
|
||||
def single(check):
|
||||
holder = subprocess.Popen(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-c",
|
||||
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
time.sleep(3)
|
||||
busy = reconcile("--apply", *FLAGS)
|
||||
finally:
|
||||
holder.wait()
|
||||
save("lock-busy.json", busy)
|
||||
check(
|
||||
"второй одновременный проход пропущен",
|
||||
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
|
||||
)
|
||||
|
||||
|
||||
def healthy(service, seconds=90):
|
||||
container = compose("ps", "-q", service).strip()
|
||||
probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container]
|
||||
deadline = time.monotonic() + seconds
|
||||
while must(probe).strip() != "healthy":
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit(f"{service} is not healthy after {seconds} s")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def interrupt(check):
|
||||
name = f"hlpipe-lab5-reconcile-sigterm-{os.getpid()}"
|
||||
compose("pause", "s3")
|
||||
try:
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"]
|
||||
runner = subprocess.Popen(
|
||||
[*cmd, "--apply", *FLAGS],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
time.sleep(4)
|
||||
state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name])
|
||||
started = time.monotonic()
|
||||
must(["docker", "stop", name])
|
||||
stopped = round(time.monotonic() - started, 2)
|
||||
code = runner.wait(timeout=60)
|
||||
finally:
|
||||
compose("unpause", "s3")
|
||||
healthy("s3")
|
||||
after = reconcile("--dry-run", *FLAGS)
|
||||
status, init = state.split()
|
||||
report = {
|
||||
"s3": "paused",
|
||||
"state_before_stop": status,
|
||||
"init": init,
|
||||
"stop_seconds": stopped,
|
||||
"exit_code": code,
|
||||
"next_pass_skipped": after["skipped"],
|
||||
}
|
||||
save("sigterm.json", report)
|
||||
check(
|
||||
"SIGTERM: идущий проход останавливается за секунды, блокировка освобождена",
|
||||
status == "running" and code == 143 and stopped < 5 and not after["skipped"],
|
||||
**report,
|
||||
)
|
||||
|
||||
|
||||
def outage(check):
|
||||
before = oracle("--grace", str(GRACE))
|
||||
compose("stop", "s3")
|
||||
try:
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
|
||||
code, out, err, wall = run([*cmd, *FLAGS])
|
||||
finally:
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3")
|
||||
after = oracle("--grace", str(GRACE))
|
||||
log = [line for line in err.splitlines() if "reconcile." in line]
|
||||
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
|
||||
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
|
||||
check(
|
||||
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
|
||||
code == 1 and wall < 30 and not out.strip() and same,
|
||||
exit_code=code,
|
||||
wall_seconds=wall,
|
||||
)
|
||||
|
||||
|
||||
def uploader(base, token, chat, *, deadline, results, lock):
|
||||
host = urlsplit(base).netloc
|
||||
conn = http.client.HTTPConnection(host, timeout=30)
|
||||
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
||||
sent = 0
|
||||
while time.monotonic() < deadline:
|
||||
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
|
||||
try:
|
||||
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
|
||||
response = conn.getresponse()
|
||||
status, data = response.status, response.read()
|
||||
if status == 201 and sent % 10 == 0:
|
||||
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
|
||||
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
|
||||
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
|
||||
linked = conn.getresponse()
|
||||
linked.read()
|
||||
status = 201 if linked.status == 201 else -linked.status
|
||||
except (OSError, http.client.HTTPException):
|
||||
conn.close()
|
||||
conn = http.client.HTTPConnection(host, timeout=30)
|
||||
status, data = 0, b""
|
||||
with lock:
|
||||
results.append((status, json.loads(data) if status == 201 else None))
|
||||
sent += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def under_load(c, check, base, seconds, threads):
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
ready = evidence_root() / "ready-load.json"
|
||||
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
|
||||
leftover = {
|
||||
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
|
||||
"pending_stale": left["pending_fresh"]
|
||||
+ left["pending_stale_with_object"]
|
||||
+ left["pending_stale_without_object"],
|
||||
}
|
||||
orphans = inject("orphan", 3)
|
||||
intents = inject("stale-pending", 3, age=INTENT_AGE)
|
||||
results, lock = [], threading.Lock()
|
||||
deadline = time.monotonic() + seconds
|
||||
workers = [
|
||||
threading.Thread(
|
||||
target=uploader,
|
||||
args=(base, alice["token"], chat["id"]),
|
||||
kwargs={"deadline": deadline, "results": results, "lock": lock},
|
||||
)
|
||||
for _ in range(threads)
|
||||
]
|
||||
for worker in workers:
|
||||
worker.start()
|
||||
runs = []
|
||||
while time.monotonic() < deadline:
|
||||
runs.append(reconcile("--apply", *FLAGS))
|
||||
for worker in workers:
|
||||
worker.join()
|
||||
runs.append(reconcile("--apply", *FLAGS))
|
||||
created = {att["id"]: att["sha256"] for status, att in results if att}
|
||||
expected = dict(json.loads(ready.read_text()), **created)
|
||||
save("ready-load.json", expected, stamp=False)
|
||||
after = oracle(
|
||||
"--grace",
|
||||
str(GRACE),
|
||||
"--deep",
|
||||
"--expect-ready",
|
||||
str(ready),
|
||||
"--output",
|
||||
str(evidence_root() / "check-under-load.json"),
|
||||
)
|
||||
statuses = {}
|
||||
for status, _ in results:
|
||||
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
||||
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
|
||||
rechecked = {}
|
||||
for r in runs:
|
||||
for name, count in r["rechecked"].items():
|
||||
rechecked[name] = rechecked.get(name, 0) + count
|
||||
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
|
||||
summary = {
|
||||
"seconds": seconds,
|
||||
"upload_threads": threads,
|
||||
"uploads": statuses,
|
||||
"created": len(created),
|
||||
"linked_every": 10,
|
||||
"reconciler_runs": len(runs),
|
||||
"run_seconds": [r["cost"]["seconds"] for r in runs],
|
||||
"fixed_total": fixed,
|
||||
"fresh_seen_total": seen,
|
||||
"rechecked_total": rechecked,
|
||||
"errors": sum(r["errors"] for r in runs),
|
||||
"skipped": sum(1 for r in runs if r["skipped"]),
|
||||
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
|
||||
"left_from_faults": leftover,
|
||||
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
|
||||
}
|
||||
save("under-load.json", summary)
|
||||
check(
|
||||
"под нагрузкой: исправлены только внедрённые, ошибок нет",
|
||||
fixed
|
||||
== {
|
||||
**dict.fromkeys(FIXABLE, 0),
|
||||
"orphan_stale": leftover["orphan_stale"] + len(orphans),
|
||||
"pending_stale": leftover["pending_stale"] + len(intents),
|
||||
}
|
||||
and not summary["errors"]
|
||||
and not summary["skipped"],
|
||||
fixed=fixed,
|
||||
runs=len(runs),
|
||||
)
|
||||
check(
|
||||
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
|
||||
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
|
||||
uploads=statuses,
|
||||
lost_ready=after["counts"]["lost_ready"],
|
||||
)
|
||||
|
||||
|
||||
def demo(args):
|
||||
evidence_root().mkdir(parents=True, exist_ok=False)
|
||||
c = client.Client(args.base_url, timeout=30)
|
||||
check = Checks()
|
||||
pre = oracle("--grace", str(GRACE))
|
||||
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
|
||||
if dirty:
|
||||
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
|
||||
started = time.monotonic()
|
||||
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
|
||||
result["faults"] = faults(c, check, pre)
|
||||
single(check)
|
||||
interrupt(check)
|
||||
outage(check)
|
||||
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
|
||||
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
|
||||
save("demo.json", result)
|
||||
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
|
||||
sys.exit(0 if check.ok else 1)
|
||||
|
||||
|
||||
def pins(args):
|
||||
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
|
||||
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
|
||||
if args.no_pin or cpus < 8:
|
||||
return sized
|
||||
return dict(
|
||||
sized,
|
||||
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
|
||||
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
|
||||
RECONCILE_CPUSET=str(cpus - 1),
|
||||
)
|
||||
|
||||
|
||||
def cpu_set(env):
|
||||
cpus = set()
|
||||
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
|
||||
for part in filter(None, env.get(name, "").split(",")):
|
||||
low, _, high = part.partition("-")
|
||||
cpus.update(range(int(low), int(high or low) + 1))
|
||||
return cpus
|
||||
|
||||
|
||||
def limits():
|
||||
ids = compose("ps", "-q", "s3", "postgres").split()
|
||||
fmt = (
|
||||
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
|
||||
" cpuset={{.HostConfig.CpusetCpus}}"
|
||||
)
|
||||
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
|
||||
|
||||
|
||||
def host(title):
|
||||
stats = must(
|
||||
[
|
||||
"docker",
|
||||
"stats",
|
||||
"--no-stream",
|
||||
"--all",
|
||||
"--format",
|
||||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
|
||||
]
|
||||
)
|
||||
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
|
||||
return (
|
||||
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
|
||||
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
|
||||
)
|
||||
|
||||
|
||||
def usage(text):
|
||||
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
|
||||
|
||||
|
||||
def sample():
|
||||
vm = compose(
|
||||
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
|
||||
)
|
||||
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
|
||||
busy = {}
|
||||
for line in vm.splitlines()[1:]:
|
||||
name, *values = line.split() or [""]
|
||||
if name.startswith("cpu") and name[3:].isdigit():
|
||||
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
|
||||
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
|
||||
|
||||
|
||||
def screen(first, last, own, pinned, args, *, started=True):
|
||||
window = max(last[0] - first[0], 0.01)
|
||||
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
|
||||
ours = last[2] - first[2] + own
|
||||
other = sum(busy.values()) - ours - args.start_other * started
|
||||
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
|
||||
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
|
||||
return {
|
||||
"window_seconds": round(window, 2),
|
||||
"ours_cores": round(ours / window, 2),
|
||||
"other_cores": round(other, 2),
|
||||
"other_on_pinned_cores": round(on_pinned, 2),
|
||||
"host_loadavg_1m": round(os.getloadavg()[0], 2),
|
||||
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
|
||||
}
|
||||
|
||||
|
||||
def wait_quiet(args, pinned):
|
||||
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
||||
while calm < 3 and time.monotonic() < deadline:
|
||||
first = sample()
|
||||
time.sleep(5)
|
||||
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
|
||||
calm = 0 if quiet["disturbed"] else calm + 1
|
||||
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
|
||||
return calm == 3
|
||||
|
||||
|
||||
def measure(flags, env, args, pinned):
|
||||
rejected = []
|
||||
for attempt in range(args.retries + 1):
|
||||
first = sample()
|
||||
report = reconcile(*flags, "--samples", "3", env=env)
|
||||
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
|
||||
if not report["screen"]["disturbed"]:
|
||||
return report, rejected
|
||||
rejected.append(report)
|
||||
if attempt < args.retries and not wait_quiet(args, pinned):
|
||||
break
|
||||
return None, rejected
|
||||
|
||||
|
||||
def seed(count, env):
|
||||
if count <= 0:
|
||||
return {"added": 0}
|
||||
env = dict(env, API_CPUS="4", API_MEMORY="512m")
|
||||
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
||||
out = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
tools,
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"api",
|
||||
"/tools/seed_storage.py",
|
||||
"--count",
|
||||
str(count),
|
||||
"--concurrency",
|
||||
"4",
|
||||
env=env,
|
||||
)
|
||||
records = [json.loads(line) for line in out.strip().splitlines()]
|
||||
return {**records[-1], "progress": records[:-1]}
|
||||
|
||||
|
||||
def median(runs, path):
|
||||
values = []
|
||||
for r in runs:
|
||||
value = r
|
||||
for key in path:
|
||||
value = value[key]
|
||||
values.append(value)
|
||||
return statistics.median(values)
|
||||
|
||||
|
||||
def summarize(results, rejected):
|
||||
rows = []
|
||||
for (n, mode), runs in sorted(results.items()):
|
||||
seconds = median(runs, ("cost", "seconds"))
|
||||
rows.append(
|
||||
{
|
||||
"objects": n,
|
||||
"mode": mode,
|
||||
"runs": len(runs),
|
||||
"seconds_median": seconds,
|
||||
"seconds_all": [r["cost"]["seconds"] for r in runs],
|
||||
"wall_seconds_median": median(runs, ("wall_seconds",)),
|
||||
"us_per_object": round(seconds / n * 1e6, 1),
|
||||
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
|
||||
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
|
||||
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
|
||||
"requests": runs[0]["cost"]["requests"],
|
||||
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
|
||||
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
|
||||
"other_on_pinned_cores_max": max(
|
||||
r["screen"]["other_on_pinned_cores"] for r in runs
|
||||
),
|
||||
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
|
||||
"rejected": rejected.get((n, mode), 0),
|
||||
"scanned": runs[0]["scanned"],
|
||||
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def table(rows):
|
||||
head = (
|
||||
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
|
||||
" gets rss,MiB other pinned rejected"
|
||||
)
|
||||
lines = [head]
|
||||
for r in rows:
|
||||
q = r["requests"]
|
||||
lines.append(
|
||||
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
|
||||
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
|
||||
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
|
||||
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
|
||||
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
|
||||
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def cost(args):
|
||||
out = evidence_root() / "cost"
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
(out / "disturbed").mkdir()
|
||||
rows = int(psql("SELECT count(*) FROM attachments"))
|
||||
if rows > args.sizes[0]:
|
||||
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
|
||||
snapshot = out / "host-docker-stats.txt"
|
||||
snapshot.write_text(host("before the series"))
|
||||
env = pins(args)
|
||||
pinned = cpu_set(env)
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
|
||||
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
|
||||
plan["screen"] = {
|
||||
"max_other_cores": args.max_other,
|
||||
"max_other_on_pinned_cores": args.max_pinned,
|
||||
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
|
||||
"retries": args.retries,
|
||||
}
|
||||
results, rejected = {}, {}
|
||||
try:
|
||||
for n in args.sizes:
|
||||
plan["seeds"].append(dict(seed(n - rows, env), target=n))
|
||||
rows = n
|
||||
time.sleep(args.settle)
|
||||
save("cost/plan.json", plan)
|
||||
if not wait_quiet(args, pinned):
|
||||
raise SystemExit("no quiet measurement window before timeout")
|
||||
modes = [("shallow", ["--dry-run"])]
|
||||
if n <= args.deep_max:
|
||||
modes.append(("deep", ["--dry-run", "--deep"]))
|
||||
if n == args.page_compare:
|
||||
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
|
||||
for mode, flags in modes:
|
||||
for attempt in range(1, args.repeats + 1):
|
||||
report, dropped = measure(flags, env, args, pinned)
|
||||
name = f"n{n}-{mode}-{attempt}"
|
||||
for number, bad in enumerate(dropped, 1):
|
||||
save(f"cost/disturbed/{name}-try{number}.json", bad)
|
||||
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
|
||||
if report is None:
|
||||
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
|
||||
if (
|
||||
report["exit_code"]
|
||||
or report["errors"]
|
||||
or report["skipped"]
|
||||
or report["scanned"] != {"objects": n, "rows": n}
|
||||
or any(report["found"].values())
|
||||
):
|
||||
save(f"cost/invalid/{name}.json", report)
|
||||
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
|
||||
save(f"cost/{name}.json", report)
|
||||
results.setdefault((n, mode), []).append(report)
|
||||
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
||||
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
|
||||
finally:
|
||||
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
|
||||
with snapshot.open("a") as handle:
|
||||
handle.write("\n" + host("after the series"))
|
||||
summary = summarize(results, rejected)
|
||||
save("cost/summary.json", {**plan, "results": summary})
|
||||
(out / "summary.txt").write_text(
|
||||
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
|
||||
)
|
||||
sys.stdout.write(table(summary))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
|
||||
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
d.add_argument("--load-seconds", type=int, default=75)
|
||||
d.add_argument("--load-threads", type=int, default=3)
|
||||
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
|
||||
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000])
|
||||
k.add_argument("--repeats", type=int, default=3)
|
||||
k.add_argument("--deep-max", type=int, default=100000)
|
||||
k.add_argument("--page-compare", type=int, default=10000)
|
||||
k.add_argument("--settle", type=float, default=10)
|
||||
k.add_argument("--s3-memory", default="3g")
|
||||
k.add_argument("--no-pin", action="store_true")
|
||||
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
|
||||
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
|
||||
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
|
||||
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
|
||||
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
|
||||
k.add_argument("--quiet-timeout", type=float, default=900.0)
|
||||
for command in (d, k):
|
||||
command.add_argument(
|
||||
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
||||
)
|
||||
args = p.parse_args()
|
||||
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
|
||||
if args.command == "demo":
|
||||
demo(args)
|
||||
else:
|
||||
cost(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,8 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
service=${API_SERVICE:-api}
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; }
|
||||
show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; }
|
||||
scripts/stand-usage.sh &
|
||||
echo "$(date -u +%T) migrate once before the first replacement"
|
||||
docker compose run --rm --no-deps -T migrate 2>&1
|
||||
old=$(docker compose ps -q "$service")
|
||||
@@ -13,8 +14,9 @@ for id in $old; do
|
||||
scripts/lb-wait.sh $((count + 1))
|
||||
new=$(docker compose ps -q "$service" | grep -v -F "$running")
|
||||
echo "$(date -u +%T) replace $(show "$id") -> $(show "$new")"
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$id" >/dev/null
|
||||
docker stop -t "${STOP_TIMEOUT:-7}" "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
done
|
||||
scripts/lb-wait.sh "$count"
|
||||
wait
|
||||
echo "$(date -u +%T) done: $count replicas replaced"
|
||||
|
||||
+30
-7
@@ -2,17 +2,30 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
undated() {
|
||||
python3 -c 'import json, sys
|
||||
def clean(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
|
||||
return [clean(v) for v in value] if isinstance(value, list) else value
|
||||
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
|
||||
while text[at:].strip():
|
||||
at += len(text[at:]) - len(text[at:].lstrip())
|
||||
value, at = decoder.raw_decode(text, at)
|
||||
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
|
||||
}
|
||||
snapshot() {
|
||||
cli sh -s <<'EOF' | grep -v '_date'
|
||||
cli sh -s <<'EOF' | undated
|
||||
ak=$(cat /run/secrets/s3_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rc $c --json
|
||||
done | sed "s/$ak/<api-key>/g"
|
||||
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
|
||||
EOF
|
||||
}
|
||||
failed=0
|
||||
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
|
||||
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse HEAD) == S3 privacy and bootstrap checks"
|
||||
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
|
||||
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
|
||||
before=$(snapshot)
|
||||
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
|
||||
cli sh -s <<'EOF' || failed=1
|
||||
b=$S3_BUCKET
|
||||
key="attachments/s3-check-$(date +%s)-$$"
|
||||
spare="$key-reconciler"
|
||||
other="s3-check-$(date +%s)-$$"
|
||||
head -c 4096 /dev/urandom > /tmp/probe
|
||||
result=0
|
||||
expect() {
|
||||
want=$1; name=$2; shift 2
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
|
||||
}
|
||||
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
|
||||
expect allow "api: HeadObject" rc object stat "api/$b/$key"
|
||||
expect allow "api: GetObject" rc object show "api/$b/$key"
|
||||
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
rc put /tmp/probe "api/$b/$spare" >/dev/null
|
||||
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
|
||||
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
|
||||
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
|
||||
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
|
||||
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
|
||||
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
|
||||
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
|
||||
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
|
||||
expect deny "api: CreateBucket" rc bucket create "api/$other"
|
||||
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
|
||||
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
|
||||
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: admin ListUsers" rc admin user list api
|
||||
rc bucket create --ignore-existing "root/$other" >/dev/null
|
||||
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
|
||||
|
||||
Executable
+524
@@ -0,0 +1,524 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import collections
|
||||
import contextlib
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from probe import ROOT, k6_metrics, lb_up, run_k6
|
||||
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
|
||||
PINNED = {
|
||||
"PG_CPUSET": "0-1",
|
||||
"LB_CPUSET": "2",
|
||||
"VALKEY_CPUSET": "3",
|
||||
"K6_CPUSET": "4-6",
|
||||
"K6_CPUS": "3",
|
||||
"K6_MEMORY": "1g",
|
||||
}
|
||||
API_FIRST_CPU = 7
|
||||
FIELDS = [
|
||||
"raw_dir",
|
||||
"variant",
|
||||
"config",
|
||||
"replicas",
|
||||
"cpus",
|
||||
"memory_mb",
|
||||
"workers",
|
||||
"pool",
|
||||
"maxconn",
|
||||
"api_cpuset",
|
||||
"rate",
|
||||
"repeat",
|
||||
"started",
|
||||
"db_mb",
|
||||
"messages",
|
||||
"k6_exit",
|
||||
"saturated",
|
||||
"requests",
|
||||
"rps",
|
||||
"ok_rps",
|
||||
"failed_ratio",
|
||||
"dropped_iterations",
|
||||
"p50_ms",
|
||||
"p95_ms",
|
||||
"p99_ms",
|
||||
"p95_post_ms",
|
||||
"p95_get_ms",
|
||||
"cpu_api",
|
||||
"cpu_postgres",
|
||||
"cpu_lb",
|
||||
"cpu_valkey",
|
||||
"cpu_k6",
|
||||
"mem_api_mb",
|
||||
"mem_postgres_mb",
|
||||
"mem_k6_mb",
|
||||
"pg_backends_max",
|
||||
"pg_active_avg",
|
||||
"pg_active_max",
|
||||
"pg_idle_avg",
|
||||
"pg_idle_tx_avg",
|
||||
"pg_lock_avg",
|
||||
"pg_lock_max",
|
||||
"pg_lwlock_avg",
|
||||
"pg_io_avg",
|
||||
"lb_scur_max",
|
||||
"lb_qcur_max",
|
||||
"lb_qtime_max_ms",
|
||||
"lb_rtime_avg_ms",
|
||||
"cpu_vm",
|
||||
"cpu_other",
|
||||
"gap_s",
|
||||
"wall_s",
|
||||
"disturbed",
|
||||
]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
command = ["docker", "compose", *args]
|
||||
return subprocess.run(
|
||||
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
|
||||
).stdout
|
||||
|
||||
|
||||
def size(text):
|
||||
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
|
||||
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
|
||||
|
||||
|
||||
def rows(path, start, end):
|
||||
with path.open() as handle:
|
||||
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
|
||||
|
||||
|
||||
def mean(values):
|
||||
return round(statistics.mean(values), 2) if values else 0.0
|
||||
|
||||
|
||||
def docker_usage(path, project, start, end):
|
||||
ticks = collections.defaultdict(collections.Counter)
|
||||
memory = collections.Counter()
|
||||
for r in rows(path, start, end):
|
||||
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
|
||||
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
|
||||
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
|
||||
cpu = {
|
||||
name: mean([t[name] for t in ticks.values() if name in t])
|
||||
for name in ("api", "postgres", "lb", "valkey", "k6")
|
||||
}
|
||||
usage = {f"cpu_{name}": value for name, value in cpu.items()}
|
||||
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
|
||||
return usage
|
||||
|
||||
|
||||
def vm_usage(path, start, end):
|
||||
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
|
||||
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
|
||||
return 0.0
|
||||
first, last = samples[0], samples[-1]
|
||||
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
|
||||
|
||||
|
||||
def gap(path, start, end):
|
||||
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
|
||||
edges = [start, *[t for t in ticks if start <= t <= end], end]
|
||||
return max(b - a for a, b in zip(edges, edges[1:]))
|
||||
|
||||
|
||||
def vm_counters():
|
||||
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
|
||||
values = [int(v) for v in lines[0].split()[1:]]
|
||||
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
|
||||
return sum(values) - values[3] - values[4], sum(values), cpus
|
||||
|
||||
|
||||
def wait_quiet(args):
|
||||
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
||||
while calm < 3 and time.monotonic() < deadline:
|
||||
first = vm_counters()
|
||||
time.sleep(5)
|
||||
last = vm_counters()
|
||||
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
|
||||
calm = calm + 1 if busy < args.max_other else 0
|
||||
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
|
||||
|
||||
|
||||
def pg_usage(path, start, end):
|
||||
ticks = collections.defaultdict(collections.Counter)
|
||||
for r in rows(path, start, end):
|
||||
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
|
||||
tick = ticks[r["ts"]]
|
||||
tick["backends"] += count
|
||||
tick["active"] += count * (state == "active")
|
||||
tick["idle"] += count * (state == "idle")
|
||||
tick["idle_tx"] += count * state.startswith("idle in transaction")
|
||||
tick["lock"] += count * (wait == "Lock")
|
||||
tick["lwlock"] += count * (wait == "LWLock")
|
||||
tick["io"] += count * (wait == "IO")
|
||||
series = list(ticks.values())
|
||||
return {
|
||||
"pg_backends_max": max((t["backends"] for t in series), default=0),
|
||||
"pg_active_avg": mean([t["active"] for t in series]),
|
||||
"pg_active_max": max((t["active"] for t in series), default=0),
|
||||
"pg_idle_avg": mean([t["idle"] for t in series]),
|
||||
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
|
||||
"pg_lock_avg": mean([t["lock"] for t in series]),
|
||||
"pg_lock_max": max((t["lock"] for t in series), default=0),
|
||||
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
|
||||
"pg_io_avg": mean([t["io"] for t in series]),
|
||||
}
|
||||
|
||||
|
||||
def lb_usage(path, start, end):
|
||||
backend = [
|
||||
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
|
||||
]
|
||||
|
||||
def numbers(field):
|
||||
return [int(r[field] or 0) for r in backend]
|
||||
|
||||
return {
|
||||
"lb_scur_max": max(numbers("scur"), default=0),
|
||||
"lb_qcur_max": max(numbers("qcur"), default=0),
|
||||
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
|
||||
"lb_rtime_avg_ms": mean(numbers("rtime")),
|
||||
}
|
||||
|
||||
|
||||
def trend(path, name):
|
||||
metrics = json.loads(Path(path).read_text())["metrics"]
|
||||
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
|
||||
return round(found.get("values", {}).get("p(95)", 0.0), 2)
|
||||
|
||||
|
||||
def sampler(*command):
|
||||
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
|
||||
|
||||
|
||||
def stop(process):
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
process.wait()
|
||||
|
||||
|
||||
def wait_lb(expected, timeout=120):
|
||||
deadline, stable = time.monotonic() + timeout, 0
|
||||
while stable < 4:
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
|
||||
stable = stable + 1 if lb_up() == expected else 0
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def layout(config, args, variant):
|
||||
name, _, extra = variant.partition(":")
|
||||
replicas, cpus = (float(v) for v in config.split("x"))
|
||||
replicas = int(replicas)
|
||||
workers = args.workers or max(1, int(cpus))
|
||||
width = max(1, math.ceil(replicas * cpus))
|
||||
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
|
||||
memory = args.api_memory or args.api_memory_total // replicas
|
||||
settings = {
|
||||
"API_REPLICAS": str(replicas),
|
||||
"API_CPUS": f"{cpus:g}",
|
||||
"API_MEMORY": f"{memory}m",
|
||||
"WEB_CONCURRENCY": str(workers),
|
||||
"DB_POOL_MAX_SIZE": str(args.pool),
|
||||
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
|
||||
"API_CPUSET": cpuset if args.pin else "",
|
||||
}
|
||||
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
|
||||
base = {
|
||||
"variant": name,
|
||||
"config": config,
|
||||
"replicas": replicas,
|
||||
"cpus": cpus,
|
||||
"memory_mb": memory,
|
||||
"workers": int(settings["WEB_CONCURRENCY"]),
|
||||
"pool": int(settings["DB_POOL_MAX_SIZE"]),
|
||||
"maxconn": int(settings["API_SERVER_MAXCONN"]),
|
||||
"api_cpuset": settings["API_CPUSET"],
|
||||
}
|
||||
return base, settings
|
||||
|
||||
|
||||
def apply(base, settings, env, out):
|
||||
env = dict(env, **settings)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
|
||||
break
|
||||
except subprocess.CalledProcessError as error:
|
||||
log = Path(out) / "compose-up-failed.txt"
|
||||
with log.open("a") as handle:
|
||||
stamp = time.strftime("%FT%TZ", time.gmtime())
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
handle.write(f"date: {stamp} commit: {commit}\n")
|
||||
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
|
||||
handle.write(error.stdout + error.stderr + "\n")
|
||||
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(30)
|
||||
wait_lb(base["replicas"])
|
||||
return env
|
||||
|
||||
|
||||
def deployment(env):
|
||||
ids = compose("ps", "-q", env=env).split()
|
||||
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", template, *ids],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
containers = []
|
||||
for line in result.stdout.splitlines():
|
||||
name, image, config = line.split(" ", 2)
|
||||
limits = json.loads(config)
|
||||
containers.append(
|
||||
dict(
|
||||
name=json.loads(name),
|
||||
image=json.loads(image),
|
||||
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
|
||||
)
|
||||
)
|
||||
return containers
|
||||
|
||||
|
||||
def provenance(folder, args, env):
|
||||
stamp = time.strftime("%FT%TZ", time.gmtime())
|
||||
metadata = {
|
||||
"date": stamp,
|
||||
"commit": args.commit,
|
||||
"project": args.project,
|
||||
"command": sys.argv,
|
||||
"api_image": env.get("API_IMAGE", "compose default"),
|
||||
"warmup_seconds": args.warmup,
|
||||
"duration_seconds": args.duration,
|
||||
"containers": args.deployment,
|
||||
}
|
||||
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
|
||||
for path in folder.rglob("*"):
|
||||
if path.suffix in {".csv", ".txt"}:
|
||||
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
|
||||
elif path.name == "summary.json":
|
||||
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
|
||||
path.write_text(json.dumps(data) + "\n")
|
||||
|
||||
|
||||
def attempt_folder(out, base, rate, repeat):
|
||||
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
||||
attempt = 1
|
||||
while (Path(out) / f"{name}_attempt{attempt}").exists():
|
||||
attempt += 1
|
||||
folder = Path(out) / f"{name}_attempt{attempt}"
|
||||
folder.mkdir(parents=True)
|
||||
return folder
|
||||
|
||||
|
||||
def measure(folder, args, rate, env):
|
||||
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
|
||||
database = subprocess.run(
|
||||
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
|
||||
).stdout.split(",")
|
||||
samplers = [
|
||||
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
|
||||
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
|
||||
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
|
||||
]
|
||||
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
|
||||
began = time.time()
|
||||
try:
|
||||
code, started = run_k6(folder / "k6", load)
|
||||
finally:
|
||||
wall = time.time() - began
|
||||
for process in samplers:
|
||||
stop(process)
|
||||
subprocess.run(
|
||||
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
|
||||
)
|
||||
if started is None:
|
||||
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
|
||||
return code, started + args.warmup, database, wall
|
||||
|
||||
|
||||
def step(base, rate, repeat, args, env):
|
||||
folder = attempt_folder(args.out, base, rate, repeat)
|
||||
code, start, database, wall = measure(folder, args, rate, env)
|
||||
end = start + args.duration
|
||||
summary = folder / "k6/summary.json"
|
||||
metrics = k6_metrics(summary, args.duration)
|
||||
saturated = (
|
||||
metrics["failed_ratio"] > args.max_error_ratio
|
||||
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
|
||||
or metrics["p95_ms"] > args.slo_p95_ms
|
||||
)
|
||||
row = dict(
|
||||
base,
|
||||
raw_dir=folder.name,
|
||||
rate=rate,
|
||||
repeat=repeat,
|
||||
k6_exit=code,
|
||||
saturated=saturated,
|
||||
**metrics,
|
||||
)
|
||||
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
|
||||
row["db_mb"] = round(int(database[0]) / 2**20, 1)
|
||||
row["messages"] = int(database[1])
|
||||
row["p95_post_ms"] = trend(summary, "post_message")
|
||||
row["p95_get_ms"] = trend(summary, "get_history")
|
||||
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
|
||||
row.update(pg_usage(folder / "pg_activity.csv", start, end))
|
||||
row.update(lb_usage(folder / "lb_stats.csv", start, end))
|
||||
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
|
||||
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
|
||||
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
|
||||
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
|
||||
row["wall_s"] = round(wall, 1)
|
||||
row["disturbed"] = (
|
||||
row["cpu_other"] > args.max_other
|
||||
or row["gap_s"] > args.max_gap
|
||||
or wall > args.warmup + args.duration + args.max_overhead
|
||||
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
|
||||
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
|
||||
or not rows(folder / "pg_activity.csv", start, end)
|
||||
or not rows(folder / "lb_stats.csv", start, end)
|
||||
)
|
||||
provenance(folder, args, env)
|
||||
if code or not metrics["requests"]:
|
||||
raise SystemExit(f"invalid k6 run: {folder}")
|
||||
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
||||
return row
|
||||
|
||||
|
||||
def rates(args, base):
|
||||
if args.rates:
|
||||
return [int(r) for r in args.rates.split(",")]
|
||||
factor, top = (float(v) for v in args.search.split(":"))
|
||||
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
|
||||
while value <= top:
|
||||
values.append(int(round(value, -1)))
|
||||
value *= factor
|
||||
return values
|
||||
|
||||
|
||||
def arguments():
|
||||
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
|
||||
parser.add_argument("--out", required=True)
|
||||
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
|
||||
parser.add_argument(
|
||||
"--variant",
|
||||
dest="variants",
|
||||
action="append",
|
||||
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
|
||||
)
|
||||
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
|
||||
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
|
||||
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
|
||||
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
|
||||
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
|
||||
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
|
||||
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
|
||||
parser.add_argument("--rates", help="comma separated offered rates, rps")
|
||||
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
|
||||
parser.add_argument("--start-per-cpu", type=float, default=100.0)
|
||||
parser.add_argument("--repeats", type=int, default=3)
|
||||
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
|
||||
parser.add_argument("--warmup", type=int, default=10)
|
||||
parser.add_argument("--duration", type=int, default=30)
|
||||
parser.add_argument("--pause", type=float, default=5.0)
|
||||
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
|
||||
parser.add_argument("--max-error-ratio", type=float, default=0.01)
|
||||
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
|
||||
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
|
||||
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
|
||||
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
|
||||
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
|
||||
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
|
||||
args = parser.parse_args()
|
||||
args.variants = args.variants or ["base"]
|
||||
if len(args.variants) > 1 and not args.rates:
|
||||
parser.error("several variants need explicit --rates")
|
||||
return args
|
||||
|
||||
|
||||
def block(args, output, config, variant, repeats):
|
||||
base, settings = layout(config, args, variant)
|
||||
current, streak = apply(base, settings, args.base_env, args.out), 0
|
||||
args.deployment = deployment(current)
|
||||
for rate in rates(args, base):
|
||||
results = []
|
||||
for repeat in repeats:
|
||||
for _ in range(args.retries + 1):
|
||||
row = step(base, rate, repeat, args, current)
|
||||
time.sleep(args.pause)
|
||||
if not row["disturbed"]:
|
||||
break
|
||||
args.rejected.writerow(row)
|
||||
wait_quiet(args)
|
||||
else:
|
||||
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
|
||||
results.append(row)
|
||||
output.writerows(results)
|
||||
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
||||
if streak == 2:
|
||||
break
|
||||
|
||||
|
||||
def series(args, output):
|
||||
every = range(args.first_repeat, args.first_repeat + args.repeats)
|
||||
configs = args.configs.split(",")
|
||||
if not args.rates:
|
||||
for config in configs:
|
||||
block(args, output, config, args.variants[0], every)
|
||||
return
|
||||
for repeat in every:
|
||||
for config in configs:
|
||||
for variant in args.variants:
|
||||
block(args, output, config, variant, [repeat])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def table(path):
|
||||
fresh = not path.exists()
|
||||
with path.open("a", newline="", buffering=1) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
||||
if fresh:
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
|
||||
writer.writeheader()
|
||||
yield writer
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments()
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
||||
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
||||
try:
|
||||
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
|
||||
series(args, writer)
|
||||
finally:
|
||||
if not args.keep:
|
||||
compose("up", "-d", "--wait", "--wait-timeout", "180")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
|
||||
put s3_root_password "$(random)"
|
||||
put s3_access_key_id "api$(random | cut -c1-16)"
|
||||
put s3_secret_access_key "$(random)"
|
||||
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
|
||||
put s3_reconciler_secret_access_key "$(random)"
|
||||
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
size="select pg_database_size(current_database()), coalesce((select n_live_tup from pg_stat_user_tables where relname = 'messages'), 0)"
|
||||
echo "$(date -u +%T) database bytes,messages: $(docker compose exec -T postgres psql -X -At -F, -U messenger -d messenger -c "$size")"
|
||||
docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)
|
||||
+1
-1
@@ -6,4 +6,4 @@ ids=$(docker compose ps -q "${API_SERVICE:-api}")
|
||||
[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; }
|
||||
target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}')
|
||||
[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null
|
||||
docker stop -t "${STOP_TIMEOUT:-7}" "$target" >/dev/null
|
||||
|
||||
@@ -77,7 +77,9 @@ def main():
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.out
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
|
||||
if any(out.iterdir()):
|
||||
p.error("output directory must be empty; choose a new --out")
|
||||
commit = seed.run(["git", "rev-parse", "HEAD"]).strip()
|
||||
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
|
||||
lines = [
|
||||
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
|
||||
@@ -102,7 +104,12 @@ def main():
|
||||
link=0,
|
||||
ids_out=None,
|
||||
)
|
||||
report = {"upload_concurrency": slots, **seed.over_http(run)}
|
||||
report = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"commit": commit,
|
||||
"upload_concurrency": slots,
|
||||
**seed.over_http(run),
|
||||
}
|
||||
report["lb"] = balancer()
|
||||
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
|
||||
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
Reference in New Issue
Block a user