- [telemetry] trace_sample_ratio (ParentBased(TraceIdRatioBased), 1 keeps ParentBased(ALWAYS_ON)) and metric_buckets (default|coarse with SLO bounds), export interval <= 30 s; otel.sdk.span.started, span.live dropped; startup records name ratio, buckets and interval - bench: telemetry services on mtusi-bench, TELEMETRY=on switch for uploads and queue, make bench_telemetry[_quick] with off/full/full-aa/head/tail/ metrics/balanced, rare events, lookup from Tempo/Loki/Prometheus, cost sampler, verdicts H1-H8 and B1-B5; collector layers and lean Prometheus in bench/otel with preflight checks - REPORT: telemetry cost hypotheses and criteria before the label Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
320 lines
12 KiB
Bash
320 lines
12 KiB
Bash
#!/usr/bin/env bash
|
||
# Сэмплер стенда (решение 7 design change scaling-bench, решение 1 design
|
||
# change profile-resources). Работает только внутри контейнера
|
||
# mtusi-bench-sampler (образ PostgreSQL стенда, --cgroupns host), не на
|
||
# хосте: его запускает bench/run.py (Stand.start_sampler). Раз в интервал
|
||
# пишет в /work/samples.log строки с отметкой часов ВМ в наносекундах
|
||
# (одна отметка на весь тик):
|
||
# cg <нс> <роль> <id12> <usage_usec> <nr_periods> <nr_throttled>
|
||
# <throttled_usec> <memory.current> <cpu.pressure some total>
|
||
# cg <нс> <роль> <id12> missing — нет каталога cgroup
|
||
# vm <нс> <занято jiffies> <всего jiffies> <PSI CPU some total, мкс>
|
||
# lb <нс> # <заголовок CSV статистики HAProxy> / lb <нс> api,<строка>
|
||
# lb <нс> missing — статистика недоступна
|
||
# Режимы — по второму аргументу:
|
||
# list (серии трека 1): <интервал, с> <роль>=<полный id контейнера>…
|
||
# Сеть mtusi-bench. Строки cg заданных контейнеров, vm и lb; в фоне
|
||
# psql с \watch пишет /work/pg-samples.log (одно соединение на всю
|
||
# ячейку): act, db, wal, io — разбор в bench/scaling.py
|
||
# (parse_samples).
|
||
# scan (профили failover и rollout): <интервал, с> scan
|
||
# Контейнер без сети. На каждом тике — обход /sys/fs/cgroup/docker/*/
|
||
# (только каталоги с именем из 64 hex — контейнеры), строка
|
||
# `cg <нс> - <id12> …` на каждый; роли назначает оркестратор. Строки
|
||
# vm — как в list. Ни psql, ни опроса LB: наблюдатель не обращается к
|
||
# стенду.
|
||
# peak (профиль uploads, design change upload-memory, решение 10):
|
||
# <интервал, с> peak <имя> <роль>=<полный id контейнера>…
|
||
# Контейнер без сети, --user 0, cgroupfs на запись. perl открывает
|
||
# memory.peak каждого контейнера на чтение и запись, пишет его прежнее
|
||
# значение (`start <роль> <байт>`) и сбрасывает пик записью в свой
|
||
# дескриптор (ядро ≥ 6.12): `reset <роль> ok|failed`. Раз в интервал —
|
||
# `pk <нс> <роль> <id12> <memory.current> <anon> <sock> <oom_kill>`
|
||
# (`anon`, `sock` — memory.stat, `oom_kill` — memory.events) или
|
||
# `pk <нс> <роль> <id12> missing`. Появился /work/<имя>.stop — perl
|
||
# читает пик через тот же дескриптор (`peak <роль> <байт>|missing`),
|
||
# строка `end <нс>`, выход 0. Всё — в /work/<имя>.log.
|
||
# queue (профиль очереди, design change track1-queue-overload, решение 10):
|
||
# <интервал, с> queue <имя очереди>
|
||
# Сеть mtusi-bench. Обход cgroup, как scan (`cg`), `vm`, `lb`; ещё `mp <нс>
|
||
# <id12> <memory.peak> <anon> <oom_kill>` на контейнер и `mq <нс> <ready>
|
||
# <unacked>` — метрики брокера HTTP/1.0 по /dev/tcp (брокер недоступен —
|
||
# `mq <нс> missing`). В фоне psql с \watch пишет /work/jobs.log (одно
|
||
# соединение): jobs|<мс>|outbox|waiting|published|processing|oldest_s|
|
||
# staging_ready. Разбор — bench/overload.py.
|
||
# cost (профиль цены телеметрии, design change track1-telemetry-cost,
|
||
# решение 10): <интервал, с> cost
|
||
# Контейнер без сети, как scan: строки `cg` всех контейнеров ВМ, `vm` и
|
||
# `mp` каждого контейнера (memory.peak, anon, oom_kill), как у queue. Ни
|
||
# psql, ни опроса LB и брокера. Разбор — bench/telemetry_cost.py.
|
||
set -u
|
||
|
||
interval=$1
|
||
shift
|
||
mode=list
|
||
if [[ ${1:-} == scan ]]; then
|
||
mode=scan
|
||
shift
|
||
elif [[ ${1:-} == cost ]]; then
|
||
mode=cost
|
||
shift
|
||
elif [[ ${1:-} == queue ]]; then
|
||
mode=queue
|
||
queue_name=$2
|
||
shift 2
|
||
elif [[ ${1:-} == peak ]]; then
|
||
mode=peak
|
||
name=$2
|
||
shift 2
|
||
fi
|
||
roles=()
|
||
ids=()
|
||
for arg in "$@"; do
|
||
roles+=("${arg%%=*}")
|
||
ids+=("${arg#*=}")
|
||
done
|
||
|
||
if [[ $mode == list ]]; then
|
||
export PGPASSWORD="$POSTGRES_PASSWORD"
|
||
psql "host=postgres user=$POSTGRES_USER dbname=$POSTGRES_DB \
|
||
application_name=bench-sampler" -XAtq -F '|' >/work/pg-samples.log 2>&1 <<SQL &
|
||
WITH t AS (
|
||
SELECT (extract(epoch FROM clock_timestamp()) * 1000)::bigint AS ts
|
||
)
|
||
SELECT 'act', (SELECT ts FROM t), coalesce(host(client_addr), '-'),
|
||
coalesce(state, '-'), coalesce(wait_event_type, 'CPU'),
|
||
coalesce(wait_event, '-'), count(*)
|
||
FROM pg_stat_activity
|
||
WHERE backend_type = 'client backend' AND pid <> pg_backend_pid()
|
||
GROUP BY 3, 4, 5, 6
|
||
UNION ALL
|
||
SELECT 'db', (SELECT ts FROM t), xact_commit::text, xact_rollback::text,
|
||
sessions::text, sessions_fatal::text, numbackends::bigint
|
||
FROM pg_stat_database WHERE datname = current_database()
|
||
UNION ALL
|
||
SELECT 'wal', (SELECT ts FROM t), wal_records::text, wal_fpi::text,
|
||
wal_bytes::text, NULL, NULL
|
||
FROM pg_stat_wal
|
||
UNION ALL
|
||
SELECT 'io', (SELECT ts FROM t), coalesce(sum(writes), 0)::text,
|
||
coalesce(sum(fsyncs), 0)::text, NULL, NULL, NULL
|
||
FROM pg_stat_io WHERE object = 'wal'
|
||
\watch $interval
|
||
SQL
|
||
fi
|
||
|
||
if [[ $mode == queue ]]; then
|
||
export PGPASSWORD="$POSTGRES_PASSWORD"
|
||
psql "host=postgres user=$POSTGRES_USER dbname=$POSTGRES_DB \
|
||
application_name=bench-sampler" -XAtq -F '|' >/work/jobs.log 2>&1 <<SQL &
|
||
SELECT 'jobs', (extract(epoch FROM clock_timestamp()) * 1000)::bigint,
|
||
count(*) FILTER (WHERE state = 'queued' AND published_at IS NULL
|
||
AND coalesce(next_attempt_at, created_at) <= clock_timestamp()),
|
||
count(*) FILTER (WHERE state = 'queued' AND published_at IS NULL
|
||
AND coalesce(next_attempt_at, created_at) > clock_timestamp()),
|
||
count(*) FILTER (WHERE state = 'queued' AND published_at IS NOT NULL),
|
||
count(*) FILTER (WHERE state = 'processing'),
|
||
coalesce(round(extract(epoch FROM clock_timestamp() - min(created_at))::numeric, 1), 0),
|
||
(SELECT count(*) FROM attachments
|
||
WHERE status = 'ready' AND staging_key IS NOT NULL)
|
||
FROM attachment_jobs WHERE state IN ('queued', 'processing')
|
||
\watch $interval
|
||
SQL
|
||
fi
|
||
|
||
if [[ $mode == peak ]]; then
|
||
log=/work/$name.log
|
||
stop=/work/$name.stop
|
||
: >"$log"
|
||
exec 4>>"$log"
|
||
# пик через свой дескриптор: сброс и чтение — в одном процессе perl, bash
|
||
# позиционировать дескриптор не умеет
|
||
perl -e '
|
||
use strict;
|
||
my ($stop, @targets) = @ARGV;
|
||
$| = 1;
|
||
my %fh;
|
||
for my $t (@targets) {
|
||
my ($role, $id) = split /=/, $t, 2;
|
||
my $path = "/sys/fs/cgroup/docker/$id/memory.peak";
|
||
my $h;
|
||
if (!open($h, "+<", $path)) {
|
||
print "start $role missing\nreset $role failed open\n";
|
||
next;
|
||
}
|
||
my $old = "";
|
||
sysread($h, $old, 64);
|
||
chomp $old;
|
||
print "start $role ", ($old =~ /^\d+$/ ? $old : "missing"), "\n";
|
||
sysseek($h, 0, 0);
|
||
if (defined syswrite($h, "reset")) {
|
||
print "reset $role ok\n";
|
||
$fh{$role} = $h;
|
||
} else {
|
||
print "reset $role failed $!\n";
|
||
}
|
||
}
|
||
select(undef, undef, undef, 0.05) until -e $stop;
|
||
for my $role (sort keys %fh) {
|
||
my $h = $fh{$role};
|
||
my $value = "";
|
||
if (sysseek($h, 0, 0) && sysread($h, $value, 64)) {
|
||
chomp $value;
|
||
}
|
||
print "peak $role ", ($value =~ /^\d+$/ ? $value : "missing"), "\n";
|
||
}
|
||
' "$stop" "$@" >&4 &
|
||
perl_pid=$!
|
||
|
||
pk_line() {
|
||
local role=$1 id=$2 dir=/sys/fs/cgroup/docker/$2
|
||
local key val mem= anon= sock= oom=
|
||
{
|
||
read -r mem <"$dir/memory.current"
|
||
while read -r key val; do
|
||
case $key in
|
||
anon) anon=$val ;;
|
||
sock) sock=$val ;;
|
||
esac
|
||
done <"$dir/memory.stat"
|
||
while read -r key val; do
|
||
[[ $key == oom_kill ]] && oom=$val
|
||
done <"$dir/memory.events"
|
||
} 2>/dev/null
|
||
if [[ -z $mem || -z $anon || -z $sock || -z $oom ]]; then
|
||
printf 'pk %s %s %s missing\n' "$ns" "$role" "${id:0:12}" >&4
|
||
return
|
||
fi
|
||
printf 'pk %s %s %s %s %s %s %s\n' "$ns" "$role" "${id:0:12}" \
|
||
"$mem" "$anon" "$sock" "$oom" >&4
|
||
}
|
||
|
||
while [[ ! -e $stop ]]; do
|
||
ns=${EPOCHREALTIME/./}000
|
||
for i in "${!ids[@]}"; do
|
||
pk_line "${roles[$i]}" "${ids[$i]}"
|
||
done
|
||
sleep "$interval"
|
||
done
|
||
wait "$perl_pid"
|
||
printf 'end %s\n' "${EPOCHREALTIME/./}000" >&4
|
||
exit 0
|
||
fi
|
||
|
||
exec 4>>/work/samples.log
|
||
|
||
cg_line() {
|
||
local role=$1 id=$2 dir=/sys/fs/cgroup/docker/$2
|
||
local key val usage= periods=0 throttled=0 thr_usec=0 mem= psi=0 rest=
|
||
if [[ ! -r $dir/cpu.stat ]]; then
|
||
printf 'cg %s %s %s missing\n' "$ns" "$role" "${id:0:12}" >&4
|
||
return
|
||
fi
|
||
# каталог может пропасть между проверкой и чтением (контейнер
|
||
# остановлен посреди тика): тогда тоже missing, а не строка нулей
|
||
{
|
||
while read -r key val; do
|
||
case $key in
|
||
usage_usec) usage=$val ;;
|
||
nr_periods) periods=$val ;;
|
||
nr_throttled) throttled=$val ;;
|
||
throttled_usec) thr_usec=$val ;;
|
||
esac
|
||
done <"$dir/cpu.stat"
|
||
read -r mem <"$dir/memory.current"
|
||
read -r _ _ _ _ rest <"$dir/cpu.pressure"
|
||
} 2>/dev/null
|
||
if [[ -z $usage || -z $mem ]]; then
|
||
printf 'cg %s %s %s missing\n' "$ns" "$role" "${id:0:12}" >&4
|
||
return
|
||
fi
|
||
psi=${rest#total=}
|
||
printf 'cg %s %s %s %s %s %s %s %s %s\n' "$ns" "$role" "${id:0:12}" \
|
||
"$usage" "$periods" "$throttled" "$thr_usec" "$mem" "$psi" >&4
|
||
}
|
||
|
||
vm_line() {
|
||
local _cpu user nice system idle iowait irq softirq steal rest total
|
||
read -r _cpu user nice system idle iowait irq softirq steal rest </proc/stat
|
||
total=$((user + nice + system + idle + iowait + irq + softirq + steal))
|
||
read -r _ _ _ _ rest </proc/pressure/cpu
|
||
printf 'vm %s %s %s %s\n' "$ns" "$((total - idle - iowait))" "$total" \
|
||
"${rest#total=}" >&4
|
||
}
|
||
|
||
lb_lines() {
|
||
local line
|
||
if ! exec 3<>/dev/tcp/haproxy/8404 2>/dev/null; then
|
||
printf 'lb %s missing\n' "$ns" >&4
|
||
return
|
||
fi
|
||
printf 'GET /stats;csv HTTP/1.0\r\n\r\n' >&3
|
||
while IFS= read -r line; do
|
||
line=${line%$'\r'}
|
||
case $line in
|
||
"# "* | api,*) printf 'lb %s %s\n' "$ns" "$line" >&4 ;;
|
||
esac
|
||
done <&3
|
||
exec 3<&-
|
||
}
|
||
|
||
mp_line() {
|
||
local id=$1 dir=/sys/fs/cgroup/docker/$1
|
||
local key val peak= anon= oom=
|
||
{
|
||
read -r peak <"$dir/memory.peak"
|
||
while read -r key val; do
|
||
[[ $key == anon ]] && { anon=$val; break; }
|
||
done <"$dir/memory.stat"
|
||
while read -r key val; do
|
||
[[ $key == oom_kill ]] && oom=$val
|
||
done <"$dir/memory.events"
|
||
} 2>/dev/null
|
||
[[ -n $peak && -n $anon && -n $oom ]] &&
|
||
printf 'mp %s %s %s %s %s\n' "$ns" "${id:0:12}" "$peak" "$anon" "$oom" >&4
|
||
}
|
||
|
||
mq_line() {
|
||
local line ready= unacked=
|
||
if ! exec 3<>/dev/tcp/rabbitmq/15692 2>/dev/null; then
|
||
printf 'mq %s missing\n' "$ns" >&4
|
||
return
|
||
fi
|
||
printf 'GET /metrics/detailed?vhost=%%2F&family=queue_coarse_metrics HTTP/1.0\r\nHost: x\r\n\r\n' >&3
|
||
while IFS= read -r line; do
|
||
line=${line%$'\r'}
|
||
case $line in
|
||
rabbitmq_detailed_queue_messages_ready\{*"$queue_name"*) ready=${line##* } ;;
|
||
rabbitmq_detailed_queue_messages_unacked\{*"$queue_name"*) unacked=${line##* } ;;
|
||
esac
|
||
done <&3
|
||
exec 3<&-
|
||
# очередь ещё не объявлена — нули, а брокер отвечает
|
||
printf 'mq %s %s %s\n' "$ns" "${ready:-0}" "${unacked:-0}" >&4
|
||
}
|
||
|
||
hex64='^[0-9a-f]{64}$'
|
||
|
||
while :; do
|
||
# EPOCHREALTIME bash 5 — часы ВМ без запуска date
|
||
ns=${EPOCHREALTIME/./}000
|
||
if [[ $mode == scan || $mode == queue || $mode == cost ]]; then
|
||
for dir in /sys/fs/cgroup/docker/*/; do
|
||
dir=${dir%/}
|
||
id=${dir##*/}
|
||
if [[ $id =~ $hex64 ]]; then
|
||
cg_line - "$id"
|
||
[[ $mode == queue || $mode == cost ]] && mp_line "$id"
|
||
fi
|
||
done
|
||
else
|
||
for i in "${!ids[@]}"; do
|
||
cg_line "${roles[$i]}" "${ids[$i]}"
|
||
done
|
||
fi
|
||
vm_line
|
||
[[ $mode == list || $mode == queue ]] && lb_lines
|
||
[[ $mode == queue ]] && mq_line
|
||
sleep "$interval"
|
||
done
|