feat:report

This commit is contained in:
Оля Мороз
2026-10-23 18:10:07 +03:00
parent ae53d39ccc
commit f4500589c6
134 changed files with 5282 additions and 237 deletions
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Графики трека 1 в формате open-loop."""
import csv
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT = Path(__file__).resolve().parents[1]
EVID = ROOT / 'evidence' / 'track1'
OUT = ROOT / 'evidence'
OUT.mkdir(exist_ok=True)
COLORS = {
'A1': '#1f77b4', 'A2': '#2ca02c', 'A4': '#d62728',
'B1': '#e377c2', 'B2': '#17becf', 'B4': '#ff7f0e',
}
# Читаем summary.csv (его пишет summarize-track1.py).
summary = ROOT / 'evidence' / 'track1-summary.csv'
rows = list(csv.DictReader(summary.open()))
def series(profile):
rs = sorted([r for r in rows if r['profile'] == profile],
key=lambda r: int(r['offered_rps']))
xs = [int(r['offered_rps']) for r in rs]
rps = [float(r['successful_rps']) for r in rs]
p95 = [float(r['p95_ms']) if r['p95_ms'] else 0.0 for r in rs]
err = [float(r['error_ratio_pct']) for r in rs]
return xs, rps, p95, err
def plot_pair(ax_rps, ax_p95, profiles, title):
for prof in profiles:
xs, rps, p95, err = series(prof)
if not xs:
continue
ax_rps.plot(xs, xs, linestyle=':', color='gray', linewidth=0.8, alpha=0.6)
ax_rps.plot(xs, rps, marker='o', color=COLORS[prof], label=prof)
ax_p95.plot(xs, p95, marker='o', color=COLORS[prof], label=prof)
ax_rps.set_xlabel('Offered requests / s')
ax_rps.set_ylabel('Successful responses / s')
ax_rps.set_title(title + ' — RPS')
ax_rps.grid(True)
ax_rps.legend()
ax_p95.set_xlabel('Offered requests / s')
ax_p95.set_ylabel('p95 latency, ms')
ax_p95.set_title(title + ' — p95')
ax_p95.axhline(y=500, linestyle='--', color='black', linewidth=0.8,
label='500 ms budget')
ax_p95.grid(True)
ax_p95.legend()
# ---------- A: fixed total ----------
fig, ax = plt.subplots(1, 2, figsize=(16, 6))
plot_pair(ax[0], ax[1], ['A1', 'A2', 'A4'],
'A: fixed total 1 CPU / 256M (1/2/4 реплики)')
fig.tight_layout()
fig.savefig(OUT / 'track1-A-fixed-total.png', dpi=140)
print('saved', OUT / 'track1-A-fixed-total.png')
# ---------- B: resources scale with replicas ----------
fig, ax = plt.subplots(1, 2, figsize=(16, 6))
plot_pair(ax[0], ax[1], ['B1', 'B2', 'B4'],
'B: 0.5 CPU / 128M на реплику (1/2/4 реплики = 0.5/1.0/2.0 CPU)')
fig.tight_layout()
fig.savefig(OUT / 'track1-B-scaling.png', dpi=140)
print('saved', OUT / 'track1-B-scaling.png')
# ---------- C: latency percentiles bar chart ----------
fig, ax = plt.subplots(figsize=(12, 5))
order = []
for prof in ['A1', 'A2', 'A4', 'B1', 'B2', 'B4']:
rs = sorted([r for r in rows if r['profile'] == prof],
key=lambda r: int(r['offered_rps']))
for r in rs:
order.append(r)
xs = list(range(len(order)))
width = 0.27
p50 = [float(r['p50_ms']) if r['p50_ms'] else 0.0 for r in order]
p95 = [float(r['p95_ms']) if r['p95_ms'] else 0.0 for r in order]
p99 = [float(r['p99_ms']) if r['p99_ms'] else 0.0 for r in order]
ax.bar([i - width for i in xs], p50, width=width, label='p50')
ax.bar(xs, p95, width=width, label='p95')
ax.bar([i + width for i in xs], p99, width=width, label='p99')
ax.set_xticks(xs)
ax.set_xticklabels(
[f"{r['profile']}-{r['offered_rps']}" for r in order],
rotation=60, ha='right', fontsize=7)
ax.set_ylabel('ms')
ax.set_title('Латентности p50/p95/p99 по профилям и offered RPS')
ax.grid(True, axis='y')
ax.legend()
fig.tight_layout()
fig.savefig(OUT / 'track1-C-latency-percentiles.png', dpi=140)
print('saved', OUT / 'track1-C-latency-percentiles.png')
# ---------- D: CPU split по профилям ----------
fig, ax = plt.subplots(figsize=(12, 5))
profiles = ['A1', 'A2', 'A4', 'B1', 'B2', 'B4']
for i, prof in enumerate(profiles):
rs = sorted([r for r in rows if r['profile'] == prof],
key=lambda r: int(r['offered_rps']))
xs = [i * 10 + j for j in range(len(rs))]
api_cpu = [float(r['api_cpu_sum_max']) for r in rs]
pg_cpu = [float(r['pg_cpu_max']) for r in rs]
lb_cpu = [float(r['lb_cpu_max']) for r in rs]
ax.plot(xs, api_cpu, marker='o', label=f'{prof} api sum', color=COLORS[prof])
ax.plot(xs, pg_cpu, marker='s', linestyle='--', color=COLORS[prof], alpha=0.6)
ax.plot(xs, lb_cpu, marker='^', linestyle=':', color=COLORS[prof], alpha=0.4)
ax.set_xlabel('profile × offered RPS')
ax.set_ylabel('CPU, cores')
ax.set_title('CPU split по профилям (точки — offered, линии — роли)')
ax.grid(True)
ax.legend(ncol=2, fontsize=8)
fig.tight_layout()
fig.savefig(OUT / 'track1-D-cpu-split.png', dpi=140)
print('saved', OUT / 'track1-D-cpu-split.png')
+19 -7
View File
@@ -1,14 +1,26 @@
#!/bin/sh
set -eu
# Перезапуск API + всех state stores, volumes сохраняются.
docker compose restart postgres redis api1 api2 lb
# Ждём готовности.
for i in $(seq 1 60); do
if docker compose exec -T lb wget -qO- http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then
echo "ready after ${i}s"
# ./scripts/restart.sh # 2 реплики (по умолчанию)
# ./scripts/restart.sh 4 # 4 реплики
REPLICAS="${1:-2}"
# Останов с сохранением volumes.
docker compose down --remove-orphans
# Пересоздание сети + контейнеров с нужным числом реплик.
API_REPLICAS="$REPLICAS" \
docker compose --env-file .env -f compose.yaml up -d --build
# Ждём готовности с хоста.
i=0
while [ "$i" -lt 90 ]; do
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
echo "ready after ${i}s (replicas=$REPLICAS)"
exit 0
fi
i=$((i + 1))
sleep 1
done
echo "stack did not become ready in 60s" >&2
echo "stack did not become ready in 90s" >&2
exit 1
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
set -eu
# Все контейнеры сервиса api (включая Exited).
# `docker compose ps -a -q api` возвращает их ID.
STOPPED=""
for id in $(docker compose ps -a -q api); do
state=$(docker inspect --format '{{.State.Status}}' "$id")
if [ "$state" = "exited" ] || [ "$state" = "created" ]; then
STOPPED="$STOPPED $id"
fi
done
if [ -z "$STOPPED" ]; then
echo "all api containers already running"
exit 0
fi
for id in $STOPPED; do
hn=$(docker inspect --format '{{.Config.Hostname}}' "$id")
docker start "$id" >/dev/null
echo "started $id ($hn)"
done
# Дождаться, пока LB увидит их снова (curl с хоста).
i=0
while [ "$i" -lt 30 ]; do
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
echo "ready after ${i}s"
exit 0
fi
i=$((i + 1))
sleep 1
done
echo "stack did not become ready in 30s" >&2
exit 1
-9
View File
@@ -1,9 +0,0 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?}"
case "$TARGET_INSTANCE_ID" in
api-1) SERVICE=api1 ;;
api-2) SERVICE=api2 ;;
*) exit 2 ;;
esac
docker compose start "$SERVICE"
+91
View File
@@ -0,0 +1,91 @@
#!/bin/sh
set -eu
DURATION="${DURATION:-60}"
WARMUP="${WARMUP:-15}"
RPS_LIST="${RPS_LIST:-100 200 300 400 500 800}"
OUTDIR="evidence/track1"
mkdir -p "$OUTDIR"
PROFILES="
A1:1:1.0:256M
A2:2:0.5:128M
A4:4:0.25:64M
B1:1:0.5:128M
B2:2:0.5:128M
B4:4:0.5:128M
"
for prof in $PROFILES; do
name=$(echo "$prof" | cut -d: -f1)
n=$(echo "$prof" | cut -d: -f2)
cpu=$(echo "$prof" | cut -d: -f3)
mem=$(echo "$prof" | cut -d: -f4)
echo "=== profile $name: replicas=$n cpu=$cpu mem=$mem"
export API_REPLICAS="$n"
export API_CPU_PER_REPLICA="$cpu"
export API_MEM_PER_REPLICA="$mem"
docker compose --env-file .env down --remove-orphans --timeout 30 </dev/null || true
docker compose --env-file .env up -d --build </dev/null
i=0
while [ "$i" -lt 90 ]; do
if curl -sf http://localhost:8080/health/ready >/dev/null 2>&1; then
echo " ready after ${i}s"
break
fi
i=$((i + 1))
sleep 1
done
replicas_up=$(docker compose --env-file .env ps -q api </dev/null | wc -l)
echo " api containers up: $replicas_up (expected $n)"
if [ "$replicas_up" -ne "$n" ]; then
echo "ERROR: expected $n api replicas, got $replicas_up" >&2
exit 1
fi
for rps in $RPS_LIST; do
echo "--- $name target_rps=$rps"
docker compose --env-file .env exec -T postgres psql -U messenger -d messenger \
-v ON_ERROR_STOP=1 < fixtures/seed.sql >/dev/null
docker compose --env-file .env exec -T postgres psql -U messenger -d messenger \
-c "ANALYZE messages; ANALYZE idempotency_keys;" >/dev/null
sleep 3
: > "$OUTDIR/${name}-rps${rps}-stats.txt"
(
while true; do
docker stats --no-stream --format \
'{{.Name}} {{.CPUPerc}} {{.MemUsage}}' >> "$OUTDIR/${name}-rps${rps}-stats.txt"
sleep 2
done
) &
STATS_PID=$!
sleep 1
python3 tests/openloop.py \
--target-rps "$rps" --duration "$DURATION" --warmup "$WARMUP" \
--output "$OUTDIR/${name}-rps${rps}.json"
kill "$STATS_PID" 2>/dev/null || true
wait "$STATS_PID" 2>/dev/null || true
docker compose --env-file .env exec -T postgres psql -U messenger -d messenger \
-At -F'|' -c "
SELECT 'state', state, count(*) FROM pg_stat_activity WHERE datname='messenger' GROUP BY state
UNION ALL
SELECT 'wait', wait_event_type, count(*) FROM pg_stat_activity WHERE datname='messenger' GROUP BY wait_event_type
UNION ALL
SELECT 'maxconns', 'setting', setting::int FROM pg_settings WHERE name='max_connections'
" > "$OUTDIR/${name}-rps${rps}-pg.txt" || true
done
done
echo "=== done, results in $OUTDIR ==="
ls -la "$OUTDIR" | tail -5
+15 -10
View File
@@ -1,15 +1,20 @@
#!/bin/sh
set -eu
# TARGET_INSTANCE_ID приходит от теста: alias обслужившей реплики.
# В compose alias = INSTANCE_ID = "api-1" / "api-2".
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
api-1) SERVICE=api1 ;;
api-2) SERVICE=api2 ;;
*) echo "unknown instance: $TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
# Находим ID контейнера, чьё имя (HOSTNAME) совпадает с TARGET_INSTANCE_ID.
CID=$(docker compose ps -q api | while read -r id; do
name=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$name" = "$TARGET_INSTANCE_ID" ]; then
echo "$id"
break
fi
done)
# Останавливаем ТОЛЬКО эту реплику, оставляем остановленной.
docker compose stop "$SERVICE"
echo "stopped $SERVICE ($TARGET_INSTANCE_ID)"
if [ -z "$CID" ]; then
echo "container with hostname $TARGET_INSTANCE_ID not found" >&2
exit 2
fi
docker stop "$CID"
echo "stopped $CID ($TARGET_INSTANCE_ID)"
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""Собирает сводную таблицу из evidence/track1/*.json (open-loop)."""
import csv
import json
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVID = ROOT / 'evidence' / 'track1'
def parse_stats(path):
"""Возвращает {'api': {...sum...}, 'postgres': {...}, 'lb': ..., 'redis': ...}."""
agg = {}
for line in path.read_text().splitlines():
# формат: <name> <cpu>% <mem> <unit> / <limit>
m = re.match(
r'^(\S+)\s+([0-9.]+)%\s+([0-9.]+)\s*(B|KiB|MiB|GiB)\s*/',
line,
)
if not m:
continue
name, cpup, memv, memunit = m.groups()
role = None
if '-api-' in name or name.endswith('-api-1') or re.search(r'-api-\d+$', name):
role = 'api'
elif '-postgres-' in name:
role = 'postgres'
elif '-redis-' in name:
role = 'redis'
elif '-lb-' in name:
role = 'lb'
if role is None:
continue
mult = {'B': 1/1024/1024, 'KiB': 1/1024, 'MiB': 1, 'GiB': 1024}[memunit]
mem_mib = float(memv) * mult
slot = agg.setdefault(role, {'cpu_max': 0.0, 'cpu_sum': 0.0, 'mem_mib': 0.0})
slot['cpu_max'] = max(slot['cpu_max'], float(cpup))
slot['cpu_sum'] += float(cpup)
slot['mem_mib'] = max(slot['mem_mib'], mem_mib)
return agg
def main():
rows = []
for path in sorted(EVID.glob('*-rps*.json')):
# имя: A1-rps100.json
stem = path.stem
profile, _, rps_part = stem.rpartition('-rps')
rps = int(rps_part)
r = json.loads(path.read_text())
# stats.txt и pg.txt могут отсутствовать (если снимали отдельно).
stats = path.with_name(stem + '-stats.txt')
agg = parse_stats(stats) if stats.exists() else {}
api = agg.get('api', {'cpu_sum': 0.0, 'cpu_max': 0.0, 'mem_mib': 0.0})
pg = agg.get('postgres', {'cpu_max': 0.0, 'mem_mib': 0.0})
lb = agg.get('lb', {'cpu_max': 0.0})
redis = agg.get('redis', {'cpu_max': 0.0})
rows.append({
'profile': profile,
'offered_rps': rps,
'successful_rps': r.get('successful_rps'),
'error_ratio_pct': round((r.get('error_ratio') or 0) * 100, 3),
'p50_ms': r['successful_latency_ms']['p50'],
'p95_ms': r['successful_latency_ms']['p95'],
'p99_ms': r['successful_latency_ms']['p99'],
'api_cpu_sum_max': round(api['cpu_sum'], 2),
'api_cpu_peak': round(api['cpu_max'], 2),
'api_mem_mib': round(api['mem_mib'], 1),
'pg_cpu_max': round(pg['cpu_max'], 2),
'pg_mem_mib': round(pg['mem_mib'], 1),
'lb_cpu_max': round(lb['cpu_max'], 2),
'redis_cpu_max': round(redis['cpu_max'], 2),
})
if not rows:
print('no json files in', EVID)
return
out_csv = ROOT / 'evidence' / 'track1-summary.csv'
with out_csv.open('w', newline='') as f:
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
print(out_csv.read_text())
print('written:', out_csv)
if __name__ == '__main__':
main()