lab3: итог

This commit is contained in:
2026-10-16 16:18:15 +03:00
parent 6cced49486
commit e42f02c016
141 changed files with 3180 additions and 4753 deletions
+116 -17
View File
@@ -214,15 +214,15 @@ def current_replicas() -> int:
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
"""Применяет конфигурацию к стенду: лимиты, число реплик, образ."""
"""Обновляет изменённые зависимости и API, не перезапуская миграции."""
save_stand_config(config)
replicas = int(config['API_REPLICAS'])
args = ['up', '-d', '--wait', '--no-deps', '--scale', f'api={replicas}']
compose('up', '-d', '--no-deps', '--wait', 'postgres', 'redis', 'haproxy', timeout=600)
args = ['up', '-d', '--no-deps', '--wait', '--scale', f'api={config["API_REPLICAS"]}']
if force_recreate:
args.append('--force-recreate')
args.append('api')
compose(*args, timeout=600)
compose(*args, 'api', timeout=600)
wait_for_lb()
verify_stand(config)
def wait_for_lb(timeout: float = 120) -> None:
@@ -306,6 +306,57 @@ def replica_map() -> dict[str, dict]:
return mapping
def observed_stand() -> dict[str, list[dict]]:
"""Только публичные параметры docker inspect; секреты Env не сохраняются."""
observed = {}
for service in ('api', 'postgres', 'redis', 'haproxy'):
ids = container_ids(service)
rows = []
if ids:
containers = json.loads(run_command(['docker', 'inspect', *ids]).stdout)
for container in containers:
row = {
'container': container['Id'][:12],
'name': container['Name'].lstrip('/'),
'cpus': container['HostConfig']['NanoCpus'] / 1e9,
'memory_bytes': container['HostConfig']['Memory'],
'image_id': container['Image'],
}
if service == 'api':
environment = dict(value.split('=', 1) for value in container['Config']['Env'] if '=' in value)
row['pool_max'] = int(environment['DATABASE_POOL_MAX'])
rows.append(row)
observed[service] = rows
return observed
def verify_stand(config: dict) -> dict:
observed = observed_stand()
expected_replicas = int(config['API_REPLICAS'])
api_cpu, api_memory = replica_limits(expected_replicas, float(config['API_TOTAL_CPUS']), int(config['API_TOTAL_MEMORY_MB']))
limits = {
'api': (api_cpu, api_memory),
'postgres': (float(config['PG_CPUS']), config['PG_MEMORY']),
'redis': (float(config['REDIS_CPUS']), config['REDIS_MEMORY']),
'haproxy': (float(config['LB_CPUS']), config['LB_MEMORY']),
}
for service, rows in observed.items():
count = expected_replicas if service == 'api' else 1
cpu, memory = limits[service]
unit = re.fullmatch(r'(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?', str(memory), re.IGNORECASE)
if not unit:
raise ValueError(f'unsupported memory limit: {memory}')
memory_bytes = int(float(unit[1]) * 1024 ** ('kmgt'.index(unit[2].lower()) + 1 if unit[2] else 0))
if len(rows) != count:
raise RuntimeError(f'{service}: expected {count} containers, observed {len(rows)}')
for row in rows:
if not math.isclose(row['cpus'], cpu, abs_tol=0.001) or row['memory_bytes'] != memory_bytes:
raise RuntimeError(f'{service}: runtime limits {row} differ from requested CPU={cpu}, RAM={memory}')
if service == 'api' and row['pool_max'] != int(config['DATABASE_POOL_MAX']):
raise RuntimeError(f'api: pool {row["pool_max"]} != requested {config["DATABASE_POOL_MAX"]}')
return observed
def image_versions() -> dict:
versions = {}
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
@@ -658,7 +709,7 @@ class Sampler:
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
" (SELECT count(*) FROM pg_stat_activity),"
" (SELECT count(*) FROM messages),"
" (SELECT reltuples::bigint FROM pg_class WHERE oid = 'messages'::regclass),"
" (SELECT pg_total_relation_size('messages'))"
)
output = psql(query)
@@ -841,6 +892,7 @@ def run_measured(
RAW_DIR.mkdir(parents=True, exist_ok=True)
stand = load_stand_config()
verify_stand(stand)
MANIFEST_VUS.clear()
if warmup:
@@ -897,6 +949,7 @@ def write_raw(
'steps': [step['rate'] for step in summary.get('steps', [])],
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
'stand': stand,
'observed_stand': verify_stand(stand),
'replicas': replica_map(),
'dataset': dataset_size(),
'versions': image_versions(),
@@ -1216,13 +1269,15 @@ def analyse_mechanism(groups: dict, labels: list[str]) -> None:
_, samples, manifest = load_raw(label)
stats = container_stats(samples)
mechanism = mechanism_stats(samples)
stand = manifest.get('stand', {})
api_limit = float(stand.get('API_CPUS_PER_REPLICA') or 0)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
generator_peak = max((value['cpu_peak'] for name, value in stats.items() if 'loadgen' in name), default=0.0)
stand = manifest.get('stand', {})
lines += [f'## {key} ({len(group)} повторов)', '']
lines += table(
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API пик %', 'CPU PG пик %', 'CPU ген. пик %'],
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра', 'CPU ген. пик, % ядра'],
[
[
step.get('achieved_rps'),
@@ -1232,7 +1287,7 @@ def analyse_mechanism(groups: dict, labels: list[str]) -> None:
(step.get('status') or {}).get('client_4xx'),
(step.get('status') or {}).get('server_5xx'),
(step.get('status') or {}).get('other'),
api_peak,
api_peak_quota,
pg_peak,
generator_peak,
]
@@ -1289,6 +1344,10 @@ def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
apply_stand(desired, force_recreate=force_recreate)
else:
wait_for_lb()
try:
verify_stand(desired)
except RuntimeError:
apply_stand(desired, force_recreate=force_recreate)
return desired
@@ -2365,10 +2424,20 @@ def analyse_growth(groups: dict, labels: list[str]) -> None:
if baseline:
rate = max((row['rate'] for row in growth_rows(growth)), default=0)
row = step_of_rate(baseline['rows'], rate)
lines.append(f'Базовая точка (серия A, N=4, тот же темп {rate} req/s): **{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms.')
lines.append(
f'Ориентир по колену (серия A, N=4, темп {rate} req/s, сборка C1, MAX_VUS=200): '
f'**{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms — генератор был ограничен пулом VU.'
)
control = groups.get('block#n4')
if control:
control_row = step_of_rate(summarise_series(control)['rows'], rate)
lines.append(
f'Контроль на том же темпе, MAX_VUS=900 (block#n4, 2 повтора): '
f'**{control_row["achieved_rps"]} req/s**, p95 {control_row["p95"]} ms.'
)
lines.append('')
lines += table(
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API пик, %', 'CPU PG пик, %'],
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра'],
[growth_row(key, group) for key, group in sorted(growth.items())],
)
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
@@ -2390,12 +2459,14 @@ def step_of_rate(rows: list[dict], rate: int) -> dict:
def growth_row(key: str, group: list[str]) -> list:
label = representative(group)
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
_, samples, _ = load_raw(label)
_, samples, manifest = load_raw(label)
step = summary['steps'][0] if summary.get('steps') else {'rate': 0, 'achieved_rps': 0, 'p95': 0, 'failed': 0}
stats = container_stats(samples)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
api_limit = float((manifest or {}).get('stand', {}).get('API_CPUS_PER_REPLICA') or 0)
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak, pg_peak]
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak_quota, pg_peak]
def make_graphs(groups: dict, labels: list[str]) -> None:
@@ -2653,18 +2724,46 @@ def cmd_track1(args) -> None:
run_measured('p-c#lb1.0', 'p-c', warmup=False)
log('=== серия роста ресурсов')
for series in ('api', 'pg', 'pool'):
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False))
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False, suffix=''))
log('=== агрегаты')
cmd_analyze(argparse.Namespace())
if args.ab_after:
log('=== A/B улучшения (p-b, 4 реплики)')
cmd_ab(argparse.Namespace(before=args.ab_before, after=args.ab_after, profile='p-b', replicas=4, reps=args.ab_reps, no_warmup=False))
cmd_ab(
argparse.Namespace(
before=args.ab_before,
after=args.ab_after,
profile='p-b',
replicas=4,
reps=args.ab_reps,
no_warmup=False,
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
pg_cpus=DEFAULT_STAND['PG_CPUS'],
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
lb_cpus=DEFAULT_STAND['LB_CPUS'],
loadgen_cpus=DEFAULT_STAND['LOADGEN_CPUS'],
loadgen_memory=DEFAULT_STAND['LOADGEN_MEMORY'],
max_vus=1200,
prefix='ab',
)
)
cmd_analyze(argparse.Namespace())
else:
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
log('=== возврат конфигурации стенда по умолчанию')
ensure_stand({'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']), 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']), 'API_IMAGE': DEFAULT_STAND['API_IMAGE'], 'BUILD_ID': DEFAULT_STAND['BUILD_ID']})
finally:
ensure_stand(
{
'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']),
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
'API_IMAGE': DEFAULT_STAND['API_IMAGE'],
'BUILD_ID': DEFAULT_STAND['BUILD_ID'],
}
)
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='down'))
+76 -35
View File
@@ -17,6 +17,7 @@ export const BASE_URL = (__ENV.BASE_URL || 'http://haproxy:8080').replace(
);
const CHAT_COUNT = Number(__ENV.CHATS || 24);
const LOAD_USERS = 3;
const SEED_MESSAGES_PER_CHAT = __ENV.MODE === 'read' ? 50 : 0;
// Бюджет VU: на 1100 req/s при p95 около 25 ms достаточно около 30 активных VU;
// 60 предвыделенных и 200 максимальных оставляют запас, чтобы узкое место
// оказалось на сервере, а не в генераторе.
@@ -137,28 +138,67 @@ export function setup() {
});
}
const chats = [];
for (let index = 0; index < CHAT_COUNT; index += 1) {
const created = http.post(
`${BASE_URL}/api/v1/chats`,
JSON.stringify({
title: `k6 ${runId} chat ${index}`,
member_ids: users.map((user) => user.id),
}),
{
headers: {
'content-type': 'application/json',
authorization: `Bearer ${users[0].token}`,
},
},
);
if (created.status !== 201)
throw new Error(
`cannot create chat ${index}: ${created.status} ${created.body}`,
);
chats.push(JSON.parse(created.body).id);
}
return { runId, users, chats, profile: __ENV.PROFILE || 'spread' };
const chats = [];
for (let index = 0; index < CHAT_COUNT; index += 1) {
const created = http.post(
`${BASE_URL}/api/v1/chats`,
JSON.stringify({
title: `k6 ${runId} chat ${index}`,
member_ids: users.map((user) => user.id),
}),
{
headers: {
'content-type': 'application/json',
authorization: `Bearer ${users[0].token}`,
},
},
);
if (created.status !== 201)
throw new Error(
`cannot create chat ${index}: ${created.status} ${created.body}`,
);
const chat = JSON.parse(created.body);
chats.push(chat.id);
if (SEED_MESSAGES_PER_CHAT === 0) continue;
for (let messageIndex = 0; messageIndex < SEED_MESSAGES_PER_CHAT; messageIndex += 1) {
const key = `seed-${runId}-${index}-${messageIndex}`;
const seeded = http.post(
`${BASE_URL}/api/v1/chats/${chat.id}/messages`,
JSON.stringify({ text: `seed ${runId} ${index} ${messageIndex}` }),
{
headers: {
'content-type': 'application/json',
authorization: `Bearer ${users[0].token}`,
'idempotency-key': key,
},
},
);
if (seeded.status !== 201)
throw new Error(
`cannot seed chat ${index} message ${messageIndex}: ${seeded.status} ${seeded.body}`,
);
}
const seededPage = http.get(
`${BASE_URL}/api/v1/chats/${chat.id}/messages?limit=50`,
{ headers: { authorization: `Bearer ${users[0].token}` } },
);
if (seededPage.status !== 200)
throw new Error(
`cannot verify chat ${index} seed: ${seededPage.status} ${seededPage.body}`,
);
const seededItems = JSON.parse(seededPage.body).items;
if (!Array.isArray(seededItems) || seededItems.length !== SEED_MESSAGES_PER_CHAT)
throw new Error(
`cannot verify chat ${index} seed count: expected ${SEED_MESSAGES_PER_CHAT}, got ${seededItems?.length ?? 'invalid'}`,
);
}
return {
runId,
users,
chats,
profile: __ENV.PROFILE || 'spread',
seed_messages_per_chat: SEED_MESSAGES_PER_CHAT,
};
}
function chatFor(data) {
@@ -171,18 +211,18 @@ function actorFor(data) {
}
function recordResponse(step, response, expected) {
step.requests.add(1);
step.duration.add(response.timings.duration);
if (response.status === expected) {
step.status2xx.add(1);
return;
}
step.failed.add(1);
if (response.status >= 200 && response.status < 300) step.status2xx.add(1);
else if (response.status >= 400 && response.status < 500) step.status4xx.add(1);
else if (response.status >= 500 && response.status < 600) step.status5xx.add(1);
else step.statusOther.add(1);
step.vus.add(exec.instance.vusActive);
step.requests.add(1);
step.duration.add(response.timings.duration);
step.vus.add(exec.instance.vusActive);
if (response.status === expected) {
step.status2xx.add(1);
return;
}
step.failed.add(1);
if (response.status >= 200 && response.status < 300) step.status2xx.add(1);
else if (response.status >= 400 && response.status < 500) step.status4xx.add(1);
else if (response.status >= 500 && response.status < 600) step.status5xx.add(1);
else step.statusOther.add(1);
}
export function writeIteration(data, metrics) {
@@ -244,7 +284,7 @@ export function summaryJson(data, metrics, rates, kind, stepSeconds) {
p99: round(values['p(99)'] ?? 0, 2),
avg: round(values.avg ?? 0, 2),
max: round(values.max ?? 0, 2),
vus_active: data.metrics[`${name}_vus`]?.values?.value ?? 0,
vus_active: data.metrics[`${name}_vus`]?.values?.max ?? 0,
};
});
const global = data.metrics.http_req_duration?.values ?? {};
@@ -252,6 +292,7 @@ export function summaryJson(data, metrics, rates, kind, stepSeconds) {
label: __ENV.LABEL || 'unlabelled',
profile: __ENV.PROFILE || 'spread',
mode: __ENV.MODE || 'write',
seed_messages_per_chat: SEED_MESSAGES_PER_CHAT,
kind,
base_url: BASE_URL,
steps,
+8 -3
View File
@@ -10,7 +10,12 @@ replicas="${API_REPLICAS:-}"
if [ -z "$replicas" ] && [ -f "$env_file" ]; then
replicas=$(sed -n 's/^API_REPLICAS=//p' "$env_file" | tail -n 1)
fi
if [ -z "$replicas" ] && [ -f .env.stand ]; then
replicas=$(sed -n 's/^API_REPLICAS=//p' .env.stand | tail -n 1)
fi
replicas="${replicas:-2}"
exec docker compose --env-file "$env_file" up -d --no-deps \
--scale "api=${replicas}" api
set -- --env-file "$env_file"
if [ -f .env.stand ]; then
set -- "$@" --env-file .env.stand
fi
exec docker compose "$@" up -d --no-deps --scale "api=${replicas}" api