forked from mtusi2026/mtusi2026lab3
lab3: итог
This commit is contained in:
+116
-17
@@ -214,15 +214,15 @@ def current_replicas() -> int:
|
||||
|
||||
|
||||
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
|
||||
"""Применяет конфигурацию к стенду: лимиты, число реплик, образ."""
|
||||
"""Обновляет изменённые зависимости и API, не перезапуская миграции."""
|
||||
save_stand_config(config)
|
||||
replicas = int(config['API_REPLICAS'])
|
||||
args = ['up', '-d', '--wait', '--no-deps', '--scale', f'api={replicas}']
|
||||
compose('up', '-d', '--no-deps', '--wait', 'postgres', 'redis', 'haproxy', timeout=600)
|
||||
args = ['up', '-d', '--no-deps', '--wait', '--scale', f'api={config["API_REPLICAS"]}']
|
||||
if force_recreate:
|
||||
args.append('--force-recreate')
|
||||
args.append('api')
|
||||
compose(*args, timeout=600)
|
||||
compose(*args, 'api', timeout=600)
|
||||
wait_for_lb()
|
||||
verify_stand(config)
|
||||
|
||||
|
||||
def wait_for_lb(timeout: float = 120) -> None:
|
||||
@@ -306,6 +306,57 @@ def replica_map() -> dict[str, dict]:
|
||||
return mapping
|
||||
|
||||
|
||||
def observed_stand() -> dict[str, list[dict]]:
|
||||
"""Только публичные параметры docker inspect; секреты Env не сохраняются."""
|
||||
observed = {}
|
||||
for service in ('api', 'postgres', 'redis', 'haproxy'):
|
||||
ids = container_ids(service)
|
||||
rows = []
|
||||
if ids:
|
||||
containers = json.loads(run_command(['docker', 'inspect', *ids]).stdout)
|
||||
for container in containers:
|
||||
row = {
|
||||
'container': container['Id'][:12],
|
||||
'name': container['Name'].lstrip('/'),
|
||||
'cpus': container['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_bytes': container['HostConfig']['Memory'],
|
||||
'image_id': container['Image'],
|
||||
}
|
||||
if service == 'api':
|
||||
environment = dict(value.split('=', 1) for value in container['Config']['Env'] if '=' in value)
|
||||
row['pool_max'] = int(environment['DATABASE_POOL_MAX'])
|
||||
rows.append(row)
|
||||
observed[service] = rows
|
||||
return observed
|
||||
|
||||
|
||||
def verify_stand(config: dict) -> dict:
|
||||
observed = observed_stand()
|
||||
expected_replicas = int(config['API_REPLICAS'])
|
||||
api_cpu, api_memory = replica_limits(expected_replicas, float(config['API_TOTAL_CPUS']), int(config['API_TOTAL_MEMORY_MB']))
|
||||
limits = {
|
||||
'api': (api_cpu, api_memory),
|
||||
'postgres': (float(config['PG_CPUS']), config['PG_MEMORY']),
|
||||
'redis': (float(config['REDIS_CPUS']), config['REDIS_MEMORY']),
|
||||
'haproxy': (float(config['LB_CPUS']), config['LB_MEMORY']),
|
||||
}
|
||||
for service, rows in observed.items():
|
||||
count = expected_replicas if service == 'api' else 1
|
||||
cpu, memory = limits[service]
|
||||
unit = re.fullmatch(r'(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?', str(memory), re.IGNORECASE)
|
||||
if not unit:
|
||||
raise ValueError(f'unsupported memory limit: {memory}')
|
||||
memory_bytes = int(float(unit[1]) * 1024 ** ('kmgt'.index(unit[2].lower()) + 1 if unit[2] else 0))
|
||||
if len(rows) != count:
|
||||
raise RuntimeError(f'{service}: expected {count} containers, observed {len(rows)}')
|
||||
for row in rows:
|
||||
if not math.isclose(row['cpus'], cpu, abs_tol=0.001) or row['memory_bytes'] != memory_bytes:
|
||||
raise RuntimeError(f'{service}: runtime limits {row} differ from requested CPU={cpu}, RAM={memory}')
|
||||
if service == 'api' and row['pool_max'] != int(config['DATABASE_POOL_MAX']):
|
||||
raise RuntimeError(f'api: pool {row["pool_max"]} != requested {config["DATABASE_POOL_MAX"]}')
|
||||
return observed
|
||||
|
||||
|
||||
def image_versions() -> dict:
|
||||
versions = {}
|
||||
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
|
||||
@@ -658,7 +709,7 @@ class Sampler:
|
||||
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
|
||||
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
|
||||
" (SELECT count(*) FROM pg_stat_activity),"
|
||||
" (SELECT count(*) FROM messages),"
|
||||
" (SELECT reltuples::bigint FROM pg_class WHERE oid = 'messages'::regclass),"
|
||||
" (SELECT pg_total_relation_size('messages'))"
|
||||
)
|
||||
output = psql(query)
|
||||
@@ -841,6 +892,7 @@ def run_measured(
|
||||
|
||||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
stand = load_stand_config()
|
||||
verify_stand(stand)
|
||||
MANIFEST_VUS.clear()
|
||||
|
||||
if warmup:
|
||||
@@ -897,6 +949,7 @@ def write_raw(
|
||||
'steps': [step['rate'] for step in summary.get('steps', [])],
|
||||
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
|
||||
'stand': stand,
|
||||
'observed_stand': verify_stand(stand),
|
||||
'replicas': replica_map(),
|
||||
'dataset': dataset_size(),
|
||||
'versions': image_versions(),
|
||||
@@ -1216,13 +1269,15 @@ def analyse_mechanism(groups: dict, labels: list[str]) -> None:
|
||||
_, samples, manifest = load_raw(label)
|
||||
stats = container_stats(samples)
|
||||
mechanism = mechanism_stats(samples)
|
||||
stand = manifest.get('stand', {})
|
||||
api_limit = float(stand.get('API_CPUS_PER_REPLICA') or 0)
|
||||
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
|
||||
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
|
||||
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
|
||||
generator_peak = max((value['cpu_peak'] for name, value in stats.items() if 'loadgen' in name), default=0.0)
|
||||
stand = manifest.get('stand', {})
|
||||
lines += [f'## {key} ({len(group)} повторов)', '']
|
||||
lines += table(
|
||||
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API пик %', 'CPU PG пик %', 'CPU ген. пик %'],
|
||||
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра', 'CPU ген. пик, % ядра'],
|
||||
[
|
||||
[
|
||||
step.get('achieved_rps'),
|
||||
@@ -1232,7 +1287,7 @@ def analyse_mechanism(groups: dict, labels: list[str]) -> None:
|
||||
(step.get('status') or {}).get('client_4xx'),
|
||||
(step.get('status') or {}).get('server_5xx'),
|
||||
(step.get('status') or {}).get('other'),
|
||||
api_peak,
|
||||
api_peak_quota,
|
||||
pg_peak,
|
||||
generator_peak,
|
||||
]
|
||||
@@ -1289,6 +1344,10 @@ def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
|
||||
apply_stand(desired, force_recreate=force_recreate)
|
||||
else:
|
||||
wait_for_lb()
|
||||
try:
|
||||
verify_stand(desired)
|
||||
except RuntimeError:
|
||||
apply_stand(desired, force_recreate=force_recreate)
|
||||
return desired
|
||||
|
||||
|
||||
@@ -2365,10 +2424,20 @@ def analyse_growth(groups: dict, labels: list[str]) -> None:
|
||||
if baseline:
|
||||
rate = max((row['rate'] for row in growth_rows(growth)), default=0)
|
||||
row = step_of_rate(baseline['rows'], rate)
|
||||
lines.append(f'Базовая точка (серия A, N=4, тот же темп {rate} req/s): **{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms.')
|
||||
lines.append(
|
||||
f'Ориентир по колену (серия A, N=4, темп {rate} req/s, сборка C1, MAX_VUS=200): '
|
||||
f'**{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms — генератор был ограничен пулом VU.'
|
||||
)
|
||||
control = groups.get('block#n4')
|
||||
if control:
|
||||
control_row = step_of_rate(summarise_series(control)['rows'], rate)
|
||||
lines.append(
|
||||
f'Контроль на том же темпе, MAX_VUS=900 (block#n4, 2 повтора): '
|
||||
f'**{control_row["achieved_rps"]} req/s**, p95 {control_row["p95"]} ms.'
|
||||
)
|
||||
lines.append('')
|
||||
lines += table(
|
||||
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API пик, %', 'CPU PG пик, %'],
|
||||
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра'],
|
||||
[growth_row(key, group) for key, group in sorted(growth.items())],
|
||||
)
|
||||
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
|
||||
@@ -2390,12 +2459,14 @@ def step_of_rate(rows: list[dict], rate: int) -> dict:
|
||||
def growth_row(key: str, group: list[str]) -> list:
|
||||
label = representative(group)
|
||||
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
|
||||
_, samples, _ = load_raw(label)
|
||||
_, samples, manifest = load_raw(label)
|
||||
step = summary['steps'][0] if summary.get('steps') else {'rate': 0, 'achieved_rps': 0, 'p95': 0, 'failed': 0}
|
||||
stats = container_stats(samples)
|
||||
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
|
||||
api_limit = float((manifest or {}).get('stand', {}).get('API_CPUS_PER_REPLICA') or 0)
|
||||
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
|
||||
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
|
||||
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak, pg_peak]
|
||||
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak_quota, pg_peak]
|
||||
|
||||
|
||||
def make_graphs(groups: dict, labels: list[str]) -> None:
|
||||
@@ -2653,18 +2724,46 @@ def cmd_track1(args) -> None:
|
||||
run_measured('p-c#lb1.0', 'p-c', warmup=False)
|
||||
log('=== серия роста ресурсов')
|
||||
for series in ('api', 'pg', 'pool'):
|
||||
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False))
|
||||
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False, suffix=''))
|
||||
log('=== агрегаты')
|
||||
cmd_analyze(argparse.Namespace())
|
||||
if args.ab_after:
|
||||
log('=== A/B улучшения (p-b, 4 реплики)')
|
||||
cmd_ab(argparse.Namespace(before=args.ab_before, after=args.ab_after, profile='p-b', replicas=4, reps=args.ab_reps, no_warmup=False))
|
||||
cmd_ab(
|
||||
argparse.Namespace(
|
||||
before=args.ab_before,
|
||||
after=args.ab_after,
|
||||
profile='p-b',
|
||||
replicas=4,
|
||||
reps=args.ab_reps,
|
||||
no_warmup=False,
|
||||
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
|
||||
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
|
||||
pg_cpus=DEFAULT_STAND['PG_CPUS'],
|
||||
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
|
||||
lb_cpus=DEFAULT_STAND['LB_CPUS'],
|
||||
loadgen_cpus=DEFAULT_STAND['LOADGEN_CPUS'],
|
||||
loadgen_memory=DEFAULT_STAND['LOADGEN_MEMORY'],
|
||||
max_vus=1200,
|
||||
prefix='ab',
|
||||
)
|
||||
)
|
||||
cmd_analyze(argparse.Namespace())
|
||||
else:
|
||||
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
|
||||
log('=== возврат конфигурации стенда по умолчанию')
|
||||
ensure_stand({'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']), 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']), 'API_IMAGE': DEFAULT_STAND['API_IMAGE'], 'BUILD_ID': DEFAULT_STAND['BUILD_ID']})
|
||||
finally:
|
||||
ensure_stand(
|
||||
{
|
||||
'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']),
|
||||
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
|
||||
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
|
||||
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
|
||||
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
|
||||
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
|
||||
'API_IMAGE': DEFAULT_STAND['API_IMAGE'],
|
||||
'BUILD_ID': DEFAULT_STAND['BUILD_ID'],
|
||||
}
|
||||
)
|
||||
if not args.no_quiesce:
|
||||
cmd_quiesce(argparse.Namespace(action='down'))
|
||||
|
||||
|
||||
+76
-35
@@ -17,6 +17,7 @@ export const BASE_URL = (__ENV.BASE_URL || 'http://haproxy:8080').replace(
|
||||
);
|
||||
const CHAT_COUNT = Number(__ENV.CHATS || 24);
|
||||
const LOAD_USERS = 3;
|
||||
const SEED_MESSAGES_PER_CHAT = __ENV.MODE === 'read' ? 50 : 0;
|
||||
// Бюджет VU: на 1100 req/s при p95 около 25 ms достаточно около 30 активных VU;
|
||||
// 60 предвыделенных и 200 максимальных оставляют запас, чтобы узкое место
|
||||
// оказалось на сервере, а не в генераторе.
|
||||
@@ -137,28 +138,67 @@ export function setup() {
|
||||
});
|
||||
}
|
||||
|
||||
const chats = [];
|
||||
for (let index = 0; index < CHAT_COUNT; index += 1) {
|
||||
const created = http.post(
|
||||
`${BASE_URL}/api/v1/chats`,
|
||||
JSON.stringify({
|
||||
title: `k6 ${runId} chat ${index}`,
|
||||
member_ids: users.map((user) => user.id),
|
||||
}),
|
||||
{
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${users[0].token}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (created.status !== 201)
|
||||
throw new Error(
|
||||
`cannot create chat ${index}: ${created.status} ${created.body}`,
|
||||
);
|
||||
chats.push(JSON.parse(created.body).id);
|
||||
}
|
||||
return { runId, users, chats, profile: __ENV.PROFILE || 'spread' };
|
||||
const chats = [];
|
||||
for (let index = 0; index < CHAT_COUNT; index += 1) {
|
||||
const created = http.post(
|
||||
`${BASE_URL}/api/v1/chats`,
|
||||
JSON.stringify({
|
||||
title: `k6 ${runId} chat ${index}`,
|
||||
member_ids: users.map((user) => user.id),
|
||||
}),
|
||||
{
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${users[0].token}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (created.status !== 201)
|
||||
throw new Error(
|
||||
`cannot create chat ${index}: ${created.status} ${created.body}`,
|
||||
);
|
||||
const chat = JSON.parse(created.body);
|
||||
chats.push(chat.id);
|
||||
if (SEED_MESSAGES_PER_CHAT === 0) continue;
|
||||
for (let messageIndex = 0; messageIndex < SEED_MESSAGES_PER_CHAT; messageIndex += 1) {
|
||||
const key = `seed-${runId}-${index}-${messageIndex}`;
|
||||
const seeded = http.post(
|
||||
`${BASE_URL}/api/v1/chats/${chat.id}/messages`,
|
||||
JSON.stringify({ text: `seed ${runId} ${index} ${messageIndex}` }),
|
||||
{
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${users[0].token}`,
|
||||
'idempotency-key': key,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (seeded.status !== 201)
|
||||
throw new Error(
|
||||
`cannot seed chat ${index} message ${messageIndex}: ${seeded.status} ${seeded.body}`,
|
||||
);
|
||||
}
|
||||
const seededPage = http.get(
|
||||
`${BASE_URL}/api/v1/chats/${chat.id}/messages?limit=50`,
|
||||
{ headers: { authorization: `Bearer ${users[0].token}` } },
|
||||
);
|
||||
if (seededPage.status !== 200)
|
||||
throw new Error(
|
||||
`cannot verify chat ${index} seed: ${seededPage.status} ${seededPage.body}`,
|
||||
);
|
||||
const seededItems = JSON.parse(seededPage.body).items;
|
||||
if (!Array.isArray(seededItems) || seededItems.length !== SEED_MESSAGES_PER_CHAT)
|
||||
throw new Error(
|
||||
`cannot verify chat ${index} seed count: expected ${SEED_MESSAGES_PER_CHAT}, got ${seededItems?.length ?? 'invalid'}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
runId,
|
||||
users,
|
||||
chats,
|
||||
profile: __ENV.PROFILE || 'spread',
|
||||
seed_messages_per_chat: SEED_MESSAGES_PER_CHAT,
|
||||
};
|
||||
}
|
||||
|
||||
function chatFor(data) {
|
||||
@@ -171,18 +211,18 @@ function actorFor(data) {
|
||||
}
|
||||
|
||||
function recordResponse(step, response, expected) {
|
||||
step.requests.add(1);
|
||||
step.duration.add(response.timings.duration);
|
||||
if (response.status === expected) {
|
||||
step.status2xx.add(1);
|
||||
return;
|
||||
}
|
||||
step.failed.add(1);
|
||||
if (response.status >= 200 && response.status < 300) step.status2xx.add(1);
|
||||
else if (response.status >= 400 && response.status < 500) step.status4xx.add(1);
|
||||
else if (response.status >= 500 && response.status < 600) step.status5xx.add(1);
|
||||
else step.statusOther.add(1);
|
||||
step.vus.add(exec.instance.vusActive);
|
||||
step.requests.add(1);
|
||||
step.duration.add(response.timings.duration);
|
||||
step.vus.add(exec.instance.vusActive);
|
||||
if (response.status === expected) {
|
||||
step.status2xx.add(1);
|
||||
return;
|
||||
}
|
||||
step.failed.add(1);
|
||||
if (response.status >= 200 && response.status < 300) step.status2xx.add(1);
|
||||
else if (response.status >= 400 && response.status < 500) step.status4xx.add(1);
|
||||
else if (response.status >= 500 && response.status < 600) step.status5xx.add(1);
|
||||
else step.statusOther.add(1);
|
||||
}
|
||||
|
||||
export function writeIteration(data, metrics) {
|
||||
@@ -244,7 +284,7 @@ export function summaryJson(data, metrics, rates, kind, stepSeconds) {
|
||||
p99: round(values['p(99)'] ?? 0, 2),
|
||||
avg: round(values.avg ?? 0, 2),
|
||||
max: round(values.max ?? 0, 2),
|
||||
vus_active: data.metrics[`${name}_vus`]?.values?.value ?? 0,
|
||||
vus_active: data.metrics[`${name}_vus`]?.values?.max ?? 0,
|
||||
};
|
||||
});
|
||||
const global = data.metrics.http_req_duration?.values ?? {};
|
||||
@@ -252,6 +292,7 @@ export function summaryJson(data, metrics, rates, kind, stepSeconds) {
|
||||
label: __ENV.LABEL || 'unlabelled',
|
||||
profile: __ENV.PROFILE || 'spread',
|
||||
mode: __ENV.MODE || 'write',
|
||||
seed_messages_per_chat: SEED_MESSAGES_PER_CHAT,
|
||||
kind,
|
||||
base_url: BASE_URL,
|
||||
steps,
|
||||
|
||||
@@ -10,7 +10,12 @@ replicas="${API_REPLICAS:-}"
|
||||
if [ -z "$replicas" ] && [ -f "$env_file" ]; then
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' "$env_file" | tail -n 1)
|
||||
fi
|
||||
if [ -z "$replicas" ] && [ -f .env.stand ]; then
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' .env.stand | tail -n 1)
|
||||
fi
|
||||
replicas="${replicas:-2}"
|
||||
|
||||
exec docker compose --env-file "$env_file" up -d --no-deps \
|
||||
--scale "api=${replicas}" api
|
||||
set -- --env-file "$env_file"
|
||||
if [ -f .env.stand ]; then
|
||||
set -- "$@" --env-file .env.stand
|
||||
fi
|
||||
exec docker compose "$@" up -d --no-deps --scale "api=${replicas}" api
|
||||
|
||||
Reference in New Issue
Block a user