lab3: итог

This commit is contained in:
2026-10-16 16:18:15 +03:00
parent 6cced49486
commit e42f02c016
141 changed files with 3180 additions and 4753 deletions
+116 -17
View File
@@ -214,15 +214,15 @@ def current_replicas() -> int:
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
"""Применяет конфигурацию к стенду: лимиты, число реплик, образ."""
"""Обновляет изменённые зависимости и API, не перезапуская миграции."""
save_stand_config(config)
replicas = int(config['API_REPLICAS'])
args = ['up', '-d', '--wait', '--no-deps', '--scale', f'api={replicas}']
compose('up', '-d', '--no-deps', '--wait', 'postgres', 'redis', 'haproxy', timeout=600)
args = ['up', '-d', '--no-deps', '--wait', '--scale', f'api={config["API_REPLICAS"]}']
if force_recreate:
args.append('--force-recreate')
args.append('api')
compose(*args, timeout=600)
compose(*args, 'api', timeout=600)
wait_for_lb()
verify_stand(config)
def wait_for_lb(timeout: float = 120) -> None:
@@ -306,6 +306,57 @@ def replica_map() -> dict[str, dict]:
return mapping
def observed_stand() -> dict[str, list[dict]]:
"""Только публичные параметры docker inspect; секреты Env не сохраняются."""
observed = {}
for service in ('api', 'postgres', 'redis', 'haproxy'):
ids = container_ids(service)
rows = []
if ids:
containers = json.loads(run_command(['docker', 'inspect', *ids]).stdout)
for container in containers:
row = {
'container': container['Id'][:12],
'name': container['Name'].lstrip('/'),
'cpus': container['HostConfig']['NanoCpus'] / 1e9,
'memory_bytes': container['HostConfig']['Memory'],
'image_id': container['Image'],
}
if service == 'api':
environment = dict(value.split('=', 1) for value in container['Config']['Env'] if '=' in value)
row['pool_max'] = int(environment['DATABASE_POOL_MAX'])
rows.append(row)
observed[service] = rows
return observed
def verify_stand(config: dict) -> dict:
observed = observed_stand()
expected_replicas = int(config['API_REPLICAS'])
api_cpu, api_memory = replica_limits(expected_replicas, float(config['API_TOTAL_CPUS']), int(config['API_TOTAL_MEMORY_MB']))
limits = {
'api': (api_cpu, api_memory),
'postgres': (float(config['PG_CPUS']), config['PG_MEMORY']),
'redis': (float(config['REDIS_CPUS']), config['REDIS_MEMORY']),
'haproxy': (float(config['LB_CPUS']), config['LB_MEMORY']),
}
for service, rows in observed.items():
count = expected_replicas if service == 'api' else 1
cpu, memory = limits[service]
unit = re.fullmatch(r'(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?', str(memory), re.IGNORECASE)
if not unit:
raise ValueError(f'unsupported memory limit: {memory}')
memory_bytes = int(float(unit[1]) * 1024 ** ('kmgt'.index(unit[2].lower()) + 1 if unit[2] else 0))
if len(rows) != count:
raise RuntimeError(f'{service}: expected {count} containers, observed {len(rows)}')
for row in rows:
if not math.isclose(row['cpus'], cpu, abs_tol=0.001) or row['memory_bytes'] != memory_bytes:
raise RuntimeError(f'{service}: runtime limits {row} differ from requested CPU={cpu}, RAM={memory}')
if service == 'api' and row['pool_max'] != int(config['DATABASE_POOL_MAX']):
raise RuntimeError(f'api: pool {row["pool_max"]} != requested {config["DATABASE_POOL_MAX"]}')
return observed
def image_versions() -> dict:
versions = {}
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
@@ -658,7 +709,7 @@ class Sampler:
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
" (SELECT count(*) FROM pg_stat_activity),"
" (SELECT count(*) FROM messages),"
" (SELECT reltuples::bigint FROM pg_class WHERE oid = 'messages'::regclass),"
" (SELECT pg_total_relation_size('messages'))"
)
output = psql(query)
@@ -841,6 +892,7 @@ def run_measured(
RAW_DIR.mkdir(parents=True, exist_ok=True)
stand = load_stand_config()
verify_stand(stand)
MANIFEST_VUS.clear()
if warmup:
@@ -897,6 +949,7 @@ def write_raw(
'steps': [step['rate'] for step in summary.get('steps', [])],
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
'stand': stand,
'observed_stand': verify_stand(stand),
'replicas': replica_map(),
'dataset': dataset_size(),
'versions': image_versions(),
@@ -1216,13 +1269,15 @@ def analyse_mechanism(groups: dict, labels: list[str]) -> None:
_, samples, manifest = load_raw(label)
stats = container_stats(samples)
mechanism = mechanism_stats(samples)
stand = manifest.get('stand', {})
api_limit = float(stand.get('API_CPUS_PER_REPLICA') or 0)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
generator_peak = max((value['cpu_peak'] for name, value in stats.items() if 'loadgen' in name), default=0.0)
stand = manifest.get('stand', {})
lines += [f'## {key} ({len(group)} повторов)', '']
lines += table(
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API пик %', 'CPU PG пик %', 'CPU ген. пик %'],
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра', 'CPU ген. пик, % ядра'],
[
[
step.get('achieved_rps'),
@@ -1232,7 +1287,7 @@ def analyse_mechanism(groups: dict, labels: list[str]) -> None:
(step.get('status') or {}).get('client_4xx'),
(step.get('status') or {}).get('server_5xx'),
(step.get('status') or {}).get('other'),
api_peak,
api_peak_quota,
pg_peak,
generator_peak,
]
@@ -1289,6 +1344,10 @@ def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
apply_stand(desired, force_recreate=force_recreate)
else:
wait_for_lb()
try:
verify_stand(desired)
except RuntimeError:
apply_stand(desired, force_recreate=force_recreate)
return desired
@@ -2365,10 +2424,20 @@ def analyse_growth(groups: dict, labels: list[str]) -> None:
if baseline:
rate = max((row['rate'] for row in growth_rows(growth)), default=0)
row = step_of_rate(baseline['rows'], rate)
lines.append(f'Базовая точка (серия A, N=4, тот же темп {rate} req/s): **{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms.')
lines.append(
f'Ориентир по колену (серия A, N=4, темп {rate} req/s, сборка C1, MAX_VUS=200): '
f'**{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms — генератор был ограничен пулом VU.'
)
control = groups.get('block#n4')
if control:
control_row = step_of_rate(summarise_series(control)['rows'], rate)
lines.append(
f'Контроль на том же темпе, MAX_VUS=900 (block#n4, 2 повтора): '
f'**{control_row["achieved_rps"]} req/s**, p95 {control_row["p95"]} ms.'
)
lines.append('')
lines += table(
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API пик, %', 'CPU PG пик, %'],
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра'],
[growth_row(key, group) for key, group in sorted(growth.items())],
)
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
@@ -2390,12 +2459,14 @@ def step_of_rate(rows: list[dict], rate: int) -> dict:
def growth_row(key: str, group: list[str]) -> list:
label = representative(group)
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
_, samples, _ = load_raw(label)
_, samples, manifest = load_raw(label)
step = summary['steps'][0] if summary.get('steps') else {'rate': 0, 'achieved_rps': 0, 'p95': 0, 'failed': 0}
stats = container_stats(samples)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
api_limit = float((manifest or {}).get('stand', {}).get('API_CPUS_PER_REPLICA') or 0)
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak, pg_peak]
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak_quota, pg_peak]
def make_graphs(groups: dict, labels: list[str]) -> None:
@@ -2653,18 +2724,46 @@ def cmd_track1(args) -> None:
run_measured('p-c#lb1.0', 'p-c', warmup=False)
log('=== серия роста ресурсов')
for series in ('api', 'pg', 'pool'):
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False))
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False, suffix=''))
log('=== агрегаты')
cmd_analyze(argparse.Namespace())
if args.ab_after:
log('=== A/B улучшения (p-b, 4 реплики)')
cmd_ab(argparse.Namespace(before=args.ab_before, after=args.ab_after, profile='p-b', replicas=4, reps=args.ab_reps, no_warmup=False))
cmd_ab(
argparse.Namespace(
before=args.ab_before,
after=args.ab_after,
profile='p-b',
replicas=4,
reps=args.ab_reps,
no_warmup=False,
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
pg_cpus=DEFAULT_STAND['PG_CPUS'],
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
lb_cpus=DEFAULT_STAND['LB_CPUS'],
loadgen_cpus=DEFAULT_STAND['LOADGEN_CPUS'],
loadgen_memory=DEFAULT_STAND['LOADGEN_MEMORY'],
max_vus=1200,
prefix='ab',
)
)
cmd_analyze(argparse.Namespace())
else:
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
log('=== возврат конфигурации стенда по умолчанию')
ensure_stand({'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']), 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']), 'API_IMAGE': DEFAULT_STAND['API_IMAGE'], 'BUILD_ID': DEFAULT_STAND['BUILD_ID']})
finally:
ensure_stand(
{
'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']),
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
'API_IMAGE': DEFAULT_STAND['API_IMAGE'],
'BUILD_ID': DEFAULT_STAND['BUILD_ID'],
}
)
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='down'))