Files
mtusi2026lab3/scripts/lab3/lab3.py
T

3231 lines
141 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Драйвер экспериментов lab3: стенд, доказательства уровней 3/4 и трек 1.
Запуск из корня репозитория: python3 scripts/lab3/lab3.py <подкоманда> ...
Подкоманды:
stand up|down|status поднять/остановить/показать стенд
check spread|instances|idempotency
разные реплики на одной сессии; карта X-Instance-Id;
фильтр e2e по идемпотентности
run --label ... один прогон профиля с сэмплером
sweep --profile p-a серия 1/2/4 реплики (интерливинг повторов)
track1-extras --part why блок 1500 req/s: контроль и серия роста ресурсов
failover --replicas all фазовый драйвер отказа реплики
rolling-update --under-load поочерёдная замена реплик под нагрузкой
ab --before T --after T интерливинг A/B двух образов
analyze агрегаты и графики из raw
quiesce up|down|status обёртка scripts/quiesce.sh
track1 полная серия трека 1 по зафиксированному плану
Только стандартная библиотека (графики — matplotlib на хосте). Данные прогонов
складываются в evidence/track1/raw: <label>.k6.json (сводка k6),
<label>.samples.jsonl.gz (сэмплер), <label>.manifest.json (условия измерения).
"""
from __future__ import annotations
import argparse
import csv
import gzip
import json
import math
import os
import re
import shutil
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
ENV_FILE = ROOT / '.env'
STAND_ENV = ROOT / '.env.stand'
STAND_STATE = ROOT / '.env.stand.json'
QUIESCE_STATE = ROOT / '.lab3-quiesce.json'
RAW_DIR = ROOT / 'evidence/track1/raw'
TRACK1_DIR = ROOT / 'evidence/track1'
GRAPH_DIR = TRACK1_DIR / 'graphs'
LEVEL3_DIR = ROOT / 'evidence/level3'
LEVEL4_DIR = ROOT / 'evidence/level4'
PROJECT = 'mtusi-lab3'
BASE_URL = os.environ.get('LAB3_BASE_URL', 'http://127.0.0.1:8080')
LB_STATS_URL = os.environ.get('LAB3_LB_STATS', 'http://127.0.0.1:8404/stats;csv')
COMPOSE = ['docker', 'compose', '--env-file', str(ENV_FILE), '--env-file', str(STAND_ENV)]
# Фиксированный бюджет серии трека 1: суммарные ресурсы API одинаковы для всех
# сравниваемых конфигураций, поэтому на реплику приходится 3/N CPU и 1.5/N GiB.
API_TOTAL_CPUS = 3.0
API_TOTAL_MEMORY_MB = 1536
DEFAULT_STAND = {
'API_TOTAL_CPUS': API_TOTAL_CPUS,
'API_TOTAL_MEMORY_MB': API_TOTAL_MEMORY_MB,
'API_REPLICAS': 2,
'PG_CPUS': 1.5,
'PG_MEMORY': '512m',
'REDIS_CPUS': 0.25,
'REDIS_MEMORY': '128m',
'LB_CPUS': 0.25,
'LB_MEMORY': '128m',
'LOADGEN_CPUS': 1.5,
'LOADGEN_MEMORY': '256m',
'API_IMAGE': 'mtusi-messenger:lab3',
'BUILD_ID': 'base',
'DATABASE_POOL_MAX': 10,
}
PROFILES = {
'p-a': {
'mode': 'write',
'spread': 'spread',
'ladder': [100, 200, 300, 400, 550, 700, 900, 1100],
},
'p-b': {
'mode': 'write',
'spread': 'hot',
'ladder': [50, 100, 150, 200, 250, 300, 350, 400],
},
'p-c': {
'mode': 'read',
'spread': 'spread',
'ladder': [200, 400, 700, 1000, 1500, 2000, 2500, 3000],
},
}
STEP_SECONDS = 15
WARMUP_SECONDS = 30
CHATS = int(os.environ.get('LAB3_CHATS', '24'))
def log(message: str) -> None:
print(f'[{datetime.now().strftime("%H:%M:%S")}] {message}', flush=True)
def now_iso() -> str:
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def run_command(
command: list[str],
*,
check: bool = True,
timeout: float | None = None,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
result = subprocess.run(
command,
cwd=ROOT,
capture_output=True,
text=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if check and result.returncode != 0:
raise SystemExit(
f'command failed ({result.returncode}): {" ".join(command)}\n'
f'{result.stdout[-2000:]}\n{result.stderr[-2000:]}'
)
return result
def compose(*args: str, check: bool = True, timeout: float | None = 300) -> subprocess.CompletedProcess:
return run_command([*COMPOSE, *args], check=check, timeout=timeout)
def read_json(path: Path, default=None):
if not path.exists():
return default
return json.loads(path.read_text())
def write_json(path: Path, payload) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=1, sort_keys=True) + '\n')
# ---------------------------------------------------------------- окружение стенда
def replica_limits(replicas: int, total_cpus: float, total_memory_mb: int) -> tuple[float, str]:
return (
round(total_cpus / replicas, 3),
f'{int(total_memory_mb // replicas)}m',
)
def render_stand_env(config: dict) -> str:
cpus, memory = replica_limits(
int(config['API_REPLICAS']),
float(config['API_TOTAL_CPUS']),
int(config['API_TOTAL_MEMORY_MB']),
)
values = {
**config,
'API_CPUS_PER_REPLICA': cpus,
'API_MEMORY_PER_REPLICA': memory,
}
lines = ['# Сгенерировано scripts/lab3/lab3.py: бюджет серии и лимиты реплик.']
for key in sorted(values, key=str):
lines.append(f'{key}={values[key]}')
return '\n'.join(lines) + '\n'
def load_stand_config() -> dict:
stored = read_json(STAND_STATE, None)
config = dict(DEFAULT_STAND)
if STAND_ENV.exists():
for line in STAND_ENV.read_text().splitlines():
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
config[key] = value
if stored:
config.update(stored)
return config
def numeric(value: str) -> float | int:
number = float(value)
return int(number) if number.is_integer() else number
def save_stand_config(config: dict) -> dict:
"""Пишет .env.stand и его машинно-читаемую копию.
Копия сохраняется уже с производными значениями (лимиты на реплику), иначе
следующие запуски подхватили бы устаревшие лимиты, а манифесты прогонов
показывали бы не те числа, что реально применены к контейнерам.
"""
rendered = render_stand_env(config)
STAND_ENV.write_text(rendered)
values = {
line.split('=', 1)[0]: line.split('=', 1)[1]
for line in rendered.splitlines()
if '=' in line and not line.startswith('#')
}
write_json(STAND_STATE, values)
return config
def current_replicas() -> int:
return int(load_stand_config().get('API_REPLICAS', 2))
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
"""Обновляет изменённые зависимости и API, не перезапуская миграции."""
save_stand_config(config)
compose('up', '-d', '--no-deps', '--wait', 'postgres', 'redis', 'haproxy', timeout=600)
args = ['up', '-d', '--no-deps', '--wait', '--scale', f'api={config["API_REPLICAS"]}']
if force_recreate:
args.append('--force-recreate')
compose(*args, 'api', timeout=600)
wait_for_lb()
verify_stand(config)
def wait_for_lb(timeout: float = 120) -> None:
deadline = time.monotonic() + timeout
last = 'no response'
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f'{BASE_URL}/health/ready', timeout=3) as response:
if response.status == 200:
return
except Exception as error: # noqa: BLE001 - диагностика ожидания важнее типа
last = str(error)
time.sleep(1)
raise SystemExit(f'stand did not become ready within {timeout}s: {last}')
def lb_backends_up() -> int:
"""Число backend-слотов LB в состоянии UP (без строк FRONTEND/BACKEND)."""
try:
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
text = response.read().decode('utf-8', 'replace')
except OSError:
return 0
lines = text.splitlines()
if lines and lines[0].startswith('#'):
lines[0] = lines[0].lstrip('#').lstrip()
return sum(
1
for row in csv.DictReader(lines)
if row.get('pxname') == 'api'
and re.match(r'^api\d+$', row.get('svname', ''))
and row.get('status') == 'UP'
)
def wait_for_replicas(expected: int, timeout: float = 180) -> list[str]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
mapping = replica_map()
if len(mapping) >= expected and lb_backends_up() >= expected:
return sorted(mapping)
time.sleep(1)
raise SystemExit(
f'ожидалось {expected} реплик, видно {len(replica_map())} контейнеров и {lb_backends_up()} UP-слотов LB'
)
def container_ids(service: str = 'api', *, include_stopped: bool = False) -> list[str]:
result = run_command(
[
'docker',
'ps',
'-q',
*(['-a'] if include_stopped else []),
'--filter',
f'label=com.docker.compose.project={PROJECT}',
'--filter',
f'label=com.docker.compose.service={service}',
]
)
return [line for line in result.stdout.split() if line]
def inspect(container: str, template: str) -> str:
return run_command(['docker', 'inspect', '-f', template, container]).stdout.strip()
def replica_map() -> dict[str, dict]:
"""X-Instance-Id -> контейнер: в Docker это hostname = короткий ID."""
mapping = {}
for container in container_ids('api'):
hostname = inspect(container, '{{.Config.Hostname}}')
mapping[hostname] = {
'container': container,
'name': inspect(container, '{{.Name}}').lstrip('/'),
'image_id': inspect(container, '{{.Image}}'),
'started_at': inspect(container, '{{.State.StartedAt}}'),
'cpus': int(inspect(container, '{{.HostConfig.NanoCpus}}')) / 1e9,
'memory_mb': int(inspect(container, '{{.HostConfig.Memory}}')) // (1024 * 1024),
}
return mapping
def observed_stand() -> dict[str, list[dict]]:
"""Только публичные параметры docker inspect; секреты Env не сохраняются."""
observed = {}
for service in ('api', 'postgres', 'redis', 'haproxy'):
ids = container_ids(service)
rows = []
if ids:
containers = json.loads(run_command(['docker', 'inspect', *ids]).stdout)
for container in containers:
row = {
'container': container['Id'][:12],
'name': container['Name'].lstrip('/'),
'cpus': container['HostConfig']['NanoCpus'] / 1e9,
'memory_bytes': container['HostConfig']['Memory'],
'image_id': container['Image'],
}
if service == 'api':
environment = dict(value.split('=', 1) for value in container['Config']['Env'] if '=' in value)
row['pool_max'] = int(environment['DATABASE_POOL_MAX'])
rows.append(row)
observed[service] = rows
return observed
def verify_stand(config: dict) -> dict:
observed = observed_stand()
expected_replicas = int(config['API_REPLICAS'])
api_cpu, api_memory = replica_limits(expected_replicas, float(config['API_TOTAL_CPUS']), int(config['API_TOTAL_MEMORY_MB']))
limits = {
'api': (api_cpu, api_memory),
'postgres': (float(config['PG_CPUS']), config['PG_MEMORY']),
'redis': (float(config['REDIS_CPUS']), config['REDIS_MEMORY']),
'haproxy': (float(config['LB_CPUS']), config['LB_MEMORY']),
}
for service, rows in observed.items():
count = expected_replicas if service == 'api' else 1
cpu, memory = limits[service]
unit = re.fullmatch(r'(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?', str(memory), re.IGNORECASE)
if not unit:
raise ValueError(f'unsupported memory limit: {memory}')
memory_bytes = int(float(unit[1]) * 1024 ** ('kmgt'.index(unit[2].lower()) + 1 if unit[2] else 0))
if len(rows) != count:
raise RuntimeError(f'{service}: expected {count} containers, observed {len(rows)}')
for row in rows:
if not math.isclose(row['cpus'], cpu, abs_tol=0.001) or row['memory_bytes'] != memory_bytes:
raise RuntimeError(f'{service}: runtime limits {row} differ from requested CPU={cpu}, RAM={memory}')
if service == 'api' and row['pool_max'] != int(config['DATABASE_POOL_MAX']):
raise RuntimeError(f'api: pool {row["pool_max"]} != requested {config["DATABASE_POOL_MAX"]}')
return observed
def image_versions() -> dict:
versions = {}
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
container = container_ids(name)
if container:
versions[name] = inspect(container[0], '{{.Config.Image}}')
versions['postgres_runtime'] = run_command(
['docker', 'exec', container_ids('postgres')[0], 'postgres', '--version'], check=False
).stdout.strip()
versions['redis_runtime'] = run_command(
['docker', 'exec', container_ids('redis')[0], 'redis-server', '--version'], check=False
).stdout.strip()
versions['haproxy_runtime'] = run_command(
['docker', 'exec', container_ids('haproxy')[0], 'haproxy', '-v'], check=False
).stdout.splitlines()[0].strip()
versions['k6'] = run_command(
['docker', 'run', '--rm', 'grafana/k6:0.54.0', 'version'], check=False, timeout=120
).stdout.strip()
if shutil.which('node'):
versions['node_host'] = run_command(['node', '--version'], check=False).stdout.strip()
return versions
def dataset_size() -> dict:
containers = container_ids('postgres')
if not containers:
return {}
query = (
"SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), "
"(SELECT count(*) FROM messages), "
"(SELECT count(*) FROM idempotency_keys), "
"(SELECT count(*) FROM chat_members)"
)
result = psql(query)
if not result:
return {}
values = result.split('|')
return dict(zip(['users', 'chats', 'messages', 'idempotency_keys', 'chat_members'], (int(v) for v in values)))
def psql(query: str) -> str:
env = psql_env()
result = run_command(
[
'docker',
'exec',
container_ids('postgres')[0],
'psql',
'-U',
env['POSTGRES_USER'],
'-d',
env['POSTGRES_DB'],
'-tA',
'-F',
'|',
'-c',
query,
],
check=False,
timeout=30,
)
return result.stdout.strip()
_PSQL_ENV: dict[str, str] = {}
def psql_env() -> dict[str, str]:
if not _PSQL_ENV:
for line in ENV_FILE.read_text().splitlines():
if '=' in line and not line.startswith('#'):
key, value = line.split('=', 1)
_PSQL_ENV[key] = value
return _PSQL_ENV
def host_background() -> dict:
memory = {}
for line in Path('/proc/meminfo').read_text().splitlines():
if line.startswith(('MemTotal:', 'MemAvailable:')):
key, value = line.split(':')
memory[key] = int(value.strip().split()[0]) // 1024
load = os.getloadavg()
return {
'load_average': [round(value, 2) for value in load],
'mem_total_mib': memory.get('MemTotal'),
'mem_available_mib': memory.get('MemAvailable'),
'quiesced': QUIESCE_STATE.exists(),
}
def bench_cpus() -> int:
return os.cpu_count() or 8
# ---------------------------------------------------------------- HTTP-клиент
class ApiError(RuntimeError):
pass
class Client:
"""Минимальный клиент для проверок и фазовых драйверов (stdlib)."""
def __init__(self, base_url: str = BASE_URL, timeout: float = 5.0):
self.base = base_url.rstrip('/')
self.timeout = timeout
def call(self, method: str, path: str, body=None, token: str | None = None, headers=None):
data = None
request_headers = dict(headers or {})
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
request_headers['content-type'] = 'application/json'
if token:
request_headers['authorization'] = f'Bearer {token}'
request = urllib.request.Request(
self.base + path, data=data, headers=request_headers, method=method
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = response.read()
status = response.status
response_headers = dict(response.headers)
except urllib.error.HTTPError as error:
payload = error.read()
status = error.code
response_headers = dict(error.headers)
parsed = json.loads(payload) if payload else None
return {'status': status, 'json': parsed, 'headers': response_headers}
def register(self, label: str) -> dict:
import uuid
username = f'lab3_{label}_{uuid.uuid4().hex[:10]}'
password = f'Lab3-{uuid.uuid4().hex}'
created = self.call(
'POST',
'/api/v1/users',
{'username': username, 'display_name': f'lab3 {label}', 'password': password},
)
if created['status'] != 201:
raise ApiError(f'register failed: {created}')
import base64
credential = base64.b64encode(f'{username}:{password}'.encode()).decode()
session = self.call(
'POST',
'/api/v1/auth/sessions',
None,
headers={'authorization': f'Basic {credential}'},
)
if session['status'] != 201:
raise ApiError(f'session failed: {session}')
return {
'id': created['json']['id'],
'user': created['json'],
'token': session['json']['token'],
}
def create_chat(self, token: str, member_ids: list[str], title: str) -> str:
created = self.call(
'POST', '/api/v1/chats', {'title': title, 'member_ids': member_ids}, token=token
)
if created['status'] != 201:
raise ApiError(f'chat failed: {created}')
return created['json']['id']
def post_message(self, token: str, chat_id: str, text: str, key: str) -> dict:
return self.call(
'POST',
f'/api/v1/chats/{chat_id}/messages',
{'text': text},
token=token,
headers={'idempotency-key': key},
)
def history(self, token: str, chat_id: str) -> list[dict]:
items: list[dict] = []
cursor = None
while True:
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
if cursor:
from urllib.parse import quote
path += '&cursor=' + quote(cursor, safe='')
page = self.call('GET', path, token=token)
if page['status'] != 200:
raise ApiError(f'history failed: {page}')
items.extend(page['json']['items'])
cursor = page['json']['next_cursor']
if cursor is None:
return items
# ---------------------------------------------------------------- сэмплер
class Sampler:
"""Фоновые наблюдения за стендом: docker stats, HAProxy, метрики, PostgreSQL."""
def __init__(self) -> None:
self.records: list[dict] = []
self._lock = threading.Lock()
self._stop = threading.Event()
self._threads: list[threading.Thread] = []
self.errors: list[str] = []
def _record(self, payload: dict) -> None:
with self._lock:
self.records.append(payload)
def _guard(self, name: str, work) -> None:
try:
work()
except Exception as error: # noqa: BLE001 - сэмплер не должен валить прогон
with self._lock:
self.errors.append(f'{name}: {type(error).__name__}: {error}')
def _loop(self, name: str, interval: float, step) -> None:
def work() -> None:
while not self._stop.is_set():
step()
self._stop.wait(interval)
self._threads.append(threading.Thread(target=lambda: self._guard(name, work), daemon=True, name=name))
def start(self) -> 'Sampler':
# docker stats в режиме потока перерисовывает экран (ANSI-последовательности)
# и склеивает JSON разных контейнеров в одну строку, поэтому опрашиваем
# его порциями --no-stream: одна строка = один JSON.
self._loop('docker_stats', 1.5, self._sample_stats)
self._loop('haproxy', 0.5, self._haproxy)
self._loop('metrics', 2.0, self._app_metrics)
self._loop('postgres', 2.0, self._postgres)
self._loop('pg_statements', 10.0, self._pg_statements)
self._loop('host', 5.0, lambda: self._record({'t': now_iso(), 'source': 'host', **host_background()}))
for thread in self._threads:
thread.start()
return self
def stop(self) -> list[dict]:
self._stop.set()
for thread in self._threads:
thread.join(timeout=10)
return self.records
def _sample_stats(self) -> None:
result = run_command(
['docker', 'stats', '--no-stream', '--format', '{{json .}}'],
check=False,
timeout=30,
)
for line in result.stdout.splitlines():
try:
row = json.loads(line)
except ValueError:
continue
self._record(
{
't': now_iso(),
'source': 'docker_stats',
'name': row.get('Name') or row.get('Container'),
'cpu_percent': float(str(row.get('CPUPerc', '0%')).rstrip('%') or 0),
'mem_mib': round(_mib(row.get('MemUsage', '0MiB / 0MiB')), 2),
'mem_limit_mib': round(_mib(str(row.get('MemUsage', '')).split('/')[-1]), 2),
'pids': _int(row.get('PIDs')),
}
)
def _haproxy(self) -> None:
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
text = response.read().decode('utf-8', 'replace')
lines = text.splitlines()
# HAProxy помечает строку заголовка как комментарий: '# pxname,svname,...'
if lines and lines[0].startswith('#'):
lines[0] = lines[0].lstrip('#').lstrip()
for row in csv.DictReader(lines):
if row.get('pxname') not in ('api', 'http'):
continue
self._record(
{
't': now_iso(),
'source': 'haproxy',
'proxy': row.get('pxname'),
'server': row.get('svname'),
'addr': row.get('addr'),
'status': row.get('status'),
'check': row.get('check_status'),
'scur': _int(row.get('scur')),
'qcur': _int(row.get('qcur')),
'stot': _int(row.get('stot')),
'ereq': _int(row.get('ereq')),
'dresp': _int(row.get('dresp')),
'wretr': _int(row.get('wretr')),
'hrsp_1xx': _int(row.get('hrsp_1xx')),
'hrsp_2xx': _int(row.get('hrsp_2xx')),
'hrsp_3xx': _int(row.get('hrsp_3xx')),
'hrsp_4xx': _int(row.get('hrsp_4xx')),
'hrsp_5xx': _int(row.get('hrsp_5xx')),
'hrsp_other': _int(row.get('hrsp_other')),
}
)
def _app_metrics(self) -> None:
containers = container_ids('api')
if not containers:
return
for container in containers:
result = run_command(
[
'docker',
'exec',
container,
'node',
'-e',
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(t=>process.stdout.write(t))",
],
check=False,
timeout=20,
)
if result.returncode != 0 or not result.stdout.strip():
continue
try:
payload = json.loads(result.stdout)
except ValueError:
continue
self._record(
{
't': now_iso(),
'source': 'metrics',
'container': container,
'instance_id': payload.get('instance_id'),
'build_id': payload.get('build_id'),
'counters': payload.get('counters', {}),
'latency_ms': payload.get('latency_ms', {}),
}
)
def _postgres(self) -> None:
query = (
"SELECT (SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT xact_rollback FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT blks_read FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT blks_hit FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
" (SELECT count(*) FROM pg_stat_activity),"
" (SELECT reltuples::bigint FROM pg_class WHERE oid = 'messages'::regclass),"
" (SELECT pg_total_relation_size('messages'))"
)
output = psql(query)
if not output:
return
fields = output.split('|')
if len(fields) != 10:
return
waits = self._wait_events()
self._record(
{
't': now_iso(),
'source': 'postgres',
'waits': waits,
**{
key: int(value)
for key, value in zip(
[
'xact_commit',
'xact_rollback',
'blks_read',
'blks_hit',
'deadlocks',
'lock_waiters',
'active',
'connections',
'messages',
'messages_bytes',
],
fields,
)
},
}
)
def _wait_events(self) -> dict:
"""Сколько сессий ждут на каждом типе события: Lock/tuple/transactionid и т.д."""
output = psql(
"SELECT coalesce(wait_event_type, 'CPU/Idle') || ':' || coalesce(wait_event, '-'), count(*)"
" FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()"
" GROUP BY 1 ORDER BY 2 DESC LIMIT 12"
)
waits = {}
for line in output.splitlines():
parts = line.split('|')
if len(parts) == 2 and parts[1].isdigit():
waits[parts[0]] = int(parts[1])
return waits
def _pg_statements(self) -> None:
output = psql(
"SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 3),"
" left(regexp_replace(query, '\\s+', ' ', 'g'), 110)"
" FROM pg_stat_statements"
" WHERE query ILIKE '%messages%' OR query ILIKE '%chats%' OR query ILIKE '%idempotency_keys%'"
" ORDER BY total_exec_time DESC LIMIT 10"
)
if not output:
return
rows = []
for line in output.splitlines():
parts = line.split('|')
if len(parts) == 4:
rows.append(
{
'calls': int(parts[0]),
'total_ms': float(parts[1]),
'mean_ms': float(parts[2]),
'query': parts[3],
}
)
self._record({'t': now_iso(), 'source': 'pg_stat_statements', 'top': rows})
def _mib(value: str) -> float:
text = str(value).strip()
match = re.match(r'([\d.]+)\s*([KMGT]i?B)?', text)
if not match:
return 0.0
number = float(match.group(1))
unit = (match.group(2) or 'B').lower()
factor = {
'b': 1 / (1024 * 1024),
'kib': 1 / 1024,
'kb': 1 / 1024,
'mib': 1,
'mb': 1,
'gib': 1024,
'gb': 1024,
'tib': 1024 * 1024,
}.get(unit, 1)
return number * factor
def _int(value) -> int:
try:
return int(str(value).strip() or 0)
except ValueError:
return 0
# ---------------------------------------------------------------- прогоны k6
def k6_run(script: str, label: str, env: dict[str, str], timeout: float) -> dict:
args = [*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps']
for key, value in sorted(env.items()):
args += ['-e', f'{key}={value}']
args += ['loadgen', 'run', f'/load/{script}']
result = run_command(args, check=False, timeout=timeout)
summary = None
for line in reversed(result.stdout.splitlines()):
stripped = line.strip()
if not stripped.startswith('{'):
continue
try:
candidate = json.loads(stripped)
except ValueError:
continue
if isinstance(candidate, dict) and 'steps' in candidate:
summary = candidate
break
if summary is None:
raise SystemExit(
f'k6 run {label} produced no summary (exit {result.returncode})\n'
f'{result.stdout[-1500:]}\n{result.stderr[-800:]}'
)
summary['exit_code'] = result.returncode
summary['stderr_tail'] = result.stderr.strip().splitlines()[-3:]
return summary
def load_env(profile: str, mode: str, spread: str, label: str, vus: dict | None = None) -> dict[str, str]:
env = {
'BASE_URL': 'http://haproxy:8080',
'PROFILE': spread,
'MODE': mode,
'CHATS': str(CHATS),
'LABEL': label,
}
max_vus = (vus or {}).get('max') or os.environ.get('LAB3_MAX_VUS')
pre_vus = (vus or {}).get('pre') or os.environ.get('LAB3_PRE_VUS')
if max_vus:
env['MAX_VUS'] = str(max_vus)
MANIFEST_VUS['max'] = str(max_vus)
if pre_vus:
env['PRE_VUS'] = str(pre_vus)
MANIFEST_VUS['pre'] = str(pre_vus)
return env
# Бюджет VU действующего прогона: попадает в манифест и в сводку k6.
MANIFEST_VUS: dict[str, str] = {}
def run_measured(
label: str,
profile: str,
*,
ladder: list[int] | None = None,
step_seconds: int = STEP_SECONDS,
rate: int | None = None,
duration: int | None = None,
mode: str | None = None,
spread: str | None = None,
warmup: bool = True,
extra: dict | None = None,
phases: dict | None = None,
vus: dict | None = None,
) -> dict:
"""Один прогон: (необязательный прогрев) + измерение + сэмплер + манифест."""
profile_config = PROFILES[profile]
mode = mode or profile_config['mode']
spread = spread or profile_config['spread']
ladder = ladder or profile_config['ladder']
if rate is not None:
ladder = [rate]
steady = rate is not None and duration is not None
script = 'write-steady.js' if steady else 'write-ladder.js'
RAW_DIR.mkdir(parents=True, exist_ok=True)
stand = load_stand_config()
verify_stand(stand)
MANIFEST_VUS.clear()
if warmup:
log(f'прогрев {label}: {WARMUP_SECONDS}s на {ladder[0]} req/s')
k6_run(
script,
f'warmup-{label}',
{
**load_env(profile, mode, spread, f'warmup-{label}', vus),
**({'RATE': str(ladder[0]), 'DURATION': str(WARMUP_SECONDS)} if steady else {'LADDER': str(ladder[0]), 'STEP_SECONDS': str(WARMUP_SECONDS)}),
},
timeout=WARMUP_SECONDS + 120,
)
sampler = Sampler().start()
started = now_iso()
env = {
**load_env(profile, mode, spread, label, vus),
**({'RATE': str(ladder[0]), 'DURATION': str(duration)} if steady else {'LADDER': ','.join(str(r) for r in ladder), 'STEP_SECONDS': str(step_seconds)}),
}
log(f'нагрузка {label}: {"steady " + str(ladder[0]) + " req/s" if steady else "ladder " + str(ladder)}')
summary = k6_run(script, label, env, timeout=(duration or len(ladder) * step_seconds) + 240)
finished = now_iso()
samples = sampler.stop()
write_raw(label, summary, samples, stand, started, finished, phases or {}, sampler.errors)
return summary
def write_raw(
label: str,
summary: dict,
samples: list[dict],
stand: dict,
started: str,
finished: str,
phases: dict,
sampler_errors: list[str],
) -> None:
RAW_DIR.mkdir(parents=True, exist_ok=True)
write_json(RAW_DIR / f'{label}.k6.json', summary)
with gzip.open(RAW_DIR / f'{label}.samples.jsonl.gz', 'wt', encoding='utf-8') as handle:
for record in samples:
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
if phases:
write_json(RAW_DIR / f'{label}.phases.json', phases)
manifest = {
'label': label,
'profile': summary.get('profile'),
'mode': summary.get('mode'),
'kind': summary.get('kind'),
'started_at': started,
'finished_at': finished,
'steps': [step['rate'] for step in summary.get('steps', [])],
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
'stand': stand,
'observed_stand': verify_stand(stand),
'replicas': replica_map(),
'dataset': dataset_size(),
'versions': image_versions(),
'host': {'cpus': bench_cpus(), 'background': host_background()},
'generator': {
'cpus': stand.get('LOADGEN_CPUS'),
'memory': stand.get('LOADGEN_MEMORY'),
'max_vus': MANIFEST_VUS.get('max') or os.environ.get('LAB3_MAX_VUS', '200'),
'pre_vus': MANIFEST_VUS.get('pre') or os.environ.get('LAB3_PRE_VUS', '60'),
},
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
'quiesce': read_json(QUIESCE_STATE, None),
'phases': phases,
'sampler_errors': sampler_errors,
'sample_count': len(samples),
}
write_json(RAW_DIR / f'{label}.manifest.json', manifest)
log(f'raw: {label} ({len(samples)} samples, errors: {len(sampler_errors)})')
def load_raw(label: str) -> tuple[dict | None, list[dict], dict | None]:
summary = read_json(RAW_DIR / f'{label}.k6.json', None)
manifest = read_json(RAW_DIR / f'{label}.manifest.json', None)
samples: list[dict] = []
path = RAW_DIR / f'{label}.samples.jsonl.gz'
if path.exists():
with gzip.open(path, 'rt', encoding='utf-8') as handle:
for line in handle:
try:
samples.append(json.loads(line))
except ValueError:
continue
return summary, samples, manifest
def raw_labels() -> list[str]:
if not RAW_DIR.exists():
return []
labels = []
for path in sorted(RAW_DIR.glob('*.k6.json')):
label = path.name[: -len('.k6.json')]
if label.startswith('warmup-'):
continue
labels.append(label)
return labels
# ---------------------------------------------------------------- анализ
def step_of(summary: dict, rate: int) -> dict | None:
for step in summary.get('steps', []):
if int(step['rate']) == int(rate):
return step
return None
def median(values: list[float]) -> float:
return round(statistics.median(values), 2) if values else 0.0
def spread(values: list[float]) -> tuple[float, float]:
return (round(min(values), 2), round(max(values), 2)) if values else (0.0, 0.0)
def group_runs(labels: list[str], predicate) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {}
for label in labels:
if predicate(label):
key = label.split('#')[0]
groups.setdefault(key, []).append(label)
return groups
def saturation(series: list[dict]) -> dict:
"""Точка сатурации: первый шаг, где успешный RPS отстал от предложенного."""
for step in series:
if step['successful_rps'] < 0.95 * step['rate']:
return {'rate': step['rate'], 'successful_rps': step['successful_rps'], 'step': step}
last = series[-1] if series else None
return {
'rate': last['rate'] if last else 0,
'successful_rps': last['successful_rps'] if last else 0,
'step': last,
}
def summarise_series(labels: list[str]) -> dict:
"""Медианы по повторам для одной конфигурации: шаги лестницы и пик.
Успешная пропускная способность (`successful_rps`) — рабочая величина:
сатурация, потолок и вердикты считаются по ней. Частота ответов
(`attempted_rps`) сохраняется отдельно, чтобы было видно, сколько запросов
сервис довёл до ответа, включая отказы.
"""
steps: dict[int, dict] = {}
for label in labels:
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
for step in summary.get('steps', []):
rate = int(step['rate'])
entry = steps.setdefault(
rate,
{'rate': rate, 'successful': [], 'attempted': [], 'p95': [], 'failed': [], 'vus': [], 'status_5xx': [], 'status_4xx': []},
)
entry['successful'].append(float(step['successful_rps']))
entry['attempted'].append(float(step['attempted_rps']))
entry['p95'].append(float(step['p95']))
entry['failed'].append(float(step['failed']))
entry['vus'].append(float(step.get('vus_active') or 0))
entry['status_5xx'].append(float((step.get('status') or {}).get('server_5xx') or 0))
entry['status_4xx'].append(float((step.get('status') or {}).get('client_4xx') or 0))
rows = []
for rate in sorted(steps):
entry = steps[rate]
rows.append(
{
'rate': rate,
'successful_rps': median(entry['successful']),
'successful_min': spread(entry['successful'])[0],
'successful_max': spread(entry['successful'])[1],
'attempted_rps': median(entry['attempted']),
'attempted_min': spread(entry['attempted'])[0],
'attempted_max': spread(entry['attempted'])[1],
'p95': median(entry['p95']),
'p95_max': spread(entry['p95'])[1],
'failed': sum(entry['failed']),
'vus_active': max(entry['vus']) if entry['vus'] else 0,
'status_5xx': int(max(entry['status_5xx'])) if entry['status_5xx'] else 0,
'status_4xx': int(max(entry['status_4xx'])) if entry['status_4xx'] else 0,
'reps': len(entry['successful']),
}
)
ceiling = max((row['successful_rps'] for row in rows), default=0.0)
return {'rows': rows, 'ceiling_rps': ceiling, 'saturation': saturation(rows), 'labels': labels}
def container_stats(samples: list[dict], prefix: str = 'mtusi-lab3-') -> dict:
stats: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'docker_stats':
continue
name = record.get('name') or ''
if not name.startswith(prefix):
continue
entry = stats.setdefault(name, {'cpu': [], 'mem': [], 'limit': 0.0})
entry['cpu'].append(float(record.get('cpu_percent') or 0))
entry['mem'].append(float(record.get('mem_mib') or 0))
entry['limit'] = max(entry['limit'], float(record.get('mem_limit_mib') or 0))
return {
name: {
'cpu_mean': round(statistics.fmean(entry['cpu']), 2) if entry['cpu'] else 0.0,
'cpu_peak': round(max(entry['cpu']), 2) if entry['cpu'] else 0.0,
'mem_mean': round(statistics.fmean(entry['mem']), 1) if entry['mem'] else 0.0,
'mem_peak': round(max(entry['mem']), 1) if entry['mem'] else 0.0,
'mem_limit': entry['limit'],
}
for name, entry in sorted(stats.items())
}
def postgres_stats(samples: list[dict]) -> dict:
rows = [record for record in samples if record.get('source') == 'postgres']
if len(rows) < 2:
return {}
first, last = rows[0], rows[-1]
seconds = max(1.0, (_timestamp(last['t']) - _timestamp(first['t'])))
hits = last['blks_hit'] - first['blks_hit']
reads = last['blks_read'] - first['blks_read']
return {
'xact_per_s': round((last['xact_commit'] - first['xact_commit']) / seconds, 1),
'rollback_per_s': round((last['xact_rollback'] - first['xact_rollback']) / seconds, 2),
'blks_hit_ratio': round(hits / (hits + reads), 4) if hits + reads else 0.0,
'lock_waiters_peak': max(row['lock_waiters'] for row in rows),
'active_peak': max(row['active'] for row in rows),
'connections_peak': max(row['connections'] for row in rows),
'deadlocks': last['deadlocks'] - first['deadlocks'],
'messages': last['messages'],
'messages_mib': round(last['messages_bytes'] / (1024 * 1024), 1),
}
def haproxy_stats(samples: list[dict]) -> dict:
servers: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'haproxy':
continue
entry = servers.setdefault(
record['server'],
{
'requests': [],
'errors': [],
'dresp': [],
'qcur': [],
'scur': [],
'codes': {},
'statuses': set(),
'addr': set(),
'events': [],
},
)
entry['requests'].append((record['t'], record['stot']))
entry['errors'].append(record['ereq'])
entry['dresp'].append(record.get('dresp') or 0)
entry['qcur'].append(record['qcur'])
entry['scur'].append(record['scur'])
for code in ('hrsp_1xx', 'hrsp_2xx', 'hrsp_3xx', 'hrsp_4xx', 'hrsp_5xx', 'hrsp_other'):
entry['codes'].setdefault(code, []).append(record.get(code) or 0)
entry['statuses'].add(record['status'])
entry['addr'].add(record['addr'])
entry['events'].append((record['t'], record['status'], record['check']))
result = {}
for name, entry in servers.items():
if not entry['requests']:
continue
first, last = entry['requests'][0], entry['requests'][-1]
seconds = max(1.0, _timestamp(last[0]) - _timestamp(first[0]))
transitions = []
previous = None
for moment, status, check in entry['events']:
if status != previous:
transitions.append({'t': moment, 'status': status, 'check': check})
previous = status
result[name] = {
'requests': last[1] - first[1],
'rps': round((last[1] - first[1]) / seconds, 2),
'errors': entry['errors'][-1] - entry['errors'][0] if entry['errors'] else 0,
'dresp': entry['dresp'][-1] - entry['dresp'][0] if entry['dresp'] else 0,
'response_codes': {
code: (values[-1] - values[0]) if values else 0
for code, values in entry['codes'].items()
},
'qcur_peak': max(entry['qcur']),
'scur_peak': max(entry['scur']),
'statuses': sorted(entry['statuses']),
'addresses': sorted(entry['addr']),
'transitions': transitions,
}
return result
def app_metrics_delta(samples: list[dict]) -> dict:
per_container: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'metrics':
continue
per_container.setdefault(record['container'], []).append(record)
result = {}
for container, rows in per_container.items():
if len(rows) < 2:
continue
first, last = rows[0], rows[-1]
counters = {}
for name, value in last['counters'].items():
counters[name] = value - first['counters'].get(name, 0)
result[container] = {
'instance_id': last.get('instance_id'),
'build_id': last.get('build_id'),
'counters': {name: value for name, value in sorted(counters.items()) if value},
'latency_ms': last.get('latency_ms', {}),
'samples': len(rows),
}
return result
def _timestamp(value: str) -> float:
return datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()
def mechanism_stats(samples: list[dict]) -> dict:
"""Где теряется время: ожидания сессий, дельты pg_stat_statements, блокировки.
Даёт не «упираемся в ресурс», а разрез по причинам: сколько сессий стоит в
Lock, какие операторы съели время за прогон, сколько запросов прошло и как
распределились ответы LB.
"""
waits: dict[str, int] = {}
lock_peak = 0
active_peak = 0
for record in samples:
if record.get('source') != 'postgres':
continue
for key, value in (record.get('waits') or {}).items():
waits[key] = max(waits.get(key, 0), value)
lock_peak = max(lock_peak, record.get('lock_waiters') or 0)
active_peak = max(active_peak, record.get('active') or 0)
statements: list[dict] = []
ticks = [record for record in samples if record.get('source') == 'pg_stat_statements' and record.get('top')]
if len(ticks) >= 2:
first, last = ticks[0]['top'], ticks[-1]['top']
before = {row['query'][:60]: row for row in first}
for row in last:
previous = before.get(row['query'][:60])
calls = row['calls'] - (previous['calls'] if previous else 0)
total_ms = row['total_ms'] - (previous['total_ms'] if previous else 0)
if calls > 0:
statements.append(
{
'query': row['query'][:60],
'calls': calls,
'total_ms': round(total_ms, 1),
'mean_ms': round(total_ms / calls, 3),
}
)
statements.sort(key=lambda row: -row['total_ms'])
return {
'waits_peak': dict(sorted(waits.items(), key=lambda item: -item[1])[:6]),
'lock_waiters_peak': lock_peak,
'active_peak': active_peak,
'statements': statements[:6],
}
def analyse_mechanism(groups: dict, labels: list[str]) -> None:
"""Разрез «почему»: один темп, разные конфигурации/реплики, почему растёт не всё."""
block = {
key: group for key, group in groups.items() if key.startswith(('block#', 'why#'))
}
if not block:
return
lines = [
'Конфигурации одного блока измерены на одном темпе (steady, 60 s, близкий',
'объём данных, интерливинг повторов), поэтому различия относятся к',
'распределению ресурсов, а не к разному времени суток или профилю нагрузки.',
'Блоки различаются темпом (block# — 900 req/s, why# — за коленом).',
'',
]
for key, group in sorted(block.items()):
label = representative(group)
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
step = summary['steps'][0] if summary.get('steps') else {}
_, samples, manifest = load_raw(label)
stats = container_stats(samples)
mechanism = mechanism_stats(samples)
stand = manifest.get('stand', {})
api_limit = float(stand.get('API_CPUS_PER_REPLICA') or 0)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
generator_peak = max((value['cpu_peak'] for name, value in stats.items() if 'loadgen' in name), default=0.0)
lines += [f'## {key} ({len(group)} повторов)', '']
lines += table(
['успешный RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра', 'CPU ген. пик, % ядра'],
[
[
step['successful_rps'],
step.get('p95'),
step.get('failed'),
(step.get('status') or {}).get('ok'),
(step.get('status') or {}).get('client_4xx'),
(step.get('status') or {}).get('server_5xx'),
(step.get('status') or {}).get('other'),
api_peak_quota,
pg_peak,
generator_peak,
]
],
)
lines.append(
f'Лимиты: реплик {stand.get("API_REPLICAS")} x {stand.get("API_CPUS_PER_REPLICA")} CPU '
f'({stand.get("API_MEMORY_PER_REPLICA")}), PG {stand.get("PG_CPUS")} CPU, пул {stand.get("DATABASE_POOL_MAX")}, '
f'LB {stand.get("LB_CPUS")} CPU, генератор {stand.get("LOADGEN_CPUS")} CPU.'
)
lines.append('')
lines += table(
['активных сессий пик', 'Lock-ожиданий пик', 'ожидания (тип: событие -> сессий)'],
[
[
mechanism['active_peak'],
mechanism['lock_waiters_peak'],
', '.join(f'{name}: {count}' for name, count in mechanism['waits_peak'].items()) or '—',
]
],
)
lines += table(
['SQL (начало) по убыванию времени', 'вызовов за прогон', 'всего, ms', 'среднее, ms'],
[[row['query'], row['calls'], row['total_ms'], row['mean_ms']] for row in mechanism['statements']],
)
app = app_metrics_delta(samples)
for container, delta in sorted(app.items()):
lines.append(
f'- реплика {delta["instance_id"]}: latency p95 {delta["latency_ms"].get("p95")} ms, '
f'счётчики {json.dumps({name: value for name, value in delta["counters"].items() if name.startswith(("messages", "idempotency", "request_count"))}, ensure_ascii=False)}'
)
lines.append('')
write_markdown(TRACK1_DIR / 'summary-mechanism.md', 'Разрез «почему»: где теряется время на одном темпе', lines)
def write_markdown(path: Path, title: str, lines: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f'# {title}\n\n' + '\n'.join(lines) + '\n', encoding='utf-8')
def table(header: list[str], rows: list[list]) -> list[str]:
lines = ['| ' + ' | '.join(header) + ' |', '| ' + ' | '.join('---' for _ in header) + ' |']
for row in rows:
lines.append('| ' + ' | '.join(str(value) for value in row) + ' |')
return lines + ['']
def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
"""Приводит стенд к нужной конфигурации; пересоздаёт только при изменении."""
desired = {**load_stand_config(), **{key: str(value) for key, value in overrides.items()}}
rendered = render_stand_env(desired)
if not STAND_ENV.exists() or STAND_ENV.read_text() != rendered:
log(f'перенастройка стенда: {overrides}')
apply_stand(desired, force_recreate=force_recreate)
else:
wait_for_lb()
try:
verify_stand(desired)
except RuntimeError:
apply_stand(desired, force_recreate=force_recreate)
return desired
# ---------------------------------------------------------------- подкоманды
def cmd_stand(args) -> None:
config = load_stand_config()
if args.action == 'up':
if args.replicas:
config['API_REPLICAS'] = str(args.replicas)
save_stand_config(config)
compose('up', '-d', '--build', '--wait', timeout=1800)
wait_for_lb()
log(f'стенд поднят: {replica_map().keys()}')
elif args.action == 'down':
compose('down')
else:
print(compose('ps').stdout)
def header(headers: dict, name: str) -> str | None:
for key, value in headers.items():
if key.lower() == name.lower():
return value
return None
def cmd_check(args) -> None:
client = Client()
if args.what == 'spread':
token = client.register('spread')['token']
counts: dict[str, int] = {}
for _ in range(args.requests):
response = client.call('GET', '/api/v1/users/me', token=token)
instance = header(response['headers'], 'X-Instance-Id')
counts[instance] = counts.get(instance, 0) + 1
payload = {
'requests': args.requests,
'distinct_instances': len(counts),
'instances': counts,
'replicas': {name: info for name, info in replica_map().items()},
'captured_at': now_iso(),
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
}
print(json.dumps(payload, ensure_ascii=False, indent=1))
write_json(LEVEL4_DIR / 'spread.json', payload)
elif args.what == 'instances':
print(json.dumps({'replicas': replica_map(), 'haproxy': haproxy_stats(Sampler()._haproxy() or [])}, ensure_ascii=False, indent=1))
elif args.what == 'idempotency':
LEVEL4_DIR.mkdir(parents=True, exist_ok=True)
result = subprocess.run(
['npm', 'run', 'test:e2e', '--', '-t', 'idempotency'],
cwd=ROOT / 'app',
capture_output=True,
text=True,
timeout=900,
env={**os.environ, 'API_BASE_URL': BASE_URL},
)
output = result.stdout + result.stderr
stamp = (
f'=== e2e-фильтр идемпотентности (конкурентный повтор через LB)\n'
f'captured_at {now_iso()}\n'
f'commit {run_command(["git", "rev-parse", "HEAD"], check=False).stdout.strip()}\n\n'
)
(LEVEL4_DIR / 'idempotency-concurrency.txt').write_text(stamp + output, encoding='utf-8')
print(output[-1500:])
if result.returncode != 0:
raise SystemExit('idempotency concurrency suite failed')
else:
raise SystemExit(f'unknown check: {args.what}')
def cmd_run(args) -> None:
overrides: dict = {}
if args.replicas:
overrides['API_REPLICAS'] = str(args.replicas)
if args.pool:
overrides['DATABASE_POOL_MAX'] = str(args.pool)
if args.loadgen_cpus:
overrides['LOADGEN_CPUS'] = str(args.loadgen_cpus)
if args.lb_cpus:
overrides['LB_CPUS'] = str(args.lb_cpus)
if args.api_total_cpus:
overrides['API_TOTAL_CPUS'] = str(args.api_total_cpus)
if args.pg_cpus:
overrides['PG_CPUS'] = str(args.pg_cpus)
overrides['LOADGEN_MEMORY'] = args.loadgen_memory
ensure_stand(overrides)
profile = args.profile
ladder = [int(value) for value in args.ladder.split(',')] if args.ladder else None
summary = run_measured(
args.label,
profile,
ladder=ladder,
step_seconds=args.step or STEP_SECONDS,
rate=args.peak,
duration=args.duration,
mode='read' if args.mode == 'read' else None,
spread=args.spread,
warmup=not args.no_warmup,
vus={'max': args.max_vus, 'pre': args.pre_vus},
)
print(json.dumps(summary, ensure_ascii=False, indent=1))
def cmd_sweep(args) -> None:
replicas = [int(value) for value in args.replicas.split(',')]
label_prefix = args.prefix or args.profile
for rep in range(1, args.reps + 1):
order = replicas if rep % 2 else list(reversed(replicas))
for count in order:
ensure_stand({'API_REPLICAS': str(count)})
run_measured(f'{label_prefix}#n{count}#rep{rep}', args.profile, warmup=not args.no_warmup)
report_series(args.profile, label_prefix)
if args.extra_replica:
for extra in range(1, args.extra_reps + 1):
count = args.extra_replica
ensure_stand({'API_REPLICAS': str(count)})
run_measured(f'{label_prefix}#n{count}#rep{args.reps + extra}', args.profile, warmup=not args.no_warmup)
report_series(args.profile, label_prefix)
def report_series(profile: str, prefix: str) -> None:
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#')]
groups = group_configurations(labels)
lines = []
for key, group in groups.items():
series = summarise_series(group)
lines += table(
['темп', 'успешный RPS', 'min..max', 'ответов RPS', 'p95', 'ошибок'],
[
[
row['rate'],
row['successful_rps'],
f'{row["successful_min"]}..{row["successful_max"]}',
row['attempted_rps'],
row['p95'],
row['failed'],
]
for row in series['rows']
],
)
lines.append(f'Потолок {key}: {series["ceiling_rps"]} req/s успешных, сатурация на {series["saturation"]["rate"]} req/s ({len(group)} повторов)')
lines.append('')
write_markdown(TRACK1_DIR / f'summary-live-{profile}.md', f'Текущие агрегаты {profile}', lines)
def configuration_key(label: str) -> str:
"""Ключ конфигурации: метки вида <профиль>#<конфигурация>#rep<N>.
Без второго сегмента повторные прогоны разных конфигураций слились бы в одну
таблицу (например, все N серии A), а A/B-стороны 'ab#A'/'ab#B' — в одну
группу 'ab'.
"""
parts = label.split('#')
return '#'.join(parts[:2]) if len(parts) > 1 else parts[0]
def group_configurations(labels: list[str]) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {}
for label in labels:
groups.setdefault(configuration_key(label), []).append(label)
return {key: sorted(value) for key, value in sorted(groups.items())}
class MessageProbe(threading.Thread):
"""Последовательный клиент: измеряет успешность записи и повторяет
неопределённый POST тем же ключом (как это делает реальный клиент)."""
def __init__(self, client: Client, token: str, chat_id: str, interval: float, tag: str):
super().__init__(daemon=True)
self.client = client
self.token = token
self.chat_id = chat_id
self.interval = interval
self.tag = tag
self.stop_event = threading.Event()
self.samples: list[dict] = []
self.confirmed: list[dict] = []
self.started_monotonic = time.monotonic()
def run(self) -> None:
import uuid
sequence = 0
started = self.started_monotonic
while not self.stop_event.is_set():
sequence += 1
key = uuid.uuid4().hex
text = f'{self.tag} {sequence}'
attempts = 0
while attempts < 3:
attempts += 1
moment = time.monotonic() - started
try:
response = self.client.post_message(self.token, self.chat_id, text, key)
except (urllib.error.URLError, TimeoutError, OSError) as error:
self.samples.append({'t': round(moment, 3), 'ok': False, 'error': type(error).__name__})
if self.stop_event.wait(0.1):
return
continue
instance = header(response['headers'], 'X-Instance-Id')
self.samples.append(
{'t': round(moment, 3), 'ok': response['status'] == 201, 'status': response['status'], 'instance': instance}
)
if response['status'] == 201:
self.confirmed.append({'text': text, 'key': key, 'id': response['json']['id']})
break
if self.stop_event.wait(self.interval):
return
def replica_ip(container: str) -> str:
return inspect(container, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
def cmd_failover(args) -> None:
config = load_stand_config()
expected = int(config['API_REPLICAS'])
ensure_stand({'API_REPLICAS': str(expected)})
replicas = wait_for_replicas(expected)
targets = replicas if args.replicas == 'all' else [args.replicas]
results = []
for target in targets:
results.append(failover_once(target, args, config))
write_failover_evidence(results)
print(json.dumps(results, ensure_ascii=False, indent=1))
def failover_once(target: str, args, config: dict) -> dict:
"""Один цикл: нагрузка -> остановка реплики -> возврат -> проверка истории.
Проба идёт постоянно и повторяет неопределённый POST тем же ключом, поэтому
«потерь» и «дублей» в истории быть не должно. Окно реального отсутствия
реплики считается от выхода контейнера (docker stop возвращает управление
уже после остановки), а не от вызова команды: дрейн длиной DRAIN_DELAY_MS
специально переживает обнаружение отказа LB и не должен выглядеть отказом.
"""
if target not in replica_map() or lb_backends_up() < len(replica_map()):
log(f'{target} не запущена или LB ещё не видит все реплики: восстанавливаю')
run_command(['scripts/start-replica.sh'])
wait_for_lb()
wait_for_replicas(len(replica_map()) or int(config['API_REPLICAS']))
if target not in replica_map():
raise SystemExit(f'replica {target} did not come back')
container = replica_map()[target]['container']
address = replica_ip(container)
client = Client(timeout=5)
alice = client.register(f'failover{target[:4]}')
chat = client.create_chat(alice['token'], [alice['id']], f'failover {target}')
load_seconds = int(args.pre + args.hold + 30)
label = f'failover#{target}'
env = {
**load_env('p-b', 'write', 'spread', label),
'RATE': str(args.rate),
'DURATION': str(load_seconds),
}
sampler = Sampler().start()
load = subprocess.Popen(
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{key}={value}'] for key, value in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
probe = MessageProbe(client, alice['token'], chat, 0.2, f'failover-{target[:6]}')
probe.start()
started = now_iso()
time.sleep(args.pre)
command_moment = time.monotonic()
stop = run_command(['scripts/stop-one.sh'], check=False, env={'TARGET_INSTANCE_ID': target}, timeout=30)
exit_moment = time.monotonic()
stop_at = now_iso()
first_success_after_exit = None
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
exit_offset = probe_offset(probe, exit_moment)
if any(sample.get('ok') and sample['t'] >= exit_offset for sample in list(probe.samples)):
first_success_after_exit = round(probe_offset(probe, time.monotonic()), 2)
break
time.sleep(0.1)
time.sleep(args.hold)
restore_moment = time.monotonic()
restore_at = now_iso()
run_command(['scripts/start-replica.sh'])
recovered = None
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
seen = {
header(client.call('GET', '/api/v1/users/me', token=alice['token'])['headers'], 'X-Instance-Id')
for _ in range(6)
}
if len(seen) >= 2:
recovered = round(time.monotonic() - restore_moment, 2)
break
time.sleep(0.5)
try:
stdout, _ = load.communicate(timeout=load_seconds + 90)
except subprocess.TimeoutExpired:
load.kill()
stdout, _ = load.communicate()
probe.stop_event.set()
probe.join(timeout=15)
samples = sampler.stop()
finished = now_iso()
summary = None
for line in reversed((stdout or '').splitlines()):
if line.strip().startswith('{'):
try:
candidate = json.loads(line)
except ValueError:
continue
if 'steps' in candidate:
summary = candidate
break
history = client.history(alice['token'], chat)
texts = [item['text'] for item in history]
duplicates = sorted({text for text in texts if texts.count(text) > 1})
missing = [item['text'] for item in probe.confirmed if item['text'] not in texts]
command_offset = probe_offset(probe, command_moment)
exit_offset = probe_offset(probe, exit_moment)
# Окно отсутствия реплики: от выхода контейнера до первого успешного ответа
# после него. Если успех не наступил (проба не восстановилась), окно тянется
# до конца наблюдения — иначе метрика «во время отказа» выродилась бы в точку.
outage_start = (
first_success_after_exit if first_success_after_exit is not None else last_offset_before_stop(probe, exit_offset)
)
last_offset = probe.samples[-1]['t'] if probe.samples else exit_offset
# Два окна, чтобы метрика не вырождалась:
# - «остановка и дрейн»: от команды docker stop до первого успеха после выхода контейнера;
# - «реплики нет»: от выхода контейнера до первого успеха после него
# (при корректном дреине это доли секунды — так и должно быть).
stop_window_end = (
first_success_after_exit if first_success_after_exit is not None else last_offset
)
phases = {
'stop_command_at': stop_at,
'stop_exit_code': stop.returncode,
'stop_command_took_s': round(exit_moment - command_moment, 2),
'first_success_after_exit_s': (
round(first_success_after_exit - exit_offset, 2) if first_success_after_exit is not None else None
),
'restore_at': restore_at,
'replica_rejoined_s': recovered,
'lb_down_at': lb_down_time(samples, address),
'stop_window_seconds': round(stop_window_end - command_offset, 2),
'absent_window_seconds': round(outage_start - exit_offset, 2),
'probe_offsets': {
'stop_command': round(command_offset, 2),
'container_exit': round(exit_offset, 2),
'restore_command': round(probe_offset(probe, restore_moment), 2),
'last_sample': round(last_offset, 2),
},
}
if summary:
write_raw(label, summary, samples, config, started, finished, phases, sampler.errors)
failures = [sample for sample in probe.samples if not sample.get('ok')]
result = {
'instance': target,
'container': container,
'address': address,
'rate': args.rate,
'phases': phases,
'probe': {
'attempts': len(probe.samples),
'successes': sum(1 for sample in probe.samples if sample.get('ok')),
'failures': len(failures),
'confirmed': len(probe.confirmed),
'rps_before': phase_rps(probe.samples, 0.0, command_offset),
'rps_stop_window': phase_rps(probe.samples, command_offset, stop_window_end),
'rps_while_stopped': phase_rps(probe.samples, exit_offset, outage_start),
'rps_after': phase_rps(probe.samples, outage_start, last_offset),
'errors_stop_window': sum(
1 for sample in failures if command_offset <= sample['t'] <= stop_window_end
),
'errors_while_stopped': sum(
1 for sample in failures if exit_offset <= sample['t'] <= outage_start
),
'timeline': [[sample['t'], 1 if sample.get('ok') else 0] for sample in probe.samples],
},
'duplicates': duplicates,
'missing_confirmed': missing,
'haproxy': haproxy_stats(samples),
'k6': {
'successful_rps': summary['steps'][0]['successful_rps'] if summary else None,
'attempted_rps': summary['steps'][0]['attempted_rps'] if summary else None,
'failed': summary['steps'][0]['failed'] if summary else None,
'dropped': summary.get('dropped_iterations') if summary else None,
},
}
log(
f'failover {target}: контейнер остановлен через {phases["stop_command_took_s"]}s, '
f'первый успех после остановки через {phases["first_success_after_exit_s"]}s, '
f'реплика вернулась через {recovered}s, дубли {len(duplicates)}, потеряно {len(missing)}'
)
return result
def probe_offset(probe: MessageProbe, moment: float) -> float:
"""Переводит момент wall-clock в шкалу времени сэмплов пробы."""
if not probe.samples:
return 0.0
return probe.samples[0]['t'] + (moment - probe.started_monotonic)
def last_offset_before_stop(probe: MessageProbe, exit_offset: float) -> float:
"""Конец наблюдения, если после выхода контейнера успехов не было."""
return probe.samples[-1]['t'] if probe.samples else exit_offset
def lb_down_time(samples: list[dict], address: str) -> str | None:
for record in samples:
if record.get('source') != 'haproxy' or record.get('server') in ('FRONTEND', 'BACKEND'):
continue
if address in (record.get('addr') or '') and record.get('status') not in ('UP',):
return record['t']
return None
def phase_rps(samples: list[dict], start: float, end: float) -> float:
window = [sample for sample in samples if start <= sample['t'] <= end]
return round(len(window) / max(0.001, end - start), 2)
def write_failover_evidence(results: list[dict]) -> None:
write_json(
LEVEL4_DIR / 'failover-sweep.json',
{
'captured_at': now_iso(),
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
'results': results,
},
)
lines = table(
[
'реплика',
'контейнер',
'попыток пробы',
'успешных',
'ошибок в отказе',
'остановка контейнера, s',
'первый успех после остановки, s',
'реплика вернулась, s',
'LB DOWN',
'RPS до',
'RPS в окне остановки',
'ошибок в окне',
'RPS без реплики',
'RPS после',
'k6 успешный RPS',
'k6 ошибок',
'дубли',
'потери',
],
[
[
result['instance'][:12],
result['container'][:12],
result['probe']['attempts'],
result['probe']['successes'],
result['probe']['errors_while_stopped'],
result['phases']['stop_command_took_s'],
result['phases']['first_success_after_exit_s'],
result['phases']['replica_rejoined_s'],
(result['phases']['lb_down_at'] or '-')[11:19],
result['probe']['rps_before'],
result['probe']['rps_stop_window'],
result['probe']['errors_stop_window'],
result['probe']['rps_while_stopped'],
result['probe']['rps_after'],
result['k6']['successful_rps'],
result['k6']['failed'],
len(result['duplicates']),
len(result['missing_confirmed']),
]
for result in results
],
)
lines += [
'',
'Проба: последовательные POST с интервалом 0.2 s, при неопределённом результате '
'повтор с тем же Idempotency-Key (как у реального клиента). «Окно остановки» — от '
'команды docker stop до первого успеха после выхода контейнера (сюда попадает дрейн); '
'«RPS без реплики» — от выхода контейнера до первого успеха после него, при корректном '
'дреине это доли секунды. Ошибок не должно быть ни в одном из окон, потери и дубли '
'проверяются по истории чата.',
]
write_markdown(LEVEL4_DIR / 'failover-sweep.md', 'Отказ каждой реплики под нагрузкой', lines)
def cmd_rolling(args) -> None:
config = load_stand_config()
replicas = int(config['API_REPLICAS'])
build_id = args.build_id or datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
before = replica_map()
migrate_before = inspect(container_ids('migrate', include_stopped=True)[0], '{{.State.StartedAt}}')
log(f'сборка образа {config["API_IMAGE"]} с BUILD_ID={build_id}')
build = run_command(
['docker', 'build', '--build-arg', f'BUILD_ID={build_id}', '-t', config['API_IMAGE'], '.'],
check=True,
timeout=1800,
)
image_id = run_command(['docker', 'image', 'inspect', '-f', '{{.Id}}', config['API_IMAGE']]).stdout.strip()
client = Client(timeout=5)
alice = client.register('rolling')
chat = client.create_chat(alice['token'], [alice['id']], 'rolling update')
probe = MessageProbe(client, alice['token'], chat, 0.2, 'rolling')
sampler = Sampler().start()
load = None
if args.under_load:
env = {**load_env('p-b', 'write', 'spread', 'rolling'), 'RATE': str(args.rate), 'DURATION': str(max(args.duration, replicas * 45))}
load = subprocess.Popen(
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{k}={v}'] for k, v in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
probe.start()
started = now_iso()
steps = []
for step in range(replicas):
step_started = time.monotonic()
current = replica_map()
known = set(current)
# Меняем самую «старую» реплику: только что созданная видна как самая молодая.
target = sorted(current, key=lambda instance: current[instance]['started_at'])[0]
container = current[target]['container']
old_build = app_build_id(container)
run_command(['docker', 'stop', '--time', '15', container], check=False, timeout=60)
run_command(['docker', 'rm', container], check=False, timeout=60)
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', f'api={replicas}', 'api', timeout=600)
new_instance, new_container = wait_for_new_replica(replicas, known)
new_build = app_build_id(new_container)
steps.append(
{
'step': step + 1,
'replacement_seconds': round(time.monotonic() - step_started, 2),
'old_instance': target,
'old_container': container[:12],
'old_build_id': old_build,
'new_instance': new_instance,
'new_container': new_container[:12],
'new_build_id': new_build,
'migrate_started_at': migrate_before,
}
)
log(f'rolling {step + 1}/{replicas}: {target[:12]} ({old_build}) -> {new_instance[:12]} ({new_build})')
time.sleep(args.pause)
summary = None
if load is not None:
try:
stdout, _ = load.communicate(timeout=max(30, args.duration))
except subprocess.TimeoutExpired:
load.terminate()
try:
stdout, _ = load.communicate(timeout=60)
except subprocess.TimeoutExpired:
load.kill()
stdout, _ = load.communicate()
probe.stop_event.set()
probe.join(timeout=10)
samples = sampler.stop()
for line in reversed((stdout or '').splitlines()):
if line.strip().startswith('{'):
try:
candidate = json.loads(line)
except ValueError:
continue
if 'steps' in candidate:
summary = candidate
break
migrate_after = inspect(container_ids('migrate', include_stopped=True)[0], '{{.State.StartedAt}}')
failures = [sample for sample in probe.samples if not sample.get('ok')]
server_failures = [sample for sample in failures if 500 <= int(sample.get('status') or 0) < 600]
finished = now_iso()
payload = {
'build_id': build_id,
'captured_at': now_iso(),
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
'replacement_seconds': [step['replacement_seconds'] for step in steps],
'image_id': image_id,
'image': config['API_IMAGE'],
'replicas': replicas,
'under_load': bool(args.under_load),
'steps': steps,
'probe': {'attempts': len(probe.samples), 'failures': len(failures), 'server_5xx': len(server_failures)},
'k6': {
'successful_rps': summary['steps'][0]['successful_rps'] if summary else None,
'attempted_rps': summary['steps'][0]['attempted_rps'] if summary else None,
'failed': summary['steps'][0]['failed'] if summary else None,
},
'migrate': {'before': migrate_before, 'after': migrate_after, 'restarted': migrate_before != migrate_after},
'build_stderr_tail': build.stderr.strip().splitlines()[-3:],
}
write_json(LEVEL4_DIR / 'rolling-update.json', payload)
if summary:
write_raw('rolling', summary, samples, config, started, finished, {'steps': steps}, sampler.errors)
lines = table(
['шаг', 'было', 'BUILD_ID до', 'стало', 'BUILD_ID после', 'замена, s'],
[
[
step['step'],
step['old_instance'][:12],
step['old_build_id'],
step['new_instance'][:12],
step['new_build_id'],
step['replacement_seconds'],
]
for step in steps
],
)
lines += [
f'5xx от API во время rolling update: {len(server_failures)} из {len(probe.samples)} попыток',
f'Миграции перезапускались: {migrate_before != migrate_after}',
]
write_markdown(LEVEL4_DIR / 'rolling-update.md', 'Rolling update под нагрузкой', lines)
print(json.dumps(payload, ensure_ascii=False, indent=1))
def app_build_id(container: str) -> str:
result = run_command(
[
'docker', 'exec', container, 'node', '-e',
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}}).then(r=>r.json())"
".then(j=>process.stdout.write(String(j.build_id)))",
],
check=False,
timeout=20,
)
return result.stdout.strip() or 'unknown'
def wait_for_new_replica(replicas: int, known: set[str], timeout: float = 180) -> tuple[str, str]:
"""Ждёт появления НОВОЙ реплики (не из known) в состоянии healthy."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
mapping = replica_map()
fresh = {instance: info for instance, info in mapping.items() if instance not in known}
if len(mapping) >= replicas and fresh:
for instance, info in sorted(fresh.items()):
status = inspect(info['container'], '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')
if status == 'healthy':
return instance, info['container']
time.sleep(1)
raise SystemExit(
f'новая реплика не появилась healthy за {timeout}s (контейнеров {len(replica_map())}, '
f'новых {len({i for i in replica_map() if i not in known})})'
)
def cmd_ab(args) -> None:
sides = {'A': args.before, 'B': args.after}
# Бюджет A/B фиксируется явно: иначе точки унаследуют конфигурацию предыдущих
# серий (именно так ранние прогоны A/B уехали на удвоенный API/PG и пул 20),
# и числа станут несравнимы с базовыми прогонами той же серии.
plan = {
'API_REPLICAS': str(args.replicas),
'API_TOTAL_CPUS': str(args.api_total_cpus),
'API_TOTAL_MEMORY_MB': str(args.api_total_memory_mb),
'PG_CPUS': str(args.pg_cpus),
'DATABASE_POOL_MAX': str(args.pool),
'LB_CPUS': str(args.lb_cpus),
'LOADGEN_CPUS': str(args.loadgen_cpus),
'LOADGEN_MEMORY': args.loadgen_memory,
}
log(f'A/B на бюджете: {plan}')
for rep in range(1, args.reps + 1):
order = ['A', 'B'] if rep % 2 else ['B', 'A']
for side in order:
ensure_stand({**plan, 'API_IMAGE': sides[side]})
run_measured(
f'{args.prefix}#{side}#rep{rep}',
args.profile,
warmup=not args.no_warmup,
vus={'max': args.max_vus},
)
report_ab(args, plan)
def ab_side_stats(row: dict) -> dict:
"""Сторона A/B в вердикте: успешный RPS медианой и диапазоном повторов.
Частота всех ответов и классы отказов остаются рядом как диагностика —
именно они объясняют расхождение, а не заменяют пропускную способность.
"""
return {
'successful_rps': row['successful_rps'],
'successful_min': row['successful_min'],
'successful_max': row['successful_max'],
'attempted_rps': row['attempted_rps'],
'p95': row['p95'],
'failed': row['failed'],
'status_5xx': row['status_5xx'],
'status_4xx': row['status_4xx'],
'reps': row['reps'],
}
def report_ab(args=None, plan: dict | None = None) -> None:
prefix = getattr(args, 'prefix', 'ab') if args else 'ab'
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#')]
groups = group_configurations(labels)
lines = []
verdicts = []
series_by_side = {
side: summarise_series(groups.get(f'{prefix}#{side}', [])) for side in ('A', 'B')
}
images = {
side: (getattr(args, side.lower(), None) if args else None) for side in ('A', 'B')
}
for side in ('A', 'B'):
if not images[side] and groups.get(f'{prefix}#{side}'):
manifest = read_json(
RAW_DIR / f'{groups[f"{prefix}#{side}"][-1]}.manifest.json', {}
)
images[side] = (manifest.get('stand') or {}).get('API_IMAGE')
rows = []
for rate in sorted({row['rate'] for row in series_by_side['A']['rows']}):
row_a = next((row for row in series_by_side['A']['rows'] if row['rate'] == rate), None)
row_b = next((row for row in series_by_side['B']['rows'] if row['rate'] == rate), None)
if not row_a or not row_b:
continue
# Δ считается по успешной пропускной способности: ответы об ошибках,
# которых у стороны A на насыщении большинство, не должны выглядеть
# как выполненная работа.
change = (
round((row_b['successful_rps'] - row_a['successful_rps']) / row_a['successful_rps'] * 100, 2)
if row_a['successful_rps']
else 0.0
)
confirmed = (
abs(change) >= 5.0
and (row_b['successful_min'] > row_a['successful_max'] or row_a['successful_min'] > row_b['successful_max'])
)
verdicts.append({
'rate': rate,
'change_percent': change,
'confirmed': confirmed,
'a': ab_side_stats(row_a),
'b': ab_side_stats(row_b),
})
lines.append(
f'{rate} req/s: A {row_a["successful_rps"]} успешных [{row_a["successful_min"]}; {row_a["successful_max"]}] '
f'(всего ответов {row_a["attempted_rps"]}) vs B {row_b["successful_rps"]} [{row_b["successful_min"]}; {row_b["successful_max"]}] '
f'(всего ответов {row_b["attempted_rps"]}) -> {change}% ({"подтверждено" if confirmed else "не подтверждено"})'
)
rows.append(
[
rate,
f'{row_a["successful_rps"]} ({row_a["successful_min"]}..{row_a["successful_max"]})',
row_a['p95'],
row_a['attempted_rps'],
f'{row_b["successful_rps"]} ({row_b["successful_min"]}..{row_b["successful_max"]})',
row_b['p95'],
row_b['attempted_rps'],
change,
'подтверждено' if confirmed else 'не подтверждено',
]
)
lines.append('')
lines += table(
[
'шаг, req/s',
'A: успешный RPS (min..max)',
'A: p95, ms',
'A: ответов RPS',
'B: успешный RPS (min..max)',
'B: p95, ms',
'B: ответов RPS',
'Δ успешных, %',
'вывод',
],
rows,
)
lines += [
f'Потолок A: {series_by_side["A"]["ceiling_rps"]} успешных req/s, потолок B: {series_by_side["B"]["ceiling_rps"]} успешных req/s',
'',
'Δ = (B − A) / A по медиане успешного RPS; «ответов RPS» — частота всех полученных ответов, '
'диагностическая величина (на насыщении она растёт, хотя полезная работа падает).',
'Правило подтверждения: медиана успешного RPS меняется ≥5 % и диапазоны min/max повторов не перекрываются.',
]
suffix = '' if prefix == 'ab' else f'-{prefix}'
write_json(
TRACK1_DIR / f'ab-verdict{suffix}.json',
{
'verdicts': verdicts,
'ceiling': {side: series_by_side[side]['ceiling_rps'] for side in ('A', 'B')},
'prefix': prefix,
'images': images,
'budget': plan
or (
read_json(RAW_DIR / f'{groups[f"{prefix}#A"][-1]}.manifest.json', {}).get('stand')
if groups.get(f'{prefix}#A')
else None
),
'rule': 'медиана успешного RPS меняется ≥5 % и диапазоны min/max повторов не пересекаются',
'captured_at': now_iso(),
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
},
)
write_markdown(
TRACK1_DIR / f'summary-ab{suffix}.md',
f'A/B: подтверждённое улучшение ({prefix})',
lines,
)
# ---------------------------------------------------------------- анализ и графики
def pyplot():
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
return plt
def representative(group: list[str]) -> str:
"""Прогон с медианным пиком — чтобы агрегаты контейнеров не брались с края."""
scored = []
for label in group:
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
steps = summary.get('steps', [])
peak = max((float(step['successful_rps']) for step in steps), default=0.0)
scored.append((peak, label))
if not scored:
raise SystemExit('нет прогонов для анализа')
scored.sort()
return scored[len(scored) // 2][1]
def ab_prefixes(labels: list[str]) -> list[str]:
"""Префиксы A/B-серий, у которых в raw есть обе стороны (ab, abdiag, …).
Иначе вердикт серии, пересобранной не последней, остался бы с прежними
числами: analyze обязан пересчитывать все вердикты из raw.
"""
prefixes = set()
for label in labels:
parts = label.split('#')
if len(parts) >= 2 and parts[1] in ('A', 'B'):
prefixes.add(parts[0])
return sorted(prefixes)
def cmd_analyze(args) -> None:
labels = raw_labels()
if not labels:
log('raw пуст: таблицы и графики будут пустыми')
groups = group_configurations(labels)
analyse_runs(groups)
analyse_series_a(groups, labels)
analyse_diagnostics(groups, labels)
analyse_growth(groups)
analyse_probe_check(groups)
analyse_mechanism(groups, labels)
for prefix in ab_prefixes(labels):
report_ab(argparse.Namespace(prefix=prefix))
make_graphs(groups)
log('analyze: таблицы и графики обновлены')
def series_kind(group: list[str]) -> str:
summary = read_json(RAW_DIR / f'{group[0]}.k6.json', {})
return summary.get('kind', 'ladder')
def analyse_runs(groups: dict) -> None:
"""Полный разбор каждого прогона: шаги, ресурсы, PostgreSQL, LB."""
lines = []
for key, group in sorted(groups.items()):
series = summarise_series(group)
label = representative(group)
_, samples, manifest = load_raw(label)
reps_note = 'одиночный прогон' if len(group) == 1 else f'{len(group)} прогонов'
lines += [f'## {key} ({reps_note}, {series_kind(group)})', '']
lines += table(
['темп, req/s', 'успешный RPS', 'min..max', 'ответов RPS', 'p95, ms', 'ошибок', 'VU', 'dropped', '5xx', '4xx'],
[
[
row['rate'],
row['successful_rps'],
f'{row["successful_min"]}..{row["successful_max"]}',
row['attempted_rps'],
row['p95'],
row['failed'],
row['vus_active'],
read_json(RAW_DIR / f'{label}.k6.json', {}).get('dropped_iterations', 0),
row.get('status_5xx', 0),
row.get('status_4xx', 0),
]
for row in series['rows']
],
)
if series_kind(group) == 'ladder':
saturation = series['saturation']
lines.append(
f'Потолок {series["ceiling_rps"]} успешных req/s, сатурация {saturation["rate"]} req/s '
f'({saturation["successful_rps"]} успешных req/s достигнуто).'
)
lines.append('')
limits = manifest.get('stand', {}) if manifest else {}
lines.append(
f'Лимиты: реплик {limits.get("API_REPLICAS")}, API {limits.get("API_CPUS_PER_REPLICA")} CPU / '
f'{limits.get("API_MEMORY_PER_REPLICA")}, PG {limits.get("PG_CPUS")} CPU, LB {limits.get("LB_CPUS")} CPU, '
f'пул {limits.get("DATABASE_POOL_MAX")}, образ {limits.get("API_IMAGE")} (BUILD_ID {limits.get("BUILD_ID")}).'
)
lines.append('')
lines += table(
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
[
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
for name, stats in container_stats(samples).items()
],
)
pg = postgres_stats(samples)
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, hit ratio {pg["blks_hit_ratio"]}, '
f'ожиданий блокировок пик {pg["lock_waiters_peak"]}, активных сессий пик {pg["active_peak"]}, '
f'соединений пик {pg["connections_peak"]}, deadlocks {pg["deadlocks"]}, messages {pg["messages_mib"]} MiB.'
)
lines.append('')
lb = haproxy_stats(samples)
total = lb.get('BACKEND')
if total:
lines.append(
f'LB: {total["requests"]} запросов ({total["rps"]} req/s), ошибок {total["errors"]}, '
f'очередь пик {total["qcur_peak"]}.'
)
for name, server in sorted(lb.items()):
if name in ('BACKEND', 'FRONTEND') or not server['requests']:
continue
lines.append(
f'- LB слот {name}: {server["requests"]} запросов ({server["rps"]} req/s), ошибок {server["errors"]}, '
f'очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}, адрес {server["addresses"]}.'
)
lines.append('')
app = app_metrics_delta(samples)
for container, delta in sorted(app.items()):
lines.append(
f'- реплика {delta["instance_id"]} (build {delta["build_id"]}, контейнер {container[:12]}): '
f'{json.dumps(delta["counters"], ensure_ascii=False)}; '
f'latency p95 {delta["latency_ms"].get("p95")} ms.'
)
lines.append('')
write_markdown(TRACK1_DIR / 'summary-runs.md', 'Разбор каждого прогона', lines)
def analyse_series_a(groups: dict, labels: list[str]) -> None:
lines = []
series_by_n = {}
for key, group in sorted(groups.items()):
if not key.startswith('p-a#n'):
continue
series = summarise_series(group)
series_by_n[key] = series
lines.append(f'## {key} ({len(group)} повторов)')
lines.append('')
lines += table(
['темп, req/s', 'успешный RPS', 'min..max', 'ответов RPS', 'p95, ms', 'p95 max', 'ошибок', 'VU'],
[
[
row['rate'],
row['successful_rps'],
f'{row["successful_min"]}..{row["successful_max"]}',
row['attempted_rps'],
row['p95'],
row['p95_max'],
row['failed'],
row['vus_active'],
]
for row in series['rows']
],
)
saturation = series['saturation']
lines.append(
f'Потолок: **{series["ceiling_rps"]} успешных req/s**; сатурация: {saturation["rate"]} req/s '
f'(достигнуто {saturation["successful_rps"]} успешных req/s).'
)
lines.append('')
label = representative(group)
_, samples, manifest = load_raw(label)
lines.append(f'Ресурсы представительного прогона `{label}` (лимиты {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")} на реплику):')
lines.append('')
lines += table(
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
[
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
for name, stats in container_stats(samples).items()
],
)
pg = postgres_stats(samples)
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, '
f'hit ratio {pg["blks_hit_ratio"]}, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
f'активных сессий пик {pg["active_peak"]}, соединений пик {pg["connections_peak"]}, '
f'таблица messages {pg["messages_mib"]} MiB.'
)
lines.append('')
lb = haproxy_stats(samples)
distribution = [
f'слот {name}: {server["requests"]} ({server["rps"]} req/s), адрес {", ".join(server["addresses"])}'
for name, server in sorted(lb.items())
if name not in ('BACKEND', 'FRONTEND') and server['requests']
]
if distribution:
lines += ['Распределение запросов LB:', ''] + [f'- {item}' for item in distribution] + ['']
lines += saturation_conclusion(series_by_n)
write_markdown(TRACK1_DIR / 'summary-series-a.md', 'Серия A: 1/2/4 реплики при фиксированном бюджете', lines)
def saturation_conclusion(series_by_n: dict) -> list[str]:
lines = ['## Точки сатурации', '']
for key, series in sorted(series_by_n.items()):
saturation = series['saturation']
lines.append(
f'- {key}: knee на {saturation["rate"]} req/s (достигнуто {saturation["successful_rps"]} успешных req/s), '
f'потолок {series["ceiling_rps"]} успешных req/s.'
)
lines.append('')
return lines
def analyse_diagnostics(groups: dict, labels: list[str]) -> None:
lines = []
for key, group in sorted(groups.items()):
if not key.startswith(('p-b', 'p-c')):
continue
series = summarise_series(group)
label = representative(group)
_, samples, manifest = load_raw(label)
pg = postgres_stats(samples)
stats = container_stats(samples)
lines.append(f'## {key} (темпы {series["rows"][0]["rate"]}..{series["rows"][-1]["rate"]} req/s)')
lines.append('')
lines += table(
['темп', 'успешный RPS', 'ответов RPS', 'p95, ms', 'ошибок'],
[[row['rate'], row['successful_rps'], row['attempted_rps'], row['p95'], row['failed']] for row in series['rows']],
)
lines.append(
f'Потолок {series["ceiling_rps"]} успешных req/s, knee на {series["saturation"]["rate"]} req/s. '
f'Лимиты: LB {manifest["stand"].get("LB_CPUS")} CPU, API {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")}, PG {manifest["stand"].get("PG_CPUS")} CPU.'
)
lines.append('')
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
f'активных сессий пик {pg["active_peak"]}, hit ratio {pg["blks_hit_ratio"]}.'
)
lines.append('')
lb = stats.get('mtusi-lab3-haproxy-1')
if lb:
lines.append(f'HAProxy: CPU средний {lb["cpu_mean"]}%, пик {lb["cpu_peak"]}%, RAM пик {lb["mem_peak"]} MiB из {lb["mem_limit"]}.')
lines.append('')
haproxy = haproxy_stats(samples)
for name, server in sorted(haproxy.items()):
if server['qcur_peak'] or server['statuses'] != ['UP']:
lines.append(f'- LB сервер {name}: очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}.')
lines.append('')
write_markdown(TRACK1_DIR / 'summary-diagnostics.md', 'Диагностика: горячий чат и путь чтения', lines)
def growth_point(key: str, group: list[str], rate: int) -> dict | None:
"""Точка серии роста ровно на этом темпе или None.
Возвращает None, если хотя бы у одного повтора нет ступени с этим темпом:
подставлять соседнюю полку в сравнение нельзя — иначе рост на 560 req/s
оказался бы рядом с 900 и «разностью» ресурсов. Успешный RPS — медиана по
повторам (как и в остальных таблицах), ресурсы — из представительного
прогона (медианного по пику).
"""
steps = []
for run in group:
step = step_of(read_json(RAW_DIR / f'{run}.k6.json', {}), rate)
if step is None:
return None
steps.append(step)
successful = [float(step['successful_rps']) for step in steps]
attempted = [float(step['attempted_rps']) for step in steps]
label = representative(group)
_, samples, manifest = load_raw(label)
manifest = manifest or {}
stand = manifest.get('stand') or {}
stats = container_stats(samples)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
api_limit = float(stand.get('API_CPUS_PER_REPLICA') or 0)
return {
'key': key,
'label': label,
'profile': manifest.get('profile'),
'mode': manifest.get('mode'),
'rate': rate,
'successful_rps': median(successful),
'successful_min': spread(successful)[0],
'successful_max': spread(successful)[1],
'attempted_rps': median(attempted),
'p95': median([float(step.get('p95') or 0) for step in steps]),
'failed': sum(step.get('failed') or 0 for step in steps),
'status_5xx': sum((step.get('status') or {}).get('server_5xx') or 0 for step in steps),
'status_4xx': sum((step.get('status') or {}).get('client_4xx') or 0 for step in steps),
'vus_active': max(step.get('vus_active') or 0 for step in steps),
'max_vus': (manifest.get('generator') or {}).get('max_vus'),
'dropped': sum(
read_json(RAW_DIR / f'{run}.k6.json', {}).get('dropped_iterations', 0) for run in group
),
'api_peak_quota': round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0,
'pg_peak': stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0),
'reps': len(group),
}
LADDER_MARKERS = ('o', 's', '^', 'D', 'v', 'P', 'X')
# Серия роста — блок why# (1500 req/s, за коленом): контроль why#n4 и точки,
# каждая из которых отличается от него ровно одним ключом бюджета.
GROWTH_CONTROL = 'why#n4'
GROWTH_POINTS = {
'why#n4-api6': ('API ×2 (6.0 CPU)', 'API_TOTAL_CPUS'),
'why#n4-pg3': ('PostgreSQL ×2 (3.0 CPU)', 'PG_CPUS'),
'why#n4-pool5': ('пул 5', 'DATABASE_POOL_MAX'),
'why#n4-pool20': ('пул 20', 'DATABASE_POOL_MAX'),
'why#n4-lb1': ('LB ×4 (1.0 CPU)', 'LB_CPUS'),
}
GROWTH_BUDGET_KEYS = ('API_REPLICAS', 'API_TOTAL_CPUS', 'PG_CPUS', 'DATABASE_POOL_MAX', 'LB_CPUS')
# Разброс повторов на насыщении (REPORT, «Границы измерения»: ±5–8 %): меньшие
# различия одиночных прогонов не считаются эффектом рычага.
GROWTH_NOISE_PERCENT = 8.0
def _budget_value(value) -> str | None:
if value is None:
return None
try:
return str(float(value))
except (TypeError, ValueError):
return str(value)
def growth_comparability(control_group: list[str], group: list[str], changed_key: str) -> str | None:
"""None, если точка сопоставима с контролем; иначе причина.
Сопоставимость проверяется по манифестам: тот же темп, профиль, режим,
коммит, образ и генератор, а из ключей бюджета отличается ровно changed_key.
"""
def facts(label: str) -> tuple[dict, dict]:
manifest = read_json(RAW_DIR / f'{label}.manifest.json', {}) or {}
summary = read_json(RAW_DIR / f'{label}.k6.json', {}) or {}
generator = manifest.get('generator') or {}
stand = manifest.get('stand') or {}
fields = {
'темп': tuple(int(step['rate']) for step in summary.get('steps', [])),
'profile': manifest.get('profile'),
'mode': manifest.get('mode'),
'commit': manifest.get('commit'),
'stand.API_IMAGE': stand.get('API_IMAGE'),
'generator.max_vus': generator.get('max_vus'),
'generator.cpus': generator.get('cpus'),
'generator.memory': generator.get('memory'),
}
return fields, stand
reference, reference_stand = facts(control_group[0])
runs = [(label, False) for label in control_group[1:]] + [(label, True) for label in group]
for label, is_point in runs:
fields, stand = facts(label)
for name, value in fields.items():
if value != reference[name]:
return f'{name}: {value} ≠ {reference[name]}'
for key in GROWTH_BUDGET_KEYS:
value = _budget_value(stand.get(key))
expected = _budget_value(reference_stand.get(key))
if is_point and key == changed_key:
if value == expected:
return f'{key}: {value} совпадает с контролем, рычаг не менялся'
elif value != expected:
return f'{key}: {value} ≠ {expected}'
return None
def growth_series(groups: dict) -> tuple[dict | None, list[dict], list[tuple[str, str, str]]]:
"""Контроль, сопоставимые точки (с Δ и выводом) и исключённые с причиной."""
control_group = groups.get(GROWTH_CONTROL)
if not control_group:
return None, [], []
rate = None
for step in read_json(RAW_DIR / f'{control_group[0]}.k6.json', {}).get('steps', []):
rate = int(step['rate'])
if rate is None:
return None, [], []
control = growth_point(GROWTH_CONTROL, control_group, rate)
if control is None:
return None, [], []
points: list[dict] = []
omitted: list[tuple[str, str, str]] = []
for key, (name, changed_key) in GROWTH_POINTS.items():
group = groups.get(key)
if not group:
omitted.append((key, name, 'не снята на этом темпе'))
continue
point = growth_point(key, group, rate)
if point is None:
omitted.append((key, name, f'нет ступени {rate} req/s'))
continue
reason = growth_comparability(control_group, group, changed_key)
if reason:
omitted.append((key, name, reason))
continue
base = control['successful_rps']
delta = round((point['successful_rps'] - base) / base * 100, 1) if base else 0.0
if delta >= GROWTH_NOISE_PERCENT:
verdict = 'рычаг поднимает потолок'
elif delta <= -GROWTH_NOISE_PERCENT:
verdict = 'ухудшает'
else:
verdict = 'в пределах разброса'
point.update(name=name, delta=delta, verdict=verdict)
points.append(point)
return control, points, omitted
def growth_table_row(point: dict, control: bool = False) -> list:
return [
point['key'],
'контроль N=4' if control else point['name'],
f'{point["successful_rps"]} ({point["successful_min"]}..{point["successful_max"]})',
'—' if control else f'{point["delta"]:+}',
point['p95'],
point['status_5xx'],
point['dropped'],
f'{point["vus_active"]} / {point["max_vus"]}',
point['api_peak_quota'],
point['pg_peak'],
'контроль' if control else point['verdict'],
]
GROWTH_TABLE_HEADER = [
'точка',
'что изменено',
'успешный RPS (min..max)',
'Δ к контролю, %',
'p95, ms',
'5xx',
'dropped',
'VU пик / бюджет',
'CPU API: пик реплики, % её квоты',
'CPU PG пик, % ядра',
'вывод',
]
def analyse_growth(groups: dict) -> None:
"""Серия роста ресурсов: блок why# на одном темпе, один ключ бюджета на точку."""
control, points, omitted = growth_series(groups)
if control is None:
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', ['Нет контроля why#n4.'])
return
manifest = read_json(RAW_DIR / f'{control["label"]}.manifest.json', {}) or {}
lines = [
f'Блок {control["rate"]} req/s, N=4, одно временное окно, MAX_VUS={control["max_vus"]}, '
f'коммит `{str(manifest.get("commit") or "")[:7]}`. Каждая точка отличается от контроля '
'одним ключом бюджета; сопоставимость проверена по манифестам.',
'',
]
rows = [growth_table_row(control, control=True)] + [growth_table_row(point) for point in points]
lines += table(GROWTH_TABLE_HEADER, rows)
if any(point['reps'] == 1 for point in [control] + points):
lines += [
f'Одиночные прогоны: различия меньше ±{GROWTH_NOISE_PERCENT:g} % не интерпретируются.',
'',
]
if omitted:
lines += ['Вне сравнения:', '']
lines += [f'- `{key}` ({name}): {reason}' for key, name, reason in omitted]
lines.append('')
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
def analyse_probe_check(groups: dict) -> None:
"""Проба готовности под перегрузкой: probecheck#n4 против эталона why#n4."""
probe = groups.get('probecheck#n4')
reference = groups.get(GROWTH_CONTROL)
if not probe or not reference:
return
rows = []
successful: dict[str, list[float]] = {}
probe_statuses: set[str] = set()
probe_5xx = 0
for key, group, check in ((GROWTH_CONTROL, reference, '/internal/lb-check'), ('probecheck#n4', probe, '/health/ready')):
for label in sorted(group):
summary, samples, manifest = load_raw(label)
manifest = manifest or {}
step = (summary or {}).get('steps', [None])[0]
if step is None:
continue
slots = {
name: value
for name, value in haproxy_stats(samples).items()
if name.startswith('api') and name[3:].isdigit() and value['requests'] > 0
}
statuses = sorted({status for value in slots.values() for status in value['statuses']})
five_xx = (step.get('status') or {}).get('server_5xx') or 0
successful.setdefault(key, []).append(float(step['successful_rps']))
if key == 'probecheck#n4':
probe_statuses.update(statuses)
probe_5xx += five_xx
rows.append(
[
label,
str(manifest.get('commit') or '')[:7],
check,
step['successful_rps'],
step.get('p95'),
five_xx,
(summary or {}).get('dropped_iterations', 0),
(manifest.get('dataset') or {}).get('messages'),
', '.join(statuses),
]
)
if 'probecheck#n4' not in successful or GROWTH_CONTROL not in successful:
return
base = median(successful[GROWTH_CONTROL])
delta = (median(successful['probecheck#n4']) - base) / base * 100 if base else 0.0
lines = table(
['прогон', 'коммит', 'проба LB', 'успешный RPS', 'p95, ms', '5xx', 'dropped', 'сообщений в БД', 'статусы слотов LB'],
rows,
)
lines += [
f'Слоты LB всё время UP: {"да" if probe_statuses == {"UP"} else "нет"}',
'',
f'Δ успешного RPS к why#n4: {delta:+.1f} % (порог разброса ±{GROWTH_NOISE_PERCENT:g} %)',
f'5xx на probecheck#n4: {probe_5xx}',
'',
]
write_markdown(TRACK1_DIR / 'summary-probe-check.md', 'Проба готовности под перегрузкой', lines)
def make_graphs(groups: dict) -> None:
if not groups:
return
plt = pyplot()
GRAPH_DIR.mkdir(parents=True, exist_ok=True)
ladders = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'ladder'}
steadys = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'steady'}
if ladders:
series = {key: summarise_series(group) for key, group in ladders.items()}
figure, axes = plt.subplots(figsize=(9, 5))
for index, (key, values) in enumerate(series.items()):
axes.plot(
[row['rate'] for row in values['rows']],
[row['successful_rps'] for row in values['rows']],
marker=LADDER_MARKERS[index % len(LADDER_MARKERS)],
color=plt.get_cmap('tab20')(index % 20),
label=key,
)
lo = min(row['rate'] for values in series.values() for row in values['rows'])
hi = max(row['rate'] for values in series.values() for row in values['rows'])
axes.plot([lo, hi], [lo, hi], linestyle='--', color='grey', zorder=0, label='предложенный темп')
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('успешный RPS (ответы ожидаемого класса)')
axes.set_title('Пропускная способность: успешный RPS от предложенного темпа')
axes.grid(alpha=0.3)
axes.legend(fontsize=7, ncol=2)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'throughput.png', dpi=120)
plt.close(figure)
figure, axes = plt.subplots(figsize=(9, 5))
for index, (key, values) in enumerate(series.items()):
color = plt.get_cmap('tab20')(index % 20)
marker = LADDER_MARKERS[index % len(LADDER_MARKERS)]
axes.plot(
[row['rate'] for row in values['rows']],
[row['p95'] for row in values['rows']],
marker=marker,
color=color,
label=f'{key}: p95, ms',
)
axes.plot(
[row['rate'] for row in values['rows']],
[row['failed'] for row in values['rows']],
linestyle=':',
marker='x',
color=color,
label=f'{key}: неуспешных ответов, шт.',
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('p95, ms / неуспешных ответов')
axes.set_title('Латентность p95 и неуспешные ответы по шагам лестницы')
axes.grid(alpha=0.3)
axes.legend(fontsize=7, ncol=2)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'latency-errors.png', dpi=120)
plt.close(figure)
figure, axes = plt.subplots(figsize=(9, 5))
for index, (key, values) in enumerate(series.items()):
axes.plot(
[row['rate'] for row in values['rows']],
[row['vus_active'] for row in values['rows']],
marker=LADDER_MARKERS[index % len(LADDER_MARKERS)],
color=plt.get_cmap('tab20')(index % 20),
label=key,
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('активных VU генератора')
axes.set_title('Занятость генератора: нехватка VU отличима от перегрузки сервера')
axes.grid(alpha=0.3)
axes.legend(fontsize=7, ncol=2)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'generator-vus.png', dpi=120)
plt.close(figure)
if steadys:
keys = list(steadys)
successful = [summarise_series(steadys[key])['ceiling_rps'] for key in keys]
offered = [summarise_series(steadys[key])['rows'][0]['rate'] for key in keys]
figure, axes = plt.subplots(figsize=(9, 5))
positions = range(len(keys))
axes.bar([p - 0.2 for p in positions], offered, width=0.4, label='предложенный темп')
axes.bar([p + 0.2 for p in positions], successful, width=0.4, label='успешный RPS')
axes.set_xticks(list(positions))
axes.set_xticklabels(keys, fontsize=8)
axes.set_ylabel('req/s (успешные ответы)')
axes.set_title('Фиксированный темп: предложено против успешного RPS')
axes.legend()
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'steady-throughput.png', dpi=120)
plt.close(figure)
for key, group in sorted(groups.items()):
plot_container_cpu(plt, representative(group))
figure, axes = plt.subplots(figsize=(9, 5))
shares = {}
for key, group in sorted(groups.items()):
_, samples, _ = load_raw(representative(group))
lb = haproxy_stats(samples)
shares[key] = {name: server['requests'] for name, server in sorted(lb.items()) if server['requests']}
names = sorted({name for share in shares.values() for name in share})
if names and shares:
width = 0.8 / max(1, len(names))
for position, name in enumerate(names):
axes.bar(
[index + position * width - 0.4 for index in range(len(shares))],
[shares[key].get(name, 0) for key in shares],
width=width,
label=name,
)
axes.set_xticks(range(len(shares)))
axes.set_xticklabels(list(shares), fontsize=7)
axes.set_ylabel('запросов за прогон')
axes.set_title('Распределение запросов по backend-слотам LB')
axes.legend(fontsize=7)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'replica-distribution.png', dpi=120)
plt.close(figure)
control, growth_points, _ = growth_series(groups)
if control is None:
log('growth.png: нет контроля why#n4')
else:
rate = control['rate']
plotted = [control] + growth_points
names = ['контроль N=4'] + [point['name'] for point in growth_points]
values = [point['successful_rps'] for point in plotted]
positions = list(range(len(plotted)))
figure, (left, right) = plt.subplots(1, 2, figsize=(11, 5))
left.bar(
positions,
values,
width=0.6,
yerr=[
[max(0.0, point['successful_rps'] - point['successful_min']) for point in plotted],
[max(0.0, point['successful_max'] - point['successful_rps']) for point in plotted],
],
capsize=4,
)
for index, point in enumerate(plotted):
label = f'{point["successful_rps"]}' if index == 0 else f'{point["successful_rps"]}\n{point["delta"]:+}%'
left.annotate(label, (index, point['successful_max']), ha='center', va='bottom', fontsize=8)
left.axhline(rate, linestyle='--', color='grey', label='предложенный темп')
left.set_xticks(positions)
left.set_xticklabels(names, fontsize=8, rotation=20, ha='right')
left.set_ylabel('успешный RPS (ответы ожидаемого класса)')
left.set_ylim(0, rate * 1.15)
left.legend(fontsize=8)
left.grid(axis='y', alpha=0.3)
right.bar(positions, [point['p95'] for point in plotted], width=0.6, color='tab:orange')
right.set_xticks(positions)
right.set_xticklabels(names, fontsize=8, rotation=20, ha='right')
right.set_ylabel('p95, ms')
right.grid(axis='y', alpha=0.3)
figure.suptitle(f'Серия роста ресурсов: {rate} req/s, N=4, одно окно (why#)')
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'growth.png', dpi=120)
plt.close(figure)
if groups.get('ab#A') and groups.get('ab#B'):
figure, axes = plt.subplots(figsize=(9, 5))
series_a = summarise_series(groups['ab#A'])
series_b = summarise_series(groups['ab#B'])
rates = [row['rate'] for row in series_a['rows']]
positions = list(range(len(rates)))
axes.bar([p - 0.2 for p in positions], [row['successful_rps'] for row in series_a['rows']], width=0.4, label='A (до)')
axes.bar([p + 0.2 for p in positions], [row['successful_rps'] for row in series_b['rows']], width=0.4, label='B (после)')
for offset, series in ((-0.2, series_a), (0.2, series_b)):
for index, row in enumerate(series['rows']):
axes.plot([index + offset, index + offset], [row['successful_min'], row['successful_max']], color='black', linewidth=1)
axes.set_xticks(positions)
axes.set_xticklabels([str(rate) for rate in rates])
axes.set_xlabel('темп, req/s')
axes.set_ylabel('успешный RPS (ответы ожидаемого класса)')
axes.set_title('A/B: медиана успешного RPS и разброс (min..max) по повторам')
axes.legend()
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'ab-comparison.png', dpi=120)
plt.close(figure)
def plot_container_cpu(plt, label: str) -> None:
_, samples, _ = load_raw(label)
series: dict[str, list[tuple[float, float]]] = {}
base = None
for record in samples:
if record.get('source') != 'docker_stats':
continue
name = record.get('name') or ''
if not name.startswith('mtusi-lab3-'):
continue
moment = _timestamp(record['t'])
base = base or moment
series.setdefault(name, []).append((moment - base, float(record.get('cpu_percent') or 0)))
if not series:
return
figure, axes = plt.subplots(figsize=(10, 5))
for name, points in sorted(series.items()):
axes.plot([point[0] for point in points], [point[1] for point in points], label=name, linewidth=1)
axes.set_xlabel('время от начала прогона, s')
axes.set_ylabel('CPU, % (100 % = одно ядро)')
axes.set_title(f'CPU контейнеров: {label}')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / f'container-cpu-{label.replace("#", "_")}.png', dpi=120)
plt.close(figure)
# ---------------------------------------------------------------- оркестрация трека
def cmd_watch(args) -> None:
"""Сэмплер без нагрузки: наблюдает стенд N секунд и пишет jsonl.gz.
Нужен там, где нагрузку создаёт внешний сценарий (публичный failover-тест,
rolling update): переходы статусов LB получают отметки времени.
"""
sampler = Sampler().start()
try:
time.sleep(args.seconds)
finally:
samples = sampler.stop()
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
with gzip.open(out, 'wt', encoding='utf-8') as handle:
for record in samples:
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
print(json.dumps({'out': str(out), 'samples': len(samples), 'errors': sampler.errors}, ensure_ascii=False))
def cmd_quiesce(args) -> None:
result = run_command(['scripts/quiesce.sh', args.action], check=False, timeout=300)
print(result.stdout + result.stderr)
if result.returncode != 0:
raise SystemExit(f'quiesce {args.action} failed')
def cmd_track1(args) -> None:
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='up'))
try:
log('=== серия A: 1/2/4 реплики, профиль p-a')
cmd_sweep(
argparse.Namespace(
profile='p-a',
replicas='1,2,4',
reps=args.reps,
no_warmup=False,
prefix='p-a',
extra_replica=2,
extra_reps=1,
)
)
log('=== серия B: горячий чат (p-b), 4 реплики')
ensure_stand({'API_REPLICAS': '4'})
run_measured('p-b#n4', 'p-b', warmup=False)
log('=== серия C: путь чтения (p-c), 4 реплики, LB 0.25 и 1.0 CPU')
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '0.25'})
run_measured('p-c#lb0.25', 'p-c', warmup=False)
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '1.0'})
run_measured('p-c#lb1.0', 'p-c', warmup=False)
log('=== блок «почему» и серия роста: 1500 req/s, интерливинг')
cmd_track1_extras(argparse.Namespace(part='why', rate=1500, duration=60, reps=args.reps, max_vus=900, prefix='why', loadgen_cpus=DEFAULT_STAND['LOADGEN_CPUS'], loadgen_memory='1024m'))
log('=== агрегаты')
cmd_analyze(argparse.Namespace())
if args.ab_after:
log('=== A/B улучшения (p-b, 4 реплики)')
cmd_ab(
argparse.Namespace(
before=args.ab_before,
after=args.ab_after,
profile='p-b',
replicas=4,
reps=args.ab_reps,
no_warmup=False,
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
pg_cpus=DEFAULT_STAND['PG_CPUS'],
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
lb_cpus=DEFAULT_STAND['LB_CPUS'],
loadgen_cpus=DEFAULT_STAND['LOADGEN_CPUS'],
loadgen_memory=DEFAULT_STAND['LOADGEN_MEMORY'],
max_vus=1200,
prefix='ab',
)
)
cmd_analyze(argparse.Namespace())
else:
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
finally:
ensure_stand(
{
'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']),
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
'API_IMAGE': DEFAULT_STAND['API_IMAGE'],
'BUILD_ID': DEFAULT_STAND['BUILD_ID'],
}
)
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='down'))
def cmd_track1_extras(args) -> None:
"""Догоняющие серии трека 1: VU-своп, блоки «почему» и контрольный A/B.
Отдельные команды, потому что они отвечают на уточняющие вопросы (потолок
сервера или генератора, где теряется время, сравнение на бюджете по умолчанию),
а не строят базовую картину серии A.
"""
base = {
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
'LOADGEN_CPUS': str(args.loadgen_cpus),
'LOADGEN_MEMORY': args.loadgen_memory,
}
if args.part in ('all', 'vuswap'):
log('=== VU-своп: та же конфигурация, меняется только MAX_VUS')
for replicas in ('1', '4'):
for max_vus in (200, 900, 2000):
ensure_stand({**base, 'API_REPLICAS': replicas, 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX'])})
run_measured(
f'vuswap#n{replicas}-vu{max_vus}',
'p-a',
ladder=[700, 900, 1100],
step_seconds=15,
warmup=False,
vus={'max': max_vus},
)
if args.part in ('all', 'why'):
log(f'=== блок «почему»: один темп {args.rate} req/s, {args.reps} повтора, интерливинг')
block = [
('n1', {'API_REPLICAS': '1', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10'}),
('n4', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10'}),
('n4-pg3', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '3.0', 'DATABASE_POOL_MAX': '10'}),
('n4-api6', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '6.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10'}),
('n4-pool20', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '20'}),
('n4-pool5', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '5'}),
('n4-lb1', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10', 'LB_CPUS': '1.0'}),
]
for rep in range(1, args.reps + 1):
order = block if rep % 2 else list(reversed(block))
for name, overrides in order:
ensure_stand({**base, **overrides})
run_measured(
f'{args.prefix}#{name}#rep{rep}',
'p-a',
rate=args.rate,
duration=args.duration,
warmup=False,
vus={'max': args.max_vus},
)
if args.part in ('all', 'abdiag'):
log('=== контрольный A/B на бюджете по умолчанию')
cmd_ab(
argparse.Namespace(
before=args.before,
after=args.after,
profile='p-b',
replicas=4,
reps=args.ab_reps,
no_warmup=False,
prefix='abdiag',
max_vus=900,
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
pg_cpus=DEFAULT_STAND['PG_CPUS'],
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
lb_cpus=DEFAULT_STAND['LB_CPUS'],
loadgen_cpus=args.loadgen_cpus,
loadgen_memory=args.loadgen_memory,
)
)
cmd_analyze(argparse.Namespace())
# ---------------------------------------------------------------- CLI
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description='Драйвер экспериментов lab3')
sub = parser.add_subparsers(dest='command', required=True)
stand = sub.add_parser('stand', help='стенд: up/down/status')
stand.add_argument('action', choices=['up', 'down', 'status'])
stand.add_argument('--replicas', type=int)
stand.set_defaults(func=cmd_stand)
check = sub.add_parser('check', help='проверки уровня 3/4')
check.add_argument('what', choices=['spread', 'instances', 'idempotency'])
check.add_argument('--requests', type=int, default=40)
check.set_defaults(func=cmd_check)
run = sub.add_parser('run', help='один прогон профиля')
run.add_argument('--label', required=True)
run.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
run.add_argument('--ladder')
run.add_argument('--step', type=int)
run.add_argument('--peak', type=int, help='одна полка фиксированного темпа')
run.add_argument('--duration', type=int, help='длительность полки, s')
run.add_argument('--replicas', type=int)
run.add_argument('--pool', type=int, help='DATABASE_POOL_MAX на реплику')
run.add_argument('--loadgen-cpus', type=float, help='лимит CPU генератора (контингенция: k6 упёрся в квоту)')
run.add_argument('--lb-cpus', type=float, help='лимит CPU HAProxy')
run.add_argument('--api-total-cpus', type=float, help='суммарный бюджет CPU API (делится на реплики)')
run.add_argument('--pg-cpus', type=float, help='лимит CPU PostgreSQL')
run.add_argument('--loadgen-memory', default='256m', help='лимит памяти генератора (k6 OOM на медленной стороне)')
run.add_argument('--max-vus', type=int, help='MAX_VUS генератора (по умолчанию 200)')
run.add_argument('--pre-vus', type=int, help='PRE_VUS генератора (по умолчанию 60)')
run.add_argument('--mode', choices=['write', 'read'])
run.add_argument('--spread', choices=['spread', 'hot'])
run.add_argument('--no-warmup', action='store_true')
run.set_defaults(func=cmd_run)
sweep = sub.add_parser('sweep', help='серия 1/2/4 реплики')
sweep.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
sweep.add_argument('--replicas', default='1,2,4')
sweep.add_argument('--reps', type=int, default=2)
sweep.add_argument('--extra-replica', type=int, default=0)
sweep.add_argument('--extra-reps', type=int, default=0)
sweep.add_argument('--prefix')
sweep.add_argument('--no-warmup', action='store_true')
sweep.set_defaults(func=cmd_sweep)
failover = sub.add_parser('failover', help='фазовый драйвер отказа реплики')
failover.add_argument('--replicas', default='all')
failover.add_argument('--rate', type=int, default=200)
failover.add_argument('--duration', type=int, default=60)
failover.add_argument('--pre', type=float, default=8.0, help='секунд нагрузки до отказа')
failover.add_argument('--hold', type=float, default=20.0, help='секунд до восстановления реплики')
failover.set_defaults(func=cmd_failover)
rolling = sub.add_parser('rolling-update', help='поочерёдная замена реплик')
rolling.add_argument('--under-load', action='store_true')
rolling.add_argument('--rate', type=int, default=200)
rolling.add_argument('--duration', type=int, default=180)
rolling.add_argument('--build-id')
rolling.add_argument('--pause', type=float, default=2.0)
rolling.set_defaults(func=cmd_rolling)
ab = sub.add_parser('ab', help='интерливинг A/B двух образов')
ab.add_argument('--before', required=True)
ab.add_argument('--after', required=True)
ab.add_argument('--profile', default='p-b')
ab.add_argument('--replicas', type=int, default=4)
ab.add_argument('--reps', type=int, default=3)
ab.add_argument('--no-warmup', action='store_true')
ab.add_argument('--api-total-cpus', type=float, default=DEFAULT_STAND['API_TOTAL_CPUS'])
ab.add_argument('--api-total-memory-mb', type=int, default=DEFAULT_STAND['API_TOTAL_MEMORY_MB'])
ab.add_argument('--pg-cpus', type=float, default=DEFAULT_STAND['PG_CPUS'])
ab.add_argument('--pool', type=int, default=DEFAULT_STAND['DATABASE_POOL_MAX'])
ab.add_argument('--lb-cpus', type=float, default=DEFAULT_STAND['LB_CPUS'])
ab.add_argument('--loadgen-cpus', type=float, default=DEFAULT_STAND['LOADGEN_CPUS'])
ab.add_argument('--loadgen-memory', default=DEFAULT_STAND['LOADGEN_MEMORY'])
ab.add_argument('--max-vus', type=int, default=1200)
ab.add_argument('--prefix', default='ab', help='префикс меток (различает серии A/B)')
ab.set_defaults(func=cmd_ab)
analyze = sub.add_parser('analyze', help='агрегаты и графики из raw')
analyze.set_defaults(func=cmd_analyze)
watch = sub.add_parser('watch', help='сэмплер без нагрузки: наблюдение за стендом')
watch.add_argument('--seconds', type=float, default=30)
watch.add_argument('--out', required=True)
watch.set_defaults(func=cmd_watch)
quiesce = sub.add_parser('quiesce', help='фон хоста: up/down/status')
quiesce.add_argument('action', choices=['up', 'down', 'status'])
quiesce.set_defaults(func=cmd_quiesce)
extras = sub.add_parser('track1-extras', help='догоняющие серии трека 1 (VU-своп, блоки «почему», A/B на бюджете по умолчанию)')
extras.add_argument('--part', choices=['all', 'vuswap', 'why', 'abdiag'], default='all')
extras.add_argument('--rate', type=int, default=1500, help='темп блока «почему»')
extras.add_argument('--duration', type=int, default=60)
extras.add_argument('--reps', type=int, default=2)
extras.add_argument('--max-vus', type=int, default=900)
extras.add_argument('--prefix', default='why')
extras.add_argument('--ab-reps', type=int, default=3)
extras.add_argument('--before', default='mtusi-messenger:lab3-before')
extras.add_argument('--after', default='mtusi-messenger:lab3-after')
extras.add_argument('--loadgen-cpus', type=float, default=DEFAULT_STAND['LOADGEN_CPUS'])
extras.add_argument('--loadgen-memory', default=DEFAULT_STAND['LOADGEN_MEMORY'],
help='память генератора: при 900 VU и секундных задержках 256 MiB мало (k6 убивается OOM)')
extras.set_defaults(func=cmd_track1_extras)
track1 = sub.add_parser('track1', help='полная серия трека 1')
track1.add_argument('--reps', type=int, default=2)
track1.add_argument('--ab-before', default='mtusi-messenger:lab3-before')
track1.add_argument('--ab-after', default=None)
track1.add_argument('--ab-reps', type=int, default=3)
track1.add_argument('--no-quiesce', action='store_true')
track1.set_defaults(func=cmd_track1)
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
RAW_DIR.mkdir(parents=True, exist_ok=True)
args.func(args)
if __name__ == '__main__':
main()