forked from mtusi2026/mtusi2026lab3
2979 lines
129 KiB
Python
Executable File
2979 lines
129 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Драйвер экспериментов lab3: стенд, доказательства уровней 3/4 и трек 1.
|
||
|
||
Запуск из корня репозитория: python3 scripts/lab3/lab3.py <подкоманда> ...
|
||
|
||
Подкоманды:
|
||
stand up|down|status поднять/остановить/показать стенд
|
||
check spread|instances|idempotency
|
||
разные реплики на одной сессии; карта X-Instance-Id;
|
||
фильтр e2e по идемпотентности
|
||
run --label ... один прогон профиля с сэмплером
|
||
sweep --profile p-a серия 1/2/4 реплики (интерливинг повторов)
|
||
growth --series api|pg|pool серия роста ресурсов на фиксированном темпе
|
||
failover --replicas all фазовый драйвер отказа реплики
|
||
rolling-update --under-load поочерёдная замена реплик под нагрузкой
|
||
ab --before T --after T интерливинг A/B двух образов
|
||
analyze агрегаты и графики из raw
|
||
quiesce up|down|status обёртка scripts/quiesce.sh
|
||
track1 полная серия трека 1 по зафиксированному плану
|
||
|
||
Только стандартная библиотека (графики — matplotlib на хосте). Данные прогонов
|
||
складываются в evidence/track1/raw: <label>.k6.json (сводка k6),
|
||
<label>.samples.jsonl.gz (сэмплер), <label>.manifest.json (условия измерения).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import gzip
|
||
import json
|
||
import math
|
||
import os
|
||
import re
|
||
import shutil
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
ENV_FILE = ROOT / '.env'
|
||
STAND_ENV = ROOT / '.env.stand'
|
||
STAND_STATE = ROOT / '.env.stand.json'
|
||
QUIESCE_STATE = ROOT / '.lab3-quiesce.json'
|
||
RAW_DIR = ROOT / 'evidence/track1/raw'
|
||
TRACK1_DIR = ROOT / 'evidence/track1'
|
||
GRAPH_DIR = TRACK1_DIR / 'graphs'
|
||
LEVEL3_DIR = ROOT / 'evidence/level3'
|
||
LEVEL4_DIR = ROOT / 'evidence/level4'
|
||
PROJECT = 'mtusi-lab3'
|
||
BASE_URL = os.environ.get('LAB3_BASE_URL', 'http://127.0.0.1:8080')
|
||
LB_STATS_URL = os.environ.get('LAB3_LB_STATS', 'http://127.0.0.1:8404/stats;csv')
|
||
COMPOSE = ['docker', 'compose', '--env-file', str(ENV_FILE), '--env-file', str(STAND_ENV)]
|
||
|
||
# Фиксированный бюджет серии трека 1: суммарные ресурсы API одинаковы для всех
|
||
# сравниваемых конфигураций, поэтому на реплику приходится 3/N CPU и 1.5/N GiB.
|
||
API_TOTAL_CPUS = 3.0
|
||
API_TOTAL_MEMORY_MB = 1536
|
||
DEFAULT_STAND = {
|
||
'API_TOTAL_CPUS': API_TOTAL_CPUS,
|
||
'API_TOTAL_MEMORY_MB': API_TOTAL_MEMORY_MB,
|
||
'API_REPLICAS': 2,
|
||
'PG_CPUS': 1.5,
|
||
'PG_MEMORY': '512m',
|
||
'REDIS_CPUS': 0.25,
|
||
'REDIS_MEMORY': '128m',
|
||
'LB_CPUS': 0.25,
|
||
'LB_MEMORY': '128m',
|
||
'LOADGEN_CPUS': 1.5,
|
||
'LOADGEN_MEMORY': '256m',
|
||
'API_IMAGE': 'mtusi-messenger:lab3',
|
||
'BUILD_ID': 'base',
|
||
'DATABASE_POOL_MAX': 10,
|
||
}
|
||
PROFILES = {
|
||
'p-a': {
|
||
'mode': 'write',
|
||
'spread': 'spread',
|
||
'ladder': [100, 200, 300, 400, 550, 700, 900, 1100],
|
||
},
|
||
'p-b': {
|
||
'mode': 'write',
|
||
'spread': 'hot',
|
||
'ladder': [50, 100, 150, 200, 250, 300, 350, 400],
|
||
},
|
||
'p-c': {
|
||
'mode': 'read',
|
||
'spread': 'spread',
|
||
'ladder': [200, 400, 700, 1000, 1500, 2000, 2500, 3000],
|
||
},
|
||
}
|
||
STEP_SECONDS = 15
|
||
WARMUP_SECONDS = 30
|
||
CHATS = int(os.environ.get('LAB3_CHATS', '24'))
|
||
|
||
|
||
def log(message: str) -> None:
|
||
print(f'[{datetime.now().strftime("%H:%M:%S")}] {message}', flush=True)
|
||
|
||
|
||
def now_iso() -> str:
|
||
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||
|
||
|
||
def run_command(
|
||
command: list[str],
|
||
*,
|
||
check: bool = True,
|
||
timeout: float | None = None,
|
||
env: dict[str, str] | None = None,
|
||
) -> subprocess.CompletedProcess:
|
||
result = subprocess.run(
|
||
command,
|
||
cwd=ROOT,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout,
|
||
env={**os.environ, **(env or {})},
|
||
)
|
||
if check and result.returncode != 0:
|
||
raise SystemExit(
|
||
f'command failed ({result.returncode}): {" ".join(command)}\n'
|
||
f'{result.stdout[-2000:]}\n{result.stderr[-2000:]}'
|
||
)
|
||
return result
|
||
|
||
|
||
def compose(*args: str, check: bool = True, timeout: float | None = 300) -> subprocess.CompletedProcess:
|
||
return run_command([*COMPOSE, *args], check=check, timeout=timeout)
|
||
|
||
|
||
def read_json(path: Path, default=None):
|
||
if not path.exists():
|
||
return default
|
||
return json.loads(path.read_text())
|
||
|
||
|
||
def write_json(path: Path, payload) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=1, sort_keys=True) + '\n')
|
||
|
||
|
||
# ---------------------------------------------------------------- окружение стенда
|
||
|
||
|
||
def replica_limits(replicas: int, total_cpus: float, total_memory_mb: int) -> tuple[float, str]:
|
||
return (
|
||
round(total_cpus / replicas, 3),
|
||
f'{int(total_memory_mb // replicas)}m',
|
||
)
|
||
|
||
|
||
def render_stand_env(config: dict) -> str:
|
||
cpus, memory = replica_limits(
|
||
int(config['API_REPLICAS']),
|
||
float(config['API_TOTAL_CPUS']),
|
||
int(config['API_TOTAL_MEMORY_MB']),
|
||
)
|
||
values = {
|
||
**config,
|
||
'API_CPUS_PER_REPLICA': cpus,
|
||
'API_MEMORY_PER_REPLICA': memory,
|
||
}
|
||
lines = ['# Сгенерировано scripts/lab3/lab3.py: бюджет серии и лимиты реплик.']
|
||
for key in sorted(values, key=str):
|
||
lines.append(f'{key}={values[key]}')
|
||
return '\n'.join(lines) + '\n'
|
||
|
||
|
||
def load_stand_config() -> dict:
|
||
stored = read_json(STAND_STATE, None)
|
||
config = dict(DEFAULT_STAND)
|
||
if STAND_ENV.exists():
|
||
for line in STAND_ENV.read_text().splitlines():
|
||
if not line or line.startswith('#') or '=' not in line:
|
||
continue
|
||
key, value = line.split('=', 1)
|
||
config[key] = value
|
||
if stored:
|
||
config.update(stored)
|
||
return config
|
||
|
||
|
||
def numeric(value: str) -> float | int:
|
||
number = float(value)
|
||
return int(number) if number.is_integer() else number
|
||
|
||
|
||
def save_stand_config(config: dict) -> dict:
|
||
"""Пишет .env.stand и его машинно-читаемую копию.
|
||
|
||
Копия сохраняется уже с производными значениями (лимиты на реплику), иначе
|
||
следующие запуски подхватили бы устаревшие лимиты, а манифесты прогонов
|
||
показывали бы не те числа, что реально применены к контейнерам.
|
||
"""
|
||
rendered = render_stand_env(config)
|
||
STAND_ENV.write_text(rendered)
|
||
values = {
|
||
line.split('=', 1)[0]: line.split('=', 1)[1]
|
||
for line in rendered.splitlines()
|
||
if '=' in line and not line.startswith('#')
|
||
}
|
||
write_json(STAND_STATE, values)
|
||
return config
|
||
|
||
|
||
def current_replicas() -> int:
|
||
return int(load_stand_config().get('API_REPLICAS', 2))
|
||
|
||
|
||
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
|
||
"""Обновляет изменённые зависимости и API, не перезапуская миграции."""
|
||
save_stand_config(config)
|
||
compose('up', '-d', '--no-deps', '--wait', 'postgres', 'redis', 'haproxy', timeout=600)
|
||
args = ['up', '-d', '--no-deps', '--wait', '--scale', f'api={config["API_REPLICAS"]}']
|
||
if force_recreate:
|
||
args.append('--force-recreate')
|
||
compose(*args, 'api', timeout=600)
|
||
wait_for_lb()
|
||
verify_stand(config)
|
||
|
||
|
||
def wait_for_lb(timeout: float = 120) -> None:
|
||
deadline = time.monotonic() + timeout
|
||
last = 'no response'
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
with urllib.request.urlopen(f'{BASE_URL}/health/ready', timeout=3) as response:
|
||
if response.status == 200:
|
||
return
|
||
except Exception as error: # noqa: BLE001 - диагностика ожидания важнее типа
|
||
last = str(error)
|
||
time.sleep(1)
|
||
raise SystemExit(f'stand did not become ready within {timeout}s: {last}')
|
||
|
||
|
||
def lb_backends_up() -> int:
|
||
"""Число backend-слотов LB в состоянии UP (без строк FRONTEND/BACKEND)."""
|
||
try:
|
||
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
|
||
text = response.read().decode('utf-8', 'replace')
|
||
except OSError:
|
||
return 0
|
||
lines = text.splitlines()
|
||
if lines and lines[0].startswith('#'):
|
||
lines[0] = lines[0].lstrip('#').lstrip()
|
||
return sum(
|
||
1
|
||
for row in csv.DictReader(lines)
|
||
if row.get('pxname') == 'api'
|
||
and re.match(r'^api\d+$', row.get('svname', ''))
|
||
and row.get('status') == 'UP'
|
||
)
|
||
|
||
|
||
def wait_for_replicas(expected: int, timeout: float = 180) -> list[str]:
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
mapping = replica_map()
|
||
if len(mapping) >= expected and lb_backends_up() >= expected:
|
||
return sorted(mapping)
|
||
time.sleep(1)
|
||
raise SystemExit(
|
||
f'ожидалось {expected} реплик, видно {len(replica_map())} контейнеров и {lb_backends_up()} UP-слотов LB'
|
||
)
|
||
|
||
|
||
def container_ids(service: str = 'api', *, include_stopped: bool = False) -> list[str]:
|
||
result = run_command(
|
||
[
|
||
'docker',
|
||
'ps',
|
||
'-q',
|
||
*(['-a'] if include_stopped else []),
|
||
'--filter',
|
||
f'label=com.docker.compose.project={PROJECT}',
|
||
'--filter',
|
||
f'label=com.docker.compose.service={service}',
|
||
]
|
||
)
|
||
return [line for line in result.stdout.split() if line]
|
||
|
||
|
||
def inspect(container: str, template: str) -> str:
|
||
return run_command(['docker', 'inspect', '-f', template, container]).stdout.strip()
|
||
|
||
|
||
def replica_map() -> dict[str, dict]:
|
||
"""X-Instance-Id -> контейнер: в Docker это hostname = короткий ID."""
|
||
mapping = {}
|
||
for container in container_ids('api'):
|
||
hostname = inspect(container, '{{.Config.Hostname}}')
|
||
mapping[hostname] = {
|
||
'container': container,
|
||
'name': inspect(container, '{{.Name}}').lstrip('/'),
|
||
'image_id': inspect(container, '{{.Image}}'),
|
||
'started_at': inspect(container, '{{.State.StartedAt}}'),
|
||
'cpus': int(inspect(container, '{{.HostConfig.NanoCpus}}')) / 1e9,
|
||
'memory_mb': int(inspect(container, '{{.HostConfig.Memory}}')) // (1024 * 1024),
|
||
}
|
||
return mapping
|
||
|
||
|
||
def observed_stand() -> dict[str, list[dict]]:
|
||
"""Только публичные параметры docker inspect; секреты Env не сохраняются."""
|
||
observed = {}
|
||
for service in ('api', 'postgres', 'redis', 'haproxy'):
|
||
ids = container_ids(service)
|
||
rows = []
|
||
if ids:
|
||
containers = json.loads(run_command(['docker', 'inspect', *ids]).stdout)
|
||
for container in containers:
|
||
row = {
|
||
'container': container['Id'][:12],
|
||
'name': container['Name'].lstrip('/'),
|
||
'cpus': container['HostConfig']['NanoCpus'] / 1e9,
|
||
'memory_bytes': container['HostConfig']['Memory'],
|
||
'image_id': container['Image'],
|
||
}
|
||
if service == 'api':
|
||
environment = dict(value.split('=', 1) for value in container['Config']['Env'] if '=' in value)
|
||
row['pool_max'] = int(environment['DATABASE_POOL_MAX'])
|
||
rows.append(row)
|
||
observed[service] = rows
|
||
return observed
|
||
|
||
|
||
def verify_stand(config: dict) -> dict:
|
||
observed = observed_stand()
|
||
expected_replicas = int(config['API_REPLICAS'])
|
||
api_cpu, api_memory = replica_limits(expected_replicas, float(config['API_TOTAL_CPUS']), int(config['API_TOTAL_MEMORY_MB']))
|
||
limits = {
|
||
'api': (api_cpu, api_memory),
|
||
'postgres': (float(config['PG_CPUS']), config['PG_MEMORY']),
|
||
'redis': (float(config['REDIS_CPUS']), config['REDIS_MEMORY']),
|
||
'haproxy': (float(config['LB_CPUS']), config['LB_MEMORY']),
|
||
}
|
||
for service, rows in observed.items():
|
||
count = expected_replicas if service == 'api' else 1
|
||
cpu, memory = limits[service]
|
||
unit = re.fullmatch(r'(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?', str(memory), re.IGNORECASE)
|
||
if not unit:
|
||
raise ValueError(f'unsupported memory limit: {memory}')
|
||
memory_bytes = int(float(unit[1]) * 1024 ** ('kmgt'.index(unit[2].lower()) + 1 if unit[2] else 0))
|
||
if len(rows) != count:
|
||
raise RuntimeError(f'{service}: expected {count} containers, observed {len(rows)}')
|
||
for row in rows:
|
||
if not math.isclose(row['cpus'], cpu, abs_tol=0.001) or row['memory_bytes'] != memory_bytes:
|
||
raise RuntimeError(f'{service}: runtime limits {row} differ from requested CPU={cpu}, RAM={memory}')
|
||
if service == 'api' and row['pool_max'] != int(config['DATABASE_POOL_MAX']):
|
||
raise RuntimeError(f'api: pool {row["pool_max"]} != requested {config["DATABASE_POOL_MAX"]}')
|
||
return observed
|
||
|
||
|
||
def image_versions() -> dict:
|
||
versions = {}
|
||
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
|
||
container = container_ids(name)
|
||
if container:
|
||
versions[name] = inspect(container[0], '{{.Config.Image}}')
|
||
versions['postgres_runtime'] = run_command(
|
||
['docker', 'exec', container_ids('postgres')[0], 'postgres', '--version'], check=False
|
||
).stdout.strip()
|
||
versions['redis_runtime'] = run_command(
|
||
['docker', 'exec', container_ids('redis')[0], 'redis-server', '--version'], check=False
|
||
).stdout.strip()
|
||
versions['haproxy_runtime'] = run_command(
|
||
['docker', 'exec', container_ids('haproxy')[0], 'haproxy', '-v'], check=False
|
||
).stdout.splitlines()[0].strip()
|
||
versions['k6'] = run_command(
|
||
['docker', 'run', '--rm', 'grafana/k6:0.54.0', 'version'], check=False, timeout=120
|
||
).stdout.strip()
|
||
if shutil.which('node'):
|
||
versions['node_host'] = run_command(['node', '--version'], check=False).stdout.strip()
|
||
return versions
|
||
|
||
|
||
def dataset_size() -> dict:
|
||
containers = container_ids('postgres')
|
||
if not containers:
|
||
return {}
|
||
query = (
|
||
"SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), "
|
||
"(SELECT count(*) FROM messages), "
|
||
"(SELECT count(*) FROM idempotency_keys), "
|
||
"(SELECT count(*) FROM chat_members)"
|
||
)
|
||
result = psql(query)
|
||
if not result:
|
||
return {}
|
||
values = result.split('|')
|
||
return dict(zip(['users', 'chats', 'messages', 'idempotency_keys', 'chat_members'], (int(v) for v in values)))
|
||
|
||
|
||
def psql(query: str) -> str:
|
||
env = psql_env()
|
||
result = run_command(
|
||
[
|
||
'docker',
|
||
'exec',
|
||
container_ids('postgres')[0],
|
||
'psql',
|
||
'-U',
|
||
env['POSTGRES_USER'],
|
||
'-d',
|
||
env['POSTGRES_DB'],
|
||
'-tA',
|
||
'-F',
|
||
'|',
|
||
'-c',
|
||
query,
|
||
],
|
||
check=False,
|
||
timeout=30,
|
||
)
|
||
return result.stdout.strip()
|
||
|
||
|
||
_PSQL_ENV: dict[str, str] = {}
|
||
|
||
|
||
def psql_env() -> dict[str, str]:
|
||
if not _PSQL_ENV:
|
||
for line in ENV_FILE.read_text().splitlines():
|
||
if '=' in line and not line.startswith('#'):
|
||
key, value = line.split('=', 1)
|
||
_PSQL_ENV[key] = value
|
||
return _PSQL_ENV
|
||
|
||
|
||
def host_background() -> dict:
|
||
memory = {}
|
||
for line in Path('/proc/meminfo').read_text().splitlines():
|
||
if line.startswith(('MemTotal:', 'MemAvailable:')):
|
||
key, value = line.split(':')
|
||
memory[key] = int(value.strip().split()[0]) // 1024
|
||
load = os.getloadavg()
|
||
return {
|
||
'load_average': [round(value, 2) for value in load],
|
||
'mem_total_mib': memory.get('MemTotal'),
|
||
'mem_available_mib': memory.get('MemAvailable'),
|
||
'quiesced': QUIESCE_STATE.exists(),
|
||
}
|
||
|
||
|
||
def bench_cpus() -> int:
|
||
return os.cpu_count() or 8
|
||
|
||
|
||
# ---------------------------------------------------------------- HTTP-клиент
|
||
|
||
|
||
class ApiError(RuntimeError):
|
||
pass
|
||
|
||
|
||
class Client:
|
||
"""Минимальный клиент для проверок и фазовых драйверов (stdlib)."""
|
||
|
||
def __init__(self, base_url: str = BASE_URL, timeout: float = 5.0):
|
||
self.base = base_url.rstrip('/')
|
||
self.timeout = timeout
|
||
|
||
def call(self, method: str, path: str, body=None, token: str | None = None, headers=None):
|
||
data = None
|
||
request_headers = dict(headers or {})
|
||
if body is not None:
|
||
data = json.dumps(body, ensure_ascii=False).encode()
|
||
request_headers['content-type'] = 'application/json'
|
||
if token:
|
||
request_headers['authorization'] = f'Bearer {token}'
|
||
request = urllib.request.Request(
|
||
self.base + path, data=data, headers=request_headers, method=method
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||
payload = response.read()
|
||
status = response.status
|
||
response_headers = dict(response.headers)
|
||
except urllib.error.HTTPError as error:
|
||
payload = error.read()
|
||
status = error.code
|
||
response_headers = dict(error.headers)
|
||
parsed = json.loads(payload) if payload else None
|
||
return {'status': status, 'json': parsed, 'headers': response_headers}
|
||
|
||
def register(self, label: str) -> dict:
|
||
import uuid
|
||
|
||
username = f'lab3_{label}_{uuid.uuid4().hex[:10]}'
|
||
password = f'Lab3-{uuid.uuid4().hex}'
|
||
created = self.call(
|
||
'POST',
|
||
'/api/v1/users',
|
||
{'username': username, 'display_name': f'lab3 {label}', 'password': password},
|
||
)
|
||
if created['status'] != 201:
|
||
raise ApiError(f'register failed: {created}')
|
||
import base64
|
||
|
||
credential = base64.b64encode(f'{username}:{password}'.encode()).decode()
|
||
session = self.call(
|
||
'POST',
|
||
'/api/v1/auth/sessions',
|
||
None,
|
||
headers={'authorization': f'Basic {credential}'},
|
||
)
|
||
if session['status'] != 201:
|
||
raise ApiError(f'session failed: {session}')
|
||
return {
|
||
'id': created['json']['id'],
|
||
'user': created['json'],
|
||
'token': session['json']['token'],
|
||
}
|
||
|
||
def create_chat(self, token: str, member_ids: list[str], title: str) -> str:
|
||
created = self.call(
|
||
'POST', '/api/v1/chats', {'title': title, 'member_ids': member_ids}, token=token
|
||
)
|
||
if created['status'] != 201:
|
||
raise ApiError(f'chat failed: {created}')
|
||
return created['json']['id']
|
||
|
||
def post_message(self, token: str, chat_id: str, text: str, key: str) -> dict:
|
||
return self.call(
|
||
'POST',
|
||
f'/api/v1/chats/{chat_id}/messages',
|
||
{'text': text},
|
||
token=token,
|
||
headers={'idempotency-key': key},
|
||
)
|
||
|
||
def history(self, token: str, chat_id: str) -> list[dict]:
|
||
items: list[dict] = []
|
||
cursor = None
|
||
while True:
|
||
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
|
||
if cursor:
|
||
from urllib.parse import quote
|
||
|
||
path += '&cursor=' + quote(cursor, safe='')
|
||
page = self.call('GET', path, token=token)
|
||
if page['status'] != 200:
|
||
raise ApiError(f'history failed: {page}')
|
||
items.extend(page['json']['items'])
|
||
cursor = page['json']['next_cursor']
|
||
if cursor is None:
|
||
return items
|
||
|
||
|
||
# ---------------------------------------------------------------- сэмплер
|
||
|
||
|
||
class Sampler:
|
||
"""Фоновые наблюдения за стендом: docker stats, HAProxy, метрики, PostgreSQL."""
|
||
|
||
def __init__(self) -> None:
|
||
self.records: list[dict] = []
|
||
self._lock = threading.Lock()
|
||
self._stop = threading.Event()
|
||
self._threads: list[threading.Thread] = []
|
||
self.errors: list[str] = []
|
||
|
||
def _record(self, payload: dict) -> None:
|
||
with self._lock:
|
||
self.records.append(payload)
|
||
|
||
def _guard(self, name: str, work) -> None:
|
||
try:
|
||
work()
|
||
except Exception as error: # noqa: BLE001 - сэмплер не должен валить прогон
|
||
with self._lock:
|
||
self.errors.append(f'{name}: {type(error).__name__}: {error}')
|
||
|
||
def _loop(self, name: str, interval: float, step) -> None:
|
||
def work() -> None:
|
||
while not self._stop.is_set():
|
||
step()
|
||
self._stop.wait(interval)
|
||
|
||
self._threads.append(threading.Thread(target=lambda: self._guard(name, work), daemon=True, name=name))
|
||
|
||
def start(self) -> 'Sampler':
|
||
# docker stats в режиме потока перерисовывает экран (ANSI-последовательности)
|
||
# и склеивает JSON разных контейнеров в одну строку, поэтому опрашиваем
|
||
# его порциями --no-stream: одна строка = один JSON.
|
||
self._loop('docker_stats', 1.5, self._sample_stats)
|
||
self._loop('haproxy', 0.5, self._haproxy)
|
||
self._loop('metrics', 2.0, self._app_metrics)
|
||
self._loop('postgres', 2.0, self._postgres)
|
||
self._loop('pg_statements', 10.0, self._pg_statements)
|
||
self._loop('host', 5.0, lambda: self._record({'t': now_iso(), 'source': 'host', **host_background()}))
|
||
for thread in self._threads:
|
||
thread.start()
|
||
return self
|
||
|
||
def stop(self) -> list[dict]:
|
||
self._stop.set()
|
||
for thread in self._threads:
|
||
thread.join(timeout=10)
|
||
return self.records
|
||
|
||
def _sample_stats(self) -> None:
|
||
result = run_command(
|
||
['docker', 'stats', '--no-stream', '--format', '{{json .}}'],
|
||
check=False,
|
||
timeout=30,
|
||
)
|
||
for line in result.stdout.splitlines():
|
||
try:
|
||
row = json.loads(line)
|
||
except ValueError:
|
||
continue
|
||
self._record(
|
||
{
|
||
't': now_iso(),
|
||
'source': 'docker_stats',
|
||
'name': row.get('Name') or row.get('Container'),
|
||
'cpu_percent': float(str(row.get('CPUPerc', '0%')).rstrip('%') or 0),
|
||
'mem_mib': round(_mib(row.get('MemUsage', '0MiB / 0MiB')), 2),
|
||
'mem_limit_mib': round(_mib(str(row.get('MemUsage', '')).split('/')[-1]), 2),
|
||
'pids': _int(row.get('PIDs')),
|
||
}
|
||
)
|
||
|
||
def _haproxy(self) -> None:
|
||
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
|
||
text = response.read().decode('utf-8', 'replace')
|
||
lines = text.splitlines()
|
||
# HAProxy помечает строку заголовка как комментарий: '# pxname,svname,...'
|
||
if lines and lines[0].startswith('#'):
|
||
lines[0] = lines[0].lstrip('#').lstrip()
|
||
for row in csv.DictReader(lines):
|
||
if row.get('pxname') not in ('api', 'http'):
|
||
continue
|
||
self._record(
|
||
{
|
||
't': now_iso(),
|
||
'source': 'haproxy',
|
||
'proxy': row.get('pxname'),
|
||
'server': row.get('svname'),
|
||
'addr': row.get('addr'),
|
||
'status': row.get('status'),
|
||
'check': row.get('check_status'),
|
||
'scur': _int(row.get('scur')),
|
||
'qcur': _int(row.get('qcur')),
|
||
'stot': _int(row.get('stot')),
|
||
'ereq': _int(row.get('ereq')),
|
||
'dresp': _int(row.get('dresp')),
|
||
'wretr': _int(row.get('wretr')),
|
||
'hrsp_1xx': _int(row.get('hrsp_1xx')),
|
||
'hrsp_2xx': _int(row.get('hrsp_2xx')),
|
||
'hrsp_3xx': _int(row.get('hrsp_3xx')),
|
||
'hrsp_4xx': _int(row.get('hrsp_4xx')),
|
||
'hrsp_5xx': _int(row.get('hrsp_5xx')),
|
||
'hrsp_other': _int(row.get('hrsp_other')),
|
||
}
|
||
)
|
||
|
||
def _app_metrics(self) -> None:
|
||
containers = container_ids('api')
|
||
if not containers:
|
||
return
|
||
for container in containers:
|
||
result = run_command(
|
||
[
|
||
'docker',
|
||
'exec',
|
||
container,
|
||
'node',
|
||
'-e',
|
||
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
|
||
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
|
||
".then(r=>r.text()).then(t=>process.stdout.write(t))",
|
||
],
|
||
check=False,
|
||
timeout=20,
|
||
)
|
||
if result.returncode != 0 or not result.stdout.strip():
|
||
continue
|
||
try:
|
||
payload = json.loads(result.stdout)
|
||
except ValueError:
|
||
continue
|
||
self._record(
|
||
{
|
||
't': now_iso(),
|
||
'source': 'metrics',
|
||
'container': container,
|
||
'instance_id': payload.get('instance_id'),
|
||
'build_id': payload.get('build_id'),
|
||
'counters': payload.get('counters', {}),
|
||
'latency_ms': payload.get('latency_ms', {}),
|
||
}
|
||
)
|
||
|
||
def _postgres(self) -> None:
|
||
query = (
|
||
"SELECT (SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()),"
|
||
" (SELECT xact_rollback FROM pg_stat_database WHERE datname = current_database()),"
|
||
" (SELECT blks_read FROM pg_stat_database WHERE datname = current_database()),"
|
||
" (SELECT blks_hit FROM pg_stat_database WHERE datname = current_database()),"
|
||
" (SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()),"
|
||
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
|
||
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
|
||
" (SELECT count(*) FROM pg_stat_activity),"
|
||
" (SELECT reltuples::bigint FROM pg_class WHERE oid = 'messages'::regclass),"
|
||
" (SELECT pg_total_relation_size('messages'))"
|
||
)
|
||
output = psql(query)
|
||
if not output:
|
||
return
|
||
fields = output.split('|')
|
||
if len(fields) != 10:
|
||
return
|
||
waits = self._wait_events()
|
||
self._record(
|
||
{
|
||
't': now_iso(),
|
||
'source': 'postgres',
|
||
'waits': waits,
|
||
**{
|
||
key: int(value)
|
||
for key, value in zip(
|
||
[
|
||
'xact_commit',
|
||
'xact_rollback',
|
||
'blks_read',
|
||
'blks_hit',
|
||
'deadlocks',
|
||
'lock_waiters',
|
||
'active',
|
||
'connections',
|
||
'messages',
|
||
'messages_bytes',
|
||
],
|
||
fields,
|
||
)
|
||
},
|
||
}
|
||
)
|
||
|
||
def _wait_events(self) -> dict:
|
||
"""Сколько сессий ждут на каждом типе события: Lock/tuple/transactionid и т.д."""
|
||
output = psql(
|
||
"SELECT coalesce(wait_event_type, 'CPU/Idle') || ':' || coalesce(wait_event, '-'), count(*)"
|
||
" FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()"
|
||
" GROUP BY 1 ORDER BY 2 DESC LIMIT 12"
|
||
)
|
||
waits = {}
|
||
for line in output.splitlines():
|
||
parts = line.split('|')
|
||
if len(parts) == 2 and parts[1].isdigit():
|
||
waits[parts[0]] = int(parts[1])
|
||
return waits
|
||
|
||
def _pg_statements(self) -> None:
|
||
output = psql(
|
||
"SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 3),"
|
||
" left(regexp_replace(query, '\\s+', ' ', 'g'), 110)"
|
||
" FROM pg_stat_statements"
|
||
" WHERE query ILIKE '%messages%' OR query ILIKE '%chats%' OR query ILIKE '%idempotency_keys%'"
|
||
" ORDER BY total_exec_time DESC LIMIT 10"
|
||
)
|
||
if not output:
|
||
return
|
||
rows = []
|
||
for line in output.splitlines():
|
||
parts = line.split('|')
|
||
if len(parts) == 4:
|
||
rows.append(
|
||
{
|
||
'calls': int(parts[0]),
|
||
'total_ms': float(parts[1]),
|
||
'mean_ms': float(parts[2]),
|
||
'query': parts[3],
|
||
}
|
||
)
|
||
self._record({'t': now_iso(), 'source': 'pg_stat_statements', 'top': rows})
|
||
|
||
|
||
def _mib(value: str) -> float:
|
||
text = str(value).strip()
|
||
match = re.match(r'([\d.]+)\s*([KMGT]i?B)?', text)
|
||
if not match:
|
||
return 0.0
|
||
number = float(match.group(1))
|
||
unit = (match.group(2) or 'B').lower()
|
||
factor = {
|
||
'b': 1 / (1024 * 1024),
|
||
'kib': 1 / 1024,
|
||
'kb': 1 / 1024,
|
||
'mib': 1,
|
||
'mb': 1,
|
||
'gib': 1024,
|
||
'gb': 1024,
|
||
'tib': 1024 * 1024,
|
||
}.get(unit, 1)
|
||
return number * factor
|
||
|
||
|
||
def _int(value) -> int:
|
||
try:
|
||
return int(str(value).strip() or 0)
|
||
except ValueError:
|
||
return 0
|
||
|
||
|
||
# ---------------------------------------------------------------- прогоны k6
|
||
|
||
|
||
def k6_run(script: str, label: str, env: dict[str, str], timeout: float) -> dict:
|
||
args = [*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps']
|
||
for key, value in sorted(env.items()):
|
||
args += ['-e', f'{key}={value}']
|
||
args += ['loadgen', 'run', f'/load/{script}']
|
||
result = run_command(args, check=False, timeout=timeout)
|
||
summary = None
|
||
for line in reversed(result.stdout.splitlines()):
|
||
stripped = line.strip()
|
||
if not stripped.startswith('{'):
|
||
continue
|
||
try:
|
||
candidate = json.loads(stripped)
|
||
except ValueError:
|
||
continue
|
||
if isinstance(candidate, dict) and 'steps' in candidate:
|
||
summary = candidate
|
||
break
|
||
if summary is None:
|
||
raise SystemExit(
|
||
f'k6 run {label} produced no summary (exit {result.returncode})\n'
|
||
f'{result.stdout[-1500:]}\n{result.stderr[-800:]}'
|
||
)
|
||
summary['exit_code'] = result.returncode
|
||
summary['stderr_tail'] = result.stderr.strip().splitlines()[-3:]
|
||
return summary
|
||
|
||
|
||
def load_env(profile: str, mode: str, spread: str, label: str, vus: dict | None = None) -> dict[str, str]:
|
||
env = {
|
||
'BASE_URL': 'http://haproxy:8080',
|
||
'PROFILE': spread,
|
||
'MODE': mode,
|
||
'CHATS': str(CHATS),
|
||
'LABEL': label,
|
||
}
|
||
max_vus = (vus or {}).get('max') or os.environ.get('LAB3_MAX_VUS')
|
||
pre_vus = (vus or {}).get('pre') or os.environ.get('LAB3_PRE_VUS')
|
||
if max_vus:
|
||
env['MAX_VUS'] = str(max_vus)
|
||
MANIFEST_VUS['max'] = str(max_vus)
|
||
if pre_vus:
|
||
env['PRE_VUS'] = str(pre_vus)
|
||
MANIFEST_VUS['pre'] = str(pre_vus)
|
||
return env
|
||
|
||
|
||
# Бюджет VU действующего прогона: попадает в манифест и в сводку k6.
|
||
MANIFEST_VUS: dict[str, str] = {}
|
||
|
||
|
||
def run_measured(
|
||
label: str,
|
||
profile: str,
|
||
*,
|
||
ladder: list[int] | None = None,
|
||
step_seconds: int = STEP_SECONDS,
|
||
rate: int | None = None,
|
||
duration: int | None = None,
|
||
mode: str | None = None,
|
||
spread: str | None = None,
|
||
warmup: bool = True,
|
||
extra: dict | None = None,
|
||
phases: dict | None = None,
|
||
vus: dict | None = None,
|
||
) -> dict:
|
||
"""Один прогон: (необязательный прогрев) + измерение + сэмплер + манифест."""
|
||
profile_config = PROFILES[profile]
|
||
mode = mode or profile_config['mode']
|
||
spread = spread or profile_config['spread']
|
||
ladder = ladder or profile_config['ladder']
|
||
if rate is not None:
|
||
ladder = [rate]
|
||
steady = rate is not None and duration is not None
|
||
script = 'write-steady.js' if steady else 'write-ladder.js'
|
||
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
stand = load_stand_config()
|
||
verify_stand(stand)
|
||
MANIFEST_VUS.clear()
|
||
|
||
if warmup:
|
||
log(f'прогрев {label}: {WARMUP_SECONDS}s на {ladder[0]} req/s')
|
||
k6_run(
|
||
script,
|
||
f'warmup-{label}',
|
||
{
|
||
**load_env(profile, mode, spread, f'warmup-{label}', vus),
|
||
**({'RATE': str(ladder[0]), 'DURATION': str(WARMUP_SECONDS)} if steady else {'LADDER': str(ladder[0]), 'STEP_SECONDS': str(WARMUP_SECONDS)}),
|
||
},
|
||
timeout=WARMUP_SECONDS + 120,
|
||
)
|
||
|
||
sampler = Sampler().start()
|
||
started = now_iso()
|
||
env = {
|
||
**load_env(profile, mode, spread, label, vus),
|
||
**({'RATE': str(ladder[0]), 'DURATION': str(duration)} if steady else {'LADDER': ','.join(str(r) for r in ladder), 'STEP_SECONDS': str(step_seconds)}),
|
||
}
|
||
log(f'нагрузка {label}: {"steady " + str(ladder[0]) + " req/s" if steady else "ladder " + str(ladder)}')
|
||
summary = k6_run(script, label, env, timeout=(duration or len(ladder) * step_seconds) + 240)
|
||
finished = now_iso()
|
||
samples = sampler.stop()
|
||
|
||
write_raw(label, summary, samples, stand, started, finished, phases or {}, sampler.errors)
|
||
return summary
|
||
|
||
|
||
def write_raw(
|
||
label: str,
|
||
summary: dict,
|
||
samples: list[dict],
|
||
stand: dict,
|
||
started: str,
|
||
finished: str,
|
||
phases: dict,
|
||
sampler_errors: list[str],
|
||
) -> None:
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
write_json(RAW_DIR / f'{label}.k6.json', summary)
|
||
with gzip.open(RAW_DIR / f'{label}.samples.jsonl.gz', 'wt', encoding='utf-8') as handle:
|
||
for record in samples:
|
||
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
|
||
if phases:
|
||
write_json(RAW_DIR / f'{label}.phases.json', phases)
|
||
manifest = {
|
||
'label': label,
|
||
'profile': summary.get('profile'),
|
||
'mode': summary.get('mode'),
|
||
'kind': summary.get('kind'),
|
||
'started_at': started,
|
||
'finished_at': finished,
|
||
'steps': [step['rate'] for step in summary.get('steps', [])],
|
||
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
|
||
'stand': stand,
|
||
'observed_stand': verify_stand(stand),
|
||
'replicas': replica_map(),
|
||
'dataset': dataset_size(),
|
||
'versions': image_versions(),
|
||
'host': {'cpus': bench_cpus(), 'background': host_background()},
|
||
'generator': {
|
||
'cpus': stand.get('LOADGEN_CPUS'),
|
||
'memory': stand.get('LOADGEN_MEMORY'),
|
||
'max_vus': MANIFEST_VUS.get('max') or os.environ.get('LAB3_MAX_VUS', '200'),
|
||
'pre_vus': MANIFEST_VUS.get('pre') or os.environ.get('LAB3_PRE_VUS', '60'),
|
||
},
|
||
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
|
||
'quiesce': read_json(QUIESCE_STATE, None),
|
||
'phases': phases,
|
||
'sampler_errors': sampler_errors,
|
||
'sample_count': len(samples),
|
||
}
|
||
write_json(RAW_DIR / f'{label}.manifest.json', manifest)
|
||
log(f'raw: {label} ({len(samples)} samples, errors: {len(sampler_errors)})')
|
||
|
||
|
||
def load_raw(label: str) -> tuple[dict | None, list[dict], dict | None]:
|
||
summary = read_json(RAW_DIR / f'{label}.k6.json', None)
|
||
manifest = read_json(RAW_DIR / f'{label}.manifest.json', None)
|
||
samples: list[dict] = []
|
||
path = RAW_DIR / f'{label}.samples.jsonl.gz'
|
||
if path.exists():
|
||
with gzip.open(path, 'rt', encoding='utf-8') as handle:
|
||
for line in handle:
|
||
try:
|
||
samples.append(json.loads(line))
|
||
except ValueError:
|
||
continue
|
||
return summary, samples, manifest
|
||
|
||
|
||
def raw_labels() -> list[str]:
|
||
if not RAW_DIR.exists():
|
||
return []
|
||
labels = []
|
||
for path in sorted(RAW_DIR.glob('*.k6.json')):
|
||
label = path.name[: -len('.k6.json')]
|
||
if label.startswith('warmup-'):
|
||
continue
|
||
labels.append(label)
|
||
return labels
|
||
|
||
|
||
# ---------------------------------------------------------------- анализ
|
||
|
||
|
||
def step_of(summary: dict, rate: int) -> dict | None:
|
||
for step in summary.get('steps', []):
|
||
if int(step['rate']) == int(rate):
|
||
return step
|
||
return None
|
||
|
||
|
||
def median(values: list[float]) -> float:
|
||
return round(statistics.median(values), 2) if values else 0.0
|
||
|
||
|
||
def spread(values: list[float]) -> tuple[float, float]:
|
||
return (round(min(values), 2), round(max(values), 2)) if values else (0.0, 0.0)
|
||
|
||
|
||
def group_runs(labels: list[str], predicate) -> dict[str, list[str]]:
|
||
groups: dict[str, list[str]] = {}
|
||
for label in labels:
|
||
if predicate(label):
|
||
key = label.split('#')[0]
|
||
groups.setdefault(key, []).append(label)
|
||
return groups
|
||
|
||
|
||
def saturation(series: list[dict]) -> dict:
|
||
"""Точка сатурации: первый шаг, где достигнутый RPS отстал от предложенного."""
|
||
for step in series:
|
||
if step['achieved_rps'] < 0.95 * step['rate']:
|
||
return {'rate': step['rate'], 'achieved_rps': step['achieved_rps'], 'step': step}
|
||
return {'rate': series[-1]['rate'] if series else 0, 'achieved_rps': series[-1]['achieved_rps'] if series else 0, 'step': series[-1] if series else None}
|
||
|
||
|
||
def summarise_series(labels: list[str]) -> dict:
|
||
"""Медианы по повторам для одной конфигурации: шаги лестницы и пик."""
|
||
steps: dict[int, dict] = {}
|
||
for label in labels:
|
||
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
|
||
for step in summary.get('steps', []):
|
||
rate = int(step['rate'])
|
||
entry = steps.setdefault(
|
||
rate,
|
||
{'rate': rate, 'achieved': [], 'p95': [], 'failed': [], 'vus': [], 'offered': [], 'status_5xx': [], 'status_4xx': []},
|
||
)
|
||
entry['achieved'].append(float(step['achieved_rps']))
|
||
entry['p95'].append(float(step['p95']))
|
||
entry['failed'].append(float(step['failed']))
|
||
entry['vus'].append(float(step.get('vus_active') or 0))
|
||
entry['offered'].append(float(step['rate']))
|
||
entry['status_5xx'].append(float((step.get('status') or {}).get('server_5xx') or 0))
|
||
entry['status_4xx'].append(float((step.get('status') or {}).get('client_4xx') or 0))
|
||
rows = []
|
||
for rate in sorted(steps):
|
||
entry = steps[rate]
|
||
rows.append(
|
||
{
|
||
'rate': rate,
|
||
'achieved_rps': median(entry['achieved']),
|
||
'achieved_min': spread(entry['achieved'])[0],
|
||
'achieved_max': spread(entry['achieved'])[1],
|
||
'p95': median(entry['p95']),
|
||
'p95_max': spread(entry['p95'])[1],
|
||
'failed': sum(entry['failed']),
|
||
'vus_active': max(entry['vus']) if entry['vus'] else 0,
|
||
'status_5xx': int(max(entry['status_5xx'])) if entry['status_5xx'] else 0,
|
||
'status_4xx': int(max(entry['status_4xx'])) if entry['status_4xx'] else 0,
|
||
'reps': len(entry['achieved']),
|
||
}
|
||
)
|
||
ceiling = max((row['achieved_rps'] for row in rows), default=0.0)
|
||
return {'rows': rows, 'ceiling_rps': ceiling, 'saturation': saturation(rows), 'labels': labels}
|
||
|
||
|
||
def container_stats(samples: list[dict], prefix: str = 'mtusi-lab3-') -> dict:
|
||
stats: dict[str, dict] = {}
|
||
for record in samples:
|
||
if record.get('source') != 'docker_stats':
|
||
continue
|
||
name = record.get('name') or ''
|
||
if not name.startswith(prefix):
|
||
continue
|
||
entry = stats.setdefault(name, {'cpu': [], 'mem': [], 'limit': 0.0})
|
||
entry['cpu'].append(float(record.get('cpu_percent') or 0))
|
||
entry['mem'].append(float(record.get('mem_mib') or 0))
|
||
entry['limit'] = max(entry['limit'], float(record.get('mem_limit_mib') or 0))
|
||
return {
|
||
name: {
|
||
'cpu_mean': round(statistics.fmean(entry['cpu']), 2) if entry['cpu'] else 0.0,
|
||
'cpu_peak': round(max(entry['cpu']), 2) if entry['cpu'] else 0.0,
|
||
'mem_mean': round(statistics.fmean(entry['mem']), 1) if entry['mem'] else 0.0,
|
||
'mem_peak': round(max(entry['mem']), 1) if entry['mem'] else 0.0,
|
||
'mem_limit': entry['limit'],
|
||
}
|
||
for name, entry in sorted(stats.items())
|
||
}
|
||
|
||
|
||
def postgres_stats(samples: list[dict]) -> dict:
|
||
rows = [record for record in samples if record.get('source') == 'postgres']
|
||
if len(rows) < 2:
|
||
return {}
|
||
first, last = rows[0], rows[-1]
|
||
seconds = max(1.0, (_timestamp(last['t']) - _timestamp(first['t'])))
|
||
hits = last['blks_hit'] - first['blks_hit']
|
||
reads = last['blks_read'] - first['blks_read']
|
||
return {
|
||
'xact_per_s': round((last['xact_commit'] - first['xact_commit']) / seconds, 1),
|
||
'rollback_per_s': round((last['xact_rollback'] - first['xact_rollback']) / seconds, 2),
|
||
'blks_hit_ratio': round(hits / (hits + reads), 4) if hits + reads else 0.0,
|
||
'lock_waiters_peak': max(row['lock_waiters'] for row in rows),
|
||
'active_peak': max(row['active'] for row in rows),
|
||
'connections_peak': max(row['connections'] for row in rows),
|
||
'deadlocks': last['deadlocks'] - first['deadlocks'],
|
||
'messages': last['messages'],
|
||
'messages_mib': round(last['messages_bytes'] / (1024 * 1024), 1),
|
||
}
|
||
|
||
|
||
def haproxy_stats(samples: list[dict]) -> dict:
|
||
servers: dict[str, dict] = {}
|
||
for record in samples:
|
||
if record.get('source') != 'haproxy':
|
||
continue
|
||
entry = servers.setdefault(
|
||
record['server'],
|
||
{
|
||
'requests': [],
|
||
'errors': [],
|
||
'dresp': [],
|
||
'qcur': [],
|
||
'scur': [],
|
||
'codes': {},
|
||
'statuses': set(),
|
||
'addr': set(),
|
||
'events': [],
|
||
},
|
||
)
|
||
entry['requests'].append((record['t'], record['stot']))
|
||
entry['errors'].append(record['ereq'])
|
||
entry['dresp'].append(record.get('dresp') or 0)
|
||
entry['qcur'].append(record['qcur'])
|
||
entry['scur'].append(record['scur'])
|
||
for code in ('hrsp_1xx', 'hrsp_2xx', 'hrsp_3xx', 'hrsp_4xx', 'hrsp_5xx', 'hrsp_other'):
|
||
entry['codes'].setdefault(code, []).append(record.get(code) or 0)
|
||
entry['statuses'].add(record['status'])
|
||
entry['addr'].add(record['addr'])
|
||
entry['events'].append((record['t'], record['status'], record['check']))
|
||
result = {}
|
||
for name, entry in servers.items():
|
||
if not entry['requests']:
|
||
continue
|
||
first, last = entry['requests'][0], entry['requests'][-1]
|
||
seconds = max(1.0, _timestamp(last[0]) - _timestamp(first[0]))
|
||
transitions = []
|
||
previous = None
|
||
for moment, status, check in entry['events']:
|
||
if status != previous:
|
||
transitions.append({'t': moment, 'status': status, 'check': check})
|
||
previous = status
|
||
result[name] = {
|
||
'requests': last[1] - first[1],
|
||
'rps': round((last[1] - first[1]) / seconds, 2),
|
||
'errors': entry['errors'][-1] - entry['errors'][0] if entry['errors'] else 0,
|
||
'dresp': entry['dresp'][-1] - entry['dresp'][0] if entry['dresp'] else 0,
|
||
'response_codes': {
|
||
code: (values[-1] - values[0]) if values else 0
|
||
for code, values in entry['codes'].items()
|
||
},
|
||
'qcur_peak': max(entry['qcur']),
|
||
'scur_peak': max(entry['scur']),
|
||
'statuses': sorted(entry['statuses']),
|
||
'addresses': sorted(entry['addr']),
|
||
'transitions': transitions,
|
||
}
|
||
return result
|
||
|
||
|
||
def app_metrics_delta(samples: list[dict]) -> dict:
|
||
per_container: dict[str, dict] = {}
|
||
for record in samples:
|
||
if record.get('source') != 'metrics':
|
||
continue
|
||
per_container.setdefault(record['container'], []).append(record)
|
||
result = {}
|
||
for container, rows in per_container.items():
|
||
if len(rows) < 2:
|
||
continue
|
||
first, last = rows[0], rows[-1]
|
||
counters = {}
|
||
for name, value in last['counters'].items():
|
||
counters[name] = value - first['counters'].get(name, 0)
|
||
result[container] = {
|
||
'instance_id': last.get('instance_id'),
|
||
'build_id': last.get('build_id'),
|
||
'counters': {name: value for name, value in sorted(counters.items()) if value},
|
||
'latency_ms': last.get('latency_ms', {}),
|
||
'samples': len(rows),
|
||
}
|
||
return result
|
||
|
||
|
||
def _timestamp(value: str) -> float:
|
||
return datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()
|
||
|
||
|
||
def mechanism_stats(samples: list[dict]) -> dict:
|
||
"""Где теряется время: ожидания сессий, дельты pg_stat_statements, блокировки.
|
||
|
||
Даёт не «упираемся в ресурс», а разрез по причинам: сколько сессий стоит в
|
||
Lock, какие операторы съели время за прогон, сколько запросов прошло и как
|
||
распределились ответы LB.
|
||
"""
|
||
waits: dict[str, int] = {}
|
||
lock_peak = 0
|
||
active_peak = 0
|
||
for record in samples:
|
||
if record.get('source') != 'postgres':
|
||
continue
|
||
for key, value in (record.get('waits') or {}).items():
|
||
waits[key] = max(waits.get(key, 0), value)
|
||
lock_peak = max(lock_peak, record.get('lock_waiters') or 0)
|
||
active_peak = max(active_peak, record.get('active') or 0)
|
||
statements: list[dict] = []
|
||
ticks = [record for record in samples if record.get('source') == 'pg_stat_statements' and record.get('top')]
|
||
if len(ticks) >= 2:
|
||
first, last = ticks[0]['top'], ticks[-1]['top']
|
||
before = {row['query'][:60]: row for row in first}
|
||
for row in last:
|
||
previous = before.get(row['query'][:60])
|
||
calls = row['calls'] - (previous['calls'] if previous else 0)
|
||
total_ms = row['total_ms'] - (previous['total_ms'] if previous else 0)
|
||
if calls > 0:
|
||
statements.append(
|
||
{
|
||
'query': row['query'][:60],
|
||
'calls': calls,
|
||
'total_ms': round(total_ms, 1),
|
||
'mean_ms': round(total_ms / calls, 3),
|
||
}
|
||
)
|
||
statements.sort(key=lambda row: -row['total_ms'])
|
||
return {
|
||
'waits_peak': dict(sorted(waits.items(), key=lambda item: -item[1])[:6]),
|
||
'lock_waiters_peak': lock_peak,
|
||
'active_peak': active_peak,
|
||
'statements': statements[:6],
|
||
}
|
||
|
||
|
||
def analyse_mechanism(groups: dict, labels: list[str]) -> None:
|
||
"""Разрез «почему»: один темп, разные конфигурации/реплики, почему растёт не всё."""
|
||
block = {
|
||
key: group for key, group in groups.items() if key.startswith(('block#', 'why#'))
|
||
}
|
||
if not block:
|
||
return
|
||
lines = [
|
||
'Конфигурации одного блока измерены на одном темпе (steady, 60 s, близкий',
|
||
'объём данных, интерливинг повторов), поэтому различия относятся к',
|
||
'распределению ресурсов, а не к разному времени суток или профилю нагрузки.',
|
||
'Блоки различаются темпом (block# — 900 req/s, why# — за коленом).',
|
||
'',
|
||
]
|
||
for key, group in sorted(block.items()):
|
||
label = representative(group)
|
||
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
|
||
step = summary['steps'][0] if summary.get('steps') else {}
|
||
_, samples, manifest = load_raw(label)
|
||
stats = container_stats(samples)
|
||
mechanism = mechanism_stats(samples)
|
||
stand = manifest.get('stand', {})
|
||
api_limit = float(stand.get('API_CPUS_PER_REPLICA') or 0)
|
||
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
|
||
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
|
||
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
|
||
generator_peak = max((value['cpu_peak'] for name, value in stats.items() if 'loadgen' in name), default=0.0)
|
||
lines += [f'## {key} ({len(group)} повторов)', '']
|
||
lines += table(
|
||
['достигнуто RPS', 'p95, ms', 'failed', '2xx', '4xx', '5xx', 'прочее', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра', 'CPU ген. пик, % ядра'],
|
||
[
|
||
[
|
||
step.get('achieved_rps'),
|
||
step.get('p95'),
|
||
step.get('failed'),
|
||
(step.get('status') or {}).get('ok'),
|
||
(step.get('status') or {}).get('client_4xx'),
|
||
(step.get('status') or {}).get('server_5xx'),
|
||
(step.get('status') or {}).get('other'),
|
||
api_peak_quota,
|
||
pg_peak,
|
||
generator_peak,
|
||
]
|
||
],
|
||
)
|
||
lines.append(
|
||
f'Лимиты: реплик {stand.get("API_REPLICAS")} x {stand.get("API_CPUS_PER_REPLICA")} CPU '
|
||
f'({stand.get("API_MEMORY_PER_REPLICA")}), PG {stand.get("PG_CPUS")} CPU, пул {stand.get("DATABASE_POOL_MAX")}, '
|
||
f'LB {stand.get("LB_CPUS")} CPU, генератор {stand.get("LOADGEN_CPUS")} CPU.'
|
||
)
|
||
lines.append('')
|
||
lines += table(
|
||
['активных сессий пик', 'Lock-ожиданий пик', 'ожидания (тип: событие -> сессий)'],
|
||
[
|
||
[
|
||
mechanism['active_peak'],
|
||
mechanism['lock_waiters_peak'],
|
||
', '.join(f'{name}: {count}' for name, count in mechanism['waits_peak'].items()) or '—',
|
||
]
|
||
],
|
||
)
|
||
lines += table(
|
||
['SQL (начало) по убыванию времени', 'вызовов за прогон', 'всего, ms', 'среднее, ms'],
|
||
[[row['query'], row['calls'], row['total_ms'], row['mean_ms']] for row in mechanism['statements']],
|
||
)
|
||
app = app_metrics_delta(samples)
|
||
for container, delta in sorted(app.items()):
|
||
lines.append(
|
||
f'- реплика {delta["instance_id"]}: latency p95 {delta["latency_ms"].get("p95")} ms, '
|
||
f'счётчики {json.dumps({name: value for name, value in delta["counters"].items() if name.startswith(("messages", "idempotency", "request_count"))}, ensure_ascii=False)}'
|
||
)
|
||
lines.append('')
|
||
write_markdown(TRACK1_DIR / 'summary-mechanism.md', 'Разрез «почему»: где теряется время на одном темпе', lines)
|
||
|
||
|
||
def write_markdown(path: Path, title: str, lines: list[str]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(f'# {title}\n\n' + '\n'.join(lines) + '\n', encoding='utf-8')
|
||
|
||
|
||
def table(header: list[str], rows: list[list]) -> list[str]:
|
||
lines = ['| ' + ' | '.join(header) + ' |', '| ' + ' | '.join('---' for _ in header) + ' |']
|
||
for row in rows:
|
||
lines.append('| ' + ' | '.join(str(value) for value in row) + ' |')
|
||
return lines + ['']
|
||
|
||
|
||
def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
|
||
"""Приводит стенд к нужной конфигурации; пересоздаёт только при изменении."""
|
||
desired = {**load_stand_config(), **{key: str(value) for key, value in overrides.items()}}
|
||
rendered = render_stand_env(desired)
|
||
if not STAND_ENV.exists() or STAND_ENV.read_text() != rendered:
|
||
log(f'перенастройка стенда: {overrides}')
|
||
apply_stand(desired, force_recreate=force_recreate)
|
||
else:
|
||
wait_for_lb()
|
||
try:
|
||
verify_stand(desired)
|
||
except RuntimeError:
|
||
apply_stand(desired, force_recreate=force_recreate)
|
||
return desired
|
||
|
||
|
||
# ---------------------------------------------------------------- подкоманды
|
||
|
||
|
||
def cmd_stand(args) -> None:
|
||
config = load_stand_config()
|
||
if args.action == 'up':
|
||
if args.replicas:
|
||
config['API_REPLICAS'] = str(args.replicas)
|
||
save_stand_config(config)
|
||
compose('up', '-d', '--build', '--wait', timeout=1800)
|
||
wait_for_lb()
|
||
log(f'стенд поднят: {replica_map().keys()}')
|
||
elif args.action == 'down':
|
||
compose('down')
|
||
else:
|
||
print(compose('ps').stdout)
|
||
|
||
|
||
def header(headers: dict, name: str) -> str | None:
|
||
for key, value in headers.items():
|
||
if key.lower() == name.lower():
|
||
return value
|
||
return None
|
||
|
||
|
||
def cmd_check(args) -> None:
|
||
client = Client()
|
||
if args.what == 'spread':
|
||
token = client.register('spread')['token']
|
||
counts: dict[str, int] = {}
|
||
for _ in range(args.requests):
|
||
response = client.call('GET', '/api/v1/users/me', token=token)
|
||
instance = header(response['headers'], 'X-Instance-Id')
|
||
counts[instance] = counts.get(instance, 0) + 1
|
||
payload = {
|
||
'requests': args.requests,
|
||
'distinct_instances': len(counts),
|
||
'instances': counts,
|
||
'replicas': {name: info for name, info in replica_map().items()},
|
||
'captured_at': now_iso(),
|
||
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
|
||
}
|
||
print(json.dumps(payload, ensure_ascii=False, indent=1))
|
||
write_json(LEVEL4_DIR / 'spread.json', payload)
|
||
elif args.what == 'instances':
|
||
print(json.dumps({'replicas': replica_map(), 'haproxy': haproxy_stats(Sampler()._haproxy() or [])}, ensure_ascii=False, indent=1))
|
||
elif args.what == 'idempotency':
|
||
LEVEL4_DIR.mkdir(parents=True, exist_ok=True)
|
||
result = subprocess.run(
|
||
['npm', 'run', 'test:e2e', '--', '-t', 'idempotency'],
|
||
cwd=ROOT / 'app',
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=900,
|
||
env={**os.environ, 'API_BASE_URL': BASE_URL},
|
||
)
|
||
output = result.stdout + result.stderr
|
||
stamp = (
|
||
f'=== e2e-фильтр идемпотентности (конкурентный повтор через LB)\n'
|
||
f'captured_at {now_iso()}\n'
|
||
f'commit {run_command(["git", "rev-parse", "HEAD"], check=False).stdout.strip()}\n\n'
|
||
)
|
||
(LEVEL4_DIR / 'idempotency-concurrency.txt').write_text(stamp + output, encoding='utf-8')
|
||
print(output[-1500:])
|
||
if result.returncode != 0:
|
||
raise SystemExit('idempotency concurrency suite failed')
|
||
else:
|
||
raise SystemExit(f'unknown check: {args.what}')
|
||
|
||
|
||
def cmd_run(args) -> None:
|
||
overrides: dict = {}
|
||
if args.replicas:
|
||
overrides['API_REPLICAS'] = str(args.replicas)
|
||
if args.pool:
|
||
overrides['DATABASE_POOL_MAX'] = str(args.pool)
|
||
if args.loadgen_cpus:
|
||
overrides['LOADGEN_CPUS'] = str(args.loadgen_cpus)
|
||
if args.lb_cpus:
|
||
overrides['LB_CPUS'] = str(args.lb_cpus)
|
||
if args.api_total_cpus:
|
||
overrides['API_TOTAL_CPUS'] = str(args.api_total_cpus)
|
||
if args.pg_cpus:
|
||
overrides['PG_CPUS'] = str(args.pg_cpus)
|
||
overrides['LOADGEN_MEMORY'] = args.loadgen_memory
|
||
ensure_stand(overrides)
|
||
profile = args.profile
|
||
ladder = [int(value) for value in args.ladder.split(',')] if args.ladder else None
|
||
summary = run_measured(
|
||
args.label,
|
||
profile,
|
||
ladder=ladder,
|
||
step_seconds=args.step or STEP_SECONDS,
|
||
rate=args.peak,
|
||
duration=args.duration,
|
||
mode='read' if args.mode == 'read' else None,
|
||
spread=args.spread,
|
||
warmup=not args.no_warmup,
|
||
vus={'max': args.max_vus, 'pre': args.pre_vus},
|
||
)
|
||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||
|
||
|
||
def cmd_sweep(args) -> None:
|
||
replicas = [int(value) for value in args.replicas.split(',')]
|
||
label_prefix = args.prefix or args.profile
|
||
for rep in range(1, args.reps + 1):
|
||
order = replicas if rep % 2 else list(reversed(replicas))
|
||
for count in order:
|
||
ensure_stand({'API_REPLICAS': str(count)})
|
||
run_measured(f'{label_prefix}#n{count}#rep{rep}', args.profile, warmup=not args.no_warmup)
|
||
report_series(args.profile, label_prefix)
|
||
if args.extra_replica:
|
||
for extra in range(1, args.extra_reps + 1):
|
||
count = args.extra_replica
|
||
ensure_stand({'API_REPLICAS': str(count)})
|
||
run_measured(f'{label_prefix}#n{count}#rep{args.reps + extra}', args.profile, warmup=not args.no_warmup)
|
||
report_series(args.profile, label_prefix)
|
||
|
||
|
||
def report_series(profile: str, prefix: str) -> None:
|
||
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#')]
|
||
groups = group_configurations(labels)
|
||
lines = []
|
||
for key, group in groups.items():
|
||
series = summarise_series(group)
|
||
lines += table(
|
||
['темп', 'достигнуто RPS', 'min', 'max', 'p95', 'ошибок'],
|
||
[
|
||
[row['rate'], row['achieved_rps'], row['achieved_min'], row['achieved_max'], row['p95'], row['failed']]
|
||
for row in series['rows']
|
||
],
|
||
)
|
||
lines.append(f'Потолок {key}: {series["ceiling_rps"]} req/s, сатурация на {series["saturation"]["rate"]} req/s ({len(group)} повторов)')
|
||
lines.append('')
|
||
write_markdown(TRACK1_DIR / f'summary-live-{profile}.md', f'Текущие агрегаты {profile}', lines)
|
||
|
||
|
||
def configuration_key(label: str) -> str:
|
||
"""Ключ конфигурации: метки вида <профиль>#<конфигурация>#rep<N>.
|
||
|
||
Без второго сегмента повторные прогоны разных конфигураций слились бы в одну
|
||
таблицу (например, все N серии A), а A/B-стороны 'ab#A'/'ab#B' — в одну
|
||
группу 'ab'.
|
||
"""
|
||
parts = label.split('#')
|
||
return '#'.join(parts[:2]) if len(parts) > 1 else parts[0]
|
||
|
||
|
||
def group_configurations(labels: list[str]) -> dict[str, list[str]]:
|
||
groups: dict[str, list[str]] = {}
|
||
for label in labels:
|
||
groups.setdefault(configuration_key(label), []).append(label)
|
||
return {key: sorted(value) for key, value in sorted(groups.items())}
|
||
|
||
|
||
def knee_rate(profile: str, prefix: str = 'p-a') -> int:
|
||
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#n4#')]
|
||
if not labels:
|
||
return 400
|
||
series = summarise_series(labels)
|
||
rows = series['rows']
|
||
for row in rows:
|
||
if row['achieved_rps'] < 0.95 * row['rate']:
|
||
previous = rows[max(0, rows.index(row) - 1)]
|
||
return int(previous['rate'])
|
||
return int(rows[-1]['rate']) if rows else 400
|
||
|
||
|
||
def cmd_growth(args) -> None:
|
||
base = load_stand_config()
|
||
rate = args.rate or max(1, int(knee_rate(args.base_profile) * 0.8))
|
||
points = {
|
||
'api': [
|
||
{'API_TOTAL_CPUS': str(float(base.get('API_TOTAL_CPUS', API_TOTAL_CPUS)) * 2), 'label': 'api-total-6'},
|
||
],
|
||
'pg': [
|
||
{'PG_CPUS': '3.0', 'label': 'pg-3.0'},
|
||
],
|
||
'pool': [
|
||
{'DATABASE_POOL_MAX': '5', 'label': 'pool-5'},
|
||
{'DATABASE_POOL_MAX': '20', 'label': 'pool-20'},
|
||
],
|
||
}
|
||
log(f'серия роста {args.series} на темпе {rate} req/s (80% от колена {args.base_profile})')
|
||
# Каждая точка отличается от базовой конфигурации ровно одним ресурсом:
|
||
# остальные значения прибиты к базовым, иначе после точки «API x2» следующий
|
||
# ресурс мерился бы уже с удвоенным бюджетом API.
|
||
baseline = {
|
||
'API_REPLICAS': '4',
|
||
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
|
||
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
|
||
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
|
||
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
|
||
}
|
||
for point in points[args.series]:
|
||
overrides = {
|
||
**baseline,
|
||
**{key: value for key, value in point.items() if key != 'label'},
|
||
}
|
||
ensure_stand(overrides)
|
||
run_measured(
|
||
f'growth#{point["label"]}{args.suffix}',
|
||
args.base_profile,
|
||
rate=rate,
|
||
duration=args.duration,
|
||
warmup=not args.no_warmup,
|
||
)
|
||
|
||
|
||
class MessageProbe(threading.Thread):
|
||
"""Последовательный клиент: измеряет успешность записи и повторяет
|
||
неопределённый POST тем же ключом (как это делает реальный клиент)."""
|
||
|
||
def __init__(self, client: Client, token: str, chat_id: str, interval: float, tag: str):
|
||
super().__init__(daemon=True)
|
||
self.client = client
|
||
self.token = token
|
||
self.chat_id = chat_id
|
||
self.interval = interval
|
||
self.tag = tag
|
||
self.stop_event = threading.Event()
|
||
self.samples: list[dict] = []
|
||
self.confirmed: list[dict] = []
|
||
self.started_monotonic = time.monotonic()
|
||
|
||
def run(self) -> None:
|
||
import uuid
|
||
|
||
sequence = 0
|
||
started = self.started_monotonic
|
||
while not self.stop_event.is_set():
|
||
sequence += 1
|
||
key = uuid.uuid4().hex
|
||
text = f'{self.tag} {sequence}'
|
||
attempts = 0
|
||
while attempts < 3:
|
||
attempts += 1
|
||
moment = time.monotonic() - started
|
||
try:
|
||
response = self.client.post_message(self.token, self.chat_id, text, key)
|
||
except (urllib.error.URLError, TimeoutError, OSError) as error:
|
||
self.samples.append({'t': round(moment, 3), 'ok': False, 'error': type(error).__name__})
|
||
if self.stop_event.wait(0.1):
|
||
return
|
||
continue
|
||
instance = header(response['headers'], 'X-Instance-Id')
|
||
self.samples.append(
|
||
{'t': round(moment, 3), 'ok': response['status'] == 201, 'status': response['status'], 'instance': instance}
|
||
)
|
||
if response['status'] == 201:
|
||
self.confirmed.append({'text': text, 'key': key, 'id': response['json']['id']})
|
||
break
|
||
if self.stop_event.wait(self.interval):
|
||
return
|
||
|
||
|
||
def replica_ip(container: str) -> str:
|
||
return inspect(container, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
|
||
|
||
|
||
def cmd_failover(args) -> None:
|
||
config = load_stand_config()
|
||
expected = int(config['API_REPLICAS'])
|
||
ensure_stand({'API_REPLICAS': str(expected)})
|
||
replicas = wait_for_replicas(expected)
|
||
targets = replicas if args.replicas == 'all' else [args.replicas]
|
||
results = []
|
||
for target in targets:
|
||
results.append(failover_once(target, args, config))
|
||
write_failover_evidence(results)
|
||
print(json.dumps(results, ensure_ascii=False, indent=1))
|
||
|
||
|
||
def failover_once(target: str, args, config: dict) -> dict:
|
||
"""Один цикл: нагрузка -> остановка реплики -> возврат -> проверка истории.
|
||
|
||
Проба идёт постоянно и повторяет неопределённый POST тем же ключом, поэтому
|
||
«потерь» и «дублей» в истории быть не должно. Окно реального отсутствия
|
||
реплики считается от выхода контейнера (docker stop возвращает управление
|
||
уже после остановки), а не от вызова команды: дрейн длиной DRAIN_DELAY_MS
|
||
специально переживает обнаружение отказа LB и не должен выглядеть отказом.
|
||
"""
|
||
if target not in replica_map() or lb_backends_up() < len(replica_map()):
|
||
log(f'{target} не запущена или LB ещё не видит все реплики: восстанавливаю')
|
||
run_command(['scripts/start-replica.sh'])
|
||
wait_for_lb()
|
||
wait_for_replicas(len(replica_map()) or int(config['API_REPLICAS']))
|
||
if target not in replica_map():
|
||
raise SystemExit(f'replica {target} did not come back')
|
||
container = replica_map()[target]['container']
|
||
address = replica_ip(container)
|
||
|
||
client = Client(timeout=5)
|
||
alice = client.register(f'failover{target[:4]}')
|
||
chat = client.create_chat(alice['token'], [alice['id']], f'failover {target}')
|
||
|
||
load_seconds = int(args.pre + args.hold + 30)
|
||
label = f'failover#{target}'
|
||
env = {
|
||
**load_env('p-b', 'write', 'spread', label),
|
||
'RATE': str(args.rate),
|
||
'DURATION': str(load_seconds),
|
||
}
|
||
sampler = Sampler().start()
|
||
load = subprocess.Popen(
|
||
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{key}={value}'] for key, value in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
|
||
cwd=ROOT,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.DEVNULL,
|
||
text=True,
|
||
)
|
||
probe = MessageProbe(client, alice['token'], chat, 0.2, f'failover-{target[:6]}')
|
||
probe.start()
|
||
started = now_iso()
|
||
|
||
time.sleep(args.pre)
|
||
command_moment = time.monotonic()
|
||
stop = run_command(['scripts/stop-one.sh'], check=False, env={'TARGET_INSTANCE_ID': target}, timeout=30)
|
||
exit_moment = time.monotonic()
|
||
stop_at = now_iso()
|
||
|
||
first_success_after_exit = None
|
||
deadline = time.monotonic() + 30
|
||
while time.monotonic() < deadline:
|
||
exit_offset = probe_offset(probe, exit_moment)
|
||
if any(sample.get('ok') and sample['t'] >= exit_offset for sample in list(probe.samples)):
|
||
first_success_after_exit = round(probe_offset(probe, time.monotonic()), 2)
|
||
break
|
||
time.sleep(0.1)
|
||
time.sleep(args.hold)
|
||
restore_moment = time.monotonic()
|
||
restore_at = now_iso()
|
||
run_command(['scripts/start-replica.sh'])
|
||
recovered = None
|
||
deadline = time.monotonic() + 90
|
||
while time.monotonic() < deadline:
|
||
seen = {
|
||
header(client.call('GET', '/api/v1/users/me', token=alice['token'])['headers'], 'X-Instance-Id')
|
||
for _ in range(6)
|
||
}
|
||
if len(seen) >= 2:
|
||
recovered = round(time.monotonic() - restore_moment, 2)
|
||
break
|
||
time.sleep(0.5)
|
||
try:
|
||
stdout, _ = load.communicate(timeout=load_seconds + 90)
|
||
except subprocess.TimeoutExpired:
|
||
load.kill()
|
||
stdout, _ = load.communicate()
|
||
probe.stop_event.set()
|
||
probe.join(timeout=15)
|
||
samples = sampler.stop()
|
||
finished = now_iso()
|
||
|
||
summary = None
|
||
for line in reversed((stdout or '').splitlines()):
|
||
if line.strip().startswith('{'):
|
||
try:
|
||
candidate = json.loads(line)
|
||
except ValueError:
|
||
continue
|
||
if 'steps' in candidate:
|
||
summary = candidate
|
||
break
|
||
|
||
history = client.history(alice['token'], chat)
|
||
texts = [item['text'] for item in history]
|
||
duplicates = sorted({text for text in texts if texts.count(text) > 1})
|
||
missing = [item['text'] for item in probe.confirmed if item['text'] not in texts]
|
||
|
||
command_offset = probe_offset(probe, command_moment)
|
||
exit_offset = probe_offset(probe, exit_moment)
|
||
# Окно отсутствия реплики: от выхода контейнера до первого успешного ответа
|
||
# после него. Если успех не наступил (проба не восстановилась), окно тянется
|
||
# до конца наблюдения — иначе метрика «во время отказа» выродилась бы в точку.
|
||
outage_start = (
|
||
first_success_after_exit if first_success_after_exit is not None else last_offset_before_stop(probe, exit_offset)
|
||
)
|
||
last_offset = probe.samples[-1]['t'] if probe.samples else exit_offset
|
||
# Два окна, чтобы метрика не вырождалась:
|
||
# - «остановка и дрейн»: от команды docker stop до первого успеха после выхода контейнера;
|
||
# - «реплики нет»: от выхода контейнера до первого успеха после него
|
||
# (при корректном дреине это доли секунды — так и должно быть).
|
||
stop_window_end = (
|
||
first_success_after_exit if first_success_after_exit is not None else last_offset
|
||
)
|
||
phases = {
|
||
'stop_command_at': stop_at,
|
||
'stop_exit_code': stop.returncode,
|
||
'stop_command_took_s': round(exit_moment - command_moment, 2),
|
||
'first_success_after_exit_s': (
|
||
round(first_success_after_exit - exit_offset, 2) if first_success_after_exit is not None else None
|
||
),
|
||
'restore_at': restore_at,
|
||
'replica_rejoined_s': recovered,
|
||
'lb_down_at': lb_down_time(samples, address),
|
||
'stop_window_seconds': round(stop_window_end - command_offset, 2),
|
||
'absent_window_seconds': round(outage_start - exit_offset, 2),
|
||
'probe_offsets': {
|
||
'stop_command': round(command_offset, 2),
|
||
'container_exit': round(exit_offset, 2),
|
||
'restore_command': round(probe_offset(probe, restore_moment), 2),
|
||
'last_sample': round(last_offset, 2),
|
||
},
|
||
}
|
||
if summary:
|
||
write_raw(label, summary, samples, config, started, finished, phases, sampler.errors)
|
||
failures = [sample for sample in probe.samples if not sample.get('ok')]
|
||
result = {
|
||
'instance': target,
|
||
'container': container,
|
||
'address': address,
|
||
'rate': args.rate,
|
||
'phases': phases,
|
||
'probe': {
|
||
'attempts': len(probe.samples),
|
||
'successes': sum(1 for sample in probe.samples if sample.get('ok')),
|
||
'failures': len(failures),
|
||
'confirmed': len(probe.confirmed),
|
||
'rps_before': phase_rps(probe.samples, 0.0, command_offset),
|
||
'rps_stop_window': phase_rps(probe.samples, command_offset, stop_window_end),
|
||
'rps_while_stopped': phase_rps(probe.samples, exit_offset, outage_start),
|
||
'rps_after': phase_rps(probe.samples, outage_start, last_offset),
|
||
'errors_stop_window': sum(
|
||
1 for sample in failures if command_offset <= sample['t'] <= stop_window_end
|
||
),
|
||
'errors_while_stopped': sum(
|
||
1 for sample in failures if exit_offset <= sample['t'] <= outage_start
|
||
),
|
||
'timeline': [[sample['t'], 1 if sample.get('ok') else 0] for sample in probe.samples],
|
||
},
|
||
'duplicates': duplicates,
|
||
'missing_confirmed': missing,
|
||
'haproxy': haproxy_stats(samples),
|
||
'k6': {
|
||
'achieved_rps': summary['steps'][0]['achieved_rps'] if summary else None,
|
||
'failed': summary['steps'][0]['failed'] if summary else None,
|
||
'dropped': summary.get('dropped_iterations') if summary else None,
|
||
},
|
||
}
|
||
log(
|
||
f'failover {target}: контейнер остановлен через {phases["stop_command_took_s"]}s, '
|
||
f'первый успех после остановки через {phases["first_success_after_exit_s"]}s, '
|
||
f'реплика вернулась через {recovered}s, дубли {len(duplicates)}, потеряно {len(missing)}'
|
||
)
|
||
return result
|
||
|
||
|
||
def probe_offset(probe: MessageProbe, moment: float) -> float:
|
||
"""Переводит момент wall-clock в шкалу времени сэмплов пробы."""
|
||
if not probe.samples:
|
||
return 0.0
|
||
return probe.samples[0]['t'] + (moment - probe.started_monotonic)
|
||
|
||
|
||
def last_offset_before_stop(probe: MessageProbe, exit_offset: float) -> float:
|
||
"""Конец наблюдения, если после выхода контейнера успехов не было."""
|
||
return probe.samples[-1]['t'] if probe.samples else exit_offset
|
||
|
||
|
||
def lb_down_time(samples: list[dict], address: str) -> str | None:
|
||
for record in samples:
|
||
if record.get('source') != 'haproxy' or record.get('server') in ('FRONTEND', 'BACKEND'):
|
||
continue
|
||
if address in (record.get('addr') or '') and record.get('status') not in ('UP',):
|
||
return record['t']
|
||
return None
|
||
|
||
|
||
def phase_rps(samples: list[dict], start: float, end: float) -> float:
|
||
window = [sample for sample in samples if start <= sample['t'] <= end]
|
||
return round(len(window) / max(0.001, end - start), 2)
|
||
|
||
|
||
def write_failover_evidence(results: list[dict]) -> None:
|
||
write_json(
|
||
LEVEL4_DIR / 'failover-sweep.json',
|
||
{
|
||
'captured_at': now_iso(),
|
||
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
|
||
'results': results,
|
||
},
|
||
)
|
||
lines = table(
|
||
[
|
||
'реплика',
|
||
'контейнер',
|
||
'попыток пробы',
|
||
'успешных',
|
||
'ошибок в отказе',
|
||
'остановка контейнера, s',
|
||
'первый успех после остановки, s',
|
||
'реплика вернулась, s',
|
||
'LB DOWN',
|
||
'RPS до',
|
||
'RPS в окне остановки',
|
||
'ошибок в окне',
|
||
'RPS без реплики',
|
||
'RPS после',
|
||
'k6 RPS',
|
||
'k6 ошибок',
|
||
'дубли',
|
||
'потери',
|
||
],
|
||
[
|
||
[
|
||
result['instance'][:12],
|
||
result['container'][:12],
|
||
result['probe']['attempts'],
|
||
result['probe']['successes'],
|
||
result['probe']['errors_while_stopped'],
|
||
result['phases']['stop_command_took_s'],
|
||
result['phases']['first_success_after_exit_s'],
|
||
result['phases']['replica_rejoined_s'],
|
||
(result['phases']['lb_down_at'] or '-')[11:19],
|
||
result['probe']['rps_before'],
|
||
result['probe']['rps_stop_window'],
|
||
result['probe']['errors_stop_window'],
|
||
result['probe']['rps_while_stopped'],
|
||
result['probe']['rps_after'],
|
||
result['k6']['achieved_rps'],
|
||
result['k6']['failed'],
|
||
len(result['duplicates']),
|
||
len(result['missing_confirmed']),
|
||
]
|
||
for result in results
|
||
],
|
||
)
|
||
lines += [
|
||
'',
|
||
'Проба: последовательные POST с интервалом 0.2 s, при неопределённом результате '
|
||
'повтор с тем же Idempotency-Key (как у реального клиента). «Окно остановки» — от '
|
||
'команды docker stop до первого успеха после выхода контейнера (сюда попадает дрейн); '
|
||
'«RPS без реплики» — от выхода контейнера до первого успеха после него, при корректном '
|
||
'дреине это доли секунды. Ошибок не должно быть ни в одном из окон, потери и дубли '
|
||
'проверяются по истории чата.',
|
||
]
|
||
write_markdown(LEVEL4_DIR / 'failover-sweep.md', 'Отказ каждой реплики под нагрузкой', lines)
|
||
|
||
|
||
def cmd_rolling(args) -> None:
|
||
config = load_stand_config()
|
||
replicas = int(config['API_REPLICAS'])
|
||
build_id = args.build_id or datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||
before = replica_map()
|
||
migrate_before = inspect(container_ids('migrate', include_stopped=True)[0], '{{.State.StartedAt}}')
|
||
log(f'сборка образа {config["API_IMAGE"]} с BUILD_ID={build_id}')
|
||
build = run_command(
|
||
['docker', 'build', '--build-arg', f'BUILD_ID={build_id}', '-t', config['API_IMAGE'], '.'],
|
||
check=True,
|
||
timeout=1800,
|
||
)
|
||
image_id = run_command(['docker', 'image', 'inspect', '-f', '{{.Id}}', config['API_IMAGE']]).stdout.strip()
|
||
|
||
client = Client(timeout=5)
|
||
alice = client.register('rolling')
|
||
chat = client.create_chat(alice['token'], [alice['id']], 'rolling update')
|
||
probe = MessageProbe(client, alice['token'], chat, 0.2, 'rolling')
|
||
sampler = Sampler().start()
|
||
load = None
|
||
if args.under_load:
|
||
env = {**load_env('p-b', 'write', 'spread', 'rolling'), 'RATE': str(args.rate), 'DURATION': str(max(args.duration, replicas * 45))}
|
||
load = subprocess.Popen(
|
||
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{k}={v}'] for k, v in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
|
||
cwd=ROOT,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.DEVNULL,
|
||
text=True,
|
||
)
|
||
probe.start()
|
||
started = now_iso()
|
||
steps = []
|
||
for step in range(replicas):
|
||
step_started = time.monotonic()
|
||
current = replica_map()
|
||
known = set(current)
|
||
# Меняем самую «старую» реплику: только что созданная видна как самая молодая.
|
||
target = sorted(current, key=lambda instance: current[instance]['started_at'])[0]
|
||
container = current[target]['container']
|
||
old_build = app_build_id(container)
|
||
run_command(['docker', 'stop', '--time', '15', container], check=False, timeout=60)
|
||
run_command(['docker', 'rm', container], check=False, timeout=60)
|
||
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', f'api={replicas}', 'api', timeout=600)
|
||
new_instance, new_container = wait_for_new_replica(replicas, known)
|
||
new_build = app_build_id(new_container)
|
||
steps.append(
|
||
{
|
||
'step': step + 1,
|
||
'replacement_seconds': round(time.monotonic() - step_started, 2),
|
||
'old_instance': target,
|
||
'old_container': container[:12],
|
||
'old_build_id': old_build,
|
||
'new_instance': new_instance,
|
||
'new_container': new_container[:12],
|
||
'new_build_id': new_build,
|
||
'migrate_started_at': migrate_before,
|
||
}
|
||
)
|
||
log(f'rolling {step + 1}/{replicas}: {target[:12]} ({old_build}) -> {new_instance[:12]} ({new_build})')
|
||
time.sleep(args.pause)
|
||
summary = None
|
||
if load is not None:
|
||
try:
|
||
stdout, _ = load.communicate(timeout=max(30, args.duration))
|
||
except subprocess.TimeoutExpired:
|
||
load.terminate()
|
||
try:
|
||
stdout, _ = load.communicate(timeout=60)
|
||
except subprocess.TimeoutExpired:
|
||
load.kill()
|
||
stdout, _ = load.communicate()
|
||
probe.stop_event.set()
|
||
probe.join(timeout=10)
|
||
samples = sampler.stop()
|
||
for line in reversed((stdout or '').splitlines()):
|
||
if line.strip().startswith('{'):
|
||
try:
|
||
candidate = json.loads(line)
|
||
except ValueError:
|
||
continue
|
||
if 'steps' in candidate:
|
||
summary = candidate
|
||
break
|
||
migrate_after = inspect(container_ids('migrate', include_stopped=True)[0], '{{.State.StartedAt}}')
|
||
failures = [sample for sample in probe.samples if not sample.get('ok')]
|
||
server_failures = [sample for sample in failures if 500 <= int(sample.get('status') or 0) < 600]
|
||
finished = now_iso()
|
||
payload = {
|
||
'build_id': build_id,
|
||
'captured_at': now_iso(),
|
||
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
|
||
'replacement_seconds': [step['replacement_seconds'] for step in steps],
|
||
'image_id': image_id,
|
||
'image': config['API_IMAGE'],
|
||
'replicas': replicas,
|
||
'under_load': bool(args.under_load),
|
||
'steps': steps,
|
||
'probe': {'attempts': len(probe.samples), 'failures': len(failures), 'server_5xx': len(server_failures)},
|
||
'k6': {'achieved_rps': summary['steps'][0]['achieved_rps'] if summary else None, 'failed': summary['steps'][0]['failed'] if summary else None},
|
||
'migrate': {'before': migrate_before, 'after': migrate_after, 'restarted': migrate_before != migrate_after},
|
||
'build_stderr_tail': build.stderr.strip().splitlines()[-3:],
|
||
}
|
||
write_json(LEVEL4_DIR / 'rolling-update.json', payload)
|
||
if summary:
|
||
write_raw('rolling', summary, samples, config, started, finished, {'steps': steps}, sampler.errors)
|
||
lines = table(
|
||
['шаг', 'было', 'BUILD_ID до', 'стало', 'BUILD_ID после', 'замена, s'],
|
||
[
|
||
[
|
||
step['step'],
|
||
step['old_instance'][:12],
|
||
step['old_build_id'],
|
||
step['new_instance'][:12],
|
||
step['new_build_id'],
|
||
step['replacement_seconds'],
|
||
]
|
||
for step in steps
|
||
],
|
||
)
|
||
lines += [
|
||
f'5xx от API во время rolling update: {len(server_failures)} из {len(probe.samples)} попыток',
|
||
f'Миграции перезапускались: {migrate_before != migrate_after}',
|
||
]
|
||
write_markdown(LEVEL4_DIR / 'rolling-update.md', 'Rolling update под нагрузкой', lines)
|
||
print(json.dumps(payload, ensure_ascii=False, indent=1))
|
||
|
||
|
||
def app_build_id(container: str) -> str:
|
||
result = run_command(
|
||
[
|
||
'docker', 'exec', container, 'node', '-e',
|
||
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
|
||
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}}).then(r=>r.json())"
|
||
".then(j=>process.stdout.write(String(j.build_id)))",
|
||
],
|
||
check=False,
|
||
timeout=20,
|
||
)
|
||
return result.stdout.strip() or 'unknown'
|
||
|
||
|
||
def wait_for_new_replica(replicas: int, known: set[str], timeout: float = 180) -> tuple[str, str]:
|
||
"""Ждёт появления НОВОЙ реплики (не из known) в состоянии healthy."""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
mapping = replica_map()
|
||
fresh = {instance: info for instance, info in mapping.items() if instance not in known}
|
||
if len(mapping) >= replicas and fresh:
|
||
for instance, info in sorted(fresh.items()):
|
||
status = inspect(info['container'], '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')
|
||
if status == 'healthy':
|
||
return instance, info['container']
|
||
time.sleep(1)
|
||
raise SystemExit(
|
||
f'новая реплика не появилась healthy за {timeout}s (контейнеров {len(replica_map())}, '
|
||
f'новых {len({i for i in replica_map() if i not in known})})'
|
||
)
|
||
|
||
|
||
def cmd_ab(args) -> None:
|
||
sides = {'A': args.before, 'B': args.after}
|
||
# Бюджет A/B фиксируется явно: иначе точки унаследуют конфигурацию предыдущих
|
||
# серий (именно так ранние прогоны A/B уехали на удвоенный API/PG и пул 20),
|
||
# и числа станут несравнимы с базовыми прогонами той же серии.
|
||
plan = {
|
||
'API_REPLICAS': str(args.replicas),
|
||
'API_TOTAL_CPUS': str(args.api_total_cpus),
|
||
'API_TOTAL_MEMORY_MB': str(args.api_total_memory_mb),
|
||
'PG_CPUS': str(args.pg_cpus),
|
||
'DATABASE_POOL_MAX': str(args.pool),
|
||
'LB_CPUS': str(args.lb_cpus),
|
||
'LOADGEN_CPUS': str(args.loadgen_cpus),
|
||
'LOADGEN_MEMORY': args.loadgen_memory,
|
||
}
|
||
log(f'A/B на бюджете: {plan}')
|
||
for rep in range(1, args.reps + 1):
|
||
order = ['A', 'B'] if rep % 2 else ['B', 'A']
|
||
for side in order:
|
||
ensure_stand({**plan, 'API_IMAGE': sides[side]})
|
||
run_measured(
|
||
f'{args.prefix}#{side}#rep{rep}',
|
||
args.profile,
|
||
warmup=not args.no_warmup,
|
||
vus={'max': args.max_vus},
|
||
)
|
||
report_ab(args, plan)
|
||
|
||
|
||
def report_ab(args=None, plan: dict | None = None) -> None:
|
||
prefix = getattr(args, 'prefix', 'ab') if args else 'ab'
|
||
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#')]
|
||
groups = group_configurations(labels)
|
||
lines = []
|
||
verdicts = []
|
||
series_by_side = {
|
||
side: summarise_series(groups.get(f'{prefix}#{side}', [])) for side in ('A', 'B')
|
||
}
|
||
images = {
|
||
side: (getattr(args, side.lower(), None) if args else None) for side in ('A', 'B')
|
||
}
|
||
for side in ('A', 'B'):
|
||
if not images[side] and groups.get(f'{prefix}#{side}'):
|
||
manifest = read_json(
|
||
RAW_DIR / f'{groups[f"{prefix}#{side}"][-1]}.manifest.json', {}
|
||
)
|
||
images[side] = (manifest.get('stand') or {}).get('API_IMAGE')
|
||
for rate in sorted({row['rate'] for row in series_by_side['A']['rows']}):
|
||
row_a = next((row for row in series_by_side['A']['rows'] if row['rate'] == rate), None)
|
||
row_b = next((row for row in series_by_side['B']['rows'] if row['rate'] == rate), None)
|
||
if not row_a or not row_b:
|
||
continue
|
||
change = (
|
||
round((row_b['achieved_rps'] - row_a['achieved_rps']) / row_a['achieved_rps'] * 100, 2)
|
||
if row_a['achieved_rps']
|
||
else 0.0
|
||
)
|
||
confirmed = (
|
||
abs(change) >= 5.0
|
||
and (row_b['achieved_min'] > row_a['achieved_max'] or row_a['achieved_min'] > row_b['achieved_max'])
|
||
)
|
||
verdicts.append({'rate': rate, 'change_percent': change, 'confirmed': confirmed})
|
||
lines.append(
|
||
f'{rate} req/s: A {row_a["achieved_rps"]} [{row_a["achieved_min"]}; {row_a["achieved_max"]}] vs '
|
||
f'B {row_b["achieved_rps"]} [{row_b["achieved_min"]}; {row_b["achieved_max"]}] -> {change}% '
|
||
f'({"подтверждено" if confirmed else "не подтверждено"})'
|
||
)
|
||
lines.append('')
|
||
lines += table(
|
||
['шаг', 'образ A: RPS (min..max)', 'p95 A', 'образ B: RPS (min..max)', 'p95 B', 'Δ, %', 'вывод'],
|
||
[
|
||
[
|
||
verdict['rate'],
|
||
f'{next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_rps"]}'
|
||
f' ({next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_min"]}..'
|
||
f'{next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_max"]})',
|
||
next(row for row in series_by_side['A']['rows'] if row['rate'] == verdict['rate'])['p95'],
|
||
f'{next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_rps"]}'
|
||
f' ({next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_min"]}..'
|
||
f'{next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_max"]})',
|
||
next(row for row in series_by_side['B']['rows'] if row['rate'] == verdict['rate'])['p95'],
|
||
verdict['change_percent'],
|
||
'подтверждено' if verdict['confirmed'] else 'не подтверждено',
|
||
]
|
||
for verdict in verdicts
|
||
],
|
||
)
|
||
lines += [
|
||
f'Потолок A: {series_by_side["A"]["ceiling_rps"]} req/s, потолок B: {series_by_side["B"]["ceiling_rps"]} req/s',
|
||
'',
|
||
'Правило подтверждения: медиана меняется ≥5 % и диапазоны min/max повторов не перекрываются.',
|
||
]
|
||
suffix = '' if prefix == 'ab' else f'-{prefix}'
|
||
write_json(
|
||
TRACK1_DIR / f'ab-verdict{suffix}.json',
|
||
{
|
||
'verdicts': verdicts,
|
||
'ceiling': {side: series_by_side[side]['ceiling_rps'] for side in ('A', 'B')},
|
||
'prefix': prefix,
|
||
'images': images,
|
||
'budget': plan
|
||
or (
|
||
read_json(RAW_DIR / f'{groups[f"{prefix}#A"][-1]}.manifest.json', {}).get('stand')
|
||
if groups.get(f'{prefix}#A')
|
||
else None
|
||
),
|
||
'rule': 'медиана меняется ≥5 % и диапазоны min/max повторов не пересекаются',
|
||
'captured_at': now_iso(),
|
||
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
|
||
},
|
||
)
|
||
write_markdown(
|
||
TRACK1_DIR / f'summary-ab{suffix}.md',
|
||
f'A/B: подтверждённое улучшение ({prefix})',
|
||
lines,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------- анализ и графики
|
||
|
||
|
||
def pyplot():
|
||
import matplotlib
|
||
|
||
matplotlib.use('Agg')
|
||
import matplotlib.pyplot as plt
|
||
|
||
return plt
|
||
|
||
|
||
def representative(group: list[str]) -> str:
|
||
"""Прогон с медианным пиком — чтобы агрегаты контейнеров не брались с края."""
|
||
scored = []
|
||
for label in group:
|
||
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
|
||
steps = summary.get('steps', [])
|
||
peak = max((step['achieved_rps'] for step in steps), default=0.0)
|
||
scored.append((peak, label))
|
||
if not scored:
|
||
raise SystemExit('нет прогонов для анализа')
|
||
scored.sort()
|
||
return scored[len(scored) // 2][1]
|
||
|
||
|
||
def cmd_analyze(args) -> None:
|
||
labels = raw_labels()
|
||
if not labels:
|
||
log('raw пуст: таблицы и графики будут пустыми')
|
||
groups = group_configurations(labels)
|
||
analyse_runs(groups)
|
||
analyse_series_a(groups, labels)
|
||
analyse_diagnostics(groups, labels)
|
||
analyse_growth(groups, labels)
|
||
analyse_mechanism(groups, labels)
|
||
if groups.get('ab#A') or groups.get('ab#B'):
|
||
report_ab()
|
||
make_graphs(groups, labels)
|
||
log('analyze: таблицы и графики обновлены')
|
||
|
||
|
||
def series_kind(group: list[str]) -> str:
|
||
summary = read_json(RAW_DIR / f'{group[0]}.k6.json', {})
|
||
return summary.get('kind', 'ladder')
|
||
|
||
|
||
def analyse_runs(groups: dict) -> None:
|
||
"""Полный разбор каждого прогона: шаги, ресурсы, PostgreSQL, LB."""
|
||
lines = []
|
||
for key, group in sorted(groups.items()):
|
||
series = summarise_series(group)
|
||
label = representative(group)
|
||
_, samples, manifest = load_raw(label)
|
||
reps_note = 'одиночный прогон' if len(group) == 1 else f'{len(group)} прогонов'
|
||
lines += [f'## {key} ({reps_note}, {series_kind(group)})', '']
|
||
lines += table(
|
||
['темп, req/s', 'достигнуто RPS', 'min..max', 'p95, ms', 'ошибок', 'VU', 'dropped', '5xx', '4xx'],
|
||
[
|
||
[
|
||
row['rate'],
|
||
row['achieved_rps'],
|
||
f'{row["achieved_min"]}..{row["achieved_max"]}',
|
||
row['p95'],
|
||
row['failed'],
|
||
row['vus_active'],
|
||
read_json(RAW_DIR / f'{label}.k6.json', {}).get('dropped_iterations', 0),
|
||
row.get('status_5xx', 0),
|
||
row.get('status_4xx', 0),
|
||
]
|
||
for row in series['rows']
|
||
],
|
||
)
|
||
if series_kind(group) == 'ladder':
|
||
saturation = series['saturation']
|
||
lines.append(
|
||
f'Потолок {series["ceiling_rps"]} req/s, сатурация {saturation["rate"]} req/s '
|
||
f'({saturation["achieved_rps"]} req/s достигнуто).'
|
||
)
|
||
lines.append('')
|
||
limits = manifest.get('stand', {}) if manifest else {}
|
||
lines.append(
|
||
f'Лимиты: реплик {limits.get("API_REPLICAS")}, API {limits.get("API_CPUS_PER_REPLICA")} CPU / '
|
||
f'{limits.get("API_MEMORY_PER_REPLICA")}, PG {limits.get("PG_CPUS")} CPU, LB {limits.get("LB_CPUS")} CPU, '
|
||
f'пул {limits.get("DATABASE_POOL_MAX")}, образ {limits.get("API_IMAGE")} (BUILD_ID {limits.get("BUILD_ID")}).'
|
||
)
|
||
lines.append('')
|
||
lines += table(
|
||
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
|
||
[
|
||
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
|
||
for name, stats in container_stats(samples).items()
|
||
],
|
||
)
|
||
pg = postgres_stats(samples)
|
||
if pg:
|
||
lines.append(
|
||
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, hit ratio {pg["blks_hit_ratio"]}, '
|
||
f'ожиданий блокировок пик {pg["lock_waiters_peak"]}, активных сессий пик {pg["active_peak"]}, '
|
||
f'соединений пик {pg["connections_peak"]}, deadlocks {pg["deadlocks"]}, messages {pg["messages_mib"]} MiB.'
|
||
)
|
||
lines.append('')
|
||
lb = haproxy_stats(samples)
|
||
total = lb.get('BACKEND')
|
||
if total:
|
||
lines.append(
|
||
f'LB: {total["requests"]} запросов ({total["rps"]} req/s), ошибок {total["errors"]}, '
|
||
f'очередь пик {total["qcur_peak"]}.'
|
||
)
|
||
for name, server in sorted(lb.items()):
|
||
if name in ('BACKEND', 'FRONTEND') or not server['requests']:
|
||
continue
|
||
lines.append(
|
||
f'- LB слот {name}: {server["requests"]} запросов ({server["rps"]} req/s), ошибок {server["errors"]}, '
|
||
f'очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}, адрес {server["addresses"]}.'
|
||
)
|
||
lines.append('')
|
||
app = app_metrics_delta(samples)
|
||
for container, delta in sorted(app.items()):
|
||
lines.append(
|
||
f'- реплика {delta["instance_id"]} (build {delta["build_id"]}, контейнер {container[:12]}): '
|
||
f'{json.dumps(delta["counters"], ensure_ascii=False)}; '
|
||
f'latency p95 {delta["latency_ms"].get("p95")} ms.'
|
||
)
|
||
lines.append('')
|
||
write_markdown(TRACK1_DIR / 'summary-runs.md', 'Разбор каждого прогона', lines)
|
||
|
||
|
||
def analyse_series_a(groups: dict, labels: list[str]) -> None:
|
||
lines = []
|
||
series_by_n = {}
|
||
for key, group in sorted(groups.items()):
|
||
if not key.startswith('p-a#n'):
|
||
continue
|
||
series = summarise_series(group)
|
||
series_by_n[key] = series
|
||
lines.append(f'## {key} ({len(group)} повторов)')
|
||
lines.append('')
|
||
lines += table(
|
||
['темп, req/s', 'достигнуто RPS', 'min..max', 'p95, ms', 'p95 max', 'ошибок', 'VU'],
|
||
[
|
||
[
|
||
row['rate'],
|
||
row['achieved_rps'],
|
||
f'{row["achieved_min"]}..{row["achieved_max"]}',
|
||
row['p95'],
|
||
row['p95_max'],
|
||
row['failed'],
|
||
row['vus_active'],
|
||
]
|
||
for row in series['rows']
|
||
],
|
||
)
|
||
saturation = series['saturation']
|
||
lines.append(
|
||
f'Потолок: **{series["ceiling_rps"]} req/s**; сатурация: {saturation["rate"]} req/s '
|
||
f'(достигнуто {saturation["achieved_rps"]} req/s).'
|
||
)
|
||
lines.append('')
|
||
label = representative(group)
|
||
_, samples, manifest = load_raw(label)
|
||
lines.append(f'Ресурсы представительного прогона `{label}` (лимиты {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
|
||
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")} на реплику):')
|
||
lines.append('')
|
||
lines += table(
|
||
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
|
||
[
|
||
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
|
||
for name, stats in container_stats(samples).items()
|
||
],
|
||
)
|
||
pg = postgres_stats(samples)
|
||
if pg:
|
||
lines.append(
|
||
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, '
|
||
f'hit ratio {pg["blks_hit_ratio"]}, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
|
||
f'активных сессий пик {pg["active_peak"]}, соединений пик {pg["connections_peak"]}, '
|
||
f'таблица messages {pg["messages_mib"]} MiB.'
|
||
)
|
||
lines.append('')
|
||
lb = haproxy_stats(samples)
|
||
distribution = [
|
||
f'слот {name}: {server["requests"]} ({server["rps"]} req/s), адрес {", ".join(server["addresses"])}'
|
||
for name, server in sorted(lb.items())
|
||
if name not in ('BACKEND', 'FRONTEND') and server['requests']
|
||
]
|
||
if distribution:
|
||
lines += ['Распределение запросов LB:', ''] + [f'- {item}' for item in distribution] + ['']
|
||
lines += saturation_conclusion(series_by_n)
|
||
write_markdown(TRACK1_DIR / 'summary-series-a.md', 'Серия A: 1/2/4 реплики при фиксированном бюджете', lines)
|
||
|
||
|
||
def saturation_conclusion(series_by_n: dict) -> list[str]:
|
||
lines = ['## Точки сатурации', '']
|
||
for key, series in sorted(series_by_n.items()):
|
||
saturation = series['saturation']
|
||
lines.append(
|
||
f'- {key}: knee на {saturation["rate"]} req/s (достигнуто {saturation["achieved_rps"]} req/s), '
|
||
f'потолок {series["ceiling_rps"]} req/s.'
|
||
)
|
||
lines.append('')
|
||
return lines
|
||
|
||
|
||
def analyse_diagnostics(groups: dict, labels: list[str]) -> None:
|
||
lines = []
|
||
for key, group in sorted(groups.items()):
|
||
if not key.startswith(('p-b', 'p-c')):
|
||
continue
|
||
series = summarise_series(group)
|
||
label = representative(group)
|
||
_, samples, manifest = load_raw(label)
|
||
pg = postgres_stats(samples)
|
||
stats = container_stats(samples)
|
||
lines.append(f'## {key} (темпы {series["rows"][0]["rate"]}..{series["rows"][-1]["rate"]} req/s)')
|
||
lines.append('')
|
||
lines += table(
|
||
['темп', 'достигнуто RPS', 'p95, ms', 'ошибок'],
|
||
[[row['rate'], row['achieved_rps'], row['p95'], row['failed']] for row in series['rows']],
|
||
)
|
||
lines.append(
|
||
f'Потолок {series["ceiling_rps"]} req/s, knee на {series["saturation"]["rate"]} req/s. '
|
||
f'Лимиты: LB {manifest["stand"].get("LB_CPUS")} CPU, API {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
|
||
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")}, PG {manifest["stand"].get("PG_CPUS")} CPU.'
|
||
)
|
||
lines.append('')
|
||
if pg:
|
||
lines.append(
|
||
f'PostgreSQL: {pg["xact_per_s"]} xact/s, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
|
||
f'активных сессий пик {pg["active_peak"]}, hit ratio {pg["blks_hit_ratio"]}.'
|
||
)
|
||
lines.append('')
|
||
lb = stats.get('mtusi-lab3-haproxy-1')
|
||
if lb:
|
||
lines.append(f'HAProxy: CPU средний {lb["cpu_mean"]}%, пик {lb["cpu_peak"]}%, RAM пик {lb["mem_peak"]} MiB из {lb["mem_limit"]}.')
|
||
lines.append('')
|
||
haproxy = haproxy_stats(samples)
|
||
for name, server in sorted(haproxy.items()):
|
||
if server['qcur_peak'] or server['statuses'] != ['UP']:
|
||
lines.append(f'- LB сервер {name}: очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}.')
|
||
lines.append('')
|
||
write_markdown(TRACK1_DIR / 'summary-diagnostics.md', 'Диагностика: горячий чат и путь чтения', lines)
|
||
|
||
|
||
def analyse_growth(groups: dict, labels: list[str]) -> None:
|
||
growth = {key: group for key, group in groups.items() if key.startswith('growth#')}
|
||
if not growth:
|
||
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', ['Нет прогонов growth.'])
|
||
return
|
||
lines = []
|
||
baseline_labels = [label for label in labels if label.startswith('p-a#n4#')]
|
||
baseline = summarise_series(baseline_labels) if baseline_labels else None
|
||
if baseline:
|
||
rate = max((row['rate'] for row in growth_rows(growth)), default=0)
|
||
row = step_of_rate(baseline['rows'], rate)
|
||
lines.append(
|
||
f'Ориентир по колену (серия A, N=4, темп {rate} req/s, сборка C1, MAX_VUS=200): '
|
||
f'**{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms — генератор был ограничен пулом VU.'
|
||
)
|
||
control = groups.get('block#n4')
|
||
if control:
|
||
control_row = step_of_rate(summarise_series(control)['rows'], rate)
|
||
lines.append(
|
||
f'Контроль на том же темпе, MAX_VUS=900 (block#n4, 2 повтора): '
|
||
f'**{control_row["achieved_rps"]} req/s**, p95 {control_row["p95"]} ms.'
|
||
)
|
||
lines.append('')
|
||
lines += table(
|
||
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API: пик реплики, % её квоты', 'CPU PG пик, % ядра'],
|
||
[growth_row(key, group) for key, group in sorted(growth.items())],
|
||
)
|
||
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
|
||
|
||
|
||
def growth_rows(growth: dict):
|
||
for group in growth.values():
|
||
summary = read_json(RAW_DIR / f'{representative(group)}.k6.json', {})
|
||
yield from summary.get('steps', [])
|
||
|
||
|
||
def step_of_rate(rows: list[dict], rate: int) -> dict:
|
||
for row in rows:
|
||
if row['rate'] == rate:
|
||
return row
|
||
return rows[-1] if rows else {'rate': 0, 'achieved_rps': 0, 'p95': 0}
|
||
|
||
|
||
def growth_row(key: str, group: list[str]) -> list:
|
||
label = representative(group)
|
||
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
|
||
_, samples, manifest = load_raw(label)
|
||
step = summary['steps'][0] if summary.get('steps') else {'rate': 0, 'achieved_rps': 0, 'p95': 0, 'failed': 0}
|
||
stats = container_stats(samples)
|
||
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
|
||
api_limit = float((manifest or {}).get('stand', {}).get('API_CPUS_PER_REPLICA') or 0)
|
||
api_peak_quota = round(api_peak / (api_limit * 100) * 100, 2) if api_limit else 0.0
|
||
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
|
||
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak_quota, pg_peak]
|
||
|
||
|
||
def make_graphs(groups: dict, labels: list[str]) -> None:
|
||
if not labels:
|
||
return
|
||
plt = pyplot()
|
||
GRAPH_DIR.mkdir(parents=True, exist_ok=True)
|
||
ladders = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'ladder'}
|
||
steadys = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'steady'}
|
||
|
||
if ladders:
|
||
series = {key: summarise_series(group) for key, group in ladders.items()}
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
for key, values in series.items():
|
||
axes.plot(
|
||
[row['rate'] for row in values['rows']],
|
||
[row['achieved_rps'] for row in values['rows']],
|
||
marker='o',
|
||
label=key,
|
||
)
|
||
reference = next(iter(series.values()))
|
||
axes.plot(
|
||
[row['rate'] for row in reference['rows']],
|
||
[row['rate'] for row in reference['rows']],
|
||
linestyle='--',
|
||
color='grey',
|
||
label='предложенный темп',
|
||
)
|
||
axes.set_xlabel('предложенный темп, req/s')
|
||
axes.set_ylabel('достигнутый RPS')
|
||
axes.set_title('Пропускная способность: достигнутый RPS от предложенного темпа')
|
||
axes.grid(alpha=0.3)
|
||
axes.legend(fontsize=8)
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'throughput.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
for key, values in series.items():
|
||
axes.plot(
|
||
[row['rate'] for row in values['rows']],
|
||
[row['p95'] for row in values['rows']],
|
||
marker='o',
|
||
label=f'{key}: p95, ms',
|
||
)
|
||
axes.plot(
|
||
[row['rate'] for row in values['rows']],
|
||
[row['failed'] for row in values['rows']],
|
||
linestyle=':',
|
||
marker='x',
|
||
label=f'{key}: ошибок, шт.',
|
||
)
|
||
axes.set_xlabel('предложенный темп, req/s')
|
||
axes.set_ylabel('p95, ms / ошибок')
|
||
axes.set_title('Латентность p95 и ошибки по шагам лестницы')
|
||
axes.grid(alpha=0.3)
|
||
axes.legend(fontsize=8)
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'latency-errors.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
for key, values in series.items():
|
||
axes.plot(
|
||
[row['rate'] for row in values['rows']],
|
||
[row['vus_active'] for row in values['rows']],
|
||
marker='s',
|
||
label=key,
|
||
)
|
||
axes.set_xlabel('предложенный темп, req/s')
|
||
axes.set_ylabel('активных VU генератора')
|
||
axes.set_title('Занятость генератора: нехватка VU отличима от перегрузки сервера')
|
||
axes.grid(alpha=0.3)
|
||
axes.legend(fontsize=8)
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'generator-vus.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
if steadys:
|
||
keys = list(steadys)
|
||
achieved = [summarise_series(steadys[key])['ceiling_rps'] for key in keys]
|
||
offered = [summarise_series(steadys[key])['rows'][0]['rate'] for key in keys]
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
positions = range(len(keys))
|
||
axes.bar([p - 0.2 for p in positions], offered, width=0.4, label='предложенный темп')
|
||
axes.bar([p + 0.2 for p in positions], achieved, width=0.4, label='достигнутый RPS')
|
||
axes.set_xticks(list(positions))
|
||
axes.set_xticklabels(keys, fontsize=8)
|
||
axes.set_ylabel('req/s')
|
||
axes.set_title('Фиксированный темп: предложено против достигнутого')
|
||
axes.legend()
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'steady-throughput.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
for key, group in sorted(groups.items()):
|
||
plot_container_cpu(plt, representative(group))
|
||
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
shares = {}
|
||
for key, group in sorted(groups.items()):
|
||
_, samples, _ = load_raw(representative(group))
|
||
lb = haproxy_stats(samples)
|
||
shares[key] = {name: server['requests'] for name, server in sorted(lb.items()) if server['requests']}
|
||
names = sorted({name for share in shares.values() for name in share})
|
||
if names and shares:
|
||
width = 0.8 / max(1, len(names))
|
||
for position, name in enumerate(names):
|
||
axes.bar(
|
||
[index + position * width - 0.4 for index in range(len(shares))],
|
||
[shares[key].get(name, 0) for key in shares],
|
||
width=width,
|
||
label=name,
|
||
)
|
||
axes.set_xticks(range(len(shares)))
|
||
axes.set_xticklabels(list(shares), fontsize=7)
|
||
axes.set_ylabel('запросов за прогон')
|
||
axes.set_title('Распределение запросов по backend-слотам LB')
|
||
axes.legend(fontsize=7)
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'replica-distribution.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
growth = {key: group for key, group in groups.items() if key.startswith('growth#')}
|
||
if growth:
|
||
rows = [growth_row(key, group) for key, group in sorted(growth.items())]
|
||
baseline_labels = [label for label in labels if label.startswith('p-a#n4#')]
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
names = []
|
||
values = []
|
||
low = []
|
||
high = []
|
||
if baseline_labels:
|
||
series = summarise_series(baseline_labels)
|
||
row = step_of_rate(series['rows'], rows[0][1])
|
||
names.append('базовая точка (серия A)')
|
||
values.append(row['achieved_rps'])
|
||
low.append(row['achieved_min'])
|
||
high.append(row['achieved_max'])
|
||
for row in rows:
|
||
names.append(row[0].replace('growth#', ''))
|
||
values.append(row[2])
|
||
low.append(row[2])
|
||
high.append(row[2])
|
||
axes.bar(names, values, yerr=[[value - minimum for value, minimum in zip(values, low)], [maximum - value for value, maximum in zip(values, high)]], capsize=5)
|
||
axes.set_ylabel('достигнутый RPS')
|
||
axes.set_title('Серия роста ресурсов: какой ресурс является ограничителем')
|
||
axes.tick_params(axis='x', labelsize=8)
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'growth.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
if groups.get('ab#A') and groups.get('ab#B'):
|
||
figure, axes = plt.subplots(figsize=(9, 5))
|
||
series_a = summarise_series(groups['ab#A'])
|
||
series_b = summarise_series(groups['ab#B'])
|
||
rates = [row['rate'] for row in series_a['rows']]
|
||
positions = list(range(len(rates)))
|
||
axes.bar([p - 0.2 for p in positions], [row['achieved_rps'] for row in series_a['rows']], width=0.4, label='A (до)')
|
||
axes.bar([p + 0.2 for p in positions], [row['achieved_rps'] for row in series_b['rows']], width=0.4, label='B (после)')
|
||
for offset, series in ((-0.2, series_a), (0.2, series_b)):
|
||
for index, row in enumerate(series['rows']):
|
||
axes.plot([index + offset, index + offset], [row['achieved_min'], row['achieved_max']], color='black', linewidth=1)
|
||
axes.set_xticks(positions)
|
||
axes.set_xticklabels([str(rate) for rate in rates])
|
||
axes.set_xlabel('темп, req/s')
|
||
axes.set_ylabel('достигнутый RPS')
|
||
axes.set_title('A/B: медиана и разброс (min..max) по повторам')
|
||
axes.legend()
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / 'ab-comparison.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
|
||
def plot_container_cpu(plt, label: str) -> None:
|
||
_, samples, _ = load_raw(label)
|
||
series: dict[str, list[tuple[float, float]]] = {}
|
||
base = None
|
||
for record in samples:
|
||
if record.get('source') != 'docker_stats':
|
||
continue
|
||
name = record.get('name') or ''
|
||
if not name.startswith('mtusi-lab3-'):
|
||
continue
|
||
moment = _timestamp(record['t'])
|
||
base = base or moment
|
||
series.setdefault(name, []).append((moment - base, float(record.get('cpu_percent') or 0)))
|
||
if not series:
|
||
return
|
||
figure, axes = plt.subplots(figsize=(10, 5))
|
||
for name, points in sorted(series.items()):
|
||
axes.plot([point[0] for point in points], [point[1] for point in points], label=name, linewidth=1)
|
||
axes.set_xlabel('время от начала прогона, s')
|
||
axes.set_ylabel('CPU, % (100 % = одно ядро)')
|
||
axes.set_title(f'CPU контейнеров: {label}')
|
||
axes.grid(alpha=0.3)
|
||
axes.legend(fontsize=8)
|
||
figure.tight_layout()
|
||
figure.savefig(GRAPH_DIR / f'container-cpu-{label.replace("#", "_")}.png', dpi=120)
|
||
plt.close(figure)
|
||
|
||
|
||
# ---------------------------------------------------------------- оркестрация трека
|
||
|
||
|
||
def cmd_watch(args) -> None:
|
||
"""Сэмплер без нагрузки: наблюдает стенд N секунд и пишет jsonl.gz.
|
||
|
||
Нужен там, где нагрузку создаёт внешний сценарий (публичный failover-тест,
|
||
rolling update): переходы статусов LB получают отметки времени.
|
||
"""
|
||
sampler = Sampler().start()
|
||
try:
|
||
time.sleep(args.seconds)
|
||
finally:
|
||
samples = sampler.stop()
|
||
out = Path(args.out)
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
with gzip.open(out, 'wt', encoding='utf-8') as handle:
|
||
for record in samples:
|
||
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
|
||
print(json.dumps({'out': str(out), 'samples': len(samples), 'errors': sampler.errors}, ensure_ascii=False))
|
||
|
||
|
||
def cmd_quiesce(args) -> None:
|
||
result = run_command(['scripts/quiesce.sh', args.action], check=False, timeout=300)
|
||
print(result.stdout + result.stderr)
|
||
if result.returncode != 0:
|
||
raise SystemExit(f'quiesce {args.action} failed')
|
||
|
||
|
||
def cmd_track1(args) -> None:
|
||
if not args.no_quiesce:
|
||
cmd_quiesce(argparse.Namespace(action='up'))
|
||
try:
|
||
log('=== серия A: 1/2/4 реплики, профиль p-a')
|
||
cmd_sweep(
|
||
argparse.Namespace(
|
||
profile='p-a',
|
||
replicas='1,2,4',
|
||
reps=args.reps,
|
||
no_warmup=False,
|
||
prefix='p-a',
|
||
extra_replica=2,
|
||
extra_reps=1,
|
||
)
|
||
)
|
||
log('=== серия B: горячий чат (p-b), 4 реплики')
|
||
ensure_stand({'API_REPLICAS': '4'})
|
||
run_measured('p-b#n4', 'p-b', warmup=False)
|
||
log('=== серия C: путь чтения (p-c), 4 реплики, LB 0.25 и 1.0 CPU')
|
||
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '0.25'})
|
||
run_measured('p-c#lb0.25', 'p-c', warmup=False)
|
||
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '1.0'})
|
||
run_measured('p-c#lb1.0', 'p-c', warmup=False)
|
||
log('=== серия роста ресурсов')
|
||
for series in ('api', 'pg', 'pool'):
|
||
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False, suffix=''))
|
||
log('=== агрегаты')
|
||
cmd_analyze(argparse.Namespace())
|
||
if args.ab_after:
|
||
log('=== A/B улучшения (p-b, 4 реплики)')
|
||
cmd_ab(
|
||
argparse.Namespace(
|
||
before=args.ab_before,
|
||
after=args.ab_after,
|
||
profile='p-b',
|
||
replicas=4,
|
||
reps=args.ab_reps,
|
||
no_warmup=False,
|
||
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
|
||
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
|
||
pg_cpus=DEFAULT_STAND['PG_CPUS'],
|
||
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
|
||
lb_cpus=DEFAULT_STAND['LB_CPUS'],
|
||
loadgen_cpus=DEFAULT_STAND['LOADGEN_CPUS'],
|
||
loadgen_memory=DEFAULT_STAND['LOADGEN_MEMORY'],
|
||
max_vus=1200,
|
||
prefix='ab',
|
||
)
|
||
)
|
||
cmd_analyze(argparse.Namespace())
|
||
else:
|
||
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
|
||
finally:
|
||
ensure_stand(
|
||
{
|
||
'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']),
|
||
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
|
||
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
|
||
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
|
||
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
|
||
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
|
||
'API_IMAGE': DEFAULT_STAND['API_IMAGE'],
|
||
'BUILD_ID': DEFAULT_STAND['BUILD_ID'],
|
||
}
|
||
)
|
||
if not args.no_quiesce:
|
||
cmd_quiesce(argparse.Namespace(action='down'))
|
||
|
||
|
||
def cmd_track1_extras(args) -> None:
|
||
"""Догоняющие серии трека 1: VU-своп, блоки «почему» и контрольный A/B.
|
||
|
||
Отдельные команды, потому что они отвечают на уточняющие вопросы (потолок
|
||
сервера или генератора, где теряется время, сравнение на бюджете по умолчанию),
|
||
а не строят базовую картину серии A.
|
||
"""
|
||
base = {
|
||
'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']),
|
||
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
|
||
'LOADGEN_CPUS': str(args.loadgen_cpus),
|
||
'LOADGEN_MEMORY': args.loadgen_memory,
|
||
}
|
||
if args.part in ('all', 'vuswap'):
|
||
log('=== VU-своп: та же конфигурация, меняется только MAX_VUS')
|
||
for replicas in ('1', '4'):
|
||
for max_vus in (200, 900, 2000):
|
||
ensure_stand({**base, 'API_REPLICAS': replicas, 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX'])})
|
||
run_measured(
|
||
f'vuswap#n{replicas}-vu{max_vus}',
|
||
'p-a',
|
||
ladder=[700, 900, 1100],
|
||
step_seconds=15,
|
||
warmup=False,
|
||
vus={'max': max_vus},
|
||
)
|
||
if args.part in ('all', 'why'):
|
||
log(f'=== блок «почему»: один темп {args.rate} req/s, {args.reps} повтора, интерливинг')
|
||
block = [
|
||
('n1', {'API_REPLICAS': '1', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10'}),
|
||
('n4', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10'}),
|
||
('n4-pg3', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '3.0', 'DATABASE_POOL_MAX': '10'}),
|
||
('n4-api6', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '6.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '10'}),
|
||
('n4-pool20', {'API_REPLICAS': '4', 'API_TOTAL_CPUS': '3.0', 'PG_CPUS': '1.5', 'DATABASE_POOL_MAX': '20'}),
|
||
]
|
||
for rep in range(1, args.reps + 1):
|
||
order = block if rep % 2 else list(reversed(block))
|
||
for name, overrides in order:
|
||
ensure_stand({**base, **overrides})
|
||
run_measured(
|
||
f'{args.prefix}#{name}#rep{rep}',
|
||
'p-a',
|
||
rate=args.rate,
|
||
duration=args.duration,
|
||
warmup=False,
|
||
vus={'max': args.max_vus},
|
||
)
|
||
if args.part in ('all', 'abdiag'):
|
||
log('=== контрольный A/B на бюджете по умолчанию')
|
||
cmd_ab(
|
||
argparse.Namespace(
|
||
before=args.before,
|
||
after=args.after,
|
||
profile='p-b',
|
||
replicas=4,
|
||
reps=args.ab_reps,
|
||
no_warmup=False,
|
||
prefix='abdiag',
|
||
max_vus=900,
|
||
api_total_cpus=DEFAULT_STAND['API_TOTAL_CPUS'],
|
||
api_total_memory_mb=DEFAULT_STAND['API_TOTAL_MEMORY_MB'],
|
||
pg_cpus=DEFAULT_STAND['PG_CPUS'],
|
||
pool=DEFAULT_STAND['DATABASE_POOL_MAX'],
|
||
lb_cpus=DEFAULT_STAND['LB_CPUS'],
|
||
loadgen_cpus=args.loadgen_cpus,
|
||
loadgen_memory=args.loadgen_memory,
|
||
)
|
||
)
|
||
cmd_analyze(argparse.Namespace())
|
||
|
||
|
||
# ---------------------------------------------------------------- CLI
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description='Драйвер экспериментов lab3')
|
||
sub = parser.add_subparsers(dest='command', required=True)
|
||
|
||
stand = sub.add_parser('stand', help='стенд: up/down/status')
|
||
stand.add_argument('action', choices=['up', 'down', 'status'])
|
||
stand.add_argument('--replicas', type=int)
|
||
stand.set_defaults(func=cmd_stand)
|
||
|
||
check = sub.add_parser('check', help='проверки уровня 3/4')
|
||
check.add_argument('what', choices=['spread', 'instances', 'idempotency'])
|
||
check.add_argument('--requests', type=int, default=40)
|
||
check.set_defaults(func=cmd_check)
|
||
|
||
run = sub.add_parser('run', help='один прогон профиля')
|
||
run.add_argument('--label', required=True)
|
||
run.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
|
||
run.add_argument('--ladder')
|
||
run.add_argument('--step', type=int)
|
||
run.add_argument('--peak', type=int, help='одна полка фиксированного темпа')
|
||
run.add_argument('--duration', type=int, help='длительность полки, s')
|
||
run.add_argument('--replicas', type=int)
|
||
run.add_argument('--pool', type=int, help='DATABASE_POOL_MAX на реплику')
|
||
run.add_argument('--loadgen-cpus', type=float, help='лимит CPU генератора (контингенция: k6 упёрся в квоту)')
|
||
run.add_argument('--lb-cpus', type=float, help='лимит CPU HAProxy')
|
||
run.add_argument('--api-total-cpus', type=float, help='суммарный бюджет CPU API (делится на реплики)')
|
||
run.add_argument('--pg-cpus', type=float, help='лимит CPU PostgreSQL')
|
||
run.add_argument('--loadgen-memory', default='256m', help='лимит памяти генератора (k6 OOM на медленной стороне)')
|
||
run.add_argument('--max-vus', type=int, help='MAX_VUS генератора (по умолчанию 200)')
|
||
run.add_argument('--pre-vus', type=int, help='PRE_VUS генератора (по умолчанию 60)')
|
||
run.add_argument('--mode', choices=['write', 'read'])
|
||
run.add_argument('--spread', choices=['spread', 'hot'])
|
||
run.add_argument('--no-warmup', action='store_true')
|
||
run.set_defaults(func=cmd_run)
|
||
|
||
sweep = sub.add_parser('sweep', help='серия 1/2/4 реплики')
|
||
sweep.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
|
||
sweep.add_argument('--replicas', default='1,2,4')
|
||
sweep.add_argument('--reps', type=int, default=2)
|
||
sweep.add_argument('--extra-replica', type=int, default=0)
|
||
sweep.add_argument('--extra-reps', type=int, default=0)
|
||
sweep.add_argument('--prefix')
|
||
sweep.add_argument('--no-warmup', action='store_true')
|
||
sweep.set_defaults(func=cmd_sweep)
|
||
|
||
growth = sub.add_parser('growth', help='серия роста ресурсов')
|
||
growth.add_argument('--series', choices=['api', 'pg', 'pool'], required=True)
|
||
growth.add_argument('--rate', type=int)
|
||
growth.add_argument('--duration', type=int, default=60)
|
||
growth.add_argument('--base-profile', default='p-a')
|
||
growth.add_argument('--no-warmup', action='store_true')
|
||
growth.add_argument('--suffix', default='', help='суффикс метки: различает прогоны одной точки на разных темпах')
|
||
growth.set_defaults(func=cmd_growth)
|
||
|
||
failover = sub.add_parser('failover', help='фазовый драйвер отказа реплики')
|
||
failover.add_argument('--replicas', default='all')
|
||
failover.add_argument('--rate', type=int, default=200)
|
||
failover.add_argument('--duration', type=int, default=60)
|
||
failover.add_argument('--pre', type=float, default=8.0, help='секунд нагрузки до отказа')
|
||
failover.add_argument('--hold', type=float, default=20.0, help='секунд до восстановления реплики')
|
||
failover.set_defaults(func=cmd_failover)
|
||
|
||
rolling = sub.add_parser('rolling-update', help='поочерёдная замена реплик')
|
||
rolling.add_argument('--under-load', action='store_true')
|
||
rolling.add_argument('--rate', type=int, default=200)
|
||
rolling.add_argument('--duration', type=int, default=180)
|
||
rolling.add_argument('--build-id')
|
||
rolling.add_argument('--pause', type=float, default=2.0)
|
||
rolling.set_defaults(func=cmd_rolling)
|
||
|
||
ab = sub.add_parser('ab', help='интерливинг A/B двух образов')
|
||
ab.add_argument('--before', required=True)
|
||
ab.add_argument('--after', required=True)
|
||
ab.add_argument('--profile', default='p-b')
|
||
ab.add_argument('--replicas', type=int, default=4)
|
||
ab.add_argument('--reps', type=int, default=3)
|
||
ab.add_argument('--no-warmup', action='store_true')
|
||
ab.add_argument('--api-total-cpus', type=float, default=DEFAULT_STAND['API_TOTAL_CPUS'])
|
||
ab.add_argument('--api-total-memory-mb', type=int, default=DEFAULT_STAND['API_TOTAL_MEMORY_MB'])
|
||
ab.add_argument('--pg-cpus', type=float, default=DEFAULT_STAND['PG_CPUS'])
|
||
ab.add_argument('--pool', type=int, default=DEFAULT_STAND['DATABASE_POOL_MAX'])
|
||
ab.add_argument('--lb-cpus', type=float, default=DEFAULT_STAND['LB_CPUS'])
|
||
ab.add_argument('--loadgen-cpus', type=float, default=DEFAULT_STAND['LOADGEN_CPUS'])
|
||
ab.add_argument('--loadgen-memory', default=DEFAULT_STAND['LOADGEN_MEMORY'])
|
||
ab.add_argument('--max-vus', type=int, default=1200)
|
||
ab.add_argument('--prefix', default='ab', help='префикс меток (различает серии A/B)')
|
||
ab.set_defaults(func=cmd_ab)
|
||
|
||
analyze = sub.add_parser('analyze', help='агрегаты и графики из raw')
|
||
analyze.set_defaults(func=cmd_analyze)
|
||
|
||
watch = sub.add_parser('watch', help='сэмплер без нагрузки: наблюдение за стендом')
|
||
watch.add_argument('--seconds', type=float, default=30)
|
||
watch.add_argument('--out', required=True)
|
||
watch.set_defaults(func=cmd_watch)
|
||
|
||
quiesce = sub.add_parser('quiesce', help='фон хоста: up/down/status')
|
||
quiesce.add_argument('action', choices=['up', 'down', 'status'])
|
||
quiesce.set_defaults(func=cmd_quiesce)
|
||
|
||
extras = sub.add_parser('track1-extras', help='догоняющие серии трека 1 (VU-своп, блоки «почему», A/B на бюджете по умолчанию)')
|
||
extras.add_argument('--part', choices=['all', 'vuswap', 'why', 'abdiag'], default='all')
|
||
extras.add_argument('--rate', type=int, default=1500, help='темп блока «почему»')
|
||
extras.add_argument('--duration', type=int, default=60)
|
||
extras.add_argument('--reps', type=int, default=2)
|
||
extras.add_argument('--max-vus', type=int, default=900)
|
||
extras.add_argument('--prefix', default='why')
|
||
extras.add_argument('--ab-reps', type=int, default=3)
|
||
extras.add_argument('--before', default='mtusi-messenger:lab3-before')
|
||
extras.add_argument('--after', default='mtusi-messenger:lab3-after')
|
||
extras.add_argument('--loadgen-cpus', type=float, default=DEFAULT_STAND['LOADGEN_CPUS'])
|
||
extras.add_argument('--loadgen-memory', default=DEFAULT_STAND['LOADGEN_MEMORY'],
|
||
help='память генератора: при 900 VU и секундных задержках 256 MiB мало (k6 убивается OOM)')
|
||
extras.set_defaults(func=cmd_track1_extras)
|
||
|
||
track1 = sub.add_parser('track1', help='полная серия трека 1')
|
||
track1.add_argument('--reps', type=int, default=2)
|
||
track1.add_argument('--ab-before', default='mtusi-messenger:lab3-before')
|
||
track1.add_argument('--ab-after', default=None)
|
||
track1.add_argument('--ab-reps', type=int, default=3)
|
||
track1.add_argument('--no-quiesce', action='store_true')
|
||
track1.set_defaults(func=cmd_track1)
|
||
return parser
|
||
|
||
|
||
def main() -> None:
|
||
parser = build_parser()
|
||
args = parser.parse_args()
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
args.func(args)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|