Files
mtusi2026lab3/scripts/lab3/lab3.py
T
ilya.tyunin 410e1ea3ad lab3: харнесс экспериментов (k6, сэмплер, анализ) и операторские цели Makefile
- scripts/load/common.js + write-ladder.js + write-steady.js: лестница плоских
  полок constant-arrival-rate, пошаговые счётчики/тренды (k6 не создаёт
  подметрики произвольных тегов), подготовка синтетических пользователей и
  чатов, сводка JSON в stdout без файлов из контейнера; MODE=write|read.
- scripts/lab3/lab3.py (stdlib): stand, check (spread/instances/idempotency),
  run, sweep, growth, failover, rolling-update, ab, analyze, quiesce, track1.
  Сэмплер: docker stats (--no-stream, поток склеивает JSON и печатает ANSI),
  HAProxy stats CSV (заголовок помечен '#'), /internal/metrics каждой реплики,
  pg_stat_database/pg_stat_activity/pg_stat_statements, фон хоста. raw:
  <label>.k6.json, <label>.samples.jsonl.gz, <label>.manifest.json.
- Анализ: таблицы по каждому прогону, серия A, диагностика, рост, A/B и
  графики (throughput, latency/errors, VU генератора, CPU контейнеров во
  времени, распределение по слотам LB, рост, A/B с min..max).

Проверено: analyze на пустом каталоге не падает; повторный analyze даёт те же
числа; прогон run --label smoke-p-a (30 s, 100 req/s) — 100.03 достигнутых
req/s, 0 ошибок, 795 сэмплов, графики и таблицы непустые.
2026-10-15 21:25:03 +03:00

2287 lines
94 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Драйвер экспериментов lab3: стенд, доказательства уровней 3/4 и трек 1.
Запуск из корня репозитория: python3 scripts/lab3/lab3.py <подкоманда> ...
Подкоманды:
stand up|down|status поднять/остановить/показать стенд
check spread|instances|idempotency
разные реплики на одной сессии; карта X-Instance-Id;
фильтр e2e по идемпотентности
run --label ... один прогон профиля с сэмплером
sweep --profile p-a серия 1/2/4 реплики (интерливинг повторов)
growth --series api|pg|pool серия роста ресурсов на фиксированном темпе
failover --replicas all фазовый драйвер отказа реплики
rolling-update --under-load поочерёдная замена реплик под нагрузкой
ab --before T --after T интерливинг A/B двух образов
analyze агрегаты и графики из raw
quiesce up|down|status обёртка scripts/quiesce.sh
track1 полная серия трека 1 по зафиксированному плану
Только стандартная библиотека (графики — matplotlib на хосте). Данные прогонов
складываются в evidence/track1/raw: <label>.k6.json (сводка k6),
<label>.samples.jsonl.gz (сэмплер), <label>.manifest.json (условия измерения).
"""
from __future__ import annotations
import argparse
import csv
import gzip
import json
import math
import os
import re
import shutil
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
ENV_FILE = ROOT / '.env'
STAND_ENV = ROOT / '.env.stand'
QUIESCE_STATE = ROOT / '.lab3-quiesce.json'
RAW_DIR = ROOT / 'evidence/track1/raw'
TRACK1_DIR = ROOT / 'evidence/track1'
GRAPH_DIR = TRACK1_DIR / 'graphs'
LEVEL3_DIR = ROOT / 'evidence/level3'
LEVEL4_DIR = ROOT / 'evidence/level4'
PROJECT = 'mtusi-lab3'
BASE_URL = os.environ.get('LAB3_BASE_URL', 'http://127.0.0.1:8080')
LB_STATS_URL = os.environ.get('LAB3_LB_STATS', 'http://127.0.0.1:8404/stats;csv')
COMPOSE = ['docker', 'compose', '--env-file', str(ENV_FILE), '--env-file', str(STAND_ENV)]
# Фиксированный бюджет серии трека 1: суммарные ресурсы API одинаковы для всех
# сравниваемых конфигураций, поэтому на реплику приходится 3/N CPU и 1.5/N GiB.
API_TOTAL_CPUS = 3.0
API_TOTAL_MEMORY_MB = 1536
DEFAULT_STAND = {
'API_TOTAL_CPUS': API_TOTAL_CPUS,
'API_TOTAL_MEMORY_MB': API_TOTAL_MEMORY_MB,
'API_REPLICAS': 2,
'PG_CPUS': 1.5,
'PG_MEMORY': '512m',
'REDIS_CPUS': 0.25,
'REDIS_MEMORY': '128m',
'LB_CPUS': 0.25,
'LB_MEMORY': '128m',
'LOADGEN_CPUS': 1.5,
'LOADGEN_MEMORY': '256m',
'API_IMAGE': 'mtusi-messenger:lab3',
'BUILD_ID': 'base',
'DATABASE_POOL_MAX': 10,
}
PROFILES = {
'p-a': {
'mode': 'write',
'spread': 'spread',
'ladder': [100, 200, 300, 400, 550, 700, 900, 1100],
},
'p-b': {
'mode': 'write',
'spread': 'hot',
'ladder': [50, 100, 150, 200, 250, 300, 350, 400],
},
'p-c': {
'mode': 'read',
'spread': 'spread',
'ladder': [200, 400, 700, 1000, 1500, 2000, 2500, 3000],
},
}
STEP_SECONDS = 15
WARMUP_SECONDS = 30
CHATS = int(os.environ.get('LAB3_CHATS', '24'))
def log(message: str) -> None:
print(f'[{datetime.now().strftime("%H:%M:%S")}] {message}', flush=True)
def now_iso() -> str:
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def run_command(
command: list[str],
*,
check: bool = True,
timeout: float | None = None,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
result = subprocess.run(
command,
cwd=ROOT,
capture_output=True,
text=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if check and result.returncode != 0:
raise SystemExit(
f'command failed ({result.returncode}): {" ".join(command)}\n'
f'{result.stdout[-2000:]}\n{result.stderr[-2000:]}'
)
return result
def compose(*args: str, check: bool = True, timeout: float | None = 300) -> subprocess.CompletedProcess:
return run_command([*COMPOSE, *args], check=check, timeout=timeout)
def read_json(path: Path, default=None):
if not path.exists():
return default
return json.loads(path.read_text())
def write_json(path: Path, payload) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=1, sort_keys=True) + '\n')
# ---------------------------------------------------------------- окружение стенда
def replica_limits(replicas: int, total_cpus: float, total_memory_mb: int) -> tuple[float, str]:
return (
round(total_cpus / replicas, 3),
f'{int(total_memory_mb // replicas)}m',
)
def render_stand_env(config: dict) -> str:
cpus, memory = replica_limits(
int(config['API_REPLICAS']),
float(config['API_TOTAL_CPUS']),
int(config['API_TOTAL_MEMORY_MB']),
)
values = {
**config,
'API_CPUS_PER_REPLICA': cpus,
'API_MEMORY_PER_REPLICA': memory,
}
lines = ['# Сгенерировано scripts/lab3/lab3.py: бюджет серии и лимиты реплик.']
for key in sorted(values, key=str):
lines.append(f'{key}={values[key]}')
return '\n'.join(lines) + '\n'
def load_stand_config() -> dict:
stored = read_json(STAND_ENV.with_suffix('.json'), None)
config = dict(DEFAULT_STAND)
if STAND_ENV.exists():
for line in STAND_ENV.read_text().splitlines():
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
config[key] = value
if stored:
config.update(stored)
return config
def numeric(value: str) -> float | int:
number = float(value)
return int(number) if number.is_integer() else number
def save_stand_config(config: dict) -> dict:
STAND_ENV.write_text(render_stand_env(config))
write_json(
STAND_ENV.with_suffix('.json'),
{key: str(value) for key, value in config.items()},
)
return config
def current_replicas() -> int:
return int(load_stand_config().get('API_REPLICAS', 2))
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
"""Применяет конфигурацию к стенду: лимиты, число реплик, образ."""
save_stand_config(config)
replicas = int(config['API_REPLICAS'])
args = ['up', '-d', '--wait', '--no-deps', '--scale', f'api={replicas}']
if force_recreate:
args.append('--force-recreate')
args.append('api')
compose(*args, timeout=600)
wait_for_lb()
def wait_for_lb(timeout: float = 120) -> None:
deadline = time.monotonic() + timeout
last = 'no response'
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f'{BASE_URL}/health/ready', timeout=3) as response:
if response.status == 200:
return
except Exception as error: # noqa: BLE001 - диагностика ожидания важнее типа
last = str(error)
time.sleep(1)
raise SystemExit(f'stand did not become ready within {timeout}s: {last}')
def container_ids(service: str = 'api') -> list[str]:
result = run_command(
[
'docker',
'ps',
'-q',
'--filter',
f'label=com.docker.compose.project={PROJECT}',
'--filter',
f'label=com.docker.compose.service={service}',
]
)
return [line for line in result.stdout.split() if line]
def inspect(container: str, template: str) -> str:
return run_command(['docker', 'inspect', '-f', template, container]).stdout.strip()
def replica_map() -> dict[str, dict]:
"""X-Instance-Id -> контейнер: в Docker это hostname = короткий ID."""
mapping = {}
for container in container_ids('api'):
hostname = inspect(container, '{{.Config.Hostname}}')
mapping[hostname] = {
'container': container,
'name': inspect(container, '{{.Name}}').lstrip('/'),
'image_id': inspect(container, '{{.Image}}'),
'started_at': inspect(container, '{{.State.StartedAt}}'),
'cpus': int(inspect(container, '{{.HostConfig.NanoCpus}}')) / 1e9,
'memory_mb': int(inspect(container, '{{.HostConfig.Memory}}')) // (1024 * 1024),
}
return mapping
def image_versions() -> dict:
versions = {}
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
container = container_ids(name)
if container:
versions[name] = inspect(container[0], '{{.Config.Image}}')
versions['postgres_runtime'] = run_command(
['docker', 'exec', container_ids('postgres')[0], 'postgres', '--version'], check=False
).stdout.strip()
versions['redis_runtime'] = run_command(
['docker', 'exec', container_ids('redis')[0], 'redis-server', '--version'], check=False
).stdout.strip()
versions['haproxy_runtime'] = run_command(
['docker', 'exec', container_ids('haproxy')[0], 'haproxy', '-v'], check=False
).stdout.splitlines()[0].strip()
versions['k6'] = run_command(
['docker', 'run', '--rm', 'grafana/k6:0.54.0', 'version'], check=False, timeout=120
).stdout.strip()
if shutil.which('node'):
versions['node_host'] = run_command(['node', '--version'], check=False).stdout.strip()
return versions
def dataset_size() -> dict:
containers = container_ids('postgres')
if not containers:
return {}
query = (
"SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), "
"(SELECT count(*) FROM messages), "
"(SELECT count(*) FROM idempotency_keys), "
"(SELECT count(*) FROM chat_members)"
)
result = psql(query)
if not result:
return {}
values = result.split('|')
return dict(zip(['users', 'chats', 'messages', 'idempotency_keys', 'chat_members'], (int(v) for v in values)))
def psql(query: str) -> str:
env = psql_env()
result = run_command(
[
'docker',
'exec',
container_ids('postgres')[0],
'psql',
'-U',
env['POSTGRES_USER'],
'-d',
env['POSTGRES_DB'],
'-tA',
'-F',
'|',
'-c',
query,
],
check=False,
timeout=30,
)
return result.stdout.strip()
_PSQL_ENV: dict[str, str] = {}
def psql_env() -> dict[str, str]:
if not _PSQL_ENV:
for line in ENV_FILE.read_text().splitlines():
if '=' in line and not line.startswith('#'):
key, value = line.split('=', 1)
_PSQL_ENV[key] = value
return _PSQL_ENV
def host_background() -> dict:
memory = {}
for line in Path('/proc/meminfo').read_text().splitlines():
if line.startswith(('MemTotal:', 'MemAvailable:')):
key, value = line.split(':')
memory[key] = int(value.strip().split()[0]) // 1024
load = os.getloadavg()
return {
'load_average': [round(value, 2) for value in load],
'mem_total_mib': memory.get('MemTotal'),
'mem_available_mib': memory.get('MemAvailable'),
'quiesced': QUIESCE_STATE.exists(),
}
def bench_cpus() -> int:
return os.cpu_count() or 8
# ---------------------------------------------------------------- HTTP-клиент
class ApiError(RuntimeError):
pass
class Client:
"""Минимальный клиент для проверок и фазовых драйверов (stdlib)."""
def __init__(self, base_url: str = BASE_URL, timeout: float = 5.0):
self.base = base_url.rstrip('/')
self.timeout = timeout
def call(self, method: str, path: str, body=None, token: str | None = None, headers=None):
data = None
request_headers = dict(headers or {})
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
request_headers['content-type'] = 'application/json'
if token:
request_headers['authorization'] = f'Bearer {token}'
request = urllib.request.Request(
self.base + path, data=data, headers=request_headers, method=method
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = response.read()
status = response.status
response_headers = dict(response.headers)
except urllib.error.HTTPError as error:
payload = error.read()
status = error.code
response_headers = dict(error.headers)
parsed = json.loads(payload) if payload else None
return {'status': status, 'json': parsed, 'headers': response_headers}
def register(self, label: str) -> dict:
import uuid
username = f'lab3_{label}_{uuid.uuid4().hex[:10]}'
password = f'Lab3-{uuid.uuid4().hex}'
created = self.call(
'POST',
'/api/v1/users',
{'username': username, 'display_name': f'lab3 {label}', 'password': password},
)
if created['status'] != 201:
raise ApiError(f'register failed: {created}')
import base64
credential = base64.b64encode(f'{username}:{password}'.encode()).decode()
session = self.call(
'POST',
'/api/v1/auth/sessions',
None,
headers={'authorization': f'Basic {credential}'},
)
if session['status'] != 201:
raise ApiError(f'session failed: {session}')
return {
'id': created['json']['id'],
'user': created['json'],
'token': session['json']['token'],
}
def create_chat(self, token: str, member_ids: list[str], title: str) -> str:
created = self.call(
'POST', '/api/v1/chats', {'title': title, 'member_ids': member_ids}, token=token
)
if created['status'] != 201:
raise ApiError(f'chat failed: {created}')
return created['json']['id']
def post_message(self, token: str, chat_id: str, text: str, key: str) -> dict:
return self.call(
'POST',
f'/api/v1/chats/{chat_id}/messages',
{'text': text},
token=token,
headers={'idempotency-key': key},
)
def history(self, token: str, chat_id: str) -> list[dict]:
items: list[dict] = []
cursor = None
while True:
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
if cursor:
from urllib.parse import quote
path += '&cursor=' + quote(cursor, safe='')
page = self.call('GET', path, token=token)
if page['status'] != 200:
raise ApiError(f'history failed: {page}')
items.extend(page['json']['items'])
cursor = page['json']['next_cursor']
if cursor is None:
return items
# ---------------------------------------------------------------- сэмплер
class Sampler:
"""Фоновые наблюдения за стендом: docker stats, HAProxy, метрики, PostgreSQL."""
def __init__(self) -> None:
self.records: list[dict] = []
self._lock = threading.Lock()
self._stop = threading.Event()
self._threads: list[threading.Thread] = []
self.errors: list[str] = []
def _record(self, payload: dict) -> None:
with self._lock:
self.records.append(payload)
def _guard(self, name: str, work) -> None:
try:
work()
except Exception as error: # noqa: BLE001 - сэмплер не должен валить прогон
with self._lock:
self.errors.append(f'{name}: {type(error).__name__}: {error}')
def _loop(self, name: str, interval: float, step) -> None:
def work() -> None:
while not self._stop.is_set():
step()
self._stop.wait(interval)
self._threads.append(threading.Thread(target=lambda: self._guard(name, work), daemon=True, name=name))
def start(self) -> 'Sampler':
# docker stats в режиме потока перерисовывает экран (ANSI-последовательности)
# и склеивает JSON разных контейнеров в одну строку, поэтому опрашиваем
# его порциями --no-stream: одна строка = один JSON.
self._loop('docker_stats', 1.5, self._sample_stats)
self._loop('haproxy', 0.5, self._haproxy)
self._loop('metrics', 2.0, self._app_metrics)
self._loop('postgres', 2.0, self._postgres)
self._loop('pg_statements', 10.0, self._pg_statements)
self._loop('host', 5.0, lambda: self._record({'t': now_iso(), 'source': 'host', **host_background()}))
for thread in self._threads:
thread.start()
return self
def stop(self) -> list[dict]:
self._stop.set()
for thread in self._threads:
thread.join(timeout=10)
return self.records
def _sample_stats(self) -> None:
result = run_command(
['docker', 'stats', '--no-stream', '--format', '{{json .}}'],
check=False,
timeout=30,
)
for line in result.stdout.splitlines():
try:
row = json.loads(line)
except ValueError:
continue
self._record(
{
't': now_iso(),
'source': 'docker_stats',
'name': row.get('Name') or row.get('Container'),
'cpu_percent': float(str(row.get('CPUPerc', '0%')).rstrip('%') or 0),
'mem_mib': round(_mib(row.get('MemUsage', '0MiB / 0MiB')), 2),
'mem_limit_mib': round(_mib(str(row.get('MemUsage', '')).split('/')[-1]), 2),
'pids': _int(row.get('PIDs')),
}
)
def _haproxy(self) -> None:
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
text = response.read().decode('utf-8', 'replace')
lines = text.splitlines()
# HAProxy помечает строку заголовка как комментарий: '# pxname,svname,...'
if lines and lines[0].startswith('#'):
lines[0] = lines[0].lstrip('#').lstrip()
for row in csv.DictReader(lines):
if row.get('pxname') not in ('api', 'http'):
continue
self._record(
{
't': now_iso(),
'source': 'haproxy',
'proxy': row.get('pxname'),
'server': row.get('svname'),
'addr': row.get('addr'),
'status': row.get('status'),
'check': row.get('check_status'),
'scur': _int(row.get('scur')),
'qcur': _int(row.get('qcur')),
'stot': _int(row.get('stot')),
'ereq': _int(row.get('ereq')),
'dresp': _int(row.get('dresp')),
'wretr': _int(row.get('wretr')),
}
)
def _app_metrics(self) -> None:
containers = container_ids('api')
if not containers:
return
for container in containers:
result = run_command(
[
'docker',
'exec',
container,
'node',
'-e',
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(t=>process.stdout.write(t))",
],
check=False,
timeout=20,
)
if result.returncode != 0 or not result.stdout.strip():
continue
try:
payload = json.loads(result.stdout)
except ValueError:
continue
self._record(
{
't': now_iso(),
'source': 'metrics',
'container': container,
'instance_id': payload.get('instance_id'),
'build_id': payload.get('build_id'),
'counters': payload.get('counters', {}),
'latency_ms': payload.get('latency_ms', {}),
}
)
def _postgres(self) -> None:
query = (
"SELECT (SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT xact_rollback FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT blks_read FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT blks_hit FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
" (SELECT count(*) FROM pg_stat_activity),"
" (SELECT count(*) FROM messages),"
" (SELECT pg_total_relation_size('messages'))"
)
output = psql(query)
if not output:
return
fields = output.split('|')
if len(fields) != 10:
return
self._record(
{
't': now_iso(),
'source': 'postgres',
**{
key: int(value)
for key, value in zip(
[
'xact_commit',
'xact_rollback',
'blks_read',
'blks_hit',
'deadlocks',
'lock_waiters',
'active',
'connections',
'messages',
'messages_bytes',
],
fields,
)
},
}
)
def _pg_statements(self) -> None:
output = psql(
"SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 3),"
" left(regexp_replace(query, '\\s+', ' ', 'g'), 90)"
" FROM pg_stat_statements WHERE query ILIKE '%messages%' OR query ILIKE '%chats%'"
" ORDER BY total_exec_time DESC LIMIT 5"
)
if not output:
return
rows = []
for line in output.splitlines():
parts = line.split('|')
if len(parts) == 4:
rows.append(
{
'calls': int(parts[0]),
'total_ms': float(parts[1]),
'mean_ms': float(parts[2]),
'query': parts[3],
}
)
self._record({'t': now_iso(), 'source': 'pg_stat_statements', 'top': rows})
def _mib(value: str) -> float:
text = str(value).strip()
match = re.match(r'([\d.]+)\s*([KMGT]i?B)?', text)
if not match:
return 0.0
number = float(match.group(1))
unit = (match.group(2) or 'B').lower()
factor = {
'b': 1 / (1024 * 1024),
'kib': 1 / 1024,
'kb': 1 / 1024,
'mib': 1,
'mb': 1,
'gib': 1024,
'gb': 1024,
'tib': 1024 * 1024,
}.get(unit, 1)
return number * factor
def _int(value) -> int:
try:
return int(str(value).strip() or 0)
except ValueError:
return 0
# ---------------------------------------------------------------- прогоны k6
def k6_run(script: str, label: str, env: dict[str, str], timeout: float) -> dict:
args = [*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps']
for key, value in sorted(env.items()):
args += ['-e', f'{key}={value}']
args += ['loadgen', 'run', f'/load/{script}']
result = run_command(args, check=False, timeout=timeout)
summary = None
for line in reversed(result.stdout.splitlines()):
stripped = line.strip()
if not stripped.startswith('{'):
continue
try:
candidate = json.loads(stripped)
except ValueError:
continue
if isinstance(candidate, dict) and 'steps' in candidate:
summary = candidate
break
if summary is None:
raise SystemExit(
f'k6 run {label} produced no summary (exit {result.returncode})\n'
f'{result.stdout[-1500:]}\n{result.stderr[-800:]}'
)
summary['exit_code'] = result.returncode
summary['stderr_tail'] = result.stderr.strip().splitlines()[-3:]
return summary
def load_env(profile: str, mode: str, spread: str, label: str) -> dict[str, str]:
return {
'BASE_URL': 'http://haproxy:8080',
'PROFILE': spread,
'MODE': mode,
'CHATS': str(CHATS),
'LABEL': label,
}
def run_measured(
label: str,
profile: str,
*,
ladder: list[int] | None = None,
step_seconds: int = STEP_SECONDS,
rate: int | None = None,
duration: int | None = None,
mode: str | None = None,
spread: str | None = None,
warmup: bool = True,
extra: dict | None = None,
phases: dict | None = None,
) -> dict:
"""Один прогон: (необязательный прогрев) + измерение + сэмплер + манифест."""
profile_config = PROFILES[profile]
mode = mode or profile_config['mode']
spread = spread or profile_config['spread']
ladder = ladder or profile_config['ladder']
if rate is not None:
ladder = [rate]
steady = rate is not None and duration is not None
script = 'write-steady.js' if steady else 'write-ladder.js'
RAW_DIR.mkdir(parents=True, exist_ok=True)
stand = load_stand_config()
if warmup:
log(f'прогрев {label}: {WARMUP_SECONDS}s на {ladder[0]} req/s')
k6_run(
script,
f'warmup-{label}',
{
**load_env(profile, mode, spread, f'warmup-{label}'),
**({'RATE': str(ladder[0]), 'DURATION': str(WARMUP_SECONDS)} if steady else {'LADDER': str(ladder[0]), 'STEP_SECONDS': str(WARMUP_SECONDS)}),
},
timeout=WARMUP_SECONDS + 120,
)
sampler = Sampler().start()
started = now_iso()
env = {
**load_env(profile, mode, spread, label),
**({'RATE': str(ladder[0]), 'DURATION': str(duration)} if steady else {'LADDER': ','.join(str(r) for r in ladder), 'STEP_SECONDS': str(step_seconds)}),
}
log(f'нагрузка {label}: {"steady " + str(ladder[0]) + " req/s" if steady else "ladder " + str(ladder)}')
summary = k6_run(script, label, env, timeout=(duration or len(ladder) * step_seconds) + 240)
finished = now_iso()
samples = sampler.stop()
write_raw(label, summary, samples, stand, started, finished, phases or {}, sampler.errors)
return summary
def write_raw(
label: str,
summary: dict,
samples: list[dict],
stand: dict,
started: str,
finished: str,
phases: dict,
sampler_errors: list[str],
) -> None:
RAW_DIR.mkdir(parents=True, exist_ok=True)
write_json(RAW_DIR / f'{label}.k6.json', summary)
with gzip.open(RAW_DIR / f'{label}.samples.jsonl.gz', 'wt', encoding='utf-8') as handle:
for record in samples:
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
if phases:
write_json(RAW_DIR / f'{label}.phases.json', phases)
manifest = {
'label': label,
'profile': summary.get('profile'),
'mode': summary.get('mode'),
'kind': summary.get('kind'),
'started_at': started,
'finished_at': finished,
'steps': [step['rate'] for step in summary.get('steps', [])],
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
'stand': stand,
'replicas': replica_map(),
'dataset': dataset_size(),
'versions': image_versions(),
'host': {'cpus': bench_cpus(), 'background': host_background()},
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
'quiesce': read_json(QUIESCE_STATE, None),
'phases': phases,
'sampler_errors': sampler_errors,
'sample_count': len(samples),
}
write_json(RAW_DIR / f'{label}.manifest.json', manifest)
log(f'raw: {label} ({len(samples)} samples, errors: {len(sampler_errors)})')
def load_raw(label: str) -> tuple[dict | None, list[dict], dict | None]:
summary = read_json(RAW_DIR / f'{label}.k6.json', None)
manifest = read_json(RAW_DIR / f'{label}.manifest.json', None)
samples: list[dict] = []
path = RAW_DIR / f'{label}.samples.jsonl.gz'
if path.exists():
with gzip.open(path, 'rt', encoding='utf-8') as handle:
for line in handle:
try:
samples.append(json.loads(line))
except ValueError:
continue
return summary, samples, manifest
def raw_labels() -> list[str]:
if not RAW_DIR.exists():
return []
labels = []
for path in sorted(RAW_DIR.glob('*.k6.json')):
label = path.name[: -len('.k6.json')]
if label.startswith('warmup-'):
continue
labels.append(label)
return labels
# ---------------------------------------------------------------- анализ
def step_of(summary: dict, rate: int) -> dict | None:
for step in summary.get('steps', []):
if int(step['rate']) == int(rate):
return step
return None
def median(values: list[float]) -> float:
return round(statistics.median(values), 2) if values else 0.0
def spread(values: list[float]) -> tuple[float, float]:
return (round(min(values), 2), round(max(values), 2)) if values else (0.0, 0.0)
def group_runs(labels: list[str], predicate) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {}
for label in labels:
if predicate(label):
key = label.split('#')[0]
groups.setdefault(key, []).append(label)
return groups
def saturation(series: list[dict]) -> dict:
"""Точка сатурации: первый шаг, где достигнутый RPS отстал от предложенного."""
for step in series:
if step['achieved_rps'] < 0.95 * step['rate']:
return {'rate': step['rate'], 'achieved_rps': step['achieved_rps'], 'step': step}
return {'rate': series[-1]['rate'] if series else 0, 'achieved_rps': series[-1]['achieved_rps'] if series else 0, 'step': series[-1] if series else None}
def summarise_series(labels: list[str]) -> dict:
"""Медианы по повторам для одной конфигурации: шаги лестницы и пик."""
steps: dict[int, dict] = {}
for label in labels:
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
for step in summary.get('steps', []):
rate = int(step['rate'])
entry = steps.setdefault(rate, {'rate': rate, 'achieved': [], 'p95': [], 'failed': [], 'vus': [], 'offered': []})
entry['achieved'].append(float(step['achieved_rps']))
entry['p95'].append(float(step['p95']))
entry['failed'].append(float(step['failed']))
entry['vus'].append(float(step.get('vus_active') or 0))
entry['offered'].append(float(step['rate']))
rows = []
for rate in sorted(steps):
entry = steps[rate]
rows.append(
{
'rate': rate,
'achieved_rps': median(entry['achieved']),
'achieved_min': spread(entry['achieved'])[0],
'achieved_max': spread(entry['achieved'])[1],
'p95': median(entry['p95']),
'p95_max': spread(entry['p95'])[1],
'failed': sum(entry['failed']),
'vus_active': max(entry['vus']) if entry['vus'] else 0,
'reps': len(entry['achieved']),
}
)
ceiling = max((row['achieved_rps'] for row in rows), default=0.0)
return {'rows': rows, 'ceiling_rps': ceiling, 'saturation': saturation(rows), 'labels': labels}
def container_stats(samples: list[dict], prefix: str = 'mtusi-lab3-') -> dict:
stats: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'docker_stats':
continue
name = record.get('name') or ''
if not name.startswith(prefix):
continue
entry = stats.setdefault(name, {'cpu': [], 'mem': [], 'limit': 0.0})
entry['cpu'].append(float(record.get('cpu_percent') or 0))
entry['mem'].append(float(record.get('mem_mib') or 0))
entry['limit'] = max(entry['limit'], float(record.get('mem_limit_mib') or 0))
return {
name: {
'cpu_mean': round(statistics.fmean(entry['cpu']), 2) if entry['cpu'] else 0.0,
'cpu_peak': round(max(entry['cpu']), 2) if entry['cpu'] else 0.0,
'mem_mean': round(statistics.fmean(entry['mem']), 1) if entry['mem'] else 0.0,
'mem_peak': round(max(entry['mem']), 1) if entry['mem'] else 0.0,
'mem_limit': entry['limit'],
}
for name, entry in sorted(stats.items())
}
def postgres_stats(samples: list[dict]) -> dict:
rows = [record for record in samples if record.get('source') == 'postgres']
if len(rows) < 2:
return {}
first, last = rows[0], rows[-1]
seconds = max(1.0, (_timestamp(last['t']) - _timestamp(first['t'])))
hits = last['blks_hit'] - first['blks_hit']
reads = last['blks_read'] - first['blks_read']
return {
'xact_per_s': round((last['xact_commit'] - first['xact_commit']) / seconds, 1),
'rollback_per_s': round((last['xact_rollback'] - first['xact_rollback']) / seconds, 2),
'blks_hit_ratio': round(hits / (hits + reads), 4) if hits + reads else 0.0,
'lock_waiters_peak': max(row['lock_waiters'] for row in rows),
'active_peak': max(row['active'] for row in rows),
'connections_peak': max(row['connections'] for row in rows),
'deadlocks': last['deadlocks'] - first['deadlocks'],
'messages': last['messages'],
'messages_mib': round(last['messages_bytes'] / (1024 * 1024), 1),
}
def haproxy_stats(samples: list[dict]) -> dict:
servers: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'haproxy':
continue
entry = servers.setdefault(
record['server'],
{'requests': [], 'errors': [], 'qcur': [], 'scur': [], 'statuses': set(), 'addr': set(), 'events': []},
)
entry['requests'].append((record['t'], record['stot']))
entry['errors'].append(record['ereq'])
entry['qcur'].append(record['qcur'])
entry['scur'].append(record['scur'])
entry['statuses'].add(record['status'])
entry['addr'].add(record['addr'])
entry['events'].append((record['t'], record['status'], record['check']))
result = {}
for name, entry in servers.items():
if not entry['requests']:
continue
first, last = entry['requests'][0], entry['requests'][-1]
seconds = max(1.0, _timestamp(last[0]) - _timestamp(first[0]))
transitions = []
previous = None
for moment, status, check in entry['events']:
if status != previous:
transitions.append({'t': moment, 'status': status, 'check': check})
previous = status
result[name] = {
'requests': last[1] - first[1],
'rps': round((last[1] - first[1]) / seconds, 2),
'errors': entry['errors'][-1],
'qcur_peak': max(entry['qcur']),
'scur_peak': max(entry['scur']),
'statuses': sorted(entry['statuses']),
'addresses': sorted(entry['addr']),
'transitions': transitions,
}
return result
def app_metrics_delta(samples: list[dict]) -> dict:
per_container: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'metrics':
continue
per_container.setdefault(record['container'], []).append(record)
result = {}
for container, rows in per_container.items():
if len(rows) < 2:
continue
first, last = rows[0], rows[-1]
counters = {}
for name, value in last['counters'].items():
counters[name] = value - first['counters'].get(name, 0)
result[container] = {
'instance_id': last.get('instance_id'),
'build_id': last.get('build_id'),
'counters': {name: value for name, value in sorted(counters.items()) if value},
'latency_ms': last.get('latency_ms', {}),
'samples': len(rows),
}
return result
def _timestamp(value: str) -> float:
return datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()
def write_markdown(path: Path, title: str, lines: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f'# {title}\n\n' + '\n'.join(lines) + '\n', encoding='utf-8')
def table(header: list[str], rows: list[list]) -> list[str]:
lines = ['| ' + ' | '.join(header) + ' |', '| ' + ' | '.join('---' for _ in header) + ' |']
for row in rows:
lines.append('| ' + ' | '.join(str(value) for value in row) + ' |')
return lines + ['']
def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
"""Приводит стенд к нужной конфигурации; пересоздаёт только при изменении."""
desired = {**load_stand_config(), **{key: str(value) for key, value in overrides.items()}}
rendered = render_stand_env(desired)
if not STAND_ENV.exists() or STAND_ENV.read_text() != rendered:
log(f'перенастройка стенда: {overrides}')
apply_stand(desired, force_recreate=force_recreate)
else:
wait_for_lb()
return desired
# ---------------------------------------------------------------- подкоманды
def cmd_stand(args) -> None:
config = load_stand_config()
if args.action == 'up':
if args.replicas:
config['API_REPLICAS'] = str(args.replicas)
save_stand_config(config)
compose('up', '-d', '--build', '--wait', timeout=1800)
wait_for_lb()
log(f'стенд поднят: {replica_map().keys()}')
elif args.action == 'down':
compose('down')
else:
print(compose('ps').stdout)
def header(headers: dict, name: str) -> str | None:
for key, value in headers.items():
if key.lower() == name.lower():
return value
return None
def cmd_check(args) -> None:
client = Client()
if args.what == 'spread':
token = client.register('spread')['token']
counts: dict[str, int] = {}
for _ in range(args.requests):
response = client.call('GET', '/api/v1/users/me', token=token)
instance = header(response['headers'], 'X-Instance-Id')
counts[instance] = counts.get(instance, 0) + 1
payload = {
'requests': args.requests,
'distinct_instances': len(counts),
'instances': counts,
'replicas': {name: info for name, info in replica_map().items()},
'captured_at': now_iso(),
}
print(json.dumps(payload, ensure_ascii=False, indent=1))
write_json(LEVEL4_DIR / 'spread.json', payload)
elif args.what == 'instances':
print(json.dumps({'replicas': replica_map(), 'haproxy': haproxy_stats(Sampler()._haproxy() or [])}, ensure_ascii=False, indent=1))
elif args.what == 'idempotency':
LEVEL4_DIR.mkdir(parents=True, exist_ok=True)
result = run_command(
['npx', 'vitest', 'run', '--config', './vitest.config.e2e.ts', '-t', 'idempotency'],
check=False,
timeout=600,
)
output = result.stdout + result.stderr
(LEVEL4_DIR / 'idempotency-concurrency.txt').write_text(output, encoding='utf-8')
print(output[-1500:])
if result.returncode != 0:
raise SystemExit('idempotency concurrency suite failed')
else:
raise SystemExit(f'unknown check: {args.what}')
def cmd_run(args) -> None:
overrides: dict = {}
if args.replicas:
overrides['API_REPLICAS'] = str(args.replicas)
if args.pool:
overrides['DATABASE_POOL_MAX'] = str(args.pool)
ensure_stand(overrides)
profile = args.profile
ladder = [int(value) for value in args.ladder.split(',')] if args.ladder else None
summary = run_measured(
args.label,
profile,
ladder=ladder,
step_seconds=args.step or STEP_SECONDS,
rate=args.peak,
duration=args.duration,
mode='read' if args.mode == 'read' else None,
spread=args.spread,
warmup=not args.no_warmup,
)
print(json.dumps(summary, ensure_ascii=False, indent=1))
def cmd_sweep(args) -> None:
replicas = [int(value) for value in args.replicas.split(',')]
label_prefix = args.prefix or args.profile
for rep in range(1, args.reps + 1):
order = replicas if rep % 2 else list(reversed(replicas))
for count in order:
ensure_stand({'API_REPLICAS': str(count)})
run_measured(f'{label_prefix}#n{count}#rep{rep}', args.profile, warmup=not args.no_warmup)
report_series(args.profile, label_prefix)
if args.extra_replica:
for extra in range(1, args.extra_reps + 1):
count = args.extra_replica
ensure_stand({'API_REPLICAS': str(count)})
run_measured(f'{label_prefix}#n{count}#rep{args.reps + extra}', args.profile, warmup=not args.no_warmup)
report_series(args.profile, label_prefix)
def report_series(profile: str, prefix: str) -> None:
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#')]
groups = group_configurations(labels)
lines = []
for key, group in groups.items():
series = summarise_series(group)
lines += table(
['темп', 'достигнуто RPS', 'min', 'max', 'p95', 'ошибок'],
[
[row['rate'], row['achieved_rps'], row['achieved_min'], row['achieved_max'], row['p95'], row['failed']]
for row in series['rows']
],
)
lines.append(f'Потолок {key}: {series["ceiling_rps"]} req/s, сатурация на {series["saturation"]["rate"]} req/s ({len(group)} повторов)')
lines.append('')
write_markdown(TRACK1_DIR / f'summary-live-{profile}.md', f'Текущие агрегаты {profile}', lines)
def group_configurations(labels: list[str]) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {}
for label in labels:
groups.setdefault(label.split('#')[0], []).append(label)
return {key: sorted(value) for key, value in sorted(groups.items())}
def knee_rate(profile: str, prefix: str = 'p-a') -> int:
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#n4#')]
if not labels:
return 400
series = summarise_series(labels)
rows = series['rows']
for row in rows:
if row['achieved_rps'] < 0.95 * row['rate']:
previous = rows[max(0, rows.index(row) - 1)]
return int(previous['rate'])
return int(rows[-1]['rate']) if rows else 400
def cmd_growth(args) -> None:
base = load_stand_config()
rate = args.rate or max(1, int(knee_rate(args.base_profile) * 0.8))
points = {
'api': [
{'API_TOTAL_CPUS': str(float(base.get('API_TOTAL_CPUS', API_TOTAL_CPUS)) * 2), 'label': 'api-total-6'},
],
'pg': [
{'PG_CPUS': '3.0', 'label': 'pg-3.0'},
],
'pool': [
{'DATABASE_POOL_MAX': '5', 'label': 'pool-5'},
{'DATABASE_POOL_MAX': '20', 'label': 'pool-20'},
],
}
log(f'серия роста {args.series} на темпе {rate} req/s (80% от колена {args.base_profile})')
for point in points[args.series]:
overrides = {'API_REPLICAS': '4', **{key: value for key, value in point.items() if key != 'label'}}
ensure_stand(overrides)
run_measured(
f'growth#{point["label"]}',
args.base_profile,
rate=rate,
duration=args.duration,
warmup=not args.no_warmup,
)
class MessageProbe(threading.Thread):
"""Последовательный клиент: измеряет успешность записи и повторяет
неопределённый POST тем же ключом (как это делает реальный клиент)."""
def __init__(self, client: Client, token: str, chat_id: str, interval: float, tag: str):
super().__init__(daemon=True)
self.client = client
self.token = token
self.chat_id = chat_id
self.interval = interval
self.tag = tag
self.stop_event = threading.Event()
self.samples: list[dict] = []
self.confirmed: list[dict] = []
def run(self) -> None:
import uuid
sequence = 0
started = time.monotonic()
while not self.stop_event.is_set():
sequence += 1
key = uuid.uuid4().hex
text = f'{self.tag} {sequence}'
attempts = 0
while attempts < 3:
attempts += 1
moment = time.monotonic() - started
try:
response = self.client.post_message(self.token, self.chat_id, text, key)
except (urllib.error.URLError, TimeoutError, OSError) as error:
self.samples.append({'t': round(moment, 3), 'ok': False, 'error': type(error).__name__})
if self.stop_event.wait(0.1):
return
continue
instance = header(response['headers'], 'X-Instance-Id')
self.samples.append(
{'t': round(moment, 3), 'ok': response['status'] == 201, 'status': response['status'], 'instance': instance}
)
if response['status'] == 201:
self.confirmed.append({'text': text, 'key': key, 'id': response['json']['id']})
break
if self.stop_event.wait(self.interval):
return
def replica_ip(container: str) -> str:
return inspect(container, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
def cmd_failover(args) -> None:
replicas = replica_map()
targets = sorted(replicas) if args.replicas == 'all' else [args.replicas]
config = load_stand_config()
ensure_stand({'API_REPLICAS': str(len(replicas))})
results = []
for target in targets:
results.append(failover_once(target, args, config))
write_failover_evidence(results)
print(json.dumps(results, ensure_ascii=False, indent=1))
def failover_once(target: str, args, config: dict) -> dict:
if target not in replica_map():
log(f'{target} не запущена: восстанавливаю реплику')
run_command(['scripts/start-replica.sh'])
wait_for_lb()
if target not in replica_map():
raise SystemExit(f'replica {target} did not come back')
container = replica_map()[target]['container']
address = replica_ip(container)
client = Client(timeout=5)
alice = client.register(f'failover{target[:4]}')
chat = client.create_chat(alice['token'], [alice['id']], f'failover {target}')
label = f'failover#{target}'
env = {
**load_env('p-b', 'write', 'spread', label),
'RATE': str(args.rate),
'DURATION': str(args.duration),
}
sampler = Sampler().start()
load = subprocess.Popen(
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{k}={v}'] for k, v in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
probe = MessageProbe(client, alice['token'], chat, 0.2, f'failover-{target[:6]}')
probe.start()
started = now_iso()
time.sleep(args.pre)
stop_moment = time.monotonic()
stop = run_command(
['scripts/stop-one.sh'],
check=False,
env={'TARGET_INSTANCE_ID': target},
timeout=30,
)
stop_at = now_iso()
first_success = None
while time.monotonic() - stop_moment < args.duration - args.pre:
successes = [sample for sample in list(probe.samples) if sample['t'] > 0 and sample.get('ok')]
if any(sample['t'] > (stop_moment - probe_start(probe)) for sample in successes):
first_success = round(time.monotonic() - stop_moment, 2)
break
time.sleep(0.1)
time.sleep(max(0.0, args.hold))
restore_at = now_iso()
run_command(['scripts/start-replica.sh'])
recovered = None
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
instances = set()
for _ in range(6):
response = client.call('GET', '/api/v1/users/me', token=alice['token'])
instances.add(header(response['headers'], 'X-Instance-Id'))
if len(instances) >= 2:
recovered = round(time.monotonic() - stop_moment, 2)
break
time.sleep(0.5)
remaining = max(0.0, args.duration - (time.monotonic() - stop_moment) - args.pre)
time.sleep(min(remaining, 15))
probe.stop_event.set()
probe.join(timeout=10)
load.terminate()
try:
stdout, _ = load.communicate(timeout=120)
except subprocess.TimeoutExpired:
load.kill()
stdout, _ = load.communicate()
samples = sampler.stop()
finished = now_iso()
summary = None
for line in reversed((stdout or '').splitlines()):
if line.strip().startswith('{'):
try:
candidate = json.loads(line)
except ValueError:
continue
if 'steps' in candidate:
summary = candidate
break
history = client.history(alice['token'], chat)
texts = [item['text'] for item in history]
duplicates = sorted({text for text in texts if texts.count(text) > 1})
missing = [item['text'] for item in probe.confirmed if item['text'] not in texts]
lb = haproxy_stats(samples)
detection = None
for name, server in lb.items():
if address in server['addresses']:
for transition in server['transitions']:
if transition['status'] != 'UP':
detection = transition['t']
break
phases = {
'stop_at': stop_at,
'stop_exit': stop.returncode,
'first_success_after_stop_s': first_success,
'restore_at': restore_at,
'second_replica_observed_s': recovered,
'lb_down_at': detection,
}
if summary:
write_raw(label, summary, samples, config, started, finished, phases, sampler.errors)
result = {
'instance': target,
'container': container,
'address': address,
'rate': args.rate,
'duration_s': args.duration,
'phases': phases,
'probe': {
'attempts': len(probe.samples),
'successes': sum(1 for sample in probe.samples if sample.get('ok')),
'failures': sum(1 for sample in probe.samples if not sample.get('ok')),
'rps_before': phase_rps(probe.samples, 0, probe_start(probe) + args.pre - probe_start(probe)),
'confirmed': len(probe.confirmed),
},
'duplicates': duplicates,
'missing_confirmed': missing,
'haproxy': lb,
'k6': {'achieved_rps': summary['steps'][0]['achieved_rps'] if summary else None, 'failed': summary['steps'][0]['failed'] if summary else None},
}
log(f'failover {target}: падение обнаружено, восстановление {recovered}s, дубли: {len(duplicates)}, потеряно: {len(missing)}')
return result
def probe_start(probe: MessageProbe) -> float:
return probe.samples[0]['t'] if probe.samples else 0.0
def phase_rps(samples: list[dict], start: float, end: float) -> float:
window = [sample for sample in samples if start <= sample['t'] <= end]
return round(len(window) / max(0.001, end - start), 2)
def write_failover_evidence(results: list[dict]) -> None:
write_json(LEVEL4_DIR / 'failover-sweep.json', results)
lines = table(
['реплика', 'контейнер', 'попыток', 'успешных', 'ошибок', 'первый успех, s', 'вторая реплика, s', 'дубли', 'потери'],
[
[
result['instance'][:12],
result['container'][:12],
result['probe']['attempts'],
result['probe']['successes'],
result['probe']['failures'],
result['phases']['first_success_after_stop_s'],
result['phases']['second_replica_observed_s'],
len(result['duplicates']),
len(result['missing_confirmed']),
]
for result in results
],
)
write_markdown(LEVEL4_DIR / 'failover-sweep.md', 'Отказ каждой реплики под нагрузкой', lines)
def cmd_rolling(args) -> None:
config = load_stand_config()
replicas = int(config['API_REPLICAS'])
build_id = args.build_id or datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
before = replica_map()
migrate_before = inspect(container_ids('migrate')[0], '{{.State.StartedAt}}') if container_ids('migrate') else None
log(f'сборка образа {config["API_IMAGE"]} с BUILD_ID={build_id}')
build = run_command(
['docker', 'build', '--build-arg', f'BUILD_ID={build_id}', '-t', config['API_IMAGE'], '.'],
check=True,
timeout=1800,
)
image_id = run_command(['docker', 'image', 'inspect', '-f', '{{.Id}}', config['API_IMAGE']]).stdout.strip()
client = Client(timeout=5)
alice = client.register('rolling')
chat = client.create_chat(alice['token'], [alice['id']], 'rolling update')
probe = MessageProbe(client, alice['token'], chat, 0.2, 'rolling')
sampler = Sampler().start()
load = None
if args.under_load:
env = {**load_env('p-b', 'write', 'spread', 'rolling'), 'RATE': str(args.rate), 'DURATION': str(max(args.duration, replicas * 45))}
load = subprocess.Popen(
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{k}={v}'] for k, v in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
probe.start()
started = now_iso()
steps = []
for step in range(replicas):
current = replica_map()
target = sorted(current)[step % len(current)]
container = current[target]['container']
old_build = app_build_id(container)
run_command(['docker', 'stop', '--time', '15', container], check=False, timeout=60)
run_command(['docker', 'rm', container], check=False, timeout=60)
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', f'api={replicas}', 'api', timeout=600)
new_instance, new_container = wait_for_replica_count(replicas, exclude=target)
new_build = app_build_id(new_container)
steps.append(
{
'step': step + 1,
'old_instance': target,
'old_container': container[:12],
'old_build_id': old_build,
'new_instance': new_instance,
'new_container': new_container[:12],
'new_build_id': new_build,
'migrate_started_at': migrate_before,
}
)
log(f'rolling {step + 1}/{replicas}: {target[:12]} ({old_build}) -> {new_instance[:12]} ({new_build})')
time.sleep(args.pause)
probe.stop_event.set()
probe.join(timeout=10)
samples = sampler.stop()
summary = None
if load is not None:
load.terminate()
try:
stdout, _ = load.communicate(timeout=120)
except subprocess.TimeoutExpired:
load.kill()
stdout, _ = load.communicate()
for line in reversed((stdout or '').splitlines()):
if line.strip().startswith('{'):
try:
candidate = json.loads(line)
except ValueError:
continue
if 'steps' in candidate:
summary = candidate
break
migrate_after = inspect(container_ids('migrate')[0], '{{.State.StartedAt}}') if container_ids('migrate') else None
failures = [sample for sample in probe.samples if not sample.get('ok')]
server_failures = [sample for sample in failures if 500 <= int(sample.get('status') or 0) < 600]
finished = now_iso()
payload = {
'build_id': build_id,
'image_id': image_id,
'image': config['API_IMAGE'],
'replicas': replicas,
'under_load': bool(args.under_load),
'steps': steps,
'probe': {'attempts': len(probe.samples), 'failures': len(failures), 'server_5xx': len(server_failures)},
'k6': {'achieved_rps': summary['steps'][0]['achieved_rps'] if summary else None, 'failed': summary['steps'][0]['failed'] if summary else None},
'migrate': {'before': migrate_before, 'after': migrate_after, 'restarted': migrate_before != migrate_after},
'build_stderr_tail': build.stderr.strip().splitlines()[-3:],
}
write_json(LEVEL4_DIR / 'rolling-update.json', payload)
if summary:
write_raw('rolling', summary, samples, config, started, finished, {'steps': steps}, sampler.errors)
lines = table(
['шаг', 'было', 'BUILD_ID до', 'стало', 'BUILD_ID после'],
[[step['step'], step['old_instance'][:12], step['old_build_id'], step['new_instance'][:12], step['new_build_id']] for step in steps],
)
lines += [
f'5xx от API во время rolling update: {len(server_failures)} из {len(probe.samples)} попыток',
f'Миграции перезапускались: {migrate_before != migrate_after}',
]
write_markdown(LEVEL4_DIR / 'rolling-update.md', 'Rolling update под нагрузкой', lines)
print(json.dumps(payload, ensure_ascii=False, indent=1))
def app_build_id(container: str) -> str:
result = run_command(
[
'docker', 'exec', container, 'node', '-e',
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}}).then(r=>r.json())"
".then(j=>process.stdout.write(String(j.build_id)))",
],
check=False,
timeout=20,
)
return result.stdout.strip() or 'unknown'
def wait_for_replica_count(replicas: int, exclude: str, timeout: float = 120) -> tuple[str, str]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
mapping = replica_map()
if len(mapping) >= replicas:
for instance, info in sorted(mapping.items()):
if instance != exclude:
return instance, info['container']
time.sleep(1)
raise SystemExit(f'replica count did not reach {replicas} within {timeout}s')
def cmd_ab(args) -> None:
sides = {'A': args.before, 'B': args.after}
for rep in range(1, args.reps + 1):
order = ['A', 'B'] if rep % 2 else ['B', 'A']
for side in order:
ensure_stand({'API_IMAGE': sides[side], 'API_REPLICAS': str(args.replicas)})
run_measured(f'ab#{side}#rep{rep}', args.profile, warmup=not args.no_warmup)
report_ab(args)
def report_ab(args) -> None:
labels = [label for label in raw_labels() if label.startswith('ab#')]
groups = group_configurations(labels)
lines = []
verdicts = []
series_by_side = {side: summarise_series(groups.get(side, [])) for side in ('A', 'B')}
for rate in sorted({row['rate'] for row in series_by_side['A']['rows']}):
row_a = next((row for row in series_by_side['A']['rows'] if row['rate'] == rate), None)
row_b = next((row for row in series_by_side['B']['rows'] if row['rate'] == rate), None)
if not row_a or not row_b:
continue
change = (
round((row_b['achieved_rps'] - row_a['achieved_rps']) / row_a['achieved_rps'] * 100, 2)
if row_a['achieved_rps']
else 0.0
)
confirmed = (
abs(change) >= 5.0
and (row_b['achieved_min'] > row_a['achieved_max'] or row_a['achieved_min'] > row_b['achieved_max'])
)
verdicts.append({'rate': rate, 'change_percent': change, 'confirmed': confirmed})
lines.append(
f'{rate} req/s: A {row_a["achieved_rps"]} [{row_a["achieved_min"]}; {row_a["achieved_max"]}] vs '
f'B {row_b["achieved_rps"]} [{row_b["achieved_min"]}; {row_b["achieved_max"]}] -> {change}% '
f'({"подтверждено" if confirmed else "не подтверждено"})'
)
lines.append('')
lines += table(
['шаг', 'образ A: RPS (min..max)', 'p95 A', 'образ B: RPS (min..max)', 'p95 B', 'Δ, %', 'вывод'],
[
[
verdict['rate'],
f'{next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_rps"]}'
f' ({next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_min"]}..'
f'{next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_max"]})',
next(row for row in series_by_side['A']['rows'] if row['rate'] == verdict['rate'])['p95'],
f'{next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_rps"]}'
f' ({next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_min"]}..'
f'{next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_max"]})',
next(row for row in series_by_side['B']['rows'] if row['rate'] == verdict['rate'])['p95'],
verdict['change_percent'],
'подтверждено' if verdict['confirmed'] else 'не подтверждено',
]
for verdict in verdicts
],
)
lines += [
f'Потолок A: {series_by_side["A"]["ceiling_rps"]} req/s, потолок B: {series_by_side["B"]["ceiling_rps"]} req/s',
'',
'Правило подтверждения: медиана меняется ≥5 % и диапазоны min/max повторов не перекрываются.',
]
write_json(TRACK1_DIR / 'ab-verdict.json', {'verdicts': verdicts, 'ceiling': {side: series_by_side[side]['ceiling_rps'] for side in ('A', 'B')}, 'images': {'A': args.before, 'B': args.after}})
write_markdown(TRACK1_DIR / 'summary-ab.md', 'A/B: подтверждённое улучшение', lines)
# ---------------------------------------------------------------- анализ и графики
def pyplot():
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
return plt
def representative(group: list[str]) -> str:
"""Прогон с медианным пиком — чтобы агрегаты контейнеров не брались с края."""
scored = []
for label in group:
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
steps = summary.get('steps', [])
peak = max((step['achieved_rps'] for step in steps), default=0.0)
scored.append((peak, label))
if not scored:
raise SystemExit('нет прогонов для анализа')
scored.sort()
return scored[len(scored) // 2][1]
def cmd_analyze(args) -> None:
labels = raw_labels()
if not labels:
log('raw пуст: таблицы и графики будут пустыми')
groups = group_configurations(labels)
analyse_runs(groups)
analyse_series_a(groups, labels)
analyse_diagnostics(groups, labels)
analyse_growth(groups, labels)
if groups.get('A') or groups.get('B'):
report_ab(None, None)
make_graphs(groups, labels)
log('analyze: таблицы и графики обновлены')
def series_kind(group: list[str]) -> str:
summary = read_json(RAW_DIR / f'{group[0]}.k6.json', {})
return summary.get('kind', 'ladder')
def analyse_runs(groups: dict) -> None:
"""Полный разбор каждого прогона: шаги, ресурсы, PostgreSQL, LB."""
lines = []
for key, group in sorted(groups.items()):
series = summarise_series(group)
label = representative(group)
_, samples, manifest = load_raw(label)
lines += [f'## {key} ({len(group)} прогонов, {series_kind(group)})', '']
lines += table(
['темп, req/s', 'достигнуто RPS', 'min..max', 'p95, ms', 'ошибок', 'VU', 'dropped'],
[
[
row['rate'],
row['achieved_rps'],
f'{row["achieved_min"]}..{row["achieved_max"]}',
row['p95'],
row['failed'],
row['vus_active'],
read_json(RAW_DIR / f'{label}.k6.json', {}).get('dropped_iterations', 0),
]
for row in series['rows']
],
)
if series_kind(group) == 'ladder':
saturation = series['saturation']
lines.append(
f'Потолок {series["ceiling_rps"]} req/s, сатурация {saturation["rate"]} req/s '
f'({saturation["achieved_rps"]} req/s достигнуто).'
)
lines.append('')
limits = manifest.get('stand', {}) if manifest else {}
lines.append(
f'Лимиты: реплик {limits.get("API_REPLICAS")}, API {limits.get("API_CPUS_PER_REPLICA")} CPU / '
f'{limits.get("API_MEMORY_PER_REPLICA")}, PG {limits.get("PG_CPUS")} CPU, LB {limits.get("LB_CPUS")} CPU, '
f'пул {limits.get("DATABASE_POOL_MAX")}, образ {limits.get("API_IMAGE")} (BUILD_ID {limits.get("BUILD_ID")}).'
)
lines.append('')
lines += table(
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
[
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
for name, stats in container_stats(samples).items()
],
)
pg = postgres_stats(samples)
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, hit ratio {pg["blks_hit_ratio"]}, '
f'ожиданий блокировок пик {pg["lock_waiters_peak"]}, активных сессий пик {pg["active_peak"]}, '
f'соединений пик {pg["connections_peak"]}, deadlocks {pg["deadlocks"]}, messages {pg["messages_mib"]} MiB.'
)
lines.append('')
lb = haproxy_stats(samples)
total = lb.get('BACKEND')
if total:
lines.append(
f'LB: {total["requests"]} запросов ({total["rps"]} req/s), ошибок {total["errors"]}, '
f'очередь пик {total["qcur_peak"]}.'
)
for name, server in sorted(lb.items()):
if name in ('BACKEND', 'FRONTEND') or not server['requests']:
continue
lines.append(
f'- LB слот {name}: {server["requests"]} запросов ({server["rps"]} req/s), ошибок {server["errors"]}, '
f'очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}, адрес {server["addresses"]}.'
)
lines.append('')
app = app_metrics_delta(samples)
for container, delta in sorted(app.items()):
lines.append(
f'- реплика {delta["instance_id"]} (build {delta["build_id"]}, контейнер {container[:12]}): '
f'{json.dumps(delta["counters"], ensure_ascii=False)}; '
f'latency p95 {delta["latency_ms"].get("p95")} ms.'
)
lines.append('')
write_markdown(TRACK1_DIR / 'summary-runs.md', 'Разбор каждого прогона', lines)
def analyse_series_a(groups: dict, labels: list[str]) -> None:
lines = []
series_by_n = {}
for key, group in sorted(groups.items()):
if not key.startswith('p-a#n'):
continue
series = summarise_series(group)
series_by_n[key] = series
lines.append(f'## {key} ({len(group)} повторов)')
lines.append('')
lines += table(
['темп, req/s', 'достигнуто RPS', 'min..max', 'p95, ms', 'p95 max', 'ошибок', 'VU'],
[
[
row['rate'],
row['achieved_rps'],
f'{row["achieved_min"]}..{row["achieved_max"]}',
row['p95'],
row['p95_max'],
row['failed'],
row['vus_active'],
]
for row in series['rows']
],
)
saturation = series['saturation']
lines.append(
f'Потолок: **{series["ceiling_rps"]} req/s**; сатурация: {saturation["rate"]} req/s '
f'(достигнуто {saturation["achieved_rps"]} req/s).'
)
lines.append('')
label = representative(group)
_, samples, manifest = load_raw(label)
lines.append(f'Ресурсы представительного прогона `{label}` (лимиты {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")} на реплику):')
lines.append('')
lines += table(
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
[
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
for name, stats in container_stats(samples).items()
],
)
pg = postgres_stats(samples)
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, '
f'hit ratio {pg["blks_hit_ratio"]}, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
f'активных сессий пик {pg["active_peak"]}, соединений пик {pg["connections_peak"]}, '
f'таблица messages {pg["messages_mib"]} MiB.'
)
lines.append('')
lb = haproxy_stats(samples)
distribution = [f'{name}: {server["requests"]} ({server["rps"]} req/s), адреса {", ".join(server["addresses"])}' for name, server in sorted(lb.items())]
if distribution:
lines += ['Распределение запросов LB:', ''] + [f'- {item}' for item in distribution] + ['']
lines += saturation_conclusion(series_by_n)
write_markdown(TRACK1_DIR / 'summary-series-a.md', 'Серия A: 1/2/4 реплики при фиксированном бюджете', lines)
def saturation_conclusion(series_by_n: dict) -> list[str]:
lines = ['## Точки сатурации', '']
for key, series in sorted(series_by_n.items()):
saturation = series['saturation']
lines.append(
f'- {key}: knee на {saturation["rate"]} req/s (достигнуто {saturation["achieved_rps"]} req/s), '
f'потолок {series["ceiling_rps"]} req/s.'
)
lines.append('')
return lines
def analyse_diagnostics(groups: dict, labels: list[str]) -> None:
lines = []
for key, group in sorted(groups.items()):
if not key.startswith(('p-b', 'p-c')):
continue
series = summarise_series(group)
label = representative(group)
_, samples, manifest = load_raw(label)
pg = postgres_stats(samples)
stats = container_stats(samples)
lines.append(f'## {key} (темпы {series["rows"][0]["rate"]}..{series["rows"][-1]["rate"]} req/s)')
lines.append('')
lines += table(
['темп', 'достигнуто RPS', 'p95, ms', 'ошибок'],
[[row['rate'], row['achieved_rps'], row['p95'], row['failed']] for row in series['rows']],
)
lines.append(
f'Потолок {series["ceiling_rps"]} req/s, knee на {series["saturation"]["rate"]} req/s. '
f'Лимиты: LB {manifest["stand"].get("LB_CPUS")} CPU, API {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")}, PG {manifest["stand"].get("PG_CPUS")} CPU.'
)
lines.append('')
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
f'активных сессий пик {pg["active_peak"]}, hit ratio {pg["blks_hit_ratio"]}.'
)
lines.append('')
lb = stats.get('mtusi-lab3-haproxy-1')
if lb:
lines.append(f'HAProxy: CPU средний {lb["cpu_mean"]}%, пик {lb["cpu_peak"]}%, RAM пик {lb["mem_peak"]} MiB из {lb["mem_limit"]}.')
lines.append('')
haproxy = haproxy_stats(samples)
for name, server in sorted(haproxy.items()):
if server['qcur_peak'] or server['statuses'] != ['UP']:
lines.append(f'- LB сервер {name}: очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}.')
lines.append('')
write_markdown(TRACK1_DIR / 'summary-diagnostics.md', 'Диагностика: горячий чат и путь чтения', lines)
def analyse_growth(groups: dict, labels: list[str]) -> None:
growth = {key: group for key, group in groups.items() if key.startswith('growth#')}
if not growth:
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', ['Нет прогонов growth.'])
return
lines = []
baseline_labels = [label for label in labels if label.startswith('p-a#n4#')]
baseline = summarise_series(baseline_labels) if baseline_labels else None
if baseline:
rate = max((row['rate'] for row in growth_rows(growth)), default=0)
row = step_of_rate(baseline['rows'], rate)
lines.append(f'Базовая точка (серия A, N=4, тот же темп {rate} req/s): **{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms.')
lines.append('')
lines += table(
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API пик, %', 'CPU PG пик, %'],
[growth_row(key, group) for key, group in sorted(growth.items())],
)
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
def growth_rows(growth: dict):
for group in growth.values():
summary = read_json(RAW_DIR / f'{representative(group)}.k6.json', {})
yield from summary.get('steps', [])
def step_of_rate(rows: list[dict], rate: int) -> dict:
for row in rows:
if row['rate'] == rate:
return row
return rows[-1] if rows else {'rate': 0, 'achieved_rps': 0, 'p95': 0}
def growth_row(key: str, group: list[str]) -> list:
label = representative(group)
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
_, samples, _ = load_raw(label)
step = summary['steps'][0] if summary.get('steps') else {'rate': 0, 'achieved_rps': 0, 'p95': 0, 'failed': 0}
stats = container_stats(samples)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak, pg_peak]
def make_graphs(groups: dict, labels: list[str]) -> None:
if not labels:
return
plt = pyplot()
GRAPH_DIR.mkdir(parents=True, exist_ok=True)
ladders = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'ladder'}
steadys = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'steady'}
if ladders:
series = {key: summarise_series(group) for key, group in ladders.items()}
figure, axes = plt.subplots(figsize=(9, 5))
for key, values in series.items():
axes.plot(
[row['rate'] for row in values['rows']],
[row['achieved_rps'] for row in values['rows']],
marker='o',
label=key,
)
reference = next(iter(series.values()))
axes.plot(
[row['rate'] for row in reference['rows']],
[row['rate'] for row in reference['rows']],
linestyle='--',
color='grey',
label='предложенный темп',
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('достигнутый RPS')
axes.set_title('Пропускная способность: достигнутый RPS от предложенного темпа')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'throughput.png', dpi=120)
plt.close(figure)
figure, axes = plt.subplots(figsize=(9, 5))
for key, values in series.items():
axes.plot(
[row['rate'] for row in values['rows']],
[row['p95'] for row in values['rows']],
marker='o',
label=f'{key}: p95, ms',
)
axes.plot(
[row['rate'] for row in values['rows']],
[row['failed'] for row in values['rows']],
linestyle=':',
marker='x',
label=f'{key}: ошибок, шт.',
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('p95, ms / ошибок')
axes.set_title('Латентность p95 и ошибки по шагам лестницы')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'latency-errors.png', dpi=120)
plt.close(figure)
figure, axes = plt.subplots(figsize=(9, 5))
for key, values in series.items():
axes.plot(
[row['rate'] for row in values['rows']],
[row['vus_active'] for row in values['rows']],
marker='s',
label=key,
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('активных VU генератора')
axes.set_title('Занятость генератора: нехватка VU отличима от перегрузки сервера')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'generator-vus.png', dpi=120)
plt.close(figure)
if steadys:
keys = list(steadys)
achieved = [summarise_series(steadys[key])['ceiling_rps'] for key in keys]
offered = [summarise_series(steadys[key])['rows'][0]['rate'] for key in keys]
figure, axes = plt.subplots(figsize=(9, 5))
positions = range(len(keys))
axes.bar([p - 0.2 for p in positions], offered, width=0.4, label='предложенный темп')
axes.bar([p + 0.2 for p in positions], achieved, width=0.4, label='достигнутый RPS')
axes.set_xticks(list(positions))
axes.set_xticklabels(keys, fontsize=8)
axes.set_ylabel('req/s')
axes.set_title('Фиксированный темп: предложено против достигнутого')
axes.legend()
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'steady-throughput.png', dpi=120)
plt.close(figure)
for key, group in sorted(groups.items()):
plot_container_cpu(plt, representative(group))
figure, axes = plt.subplots(figsize=(9, 5))
shares = {}
for key, group in sorted(groups.items()):
_, samples, _ = load_raw(representative(group))
lb = haproxy_stats(samples)
shares[key] = {name: server['requests'] for name, server in sorted(lb.items()) if server['requests']}
names = sorted({name for share in shares.values() for name in share})
if names and shares:
width = 0.8 / max(1, len(names))
for position, name in enumerate(names):
axes.bar(
[index + position * width - 0.4 for index in range(len(shares))],
[shares[key].get(name, 0) for key in shares],
width=width,
label=name,
)
axes.set_xticks(range(len(shares)))
axes.set_xticklabels(list(shares), fontsize=7)
axes.set_ylabel('запросов за прогон')
axes.set_title('Распределение запросов по backend-слотам LB')
axes.legend(fontsize=7)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'replica-distribution.png', dpi=120)
plt.close(figure)
growth = {key: group for key, group in groups.items() if key.startswith('growth#')}
if growth:
rows = [growth_row(key, group) for key, group in sorted(growth.items())]
baseline_labels = [label for label in labels if label.startswith('p-a#n4#')]
figure, axes = plt.subplots(figsize=(9, 5))
names = []
values = []
low = []
high = []
if baseline_labels:
series = summarise_series(baseline_labels)
row = step_of_rate(series['rows'], rows[0][1])
names.append('базовая точка (серия A)')
values.append(row['achieved_rps'])
low.append(row['achieved_min'])
high.append(row['achieved_max'])
for row in rows:
names.append(row[0].replace('growth#', ''))
values.append(row[2])
low.append(row[2])
high.append(row[2])
axes.bar(names, values, yerr=[[value - minimum for value, minimum in zip(values, low)], [maximum - value for value, maximum in zip(values, high)]], capsize=5)
axes.set_ylabel('достигнутый RPS')
axes.set_title('Серия роста ресурсов: какой ресурс является ограничителем')
axes.tick_params(axis='x', labelsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'growth.png', dpi=120)
plt.close(figure)
if groups.get('A') and groups.get('B'):
figure, axes = plt.subplots(figsize=(9, 5))
series_a = summarise_series(groups['A'])
series_b = summarise_series(groups['B'])
rates = [row['rate'] for row in series_a['rows']]
positions = list(range(len(rates)))
axes.bar([p - 0.2 for p in positions], [row['achieved_rps'] for row in series_a['rows']], width=0.4, label='A (до)')
axes.bar([p + 0.2 for p in positions], [row['achieved_rps'] for row in series_b['rows']], width=0.4, label='B (после)')
for offset, series in ((-0.2, series_a), (0.2, series_b)):
for index, row in enumerate(series['rows']):
axes.plot([index + offset, index + offset], [row['achieved_min'], row['achieved_max']], color='black', linewidth=1)
axes.set_xticks(positions)
axes.set_xticklabels([str(rate) for rate in rates])
axes.set_xlabel('темп, req/s')
axes.set_ylabel('достигнутый RPS')
axes.set_title('A/B: медиана и разброс (min..max) по повторам')
axes.legend()
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'ab-comparison.png', dpi=120)
plt.close(figure)
def plot_container_cpu(plt, label: str) -> None:
_, samples, _ = load_raw(label)
series: dict[str, list[tuple[float, float]]] = {}
base = None
for record in samples:
if record.get('source') != 'docker_stats':
continue
name = record.get('name') or ''
if not name.startswith('mtusi-lab3-'):
continue
moment = _timestamp(record['t'])
base = base or moment
series.setdefault(name, []).append((moment - base, float(record.get('cpu_percent') or 0)))
if not series:
return
figure, axes = plt.subplots(figsize=(10, 5))
for name, points in sorted(series.items()):
axes.plot([point[0] for point in points], [point[1] for point in points], label=name, linewidth=1)
axes.set_xlabel('время от начала прогона, s')
axes.set_ylabel('CPU, % (100 % = одно ядро)')
axes.set_title(f'CPU контейнеров: {label}')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / f'container-cpu-{label.replace("#", "_")}.png', dpi=120)
plt.close(figure)
# ---------------------------------------------------------------- оркестрация трека
def cmd_quiesce(args) -> None:
result = run_command(['scripts/quiesce.sh', args.action], check=False, timeout=300)
print(result.stdout + result.stderr)
if result.returncode != 0:
raise SystemExit(f'quiesce {args.action} failed')
def cmd_track1(args) -> None:
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='up'))
try:
log('=== серия A: 1/2/4 реплики, профиль p-a')
cmd_sweep(
argparse.Namespace(
profile='p-a',
replicas='1,2,4',
reps=args.reps,
no_warmup=False,
prefix='p-a',
extra_replica=2,
extra_reps=1,
)
)
log('=== серия B: горячий чат (p-b), 4 реплики')
ensure_stand({'API_REPLICAS': '4'})
run_measured('p-b#n4', 'p-b', warmup=False)
log('=== серия C: путь чтения (p-c), 4 реплики, LB 0.25 и 1.0 CPU')
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '0.25'})
run_measured('p-c#lb0.25', 'p-c', warmup=False)
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '1.0'})
run_measured('p-c#lb1.0', 'p-c', warmup=False)
log('=== серия роста ресурсов')
for series in ('api', 'pg', 'pool'):
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False))
log('=== агрегаты')
cmd_analyze(argparse.Namespace())
if args.ab_after:
log('=== A/B улучшения (p-b, 4 реплики)')
cmd_ab(argparse.Namespace(before=args.ab_before, after=args.ab_after, profile='p-b', replicas=4, reps=args.ab_reps, no_warmup=False))
cmd_analyze(argparse.Namespace())
else:
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
log('=== возврат конфигурации стенда по умолчанию')
ensure_stand({'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']), 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']), 'API_IMAGE': DEFAULT_STAND['API_IMAGE'], 'BUILD_ID': DEFAULT_STAND['BUILD_ID']})
finally:
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='down'))
# ---------------------------------------------------------------- CLI
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description='Драйвер экспериментов lab3')
sub = parser.add_subparsers(dest='command', required=True)
stand = sub.add_parser('stand', help='стенд: up/down/status')
stand.add_argument('action', choices=['up', 'down', 'status'])
stand.add_argument('--replicas', type=int)
stand.set_defaults(func=cmd_stand)
check = sub.add_parser('check', help='проверки уровня 3/4')
check.add_argument('what', choices=['spread', 'instances', 'idempotency'])
check.add_argument('--requests', type=int, default=40)
check.set_defaults(func=cmd_check)
run = sub.add_parser('run', help='один прогон профиля')
run.add_argument('--label', required=True)
run.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
run.add_argument('--ladder')
run.add_argument('--step', type=int)
run.add_argument('--peak', type=int, help='одна полка фиксированного темпа')
run.add_argument('--duration', type=int, help='длительность полки, s')
run.add_argument('--replicas', type=int)
run.add_argument('--pool', type=int, help='DATABASE_POOL_MAX на реплику')
run.add_argument('--mode', choices=['write', 'read'])
run.add_argument('--spread', choices=['spread', 'hot'])
run.add_argument('--no-warmup', action='store_true')
run.set_defaults(func=cmd_run)
sweep = sub.add_parser('sweep', help='серия 1/2/4 реплики')
sweep.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
sweep.add_argument('--replicas', default='1,2,4')
sweep.add_argument('--reps', type=int, default=2)
sweep.add_argument('--extra-replica', type=int, default=0)
sweep.add_argument('--extra-reps', type=int, default=0)
sweep.add_argument('--prefix')
sweep.add_argument('--no-warmup', action='store_true')
sweep.set_defaults(func=cmd_sweep)
growth = sub.add_parser('growth', help='серия роста ресурсов')
growth.add_argument('--series', choices=['api', 'pg', 'pool'], required=True)
growth.add_argument('--rate', type=int)
growth.add_argument('--duration', type=int, default=60)
growth.add_argument('--base-profile', default='p-a')
growth.add_argument('--no-warmup', action='store_true')
growth.set_defaults(func=cmd_growth)
failover = sub.add_parser('failover', help='фазовый драйвер отказа реплики')
failover.add_argument('--replicas', default='all')
failover.add_argument('--rate', type=int, default=200)
failover.add_argument('--duration', type=int, default=60)
failover.add_argument('--pre', type=float, default=8.0, help='секунд нагрузки до отказа')
failover.add_argument('--hold', type=float, default=20.0, help='секунд до восстановления реплики')
failover.set_defaults(func=cmd_failover)
rolling = sub.add_parser('rolling-update', help='поочерёдная замена реплик')
rolling.add_argument('--under-load', action='store_true')
rolling.add_argument('--rate', type=int, default=200)
rolling.add_argument('--duration', type=int, default=180)
rolling.add_argument('--build-id')
rolling.add_argument('--pause', type=float, default=2.0)
rolling.set_defaults(func=cmd_rolling)
ab = sub.add_parser('ab', help='интерливинг A/B двух образов')
ab.add_argument('--before', required=True)
ab.add_argument('--after', required=True)
ab.add_argument('--profile', default='p-b')
ab.add_argument('--replicas', type=int, default=4)
ab.add_argument('--reps', type=int, default=3)
ab.add_argument('--no-warmup', action='store_true')
ab.set_defaults(func=cmd_ab)
analyze = sub.add_parser('analyze', help='агрегаты и графики из raw')
analyze.set_defaults(func=cmd_analyze)
quiesce = sub.add_parser('quiesce', help='фон хоста: up/down/status')
quiesce.add_argument('action', choices=['up', 'down', 'status'])
quiesce.set_defaults(func=cmd_quiesce)
track1 = sub.add_parser('track1', help='полная серия трека 1')
track1.add_argument('--reps', type=int, default=2)
track1.add_argument('--ab-before', default='mtusi-messenger:lab3-before')
track1.add_argument('--ab-after', default=None)
track1.add_argument('--ab-reps', type=int, default=3)
track1.add_argument('--no-quiesce', action='store_true')
track1.set_defaults(func=cmd_track1)
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
RAW_DIR.mkdir(parents=True, exist_ok=True)
args.func(args)
if __name__ == '__main__':
main()