Files
mtusi2026lab3/scripts/lab3/lab3.py
T
ilya.tyunin 5a5a8fe5cb lab3: путь записи сообщения одним оператором (C2)
Владелец ключа идемпотентности больше не оркестрируется в приложении: проверка
членства, заявка ключа, вставка сообщения, инкремент версии чата (якорь
инвалидации кэша) и сохранение тела ответа выполняются одним оператором SQL,
атомарность которого обеспечивает сам оператор.

- messages.repository: createOnce() одним запросом вместо трёх (проверка
  членства + INSERT + UPDATE version) и явной транзакции; тело ответа строится
  в SQL и служит источником истины и для первого ответа, и для повторов.
- idempotency.repository: read() читает сохранённый ответ вместе с повторной
  проверкой доступа, delete/purge не изменились; claim/store внутри транзакции
  больше не нужны.
- messages.service: без DatabaseClient и без предварительного chatsService.
  getForUser в пути записи; ключ обязателен, тело проверяется до обращения к БД,
  просроченный ключ освобождается и занимается заново.

Итог: 1 round trip на запись вместо 8 и 2 вместо 3 на повтор; короче удержание
блокировки строки chats, что и проверяется A/B на профиле горячего чата.

Проверено: npm run build, tsc, unit-тесты, lint, публичный smoke и black-box
набор приложения (включая конкурентный повтор с одним ключом) на стенде с
образом mtusi-messenger:lab3-after.
2026-10-15 22:48:22 +03:00

2506 lines
104 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Драйвер экспериментов lab3: стенд, доказательства уровней 3/4 и трек 1.
Запуск из корня репозитория: python3 scripts/lab3/lab3.py <подкоманда> ...
Подкоманды:
stand up|down|status поднять/остановить/показать стенд
check spread|instances|idempotency
разные реплики на одной сессии; карта X-Instance-Id;
фильтр e2e по идемпотентности
run --label ... один прогон профиля с сэмплером
sweep --profile p-a серия 1/2/4 реплики (интерливинг повторов)
growth --series api|pg|pool серия роста ресурсов на фиксированном темпе
failover --replicas all фазовый драйвер отказа реплики
rolling-update --under-load поочерёдная замена реплик под нагрузкой
ab --before T --after T интерливинг A/B двух образов
analyze агрегаты и графики из raw
quiesce up|down|status обёртка scripts/quiesce.sh
track1 полная серия трека 1 по зафиксированному плану
Только стандартная библиотека (графики — matplotlib на хосте). Данные прогонов
складываются в evidence/track1/raw: <label>.k6.json (сводка k6),
<label>.samples.jsonl.gz (сэмплер), <label>.manifest.json (условия измерения).
"""
from __future__ import annotations
import argparse
import csv
import gzip
import json
import math
import os
import re
import shutil
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
ENV_FILE = ROOT / '.env'
STAND_ENV = ROOT / '.env.stand'
STAND_STATE = ROOT / '.env.stand.json'
QUIESCE_STATE = ROOT / '.lab3-quiesce.json'
RAW_DIR = ROOT / 'evidence/track1/raw'
TRACK1_DIR = ROOT / 'evidence/track1'
GRAPH_DIR = TRACK1_DIR / 'graphs'
LEVEL3_DIR = ROOT / 'evidence/level3'
LEVEL4_DIR = ROOT / 'evidence/level4'
PROJECT = 'mtusi-lab3'
BASE_URL = os.environ.get('LAB3_BASE_URL', 'http://127.0.0.1:8080')
LB_STATS_URL = os.environ.get('LAB3_LB_STATS', 'http://127.0.0.1:8404/stats;csv')
COMPOSE = ['docker', 'compose', '--env-file', str(ENV_FILE), '--env-file', str(STAND_ENV)]
# Фиксированный бюджет серии трека 1: суммарные ресурсы API одинаковы для всех
# сравниваемых конфигураций, поэтому на реплику приходится 3/N CPU и 1.5/N GiB.
API_TOTAL_CPUS = 3.0
API_TOTAL_MEMORY_MB = 1536
DEFAULT_STAND = {
'API_TOTAL_CPUS': API_TOTAL_CPUS,
'API_TOTAL_MEMORY_MB': API_TOTAL_MEMORY_MB,
'API_REPLICAS': 2,
'PG_CPUS': 1.5,
'PG_MEMORY': '512m',
'REDIS_CPUS': 0.25,
'REDIS_MEMORY': '128m',
'LB_CPUS': 0.25,
'LB_MEMORY': '128m',
'LOADGEN_CPUS': 1.5,
'LOADGEN_MEMORY': '256m',
'API_IMAGE': 'mtusi-messenger:lab3',
'BUILD_ID': 'base',
'DATABASE_POOL_MAX': 10,
}
PROFILES = {
'p-a': {
'mode': 'write',
'spread': 'spread',
'ladder': [100, 200, 300, 400, 550, 700, 900, 1100],
},
'p-b': {
'mode': 'write',
'spread': 'hot',
'ladder': [50, 100, 150, 200, 250, 300, 350, 400],
},
'p-c': {
'mode': 'read',
'spread': 'spread',
'ladder': [200, 400, 700, 1000, 1500, 2000, 2500, 3000],
},
}
STEP_SECONDS = 15
WARMUP_SECONDS = 30
CHATS = int(os.environ.get('LAB3_CHATS', '24'))
def log(message: str) -> None:
print(f'[{datetime.now().strftime("%H:%M:%S")}] {message}', flush=True)
def now_iso() -> str:
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
def run_command(
command: list[str],
*,
check: bool = True,
timeout: float | None = None,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
result = subprocess.run(
command,
cwd=ROOT,
capture_output=True,
text=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if check and result.returncode != 0:
raise SystemExit(
f'command failed ({result.returncode}): {" ".join(command)}\n'
f'{result.stdout[-2000:]}\n{result.stderr[-2000:]}'
)
return result
def compose(*args: str, check: bool = True, timeout: float | None = 300) -> subprocess.CompletedProcess:
return run_command([*COMPOSE, *args], check=check, timeout=timeout)
def read_json(path: Path, default=None):
if not path.exists():
return default
return json.loads(path.read_text())
def write_json(path: Path, payload) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=1, sort_keys=True) + '\n')
# ---------------------------------------------------------------- окружение стенда
def replica_limits(replicas: int, total_cpus: float, total_memory_mb: int) -> tuple[float, str]:
return (
round(total_cpus / replicas, 3),
f'{int(total_memory_mb // replicas)}m',
)
def render_stand_env(config: dict) -> str:
cpus, memory = replica_limits(
int(config['API_REPLICAS']),
float(config['API_TOTAL_CPUS']),
int(config['API_TOTAL_MEMORY_MB']),
)
values = {
**config,
'API_CPUS_PER_REPLICA': cpus,
'API_MEMORY_PER_REPLICA': memory,
}
lines = ['# Сгенерировано scripts/lab3/lab3.py: бюджет серии и лимиты реплик.']
for key in sorted(values, key=str):
lines.append(f'{key}={values[key]}')
return '\n'.join(lines) + '\n'
def load_stand_config() -> dict:
stored = read_json(STAND_STATE, None)
config = dict(DEFAULT_STAND)
if STAND_ENV.exists():
for line in STAND_ENV.read_text().splitlines():
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
config[key] = value
if stored:
config.update(stored)
return config
def numeric(value: str) -> float | int:
number = float(value)
return int(number) if number.is_integer() else number
def save_stand_config(config: dict) -> dict:
"""Пишет .env.stand и его машинно-читаемую копию.
Копия сохраняется уже с производными значениями (лимиты на реплику), иначе
следующие запуски подхватили бы устаревшие лимиты, а манифесты прогонов
показывали бы не те числа, что реально применены к контейнерам.
"""
rendered = render_stand_env(config)
STAND_ENV.write_text(rendered)
values = {
line.split('=', 1)[0]: line.split('=', 1)[1]
for line in rendered.splitlines()
if '=' in line and not line.startswith('#')
}
write_json(STAND_STATE, values)
return config
def current_replicas() -> int:
return int(load_stand_config().get('API_REPLICAS', 2))
def apply_stand(config: dict, *, force_recreate: bool = True) -> None:
"""Применяет конфигурацию к стенду: лимиты, число реплик, образ."""
save_stand_config(config)
replicas = int(config['API_REPLICAS'])
args = ['up', '-d', '--wait', '--no-deps', '--scale', f'api={replicas}']
if force_recreate:
args.append('--force-recreate')
args.append('api')
compose(*args, timeout=600)
wait_for_lb()
def wait_for_lb(timeout: float = 120) -> None:
deadline = time.monotonic() + timeout
last = 'no response'
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f'{BASE_URL}/health/ready', timeout=3) as response:
if response.status == 200:
return
except Exception as error: # noqa: BLE001 - диагностика ожидания важнее типа
last = str(error)
time.sleep(1)
raise SystemExit(f'stand did not become ready within {timeout}s: {last}')
def lb_backends_up() -> int:
"""Число backend-слотов LB в состоянии UP (без строк FRONTEND/BACKEND)."""
try:
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
text = response.read().decode('utf-8', 'replace')
except OSError:
return 0
lines = text.splitlines()
if lines and lines[0].startswith('#'):
lines[0] = lines[0].lstrip('#').lstrip()
return sum(
1
for row in csv.DictReader(lines)
if row.get('pxname') == 'api'
and re.match(r'^api\d+$', row.get('svname', ''))
and row.get('status') == 'UP'
)
def wait_for_replicas(expected: int, timeout: float = 180) -> list[str]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
mapping = replica_map()
if len(mapping) >= expected and lb_backends_up() >= expected:
return sorted(mapping)
time.sleep(1)
raise SystemExit(
f'ожидалось {expected} реплик, видно {len(replica_map())} контейнеров и {lb_backends_up()} UP-слотов LB'
)
def container_ids(service: str = 'api', *, include_stopped: bool = False) -> list[str]:
result = run_command(
[
'docker',
'ps',
'-q',
*(['-a'] if include_stopped else []),
'--filter',
f'label=com.docker.compose.project={PROJECT}',
'--filter',
f'label=com.docker.compose.service={service}',
]
)
return [line for line in result.stdout.split() if line]
def inspect(container: str, template: str) -> str:
return run_command(['docker', 'inspect', '-f', template, container]).stdout.strip()
def replica_map() -> dict[str, dict]:
"""X-Instance-Id -> контейнер: в Docker это hostname = короткий ID."""
mapping = {}
for container in container_ids('api'):
hostname = inspect(container, '{{.Config.Hostname}}')
mapping[hostname] = {
'container': container,
'name': inspect(container, '{{.Name}}').lstrip('/'),
'image_id': inspect(container, '{{.Image}}'),
'started_at': inspect(container, '{{.State.StartedAt}}'),
'cpus': int(inspect(container, '{{.HostConfig.NanoCpus}}')) / 1e9,
'memory_mb': int(inspect(container, '{{.HostConfig.Memory}}')) // (1024 * 1024),
}
return mapping
def image_versions() -> dict:
versions = {}
for name in ('postgres', 'redis', 'haproxy', 'loadgen'):
container = container_ids(name)
if container:
versions[name] = inspect(container[0], '{{.Config.Image}}')
versions['postgres_runtime'] = run_command(
['docker', 'exec', container_ids('postgres')[0], 'postgres', '--version'], check=False
).stdout.strip()
versions['redis_runtime'] = run_command(
['docker', 'exec', container_ids('redis')[0], 'redis-server', '--version'], check=False
).stdout.strip()
versions['haproxy_runtime'] = run_command(
['docker', 'exec', container_ids('haproxy')[0], 'haproxy', '-v'], check=False
).stdout.splitlines()[0].strip()
versions['k6'] = run_command(
['docker', 'run', '--rm', 'grafana/k6:0.54.0', 'version'], check=False, timeout=120
).stdout.strip()
if shutil.which('node'):
versions['node_host'] = run_command(['node', '--version'], check=False).stdout.strip()
return versions
def dataset_size() -> dict:
containers = container_ids('postgres')
if not containers:
return {}
query = (
"SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), "
"(SELECT count(*) FROM messages), "
"(SELECT count(*) FROM idempotency_keys), "
"(SELECT count(*) FROM chat_members)"
)
result = psql(query)
if not result:
return {}
values = result.split('|')
return dict(zip(['users', 'chats', 'messages', 'idempotency_keys', 'chat_members'], (int(v) for v in values)))
def psql(query: str) -> str:
env = psql_env()
result = run_command(
[
'docker',
'exec',
container_ids('postgres')[0],
'psql',
'-U',
env['POSTGRES_USER'],
'-d',
env['POSTGRES_DB'],
'-tA',
'-F',
'|',
'-c',
query,
],
check=False,
timeout=30,
)
return result.stdout.strip()
_PSQL_ENV: dict[str, str] = {}
def psql_env() -> dict[str, str]:
if not _PSQL_ENV:
for line in ENV_FILE.read_text().splitlines():
if '=' in line and not line.startswith('#'):
key, value = line.split('=', 1)
_PSQL_ENV[key] = value
return _PSQL_ENV
def host_background() -> dict:
memory = {}
for line in Path('/proc/meminfo').read_text().splitlines():
if line.startswith(('MemTotal:', 'MemAvailable:')):
key, value = line.split(':')
memory[key] = int(value.strip().split()[0]) // 1024
load = os.getloadavg()
return {
'load_average': [round(value, 2) for value in load],
'mem_total_mib': memory.get('MemTotal'),
'mem_available_mib': memory.get('MemAvailable'),
'quiesced': QUIESCE_STATE.exists(),
}
def bench_cpus() -> int:
return os.cpu_count() or 8
# ---------------------------------------------------------------- HTTP-клиент
class ApiError(RuntimeError):
pass
class Client:
"""Минимальный клиент для проверок и фазовых драйверов (stdlib)."""
def __init__(self, base_url: str = BASE_URL, timeout: float = 5.0):
self.base = base_url.rstrip('/')
self.timeout = timeout
def call(self, method: str, path: str, body=None, token: str | None = None, headers=None):
data = None
request_headers = dict(headers or {})
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
request_headers['content-type'] = 'application/json'
if token:
request_headers['authorization'] = f'Bearer {token}'
request = urllib.request.Request(
self.base + path, data=data, headers=request_headers, method=method
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = response.read()
status = response.status
response_headers = dict(response.headers)
except urllib.error.HTTPError as error:
payload = error.read()
status = error.code
response_headers = dict(error.headers)
parsed = json.loads(payload) if payload else None
return {'status': status, 'json': parsed, 'headers': response_headers}
def register(self, label: str) -> dict:
import uuid
username = f'lab3_{label}_{uuid.uuid4().hex[:10]}'
password = f'Lab3-{uuid.uuid4().hex}'
created = self.call(
'POST',
'/api/v1/users',
{'username': username, 'display_name': f'lab3 {label}', 'password': password},
)
if created['status'] != 201:
raise ApiError(f'register failed: {created}')
import base64
credential = base64.b64encode(f'{username}:{password}'.encode()).decode()
session = self.call(
'POST',
'/api/v1/auth/sessions',
None,
headers={'authorization': f'Basic {credential}'},
)
if session['status'] != 201:
raise ApiError(f'session failed: {session}')
return {
'id': created['json']['id'],
'user': created['json'],
'token': session['json']['token'],
}
def create_chat(self, token: str, member_ids: list[str], title: str) -> str:
created = self.call(
'POST', '/api/v1/chats', {'title': title, 'member_ids': member_ids}, token=token
)
if created['status'] != 201:
raise ApiError(f'chat failed: {created}')
return created['json']['id']
def post_message(self, token: str, chat_id: str, text: str, key: str) -> dict:
return self.call(
'POST',
f'/api/v1/chats/{chat_id}/messages',
{'text': text},
token=token,
headers={'idempotency-key': key},
)
def history(self, token: str, chat_id: str) -> list[dict]:
items: list[dict] = []
cursor = None
while True:
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
if cursor:
from urllib.parse import quote
path += '&cursor=' + quote(cursor, safe='')
page = self.call('GET', path, token=token)
if page['status'] != 200:
raise ApiError(f'history failed: {page}')
items.extend(page['json']['items'])
cursor = page['json']['next_cursor']
if cursor is None:
return items
# ---------------------------------------------------------------- сэмплер
class Sampler:
"""Фоновые наблюдения за стендом: docker stats, HAProxy, метрики, PostgreSQL."""
def __init__(self) -> None:
self.records: list[dict] = []
self._lock = threading.Lock()
self._stop = threading.Event()
self._threads: list[threading.Thread] = []
self.errors: list[str] = []
def _record(self, payload: dict) -> None:
with self._lock:
self.records.append(payload)
def _guard(self, name: str, work) -> None:
try:
work()
except Exception as error: # noqa: BLE001 - сэмплер не должен валить прогон
with self._lock:
self.errors.append(f'{name}: {type(error).__name__}: {error}')
def _loop(self, name: str, interval: float, step) -> None:
def work() -> None:
while not self._stop.is_set():
step()
self._stop.wait(interval)
self._threads.append(threading.Thread(target=lambda: self._guard(name, work), daemon=True, name=name))
def start(self) -> 'Sampler':
# docker stats в режиме потока перерисовывает экран (ANSI-последовательности)
# и склеивает JSON разных контейнеров в одну строку, поэтому опрашиваем
# его порциями --no-stream: одна строка = один JSON.
self._loop('docker_stats', 1.5, self._sample_stats)
self._loop('haproxy', 0.5, self._haproxy)
self._loop('metrics', 2.0, self._app_metrics)
self._loop('postgres', 2.0, self._postgres)
self._loop('pg_statements', 10.0, self._pg_statements)
self._loop('host', 5.0, lambda: self._record({'t': now_iso(), 'source': 'host', **host_background()}))
for thread in self._threads:
thread.start()
return self
def stop(self) -> list[dict]:
self._stop.set()
for thread in self._threads:
thread.join(timeout=10)
return self.records
def _sample_stats(self) -> None:
result = run_command(
['docker', 'stats', '--no-stream', '--format', '{{json .}}'],
check=False,
timeout=30,
)
for line in result.stdout.splitlines():
try:
row = json.loads(line)
except ValueError:
continue
self._record(
{
't': now_iso(),
'source': 'docker_stats',
'name': row.get('Name') or row.get('Container'),
'cpu_percent': float(str(row.get('CPUPerc', '0%')).rstrip('%') or 0),
'mem_mib': round(_mib(row.get('MemUsage', '0MiB / 0MiB')), 2),
'mem_limit_mib': round(_mib(str(row.get('MemUsage', '')).split('/')[-1]), 2),
'pids': _int(row.get('PIDs')),
}
)
def _haproxy(self) -> None:
with urllib.request.urlopen(LB_STATS_URL, timeout=3) as response:
text = response.read().decode('utf-8', 'replace')
lines = text.splitlines()
# HAProxy помечает строку заголовка как комментарий: '# pxname,svname,...'
if lines and lines[0].startswith('#'):
lines[0] = lines[0].lstrip('#').lstrip()
for row in csv.DictReader(lines):
if row.get('pxname') not in ('api', 'http'):
continue
self._record(
{
't': now_iso(),
'source': 'haproxy',
'proxy': row.get('pxname'),
'server': row.get('svname'),
'addr': row.get('addr'),
'status': row.get('status'),
'check': row.get('check_status'),
'scur': _int(row.get('scur')),
'qcur': _int(row.get('qcur')),
'stot': _int(row.get('stot')),
'ereq': _int(row.get('ereq')),
'dresp': _int(row.get('dresp')),
'wretr': _int(row.get('wretr')),
}
)
def _app_metrics(self) -> None:
containers = container_ids('api')
if not containers:
return
for container in containers:
result = run_command(
[
'docker',
'exec',
container,
'node',
'-e',
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(t=>process.stdout.write(t))",
],
check=False,
timeout=20,
)
if result.returncode != 0 or not result.stdout.strip():
continue
try:
payload = json.loads(result.stdout)
except ValueError:
continue
self._record(
{
't': now_iso(),
'source': 'metrics',
'container': container,
'instance_id': payload.get('instance_id'),
'build_id': payload.get('build_id'),
'counters': payload.get('counters', {}),
'latency_ms': payload.get('latency_ms', {}),
}
)
def _postgres(self) -> None:
query = (
"SELECT (SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT xact_rollback FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT blks_read FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT blks_hit FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()),"
" (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'),"
" (SELECT count(*) FROM pg_stat_activity WHERE state = 'active'),"
" (SELECT count(*) FROM pg_stat_activity),"
" (SELECT count(*) FROM messages),"
" (SELECT pg_total_relation_size('messages'))"
)
output = psql(query)
if not output:
return
fields = output.split('|')
if len(fields) != 10:
return
self._record(
{
't': now_iso(),
'source': 'postgres',
**{
key: int(value)
for key, value in zip(
[
'xact_commit',
'xact_rollback',
'blks_read',
'blks_hit',
'deadlocks',
'lock_waiters',
'active',
'connections',
'messages',
'messages_bytes',
],
fields,
)
},
}
)
def _pg_statements(self) -> None:
output = psql(
"SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 3),"
" left(regexp_replace(query, '\\s+', ' ', 'g'), 90)"
" FROM pg_stat_statements WHERE query ILIKE '%messages%' OR query ILIKE '%chats%'"
" ORDER BY total_exec_time DESC LIMIT 5"
)
if not output:
return
rows = []
for line in output.splitlines():
parts = line.split('|')
if len(parts) == 4:
rows.append(
{
'calls': int(parts[0]),
'total_ms': float(parts[1]),
'mean_ms': float(parts[2]),
'query': parts[3],
}
)
self._record({'t': now_iso(), 'source': 'pg_stat_statements', 'top': rows})
def _mib(value: str) -> float:
text = str(value).strip()
match = re.match(r'([\d.]+)\s*([KMGT]i?B)?', text)
if not match:
return 0.0
number = float(match.group(1))
unit = (match.group(2) or 'B').lower()
factor = {
'b': 1 / (1024 * 1024),
'kib': 1 / 1024,
'kb': 1 / 1024,
'mib': 1,
'mb': 1,
'gib': 1024,
'gb': 1024,
'tib': 1024 * 1024,
}.get(unit, 1)
return number * factor
def _int(value) -> int:
try:
return int(str(value).strip() or 0)
except ValueError:
return 0
# ---------------------------------------------------------------- прогоны k6
def k6_run(script: str, label: str, env: dict[str, str], timeout: float) -> dict:
args = [*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps']
for key, value in sorted(env.items()):
args += ['-e', f'{key}={value}']
args += ['loadgen', 'run', f'/load/{script}']
result = run_command(args, check=False, timeout=timeout)
summary = None
for line in reversed(result.stdout.splitlines()):
stripped = line.strip()
if not stripped.startswith('{'):
continue
try:
candidate = json.loads(stripped)
except ValueError:
continue
if isinstance(candidate, dict) and 'steps' in candidate:
summary = candidate
break
if summary is None:
raise SystemExit(
f'k6 run {label} produced no summary (exit {result.returncode})\n'
f'{result.stdout[-1500:]}\n{result.stderr[-800:]}'
)
summary['exit_code'] = result.returncode
summary['stderr_tail'] = result.stderr.strip().splitlines()[-3:]
return summary
def load_env(profile: str, mode: str, spread: str, label: str) -> dict[str, str]:
env = {
'BASE_URL': 'http://haproxy:8080',
'PROFILE': spread,
'MODE': mode,
'CHATS': str(CHATS),
'LABEL': label,
}
if os.environ.get('LAB3_MAX_VUS'):
env['MAX_VUS'] = os.environ['LAB3_MAX_VUS']
if os.environ.get('LAB3_PRE_VUS'):
env['PRE_VUS'] = os.environ['LAB3_PRE_VUS']
return env
def run_measured(
label: str,
profile: str,
*,
ladder: list[int] | None = None,
step_seconds: int = STEP_SECONDS,
rate: int | None = None,
duration: int | None = None,
mode: str | None = None,
spread: str | None = None,
warmup: bool = True,
extra: dict | None = None,
phases: dict | None = None,
) -> dict:
"""Один прогон: (необязательный прогрев) + измерение + сэмплер + манифест."""
profile_config = PROFILES[profile]
mode = mode or profile_config['mode']
spread = spread or profile_config['spread']
ladder = ladder or profile_config['ladder']
if rate is not None:
ladder = [rate]
steady = rate is not None and duration is not None
script = 'write-steady.js' if steady else 'write-ladder.js'
RAW_DIR.mkdir(parents=True, exist_ok=True)
stand = load_stand_config()
if warmup:
log(f'прогрев {label}: {WARMUP_SECONDS}s на {ladder[0]} req/s')
k6_run(
script,
f'warmup-{label}',
{
**load_env(profile, mode, spread, f'warmup-{label}'),
**({'RATE': str(ladder[0]), 'DURATION': str(WARMUP_SECONDS)} if steady else {'LADDER': str(ladder[0]), 'STEP_SECONDS': str(WARMUP_SECONDS)}),
},
timeout=WARMUP_SECONDS + 120,
)
sampler = Sampler().start()
started = now_iso()
env = {
**load_env(profile, mode, spread, label),
**({'RATE': str(ladder[0]), 'DURATION': str(duration)} if steady else {'LADDER': ','.join(str(r) for r in ladder), 'STEP_SECONDS': str(step_seconds)}),
}
log(f'нагрузка {label}: {"steady " + str(ladder[0]) + " req/s" if steady else "ladder " + str(ladder)}')
summary = k6_run(script, label, env, timeout=(duration or len(ladder) * step_seconds) + 240)
finished = now_iso()
samples = sampler.stop()
write_raw(label, summary, samples, stand, started, finished, phases or {}, sampler.errors)
return summary
def write_raw(
label: str,
summary: dict,
samples: list[dict],
stand: dict,
started: str,
finished: str,
phases: dict,
sampler_errors: list[str],
) -> None:
RAW_DIR.mkdir(parents=True, exist_ok=True)
write_json(RAW_DIR / f'{label}.k6.json', summary)
with gzip.open(RAW_DIR / f'{label}.samples.jsonl.gz', 'wt', encoding='utf-8') as handle:
for record in samples:
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
if phases:
write_json(RAW_DIR / f'{label}.phases.json', phases)
manifest = {
'label': label,
'profile': summary.get('profile'),
'mode': summary.get('mode'),
'kind': summary.get('kind'),
'started_at': started,
'finished_at': finished,
'steps': [step['rate'] for step in summary.get('steps', [])],
'step_seconds': summary['steps'][0]['duration_s'] if summary.get('steps') else None,
'stand': stand,
'replicas': replica_map(),
'dataset': dataset_size(),
'versions': image_versions(),
'host': {'cpus': bench_cpus(), 'background': host_background()},
'generator': {
'cpus': stand.get('LOADGEN_CPUS'),
'memory': stand.get('LOADGEN_MEMORY'),
'max_vus': os.environ.get('LAB3_MAX_VUS', '200'),
'pre_vus': os.environ.get('LAB3_PRE_VUS', '60'),
},
'commit': run_command(['git', 'rev-parse', 'HEAD'], check=False).stdout.strip(),
'quiesce': read_json(QUIESCE_STATE, None),
'phases': phases,
'sampler_errors': sampler_errors,
'sample_count': len(samples),
}
write_json(RAW_DIR / f'{label}.manifest.json', manifest)
log(f'raw: {label} ({len(samples)} samples, errors: {len(sampler_errors)})')
def load_raw(label: str) -> tuple[dict | None, list[dict], dict | None]:
summary = read_json(RAW_DIR / f'{label}.k6.json', None)
manifest = read_json(RAW_DIR / f'{label}.manifest.json', None)
samples: list[dict] = []
path = RAW_DIR / f'{label}.samples.jsonl.gz'
if path.exists():
with gzip.open(path, 'rt', encoding='utf-8') as handle:
for line in handle:
try:
samples.append(json.loads(line))
except ValueError:
continue
return summary, samples, manifest
def raw_labels() -> list[str]:
if not RAW_DIR.exists():
return []
labels = []
for path in sorted(RAW_DIR.glob('*.k6.json')):
label = path.name[: -len('.k6.json')]
if label.startswith('warmup-'):
continue
labels.append(label)
return labels
# ---------------------------------------------------------------- анализ
def step_of(summary: dict, rate: int) -> dict | None:
for step in summary.get('steps', []):
if int(step['rate']) == int(rate):
return step
return None
def median(values: list[float]) -> float:
return round(statistics.median(values), 2) if values else 0.0
def spread(values: list[float]) -> tuple[float, float]:
return (round(min(values), 2), round(max(values), 2)) if values else (0.0, 0.0)
def group_runs(labels: list[str], predicate) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {}
for label in labels:
if predicate(label):
key = label.split('#')[0]
groups.setdefault(key, []).append(label)
return groups
def saturation(series: list[dict]) -> dict:
"""Точка сатурации: первый шаг, где достигнутый RPS отстал от предложенного."""
for step in series:
if step['achieved_rps'] < 0.95 * step['rate']:
return {'rate': step['rate'], 'achieved_rps': step['achieved_rps'], 'step': step}
return {'rate': series[-1]['rate'] if series else 0, 'achieved_rps': series[-1]['achieved_rps'] if series else 0, 'step': series[-1] if series else None}
def summarise_series(labels: list[str]) -> dict:
"""Медианы по повторам для одной конфигурации: шаги лестницы и пик."""
steps: dict[int, dict] = {}
for label in labels:
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
for step in summary.get('steps', []):
rate = int(step['rate'])
entry = steps.setdefault(rate, {'rate': rate, 'achieved': [], 'p95': [], 'failed': [], 'vus': [], 'offered': []})
entry['achieved'].append(float(step['achieved_rps']))
entry['p95'].append(float(step['p95']))
entry['failed'].append(float(step['failed']))
entry['vus'].append(float(step.get('vus_active') or 0))
entry['offered'].append(float(step['rate']))
rows = []
for rate in sorted(steps):
entry = steps[rate]
rows.append(
{
'rate': rate,
'achieved_rps': median(entry['achieved']),
'achieved_min': spread(entry['achieved'])[0],
'achieved_max': spread(entry['achieved'])[1],
'p95': median(entry['p95']),
'p95_max': spread(entry['p95'])[1],
'failed': sum(entry['failed']),
'vus_active': max(entry['vus']) if entry['vus'] else 0,
'reps': len(entry['achieved']),
}
)
ceiling = max((row['achieved_rps'] for row in rows), default=0.0)
return {'rows': rows, 'ceiling_rps': ceiling, 'saturation': saturation(rows), 'labels': labels}
def container_stats(samples: list[dict], prefix: str = 'mtusi-lab3-') -> dict:
stats: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'docker_stats':
continue
name = record.get('name') or ''
if not name.startswith(prefix):
continue
entry = stats.setdefault(name, {'cpu': [], 'mem': [], 'limit': 0.0})
entry['cpu'].append(float(record.get('cpu_percent') or 0))
entry['mem'].append(float(record.get('mem_mib') or 0))
entry['limit'] = max(entry['limit'], float(record.get('mem_limit_mib') or 0))
return {
name: {
'cpu_mean': round(statistics.fmean(entry['cpu']), 2) if entry['cpu'] else 0.0,
'cpu_peak': round(max(entry['cpu']), 2) if entry['cpu'] else 0.0,
'mem_mean': round(statistics.fmean(entry['mem']), 1) if entry['mem'] else 0.0,
'mem_peak': round(max(entry['mem']), 1) if entry['mem'] else 0.0,
'mem_limit': entry['limit'],
}
for name, entry in sorted(stats.items())
}
def postgres_stats(samples: list[dict]) -> dict:
rows = [record for record in samples if record.get('source') == 'postgres']
if len(rows) < 2:
return {}
first, last = rows[0], rows[-1]
seconds = max(1.0, (_timestamp(last['t']) - _timestamp(first['t'])))
hits = last['blks_hit'] - first['blks_hit']
reads = last['blks_read'] - first['blks_read']
return {
'xact_per_s': round((last['xact_commit'] - first['xact_commit']) / seconds, 1),
'rollback_per_s': round((last['xact_rollback'] - first['xact_rollback']) / seconds, 2),
'blks_hit_ratio': round(hits / (hits + reads), 4) if hits + reads else 0.0,
'lock_waiters_peak': max(row['lock_waiters'] for row in rows),
'active_peak': max(row['active'] for row in rows),
'connections_peak': max(row['connections'] for row in rows),
'deadlocks': last['deadlocks'] - first['deadlocks'],
'messages': last['messages'],
'messages_mib': round(last['messages_bytes'] / (1024 * 1024), 1),
}
def haproxy_stats(samples: list[dict]) -> dict:
servers: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'haproxy':
continue
entry = servers.setdefault(
record['server'],
{'requests': [], 'errors': [], 'qcur': [], 'scur': [], 'statuses': set(), 'addr': set(), 'events': []},
)
entry['requests'].append((record['t'], record['stot']))
entry['errors'].append(record['ereq'])
entry['qcur'].append(record['qcur'])
entry['scur'].append(record['scur'])
entry['statuses'].add(record['status'])
entry['addr'].add(record['addr'])
entry['events'].append((record['t'], record['status'], record['check']))
result = {}
for name, entry in servers.items():
if not entry['requests']:
continue
first, last = entry['requests'][0], entry['requests'][-1]
seconds = max(1.0, _timestamp(last[0]) - _timestamp(first[0]))
transitions = []
previous = None
for moment, status, check in entry['events']:
if status != previous:
transitions.append({'t': moment, 'status': status, 'check': check})
previous = status
result[name] = {
'requests': last[1] - first[1],
'rps': round((last[1] - first[1]) / seconds, 2),
'errors': entry['errors'][-1],
'qcur_peak': max(entry['qcur']),
'scur_peak': max(entry['scur']),
'statuses': sorted(entry['statuses']),
'addresses': sorted(entry['addr']),
'transitions': transitions,
}
return result
def app_metrics_delta(samples: list[dict]) -> dict:
per_container: dict[str, dict] = {}
for record in samples:
if record.get('source') != 'metrics':
continue
per_container.setdefault(record['container'], []).append(record)
result = {}
for container, rows in per_container.items():
if len(rows) < 2:
continue
first, last = rows[0], rows[-1]
counters = {}
for name, value in last['counters'].items():
counters[name] = value - first['counters'].get(name, 0)
result[container] = {
'instance_id': last.get('instance_id'),
'build_id': last.get('build_id'),
'counters': {name: value for name, value in sorted(counters.items()) if value},
'latency_ms': last.get('latency_ms', {}),
'samples': len(rows),
}
return result
def _timestamp(value: str) -> float:
return datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()
def write_markdown(path: Path, title: str, lines: list[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f'# {title}\n\n' + '\n'.join(lines) + '\n', encoding='utf-8')
def table(header: list[str], rows: list[list]) -> list[str]:
lines = ['| ' + ' | '.join(header) + ' |', '| ' + ' | '.join('---' for _ in header) + ' |']
for row in rows:
lines.append('| ' + ' | '.join(str(value) for value in row) + ' |')
return lines + ['']
def ensure_stand(overrides: dict, *, force_recreate: bool = True) -> dict:
"""Приводит стенд к нужной конфигурации; пересоздаёт только при изменении."""
desired = {**load_stand_config(), **{key: str(value) for key, value in overrides.items()}}
rendered = render_stand_env(desired)
if not STAND_ENV.exists() or STAND_ENV.read_text() != rendered:
log(f'перенастройка стенда: {overrides}')
apply_stand(desired, force_recreate=force_recreate)
else:
wait_for_lb()
return desired
# ---------------------------------------------------------------- подкоманды
def cmd_stand(args) -> None:
config = load_stand_config()
if args.action == 'up':
if args.replicas:
config['API_REPLICAS'] = str(args.replicas)
save_stand_config(config)
compose('up', '-d', '--build', '--wait', timeout=1800)
wait_for_lb()
log(f'стенд поднят: {replica_map().keys()}')
elif args.action == 'down':
compose('down')
else:
print(compose('ps').stdout)
def header(headers: dict, name: str) -> str | None:
for key, value in headers.items():
if key.lower() == name.lower():
return value
return None
def cmd_check(args) -> None:
client = Client()
if args.what == 'spread':
token = client.register('spread')['token']
counts: dict[str, int] = {}
for _ in range(args.requests):
response = client.call('GET', '/api/v1/users/me', token=token)
instance = header(response['headers'], 'X-Instance-Id')
counts[instance] = counts.get(instance, 0) + 1
payload = {
'requests': args.requests,
'distinct_instances': len(counts),
'instances': counts,
'replicas': {name: info for name, info in replica_map().items()},
'captured_at': now_iso(),
}
print(json.dumps(payload, ensure_ascii=False, indent=1))
write_json(LEVEL4_DIR / 'spread.json', payload)
elif args.what == 'instances':
print(json.dumps({'replicas': replica_map(), 'haproxy': haproxy_stats(Sampler()._haproxy() or [])}, ensure_ascii=False, indent=1))
elif args.what == 'idempotency':
LEVEL4_DIR.mkdir(parents=True, exist_ok=True)
result = subprocess.run(
['npm', 'run', 'test:e2e', '--', '-t', 'idempotency'],
cwd=ROOT / 'app',
capture_output=True,
text=True,
timeout=900,
env={**os.environ, 'API_BASE_URL': BASE_URL},
)
output = result.stdout + result.stderr
(LEVEL4_DIR / 'idempotency-concurrency.txt').write_text(output, encoding='utf-8')
print(output[-1500:])
if result.returncode != 0:
raise SystemExit('idempotency concurrency suite failed')
else:
raise SystemExit(f'unknown check: {args.what}')
def cmd_run(args) -> None:
overrides: dict = {}
if args.replicas:
overrides['API_REPLICAS'] = str(args.replicas)
if args.pool:
overrides['DATABASE_POOL_MAX'] = str(args.pool)
if args.loadgen_cpus:
overrides['LOADGEN_CPUS'] = str(args.loadgen_cpus)
if args.lb_cpus:
overrides['LB_CPUS'] = str(args.lb_cpus)
if args.api_total_cpus:
overrides['API_TOTAL_CPUS'] = str(args.api_total_cpus)
if args.pg_cpus:
overrides['PG_CPUS'] = str(args.pg_cpus)
ensure_stand(overrides)
profile = args.profile
ladder = [int(value) for value in args.ladder.split(',')] if args.ladder else None
summary = run_measured(
args.label,
profile,
ladder=ladder,
step_seconds=args.step or STEP_SECONDS,
rate=args.peak,
duration=args.duration,
mode='read' if args.mode == 'read' else None,
spread=args.spread,
warmup=not args.no_warmup,
)
print(json.dumps(summary, ensure_ascii=False, indent=1))
def cmd_sweep(args) -> None:
replicas = [int(value) for value in args.replicas.split(',')]
label_prefix = args.prefix or args.profile
for rep in range(1, args.reps + 1):
order = replicas if rep % 2 else list(reversed(replicas))
for count in order:
ensure_stand({'API_REPLICAS': str(count)})
run_measured(f'{label_prefix}#n{count}#rep{rep}', args.profile, warmup=not args.no_warmup)
report_series(args.profile, label_prefix)
if args.extra_replica:
for extra in range(1, args.extra_reps + 1):
count = args.extra_replica
ensure_stand({'API_REPLICAS': str(count)})
run_measured(f'{label_prefix}#n{count}#rep{args.reps + extra}', args.profile, warmup=not args.no_warmup)
report_series(args.profile, label_prefix)
def report_series(profile: str, prefix: str) -> None:
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#')]
groups = group_configurations(labels)
lines = []
for key, group in groups.items():
series = summarise_series(group)
lines += table(
['темп', 'достигнуто RPS', 'min', 'max', 'p95', 'ошибок'],
[
[row['rate'], row['achieved_rps'], row['achieved_min'], row['achieved_max'], row['p95'], row['failed']]
for row in series['rows']
],
)
lines.append(f'Потолок {key}: {series["ceiling_rps"]} req/s, сатурация на {series["saturation"]["rate"]} req/s ({len(group)} повторов)')
lines.append('')
write_markdown(TRACK1_DIR / f'summary-live-{profile}.md', f'Текущие агрегаты {profile}', lines)
def configuration_key(label: str) -> str:
"""Ключ конфигурации: метки вида <профиль>#<конфигурация>#rep<N>.
Без второго сегмента повторные прогоны разных конфигураций слились бы в одну
таблицу (например, все N серии A), а A/B-стороны 'ab#A'/'ab#B' — в одну
группу 'ab'.
"""
parts = label.split('#')
return '#'.join(parts[:2]) if len(parts) > 1 else parts[0]
def group_configurations(labels: list[str]) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {}
for label in labels:
groups.setdefault(configuration_key(label), []).append(label)
return {key: sorted(value) for key, value in sorted(groups.items())}
def knee_rate(profile: str, prefix: str = 'p-a') -> int:
labels = [label for label in raw_labels() if label.startswith(f'{prefix}#n4#')]
if not labels:
return 400
series = summarise_series(labels)
rows = series['rows']
for row in rows:
if row['achieved_rps'] < 0.95 * row['rate']:
previous = rows[max(0, rows.index(row) - 1)]
return int(previous['rate'])
return int(rows[-1]['rate']) if rows else 400
def cmd_growth(args) -> None:
base = load_stand_config()
rate = args.rate or max(1, int(knee_rate(args.base_profile) * 0.8))
points = {
'api': [
{'API_TOTAL_CPUS': str(float(base.get('API_TOTAL_CPUS', API_TOTAL_CPUS)) * 2), 'label': 'api-total-6'},
],
'pg': [
{'PG_CPUS': '3.0', 'label': 'pg-3.0'},
],
'pool': [
{'DATABASE_POOL_MAX': '5', 'label': 'pool-5'},
{'DATABASE_POOL_MAX': '20', 'label': 'pool-20'},
],
}
log(f'серия роста {args.series} на темпе {rate} req/s (80% от колена {args.base_profile})')
# Каждая точка отличается от базовой конфигурации ровно одним ресурсом:
# остальные значения прибиты к базовым, иначе после точки «API x2» следующий
# ресурс мерился бы уже с удвоенным бюджетом API.
baseline = {
'API_REPLICAS': '4',
'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']),
'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']),
'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']),
'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']),
}
for point in points[args.series]:
overrides = {
**baseline,
**{key: value for key, value in point.items() if key != 'label'},
}
ensure_stand(overrides)
run_measured(
f'growth#{point["label"]}{args.suffix}',
args.base_profile,
rate=rate,
duration=args.duration,
warmup=not args.no_warmup,
)
class MessageProbe(threading.Thread):
"""Последовательный клиент: измеряет успешность записи и повторяет
неопределённый POST тем же ключом (как это делает реальный клиент)."""
def __init__(self, client: Client, token: str, chat_id: str, interval: float, tag: str):
super().__init__(daemon=True)
self.client = client
self.token = token
self.chat_id = chat_id
self.interval = interval
self.tag = tag
self.stop_event = threading.Event()
self.samples: list[dict] = []
self.confirmed: list[dict] = []
self.started_monotonic = time.monotonic()
def run(self) -> None:
import uuid
sequence = 0
started = self.started_monotonic
while not self.stop_event.is_set():
sequence += 1
key = uuid.uuid4().hex
text = f'{self.tag} {sequence}'
attempts = 0
while attempts < 3:
attempts += 1
moment = time.monotonic() - started
try:
response = self.client.post_message(self.token, self.chat_id, text, key)
except (urllib.error.URLError, TimeoutError, OSError) as error:
self.samples.append({'t': round(moment, 3), 'ok': False, 'error': type(error).__name__})
if self.stop_event.wait(0.1):
return
continue
instance = header(response['headers'], 'X-Instance-Id')
self.samples.append(
{'t': round(moment, 3), 'ok': response['status'] == 201, 'status': response['status'], 'instance': instance}
)
if response['status'] == 201:
self.confirmed.append({'text': text, 'key': key, 'id': response['json']['id']})
break
if self.stop_event.wait(self.interval):
return
def replica_ip(container: str) -> str:
return inspect(container, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
def cmd_failover(args) -> None:
config = load_stand_config()
expected = int(config['API_REPLICAS'])
ensure_stand({'API_REPLICAS': str(expected)})
replicas = wait_for_replicas(expected)
targets = replicas if args.replicas == 'all' else [args.replicas]
results = []
for target in targets:
results.append(failover_once(target, args, config))
write_failover_evidence(results)
print(json.dumps(results, ensure_ascii=False, indent=1))
def failover_once(target: str, args, config: dict) -> dict:
"""Один цикл: нагрузка -> остановка реплики -> возврат -> проверка истории.
Проба идёт постоянно и повторяет неопределённый POST тем же ключом, поэтому
«потерь» и «дублей» в истории быть не должно. Окно реального отсутствия
реплики считается от выхода контейнера (docker stop возвращает управление
уже после остановки), а не от вызова команды: дрейн длиной DRAIN_DELAY_MS
специально переживает обнаружение отказа LB и не должен выглядеть отказом.
"""
if target not in replica_map() or lb_backends_up() < len(replica_map()):
log(f'{target} не запущена или LB ещё не видит все реплики: восстанавливаю')
run_command(['scripts/start-replica.sh'])
wait_for_lb()
wait_for_replicas(len(replica_map()) or int(config['API_REPLICAS']))
if target not in replica_map():
raise SystemExit(f'replica {target} did not come back')
container = replica_map()[target]['container']
address = replica_ip(container)
client = Client(timeout=5)
alice = client.register(f'failover{target[:4]}')
chat = client.create_chat(alice['token'], [alice['id']], f'failover {target}')
load_seconds = int(args.pre + args.hold + 30)
label = f'failover#{target}'
env = {
**load_env('p-b', 'write', 'spread', label),
'RATE': str(args.rate),
'DURATION': str(load_seconds),
}
sampler = Sampler().start()
load = subprocess.Popen(
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{key}={value}'] for key, value in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
probe = MessageProbe(client, alice['token'], chat, 0.2, f'failover-{target[:6]}')
probe.start()
started = now_iso()
time.sleep(args.pre)
command_moment = time.monotonic()
stop = run_command(['scripts/stop-one.sh'], check=False, env={'TARGET_INSTANCE_ID': target}, timeout=30)
exit_moment = time.monotonic()
stop_at = now_iso()
first_success_after_exit = None
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
exit_offset = probe_offset(probe, exit_moment)
if any(sample.get('ok') and sample['t'] >= exit_offset for sample in list(probe.samples)):
first_success_after_exit = round(probe_offset(probe, time.monotonic()), 2)
break
time.sleep(0.1)
time.sleep(args.hold)
restore_moment = time.monotonic()
restore_at = now_iso()
run_command(['scripts/start-replica.sh'])
recovered = None
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
seen = {
header(client.call('GET', '/api/v1/users/me', token=alice['token'])['headers'], 'X-Instance-Id')
for _ in range(6)
}
if len(seen) >= 2:
recovered = round(time.monotonic() - restore_moment, 2)
break
time.sleep(0.5)
try:
stdout, _ = load.communicate(timeout=load_seconds + 90)
except subprocess.TimeoutExpired:
load.kill()
stdout, _ = load.communicate()
probe.stop_event.set()
probe.join(timeout=15)
samples = sampler.stop()
finished = now_iso()
summary = None
for line in reversed((stdout or '').splitlines()):
if line.strip().startswith('{'):
try:
candidate = json.loads(line)
except ValueError:
continue
if 'steps' in candidate:
summary = candidate
break
history = client.history(alice['token'], chat)
texts = [item['text'] for item in history]
duplicates = sorted({text for text in texts if texts.count(text) > 1})
missing = [item['text'] for item in probe.confirmed if item['text'] not in texts]
command_offset = probe_offset(probe, command_moment)
exit_offset = probe_offset(probe, exit_moment)
outage_start = first_success_after_exit if first_success_after_exit is None else exit_offset
last_offset = probe.samples[-1]['t'] if probe.samples else exit_offset
phases = {
'stop_command_at': stop_at,
'stop_exit_code': stop.returncode,
'stop_command_took_s': round(exit_moment - command_moment, 2),
'first_success_after_exit_s': (
round(first_success_after_exit - exit_offset, 2) if first_success_after_exit is not None else None
),
'restore_at': restore_at,
'replica_rejoined_s': recovered,
'lb_down_at': lb_down_time(samples, address),
'probe_offsets': {
'stop_command': round(command_offset, 2),
'container_exit': round(exit_offset, 2),
'restore_command': round(probe_offset(probe, restore_moment), 2),
'last_sample': round(last_offset, 2),
},
}
if summary:
write_raw(label, summary, samples, config, started, finished, phases, sampler.errors)
failures = [sample for sample in probe.samples if not sample.get('ok')]
result = {
'instance': target,
'container': container,
'address': address,
'rate': args.rate,
'phases': phases,
'probe': {
'attempts': len(probe.samples),
'successes': sum(1 for sample in probe.samples if sample.get('ok')),
'failures': len(failures),
'confirmed': len(probe.confirmed),
'rps_before': phase_rps(probe.samples, 0.0, command_offset),
'rps_while_stopped': phase_rps(probe.samples, exit_offset, outage_start),
'rps_after': phase_rps(probe.samples, outage_start, last_offset),
'errors_while_stopped': sum(
1 for sample in failures if exit_offset <= sample['t'] <= outage_start
),
'timeline': [[sample['t'], 1 if sample.get('ok') else 0] for sample in probe.samples],
},
'duplicates': duplicates,
'missing_confirmed': missing,
'haproxy': haproxy_stats(samples),
'k6': {
'achieved_rps': summary['steps'][0]['achieved_rps'] if summary else None,
'failed': summary['steps'][0]['failed'] if summary else None,
'dropped': summary.get('dropped_iterations') if summary else None,
},
}
log(
f'failover {target}: контейнер остановлен через {phases["stop_command_took_s"]}s, '
f'первый успех после остановки через {phases["first_success_after_exit_s"]}s, '
f'реплика вернулась через {recovered}s, дубли {len(duplicates)}, потеряно {len(missing)}'
)
return result
def probe_offset(probe: MessageProbe, moment: float) -> float:
"""Переводит момент wall-clock в шкалу времени сэмплов пробы."""
if not probe.samples:
return 0.0
return probe.samples[0]['t'] + (moment - probe.started_monotonic)
def lb_down_time(samples: list[dict], address: str) -> str | None:
for record in samples:
if record.get('source') != 'haproxy' or record.get('server') in ('FRONTEND', 'BACKEND'):
continue
if address in (record.get('addr') or '') and record.get('status') not in ('UP',):
return record['t']
return None
def phase_rps(samples: list[dict], start: float, end: float) -> float:
window = [sample for sample in samples if start <= sample['t'] <= end]
return round(len(window) / max(0.001, end - start), 2)
def write_failover_evidence(results: list[dict]) -> None:
write_json(LEVEL4_DIR / 'failover-sweep.json', results)
lines = table(
[
'реплика',
'контейнер',
'попыток пробы',
'успешных',
'ошибок в отказе',
'остановка контейнера, s',
'первый успех после остановки, s',
'реплика вернулась, s',
'LB DOWN',
'RPS до',
'RPS в отказе',
'RPS после',
'k6 RPS',
'k6 ошибок',
'дубли',
'потери',
],
[
[
result['instance'][:12],
result['container'][:12],
result['probe']['attempts'],
result['probe']['successes'],
result['probe']['errors_while_stopped'],
result['phases']['stop_command_took_s'],
result['phases']['first_success_after_exit_s'],
result['phases']['replica_rejoined_s'],
(result['phases']['lb_down_at'] or '-')[11:19],
result['probe']['rps_before'],
result['probe']['rps_while_stopped'],
result['probe']['rps_after'],
result['k6']['achieved_rps'],
result['k6']['failed'],
len(result['duplicates']),
len(result['missing_confirmed']),
]
for result in results
],
)
lines += [
'',
'Проба: последовательные POST с интервалом 0.2 s, при неопределённом результате '
'повтор с тем же Idempotency-Key (как у реального клиента). Окно отсутствия реплики '
'считается от выхода контейнера до первого успешного ответа после него; дрейн перед '
'остановкой специально переживает обнаружение отказа LB, поэтому клиент не обязан '
'видеть ошибки. Потери и дубли проверяются по истории чата.',
]
write_markdown(LEVEL4_DIR / 'failover-sweep.md', 'Отказ каждой реплики под нагрузкой', lines)
def cmd_rolling(args) -> None:
config = load_stand_config()
replicas = int(config['API_REPLICAS'])
build_id = args.build_id or datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
before = replica_map()
migrate_before = inspect(container_ids('migrate', include_stopped=True)[0], '{{.State.StartedAt}}')
log(f'сборка образа {config["API_IMAGE"]} с BUILD_ID={build_id}')
build = run_command(
['docker', 'build', '--build-arg', f'BUILD_ID={build_id}', '-t', config['API_IMAGE'], '.'],
check=True,
timeout=1800,
)
image_id = run_command(['docker', 'image', 'inspect', '-f', '{{.Id}}', config['API_IMAGE']]).stdout.strip()
client = Client(timeout=5)
alice = client.register('rolling')
chat = client.create_chat(alice['token'], [alice['id']], 'rolling update')
probe = MessageProbe(client, alice['token'], chat, 0.2, 'rolling')
sampler = Sampler().start()
load = None
if args.under_load:
env = {**load_env('p-b', 'write', 'spread', 'rolling'), 'RATE': str(args.rate), 'DURATION': str(max(args.duration, replicas * 45))}
load = subprocess.Popen(
[*COMPOSE, '--profile', 'load', 'run', '--rm', '--no-deps', *sum((['-e', f'{k}={v}'] for k, v in env.items()), []), 'loadgen', 'run', '/load/write-steady.js'],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
probe.start()
started = now_iso()
steps = []
for step in range(replicas):
current = replica_map()
known = set(current)
# Меняем самую «старую» реплику: только что созданная видна как самая молодая.
target = sorted(current, key=lambda instance: current[instance]['started_at'])[0]
container = current[target]['container']
old_build = app_build_id(container)
run_command(['docker', 'stop', '--time', '15', container], check=False, timeout=60)
run_command(['docker', 'rm', container], check=False, timeout=60)
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', f'api={replicas}', 'api', timeout=600)
new_instance, new_container = wait_for_new_replica(replicas, known)
new_build = app_build_id(new_container)
steps.append(
{
'step': step + 1,
'old_instance': target,
'old_container': container[:12],
'old_build_id': old_build,
'new_instance': new_instance,
'new_container': new_container[:12],
'new_build_id': new_build,
'migrate_started_at': migrate_before,
}
)
log(f'rolling {step + 1}/{replicas}: {target[:12]} ({old_build}) -> {new_instance[:12]} ({new_build})')
time.sleep(args.pause)
summary = None
if load is not None:
try:
stdout, _ = load.communicate(timeout=max(30, args.duration))
except subprocess.TimeoutExpired:
load.terminate()
try:
stdout, _ = load.communicate(timeout=60)
except subprocess.TimeoutExpired:
load.kill()
stdout, _ = load.communicate()
probe.stop_event.set()
probe.join(timeout=10)
samples = sampler.stop()
for line in reversed((stdout or '').splitlines()):
if line.strip().startswith('{'):
try:
candidate = json.loads(line)
except ValueError:
continue
if 'steps' in candidate:
summary = candidate
break
migrate_after = inspect(container_ids('migrate', include_stopped=True)[0], '{{.State.StartedAt}}')
failures = [sample for sample in probe.samples if not sample.get('ok')]
server_failures = [sample for sample in failures if 500 <= int(sample.get('status') or 0) < 600]
finished = now_iso()
payload = {
'build_id': build_id,
'image_id': image_id,
'image': config['API_IMAGE'],
'replicas': replicas,
'under_load': bool(args.under_load),
'steps': steps,
'probe': {'attempts': len(probe.samples), 'failures': len(failures), 'server_5xx': len(server_failures)},
'k6': {'achieved_rps': summary['steps'][0]['achieved_rps'] if summary else None, 'failed': summary['steps'][0]['failed'] if summary else None},
'migrate': {'before': migrate_before, 'after': migrate_after, 'restarted': migrate_before != migrate_after},
'build_stderr_tail': build.stderr.strip().splitlines()[-3:],
}
write_json(LEVEL4_DIR / 'rolling-update.json', payload)
if summary:
write_raw('rolling', summary, samples, config, started, finished, {'steps': steps}, sampler.errors)
lines = table(
['шаг', 'было', 'BUILD_ID до', 'стало', 'BUILD_ID после'],
[[step['step'], step['old_instance'][:12], step['old_build_id'], step['new_instance'][:12], step['new_build_id']] for step in steps],
)
lines += [
f'5xx от API во время rolling update: {len(server_failures)} из {len(probe.samples)} попыток',
f'Миграции перезапускались: {migrate_before != migrate_after}',
]
write_markdown(LEVEL4_DIR / 'rolling-update.md', 'Rolling update под нагрузкой', lines)
print(json.dumps(payload, ensure_ascii=False, indent=1))
def app_build_id(container: str) -> str:
result = run_command(
[
'docker', 'exec', container, 'node', '-e',
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics',"
"{headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}}).then(r=>r.json())"
".then(j=>process.stdout.write(String(j.build_id)))",
],
check=False,
timeout=20,
)
return result.stdout.strip() or 'unknown'
def wait_for_new_replica(replicas: int, known: set[str], timeout: float = 180) -> tuple[str, str]:
"""Ждёт появления НОВОЙ реплики (не из known) в состоянии healthy."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
mapping = replica_map()
fresh = {instance: info for instance, info in mapping.items() if instance not in known}
if len(mapping) >= replicas and fresh:
for instance, info in sorted(fresh.items()):
status = inspect(info['container'], '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')
if status == 'healthy':
return instance, info['container']
time.sleep(1)
raise SystemExit(
f'новая реплика не появилась healthy за {timeout}s (контейнеров {len(replica_map())}, '
f'новых {len({i for i in replica_map() if i not in known})})'
)
def cmd_ab(args) -> None:
sides = {'A': args.before, 'B': args.after}
for rep in range(1, args.reps + 1):
order = ['A', 'B'] if rep % 2 else ['B', 'A']
for side in order:
ensure_stand({'API_IMAGE': sides[side], 'API_REPLICAS': str(args.replicas)})
run_measured(f'ab#{side}#rep{rep}', args.profile, warmup=not args.no_warmup)
report_ab(args)
def report_ab(args=None) -> None:
labels = [label for label in raw_labels() if label.startswith('ab#')]
groups = group_configurations(labels)
lines = []
verdicts = []
series_by_side = {
side: summarise_series(groups.get(f'ab#{side}', [])) for side in ('A', 'B')
}
images = {
side: (getattr(args, side.lower(), None) if args else None) for side in ('A', 'B')
}
for side in ('A', 'B'):
if not images[side] and groups.get(f'ab#{side}'):
manifest = read_json(
RAW_DIR / f'{groups[f"ab#{side}"][-1]}.manifest.json', {}
)
images[side] = (manifest.get('stand') or {}).get('API_IMAGE')
for rate in sorted({row['rate'] for row in series_by_side['A']['rows']}):
row_a = next((row for row in series_by_side['A']['rows'] if row['rate'] == rate), None)
row_b = next((row for row in series_by_side['B']['rows'] if row['rate'] == rate), None)
if not row_a or not row_b:
continue
change = (
round((row_b['achieved_rps'] - row_a['achieved_rps']) / row_a['achieved_rps'] * 100, 2)
if row_a['achieved_rps']
else 0.0
)
confirmed = (
abs(change) >= 5.0
and (row_b['achieved_min'] > row_a['achieved_max'] or row_a['achieved_min'] > row_b['achieved_max'])
)
verdicts.append({'rate': rate, 'change_percent': change, 'confirmed': confirmed})
lines.append(
f'{rate} req/s: A {row_a["achieved_rps"]} [{row_a["achieved_min"]}; {row_a["achieved_max"]}] vs '
f'B {row_b["achieved_rps"]} [{row_b["achieved_min"]}; {row_b["achieved_max"]}] -> {change}% '
f'({"подтверждено" if confirmed else "не подтверждено"})'
)
lines.append('')
lines += table(
['шаг', 'образ A: RPS (min..max)', 'p95 A', 'образ B: RPS (min..max)', 'p95 B', 'Δ, %', 'вывод'],
[
[
verdict['rate'],
f'{next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_rps"]}'
f' ({next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_min"]}..'
f'{next(row for row in series_by_side["A"]["rows"] if row["rate"] == verdict["rate"])["achieved_max"]})',
next(row for row in series_by_side['A']['rows'] if row['rate'] == verdict['rate'])['p95'],
f'{next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_rps"]}'
f' ({next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_min"]}..'
f'{next(row for row in series_by_side["B"]["rows"] if row["rate"] == verdict["rate"])["achieved_max"]})',
next(row for row in series_by_side['B']['rows'] if row['rate'] == verdict['rate'])['p95'],
verdict['change_percent'],
'подтверждено' if verdict['confirmed'] else 'не подтверждено',
]
for verdict in verdicts
],
)
lines += [
f'Потолок A: {series_by_side["A"]["ceiling_rps"]} req/s, потолок B: {series_by_side["B"]["ceiling_rps"]} req/s',
'',
'Правило подтверждения: медиана меняется ≥5 % и диапазоны min/max повторов не перекрываются.',
]
write_json(
TRACK1_DIR / 'ab-verdict.json',
{
'verdicts': verdicts,
'ceiling': {side: series_by_side[side]['ceiling_rps'] for side in ('A', 'B')},
'images': images,
},
)
write_markdown(TRACK1_DIR / 'summary-ab.md', 'A/B: подтверждённое улучшение', lines)
# ---------------------------------------------------------------- анализ и графики
def pyplot():
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
return plt
def representative(group: list[str]) -> str:
"""Прогон с медианным пиком — чтобы агрегаты контейнеров не брались с края."""
scored = []
for label in group:
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
steps = summary.get('steps', [])
peak = max((step['achieved_rps'] for step in steps), default=0.0)
scored.append((peak, label))
if not scored:
raise SystemExit('нет прогонов для анализа')
scored.sort()
return scored[len(scored) // 2][1]
def cmd_analyze(args) -> None:
labels = raw_labels()
if not labels:
log('raw пуст: таблицы и графики будут пустыми')
groups = group_configurations(labels)
analyse_runs(groups)
analyse_series_a(groups, labels)
analyse_diagnostics(groups, labels)
analyse_growth(groups, labels)
if groups.get('ab#A') or groups.get('ab#B'):
report_ab()
make_graphs(groups, labels)
log('analyze: таблицы и графики обновлены')
def series_kind(group: list[str]) -> str:
summary = read_json(RAW_DIR / f'{group[0]}.k6.json', {})
return summary.get('kind', 'ladder')
def analyse_runs(groups: dict) -> None:
"""Полный разбор каждого прогона: шаги, ресурсы, PostgreSQL, LB."""
lines = []
for key, group in sorted(groups.items()):
series = summarise_series(group)
label = representative(group)
_, samples, manifest = load_raw(label)
lines += [f'## {key} ({len(group)} прогонов, {series_kind(group)})', '']
lines += table(
['темп, req/s', 'достигнуто RPS', 'min..max', 'p95, ms', 'ошибок', 'VU', 'dropped'],
[
[
row['rate'],
row['achieved_rps'],
f'{row["achieved_min"]}..{row["achieved_max"]}',
row['p95'],
row['failed'],
row['vus_active'],
read_json(RAW_DIR / f'{label}.k6.json', {}).get('dropped_iterations', 0),
]
for row in series['rows']
],
)
if series_kind(group) == 'ladder':
saturation = series['saturation']
lines.append(
f'Потолок {series["ceiling_rps"]} req/s, сатурация {saturation["rate"]} req/s '
f'({saturation["achieved_rps"]} req/s достигнуто).'
)
lines.append('')
limits = manifest.get('stand', {}) if manifest else {}
lines.append(
f'Лимиты: реплик {limits.get("API_REPLICAS")}, API {limits.get("API_CPUS_PER_REPLICA")} CPU / '
f'{limits.get("API_MEMORY_PER_REPLICA")}, PG {limits.get("PG_CPUS")} CPU, LB {limits.get("LB_CPUS")} CPU, '
f'пул {limits.get("DATABASE_POOL_MAX")}, образ {limits.get("API_IMAGE")} (BUILD_ID {limits.get("BUILD_ID")}).'
)
lines.append('')
lines += table(
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
[
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
for name, stats in container_stats(samples).items()
],
)
pg = postgres_stats(samples)
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, hit ratio {pg["blks_hit_ratio"]}, '
f'ожиданий блокировок пик {pg["lock_waiters_peak"]}, активных сессий пик {pg["active_peak"]}, '
f'соединений пик {pg["connections_peak"]}, deadlocks {pg["deadlocks"]}, messages {pg["messages_mib"]} MiB.'
)
lines.append('')
lb = haproxy_stats(samples)
total = lb.get('BACKEND')
if total:
lines.append(
f'LB: {total["requests"]} запросов ({total["rps"]} req/s), ошибок {total["errors"]}, '
f'очередь пик {total["qcur_peak"]}.'
)
for name, server in sorted(lb.items()):
if name in ('BACKEND', 'FRONTEND') or not server['requests']:
continue
lines.append(
f'- LB слот {name}: {server["requests"]} запросов ({server["rps"]} req/s), ошибок {server["errors"]}, '
f'очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}, адрес {server["addresses"]}.'
)
lines.append('')
app = app_metrics_delta(samples)
for container, delta in sorted(app.items()):
lines.append(
f'- реплика {delta["instance_id"]} (build {delta["build_id"]}, контейнер {container[:12]}): '
f'{json.dumps(delta["counters"], ensure_ascii=False)}; '
f'latency p95 {delta["latency_ms"].get("p95")} ms.'
)
lines.append('')
write_markdown(TRACK1_DIR / 'summary-runs.md', 'Разбор каждого прогона', lines)
def analyse_series_a(groups: dict, labels: list[str]) -> None:
lines = []
series_by_n = {}
for key, group in sorted(groups.items()):
if not key.startswith('p-a#n'):
continue
series = summarise_series(group)
series_by_n[key] = series
lines.append(f'## {key} ({len(group)} повторов)')
lines.append('')
lines += table(
['темп, req/s', 'достигнуто RPS', 'min..max', 'p95, ms', 'p95 max', 'ошибок', 'VU'],
[
[
row['rate'],
row['achieved_rps'],
f'{row["achieved_min"]}..{row["achieved_max"]}',
row['p95'],
row['p95_max'],
row['failed'],
row['vus_active'],
]
for row in series['rows']
],
)
saturation = series['saturation']
lines.append(
f'Потолок: **{series["ceiling_rps"]} req/s**; сатурация: {saturation["rate"]} req/s '
f'(достигнуто {saturation["achieved_rps"]} req/s).'
)
lines.append('')
label = representative(group)
_, samples, manifest = load_raw(label)
lines.append(f'Ресурсы представительного прогона `{label}` (лимиты {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")} на реплику):')
lines.append('')
lines += table(
['контейнер', 'CPU средний, %', 'CPU пик, %', 'RAM средняя, MiB', 'RAM пик, MiB', 'лимит, MiB'],
[
[name, stats['cpu_mean'], stats['cpu_peak'], stats['mem_mean'], stats['mem_peak'], stats['mem_limit']]
for name, stats in container_stats(samples).items()
],
)
pg = postgres_stats(samples)
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, откатов {pg["rollback_per_s"]}/s, '
f'hit ratio {pg["blks_hit_ratio"]}, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
f'активных сессий пик {pg["active_peak"]}, соединений пик {pg["connections_peak"]}, '
f'таблица messages {pg["messages_mib"]} MiB.'
)
lines.append('')
lb = haproxy_stats(samples)
distribution = [f'{name}: {server["requests"]} ({server["rps"]} req/s), адреса {", ".join(server["addresses"])}' for name, server in sorted(lb.items())]
if distribution:
lines += ['Распределение запросов LB:', ''] + [f'- {item}' for item in distribution] + ['']
lines += saturation_conclusion(series_by_n)
write_markdown(TRACK1_DIR / 'summary-series-a.md', 'Серия A: 1/2/4 реплики при фиксированном бюджете', lines)
def saturation_conclusion(series_by_n: dict) -> list[str]:
lines = ['## Точки сатурации', '']
for key, series in sorted(series_by_n.items()):
saturation = series['saturation']
lines.append(
f'- {key}: knee на {saturation["rate"]} req/s (достигнуто {saturation["achieved_rps"]} req/s), '
f'потолок {series["ceiling_rps"]} req/s.'
)
lines.append('')
return lines
def analyse_diagnostics(groups: dict, labels: list[str]) -> None:
lines = []
for key, group in sorted(groups.items()):
if not key.startswith(('p-b', 'p-c')):
continue
series = summarise_series(group)
label = representative(group)
_, samples, manifest = load_raw(label)
pg = postgres_stats(samples)
stats = container_stats(samples)
lines.append(f'## {key} (темпы {series["rows"][0]["rate"]}..{series["rows"][-1]["rate"]} req/s)')
lines.append('')
lines += table(
['темп', 'достигнуто RPS', 'p95, ms', 'ошибок'],
[[row['rate'], row['achieved_rps'], row['p95'], row['failed']] for row in series['rows']],
)
lines.append(
f'Потолок {series["ceiling_rps"]} req/s, knee на {series["saturation"]["rate"]} req/s. '
f'Лимиты: LB {manifest["stand"].get("LB_CPUS")} CPU, API {manifest["stand"].get("API_CPUS_PER_REPLICA")} CPU / '
f'{manifest["stand"].get("API_MEMORY_PER_REPLICA")}, PG {manifest["stand"].get("PG_CPUS")} CPU.'
)
lines.append('')
if pg:
lines.append(
f'PostgreSQL: {pg["xact_per_s"]} xact/s, ожиданий блокировок пик {pg["lock_waiters_peak"]}, '
f'активных сессий пик {pg["active_peak"]}, hit ratio {pg["blks_hit_ratio"]}.'
)
lines.append('')
lb = stats.get('mtusi-lab3-haproxy-1')
if lb:
lines.append(f'HAProxy: CPU средний {lb["cpu_mean"]}%, пик {lb["cpu_peak"]}%, RAM пик {lb["mem_peak"]} MiB из {lb["mem_limit"]}.')
lines.append('')
haproxy = haproxy_stats(samples)
for name, server in sorted(haproxy.items()):
if server['qcur_peak'] or server['statuses'] != ['UP']:
lines.append(f'- LB сервер {name}: очередь пик {server["qcur_peak"]}, статусы {server["statuses"]}.')
lines.append('')
write_markdown(TRACK1_DIR / 'summary-diagnostics.md', 'Диагностика: горячий чат и путь чтения', lines)
def analyse_growth(groups: dict, labels: list[str]) -> None:
growth = {key: group for key, group in groups.items() if key.startswith('growth#')}
if not growth:
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', ['Нет прогонов growth.'])
return
lines = []
baseline_labels = [label for label in labels if label.startswith('p-a#n4#')]
baseline = summarise_series(baseline_labels) if baseline_labels else None
if baseline:
rate = max((row['rate'] for row in growth_rows(growth)), default=0)
row = step_of_rate(baseline['rows'], rate)
lines.append(f'Базовая точка (серия A, N=4, тот же темп {rate} req/s): **{row["achieved_rps"]} req/s**, p95 {row["p95"]} ms.')
lines.append('')
lines += table(
['точка', 'темп', 'достигнуто RPS', 'p95, ms', 'ошибок', 'CPU API пик, %', 'CPU PG пик, %'],
[growth_row(key, group) for key, group in sorted(growth.items())],
)
write_markdown(TRACK1_DIR / 'summary-growth.md', 'Серия роста ресурсов', lines)
def growth_rows(growth: dict):
for group in growth.values():
summary = read_json(RAW_DIR / f'{representative(group)}.k6.json', {})
yield from summary.get('steps', [])
def step_of_rate(rows: list[dict], rate: int) -> dict:
for row in rows:
if row['rate'] == rate:
return row
return rows[-1] if rows else {'rate': 0, 'achieved_rps': 0, 'p95': 0}
def growth_row(key: str, group: list[str]) -> list:
label = representative(group)
summary = read_json(RAW_DIR / f'{label}.k6.json', {})
_, samples, _ = load_raw(label)
step = summary['steps'][0] if summary.get('steps') else {'rate': 0, 'achieved_rps': 0, 'p95': 0, 'failed': 0}
stats = container_stats(samples)
api_peak = max((value['cpu_peak'] for name, value in stats.items() if 'api' in name), default=0.0)
pg_peak = stats.get('mtusi-lab3-postgres-1', {}).get('cpu_peak', 0.0)
return [key, step['rate'], step['achieved_rps'], step['p95'], step['failed'], api_peak, pg_peak]
def make_graphs(groups: dict, labels: list[str]) -> None:
if not labels:
return
plt = pyplot()
GRAPH_DIR.mkdir(parents=True, exist_ok=True)
ladders = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'ladder'}
steadys = {key: group for key, group in sorted(groups.items()) if series_kind(group) == 'steady'}
if ladders:
series = {key: summarise_series(group) for key, group in ladders.items()}
figure, axes = plt.subplots(figsize=(9, 5))
for key, values in series.items():
axes.plot(
[row['rate'] for row in values['rows']],
[row['achieved_rps'] for row in values['rows']],
marker='o',
label=key,
)
reference = next(iter(series.values()))
axes.plot(
[row['rate'] for row in reference['rows']],
[row['rate'] for row in reference['rows']],
linestyle='--',
color='grey',
label='предложенный темп',
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('достигнутый RPS')
axes.set_title('Пропускная способность: достигнутый RPS от предложенного темпа')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'throughput.png', dpi=120)
plt.close(figure)
figure, axes = plt.subplots(figsize=(9, 5))
for key, values in series.items():
axes.plot(
[row['rate'] for row in values['rows']],
[row['p95'] for row in values['rows']],
marker='o',
label=f'{key}: p95, ms',
)
axes.plot(
[row['rate'] for row in values['rows']],
[row['failed'] for row in values['rows']],
linestyle=':',
marker='x',
label=f'{key}: ошибок, шт.',
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('p95, ms / ошибок')
axes.set_title('Латентность p95 и ошибки по шагам лестницы')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'latency-errors.png', dpi=120)
plt.close(figure)
figure, axes = plt.subplots(figsize=(9, 5))
for key, values in series.items():
axes.plot(
[row['rate'] for row in values['rows']],
[row['vus_active'] for row in values['rows']],
marker='s',
label=key,
)
axes.set_xlabel('предложенный темп, req/s')
axes.set_ylabel('активных VU генератора')
axes.set_title('Занятость генератора: нехватка VU отличима от перегрузки сервера')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'generator-vus.png', dpi=120)
plt.close(figure)
if steadys:
keys = list(steadys)
achieved = [summarise_series(steadys[key])['ceiling_rps'] for key in keys]
offered = [summarise_series(steadys[key])['rows'][0]['rate'] for key in keys]
figure, axes = plt.subplots(figsize=(9, 5))
positions = range(len(keys))
axes.bar([p - 0.2 for p in positions], offered, width=0.4, label='предложенный темп')
axes.bar([p + 0.2 for p in positions], achieved, width=0.4, label='достигнутый RPS')
axes.set_xticks(list(positions))
axes.set_xticklabels(keys, fontsize=8)
axes.set_ylabel('req/s')
axes.set_title('Фиксированный темп: предложено против достигнутого')
axes.legend()
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'steady-throughput.png', dpi=120)
plt.close(figure)
for key, group in sorted(groups.items()):
plot_container_cpu(plt, representative(group))
figure, axes = plt.subplots(figsize=(9, 5))
shares = {}
for key, group in sorted(groups.items()):
_, samples, _ = load_raw(representative(group))
lb = haproxy_stats(samples)
shares[key] = {name: server['requests'] for name, server in sorted(lb.items()) if server['requests']}
names = sorted({name for share in shares.values() for name in share})
if names and shares:
width = 0.8 / max(1, len(names))
for position, name in enumerate(names):
axes.bar(
[index + position * width - 0.4 for index in range(len(shares))],
[shares[key].get(name, 0) for key in shares],
width=width,
label=name,
)
axes.set_xticks(range(len(shares)))
axes.set_xticklabels(list(shares), fontsize=7)
axes.set_ylabel('запросов за прогон')
axes.set_title('Распределение запросов по backend-слотам LB')
axes.legend(fontsize=7)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'replica-distribution.png', dpi=120)
plt.close(figure)
growth = {key: group for key, group in groups.items() if key.startswith('growth#')}
if growth:
rows = [growth_row(key, group) for key, group in sorted(growth.items())]
baseline_labels = [label for label in labels if label.startswith('p-a#n4#')]
figure, axes = plt.subplots(figsize=(9, 5))
names = []
values = []
low = []
high = []
if baseline_labels:
series = summarise_series(baseline_labels)
row = step_of_rate(series['rows'], rows[0][1])
names.append('базовая точка (серия A)')
values.append(row['achieved_rps'])
low.append(row['achieved_min'])
high.append(row['achieved_max'])
for row in rows:
names.append(row[0].replace('growth#', ''))
values.append(row[2])
low.append(row[2])
high.append(row[2])
axes.bar(names, values, yerr=[[value - minimum for value, minimum in zip(values, low)], [maximum - value for value, maximum in zip(values, high)]], capsize=5)
axes.set_ylabel('достигнутый RPS')
axes.set_title('Серия роста ресурсов: какой ресурс является ограничителем')
axes.tick_params(axis='x', labelsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'growth.png', dpi=120)
plt.close(figure)
if groups.get('ab#A') and groups.get('ab#B'):
figure, axes = plt.subplots(figsize=(9, 5))
series_a = summarise_series(groups['ab#A'])
series_b = summarise_series(groups['ab#B'])
rates = [row['rate'] for row in series_a['rows']]
positions = list(range(len(rates)))
axes.bar([p - 0.2 for p in positions], [row['achieved_rps'] for row in series_a['rows']], width=0.4, label='A (до)')
axes.bar([p + 0.2 for p in positions], [row['achieved_rps'] for row in series_b['rows']], width=0.4, label='B (после)')
for offset, series in ((-0.2, series_a), (0.2, series_b)):
for index, row in enumerate(series['rows']):
axes.plot([index + offset, index + offset], [row['achieved_min'], row['achieved_max']], color='black', linewidth=1)
axes.set_xticks(positions)
axes.set_xticklabels([str(rate) for rate in rates])
axes.set_xlabel('темп, req/s')
axes.set_ylabel('достигнутый RPS')
axes.set_title('A/B: медиана и разброс (min..max) по повторам')
axes.legend()
figure.tight_layout()
figure.savefig(GRAPH_DIR / 'ab-comparison.png', dpi=120)
plt.close(figure)
def plot_container_cpu(plt, label: str) -> None:
_, samples, _ = load_raw(label)
series: dict[str, list[tuple[float, float]]] = {}
base = None
for record in samples:
if record.get('source') != 'docker_stats':
continue
name = record.get('name') or ''
if not name.startswith('mtusi-lab3-'):
continue
moment = _timestamp(record['t'])
base = base or moment
series.setdefault(name, []).append((moment - base, float(record.get('cpu_percent') or 0)))
if not series:
return
figure, axes = plt.subplots(figsize=(10, 5))
for name, points in sorted(series.items()):
axes.plot([point[0] for point in points], [point[1] for point in points], label=name, linewidth=1)
axes.set_xlabel('время от начала прогона, s')
axes.set_ylabel('CPU, % (100 % = одно ядро)')
axes.set_title(f'CPU контейнеров: {label}')
axes.grid(alpha=0.3)
axes.legend(fontsize=8)
figure.tight_layout()
figure.savefig(GRAPH_DIR / f'container-cpu-{label.replace("#", "_")}.png', dpi=120)
plt.close(figure)
# ---------------------------------------------------------------- оркестрация трека
def cmd_watch(args) -> None:
"""Сэмплер без нагрузки: наблюдает стенд N секунд и пишет jsonl.gz.
Нужен там, где нагрузку создаёт внешний сценарий (публичный failover-тест,
rolling update): переходы статусов LB получают отметки времени.
"""
sampler = Sampler().start()
try:
time.sleep(args.seconds)
finally:
samples = sampler.stop()
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
with gzip.open(out, 'wt', encoding='utf-8') as handle:
for record in samples:
handle.write(json.dumps(record, ensure_ascii=False) + '\n')
print(json.dumps({'out': str(out), 'samples': len(samples), 'errors': sampler.errors}, ensure_ascii=False))
def cmd_quiesce(args) -> None:
result = run_command(['scripts/quiesce.sh', args.action], check=False, timeout=300)
print(result.stdout + result.stderr)
if result.returncode != 0:
raise SystemExit(f'quiesce {args.action} failed')
def cmd_track1(args) -> None:
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='up'))
try:
log('=== серия A: 1/2/4 реплики, профиль p-a')
cmd_sweep(
argparse.Namespace(
profile='p-a',
replicas='1,2,4',
reps=args.reps,
no_warmup=False,
prefix='p-a',
extra_replica=2,
extra_reps=1,
)
)
log('=== серия B: горячий чат (p-b), 4 реплики')
ensure_stand({'API_REPLICAS': '4'})
run_measured('p-b#n4', 'p-b', warmup=False)
log('=== серия C: путь чтения (p-c), 4 реплики, LB 0.25 и 1.0 CPU')
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '0.25'})
run_measured('p-c#lb0.25', 'p-c', warmup=False)
ensure_stand({'API_REPLICAS': '4', 'LB_CPUS': '1.0'})
run_measured('p-c#lb1.0', 'p-c', warmup=False)
log('=== серия роста ресурсов')
for series in ('api', 'pg', 'pool'):
cmd_growth(argparse.Namespace(series=series, rate=None, duration=60, base_profile='p-a', no_warmup=False))
log('=== агрегаты')
cmd_analyze(argparse.Namespace())
if args.ab_after:
log('=== A/B улучшения (p-b, 4 реплики)')
cmd_ab(argparse.Namespace(before=args.ab_before, after=args.ab_after, profile='p-b', replicas=4, reps=args.ab_reps, no_warmup=False))
cmd_analyze(argparse.Namespace())
else:
log('A/B не запускался: передайте --ab-after <образ> после коммита оптимизации')
log('=== возврат конфигурации стенда по умолчанию')
ensure_stand({'API_REPLICAS': str(DEFAULT_STAND['API_REPLICAS']), 'API_TOTAL_CPUS': str(DEFAULT_STAND['API_TOTAL_CPUS']), 'API_TOTAL_MEMORY_MB': str(DEFAULT_STAND['API_TOTAL_MEMORY_MB']), 'PG_CPUS': str(DEFAULT_STAND['PG_CPUS']), 'LB_CPUS': str(DEFAULT_STAND['LB_CPUS']), 'DATABASE_POOL_MAX': str(DEFAULT_STAND['DATABASE_POOL_MAX']), 'API_IMAGE': DEFAULT_STAND['API_IMAGE'], 'BUILD_ID': DEFAULT_STAND['BUILD_ID']})
finally:
if not args.no_quiesce:
cmd_quiesce(argparse.Namespace(action='down'))
# ---------------------------------------------------------------- CLI
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description='Драйвер экспериментов lab3')
sub = parser.add_subparsers(dest='command', required=True)
stand = sub.add_parser('stand', help='стенд: up/down/status')
stand.add_argument('action', choices=['up', 'down', 'status'])
stand.add_argument('--replicas', type=int)
stand.set_defaults(func=cmd_stand)
check = sub.add_parser('check', help='проверки уровня 3/4')
check.add_argument('what', choices=['spread', 'instances', 'idempotency'])
check.add_argument('--requests', type=int, default=40)
check.set_defaults(func=cmd_check)
run = sub.add_parser('run', help='один прогон профиля')
run.add_argument('--label', required=True)
run.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
run.add_argument('--ladder')
run.add_argument('--step', type=int)
run.add_argument('--peak', type=int, help='одна полка фиксированного темпа')
run.add_argument('--duration', type=int, help='длительность полки, s')
run.add_argument('--replicas', type=int)
run.add_argument('--pool', type=int, help='DATABASE_POOL_MAX на реплику')
run.add_argument('--loadgen-cpus', type=float, help='лимит CPU генератора (контингенция: k6 упёрся в квоту)')
run.add_argument('--lb-cpus', type=float, help='лимит CPU HAProxy')
run.add_argument('--api-total-cpus', type=float, help='суммарный бюджет CPU API (делится на реплики)')
run.add_argument('--pg-cpus', type=float, help='лимит CPU PostgreSQL')
run.add_argument('--mode', choices=['write', 'read'])
run.add_argument('--spread', choices=['spread', 'hot'])
run.add_argument('--no-warmup', action='store_true')
run.set_defaults(func=cmd_run)
sweep = sub.add_parser('sweep', help='серия 1/2/4 реплики')
sweep.add_argument('--profile', choices=sorted(PROFILES), default='p-a')
sweep.add_argument('--replicas', default='1,2,4')
sweep.add_argument('--reps', type=int, default=2)
sweep.add_argument('--extra-replica', type=int, default=0)
sweep.add_argument('--extra-reps', type=int, default=0)
sweep.add_argument('--prefix')
sweep.add_argument('--no-warmup', action='store_true')
sweep.set_defaults(func=cmd_sweep)
growth = sub.add_parser('growth', help='серия роста ресурсов')
growth.add_argument('--series', choices=['api', 'pg', 'pool'], required=True)
growth.add_argument('--rate', type=int)
growth.add_argument('--duration', type=int, default=60)
growth.add_argument('--base-profile', default='p-a')
growth.add_argument('--no-warmup', action='store_true')
growth.add_argument('--suffix', default='', help='суффикс метки: различает прогоны одной точки на разных темпах')
growth.set_defaults(func=cmd_growth)
failover = sub.add_parser('failover', help='фазовый драйвер отказа реплики')
failover.add_argument('--replicas', default='all')
failover.add_argument('--rate', type=int, default=200)
failover.add_argument('--duration', type=int, default=60)
failover.add_argument('--pre', type=float, default=8.0, help='секунд нагрузки до отказа')
failover.add_argument('--hold', type=float, default=20.0, help='секунд до восстановления реплики')
failover.set_defaults(func=cmd_failover)
rolling = sub.add_parser('rolling-update', help='поочерёдная замена реплик')
rolling.add_argument('--under-load', action='store_true')
rolling.add_argument('--rate', type=int, default=200)
rolling.add_argument('--duration', type=int, default=180)
rolling.add_argument('--build-id')
rolling.add_argument('--pause', type=float, default=2.0)
rolling.set_defaults(func=cmd_rolling)
ab = sub.add_parser('ab', help='интерливинг A/B двух образов')
ab.add_argument('--before', required=True)
ab.add_argument('--after', required=True)
ab.add_argument('--profile', default='p-b')
ab.add_argument('--replicas', type=int, default=4)
ab.add_argument('--reps', type=int, default=3)
ab.add_argument('--no-warmup', action='store_true')
ab.set_defaults(func=cmd_ab)
analyze = sub.add_parser('analyze', help='агрегаты и графики из raw')
analyze.set_defaults(func=cmd_analyze)
watch = sub.add_parser('watch', help='сэмплер без нагрузки: наблюдение за стендом')
watch.add_argument('--seconds', type=float, default=30)
watch.add_argument('--out', required=True)
watch.set_defaults(func=cmd_watch)
quiesce = sub.add_parser('quiesce', help='фон хоста: up/down/status')
quiesce.add_argument('action', choices=['up', 'down', 'status'])
quiesce.set_defaults(func=cmd_quiesce)
track1 = sub.add_parser('track1', help='полная серия трека 1')
track1.add_argument('--reps', type=int, default=2)
track1.add_argument('--ab-before', default='mtusi-messenger:lab3-before')
track1.add_argument('--ab-after', default=None)
track1.add_argument('--ab-reps', type=int, default=3)
track1.add_argument('--no-quiesce', action='store_true')
track1.set_defaults(func=cmd_track1)
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
RAW_DIR.mkdir(parents=True, exist_ok=True)
args.func(args)
if __name__ == '__main__':
main()