forked from mtusi2026/mtusi2026lab3
525 lines
18 KiB
Python
525 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare API replica counts under fixed and increasing total API resources."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
|
COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
|
|
LOAD_SCRIPT = ROOT / 'tests' / 'load.py'
|
|
MEMORY_PATTERN = re.compile(r'([\d.]+)\s*(B|KiB|MiB|GiB|kB|MB|GB)', re.IGNORECASE)
|
|
os.environ.setdefault('PYTHONUTF8', '1')
|
|
|
|
|
|
def command(args, overlay=None, check=True, capture=False):
|
|
argv = COMPOSE + ['--env-file', str(ROOT / '.env')]
|
|
if overlay:
|
|
argv += ['--env-file', str(overlay)]
|
|
argv += ['-f', str(ROOT / 'compose.yaml')] + list(args)
|
|
result = subprocess.run(
|
|
argv,
|
|
cwd=ROOT,
|
|
check=False,
|
|
text=True,
|
|
capture_output=capture,
|
|
)
|
|
if check and result.returncode:
|
|
detail = (result.stderr or result.stdout or '').strip()
|
|
raise RuntimeError('Compose command failed: ' + ' '.join(args) + '\n' + detail)
|
|
return result
|
|
|
|
|
|
def wait_ready(timeout=90):
|
|
from urllib.error import URLError
|
|
from urllib.request import urlopen
|
|
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urlopen(BASE_URL.rstrip('/') + '/health/ready', timeout=2) as response:
|
|
if response.status == 200:
|
|
return
|
|
except (OSError, URLError):
|
|
time.sleep(0.5)
|
|
raise TimeoutError('Load balancer did not become ready within %s seconds' % timeout)
|
|
|
|
|
|
def sample_instances(minimum, timeout=15):
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure
|
|
|
|
client = Client(BASE_URL, timeout=5)
|
|
instances = set()
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
for _ in range(40):
|
|
try:
|
|
response = client.call(
|
|
'GET',
|
|
'/api/v1/users/me',
|
|
expected=401,
|
|
schema='Error',
|
|
)
|
|
instances.add(response['headers']['X-Instance-Id'])
|
|
except Failure:
|
|
time.sleep(0.1)
|
|
if len(instances) >= minimum:
|
|
return instances
|
|
time.sleep(1)
|
|
raise TimeoutError(
|
|
'Expected %d API instance IDs through Nginx; observed %d'
|
|
% (minimum, len(instances))
|
|
)
|
|
|
|
|
|
def parse_memory(value):
|
|
match = MEMORY_PATTERN.search(value)
|
|
if not match:
|
|
return 0
|
|
amount = float(match.group(1))
|
|
unit = match.group(2).lower()
|
|
factors = {
|
|
'b': 1,
|
|
'kib': 1024,
|
|
'mib': 1024 ** 2,
|
|
'gib': 1024 ** 3,
|
|
'kb': 1000,
|
|
'mb': 1000 ** 2,
|
|
'gb': 1000 ** 3,
|
|
}
|
|
return int(amount * factors[unit])
|
|
|
|
|
|
def collect_stats():
|
|
result = command(['ps', '-q'], check=True, capture=True)
|
|
container_ids = result.stdout.split()
|
|
if not container_ids:
|
|
return {}
|
|
stats = subprocess.run(
|
|
[
|
|
'docker',
|
|
'stats',
|
|
'--no-stream',
|
|
'--format',
|
|
'{{.Name}};{{.CPUPerc}};{{.MemUsage}}',
|
|
*container_ids,
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if stats.returncode:
|
|
raise RuntimeError('docker stats failed: ' + (stats.stderr or stats.stdout).strip())
|
|
values = {}
|
|
for line in stats.stdout.splitlines():
|
|
parts = line.split(';', 2)
|
|
if len(parts) != 3:
|
|
continue
|
|
name, cpu, memory = parts
|
|
service = next(
|
|
(item for item in ('api', 'postgres', 'nginx') if '-' + item + '-' in name),
|
|
None,
|
|
)
|
|
if service:
|
|
values.setdefault(service, []).append(
|
|
{
|
|
'cpu_percent': float(cpu.rstrip('%') or 0),
|
|
'memory_bytes': parse_memory(memory.split('/')[0]),
|
|
}
|
|
)
|
|
return values
|
|
|
|
|
|
def run_load(duration, concurrency, output=None):
|
|
args = [
|
|
sys.executable,
|
|
str(LOAD_SCRIPT),
|
|
'--base-url',
|
|
BASE_URL,
|
|
'--duration',
|
|
str(duration),
|
|
'--concurrency',
|
|
str(concurrency),
|
|
]
|
|
if output:
|
|
args += ['--output', str(output)]
|
|
return subprocess.run(
|
|
args,
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def measure(duration, concurrency, output):
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
str(LOAD_SCRIPT),
|
|
'--base-url',
|
|
BASE_URL,
|
|
'--duration',
|
|
str(duration),
|
|
'--concurrency',
|
|
str(concurrency),
|
|
'--output',
|
|
str(output),
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
peaks = {}
|
|
try:
|
|
while process.poll() is None:
|
|
for service, entries in collect_stats().items():
|
|
values = peaks.setdefault(
|
|
service, {'cpu_percent': 0.0, 'memory_bytes': 0}
|
|
)
|
|
values['cpu_percent'] = max(
|
|
values['cpu_percent'],
|
|
sum(entry['cpu_percent'] for entry in entries),
|
|
)
|
|
values['memory_bytes'] = max(
|
|
values['memory_bytes'],
|
|
sum(entry['memory_bytes'] for entry in entries),
|
|
)
|
|
time.sleep(1)
|
|
stdout, stderr = process.communicate()
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=5)
|
|
if process.returncode:
|
|
raise RuntimeError('Load run failed:\n' + (stderr or stdout).strip())
|
|
return json.loads(output.read_text(encoding='utf-8')), peaks
|
|
|
|
|
|
def run_scenario(name, replicas, cpu_per_replica, memory_per_replica, args, output_dir):
|
|
with tempfile.NamedTemporaryFile(
|
|
mode='w',
|
|
encoding='ascii',
|
|
suffix='.env',
|
|
prefix='lab3-track1-',
|
|
delete=False,
|
|
) as handle:
|
|
handle.write('API_REPLICAS=%s\n' % replicas)
|
|
handle.write('API_CPU_LIMIT=%s\n' % cpu_per_replica)
|
|
handle.write('API_MEMORY_LIMIT=%sM\n' % memory_per_replica)
|
|
override = Path(handle.name)
|
|
try:
|
|
command(
|
|
['up', '--detach', '--build', '--scale', 'api=%d' % replicas],
|
|
overlay=override,
|
|
capture=True,
|
|
)
|
|
wait_ready()
|
|
instances = sample_instances(min(2, replicas))
|
|
running = command(['ps', '-q', 'api'], capture=True).stdout.split()
|
|
if len(running) != replicas:
|
|
raise RuntimeError(
|
|
'Expected %d running API replicas, found %d' % (replicas, len(running))
|
|
)
|
|
|
|
warmup = run_load(args.warmup, args.concurrency)
|
|
if warmup.returncode:
|
|
raise RuntimeError('Warm-up load failed: ' + warmup.stderr.strip())
|
|
|
|
runs = []
|
|
for repeat in range(1, args.repeats + 1):
|
|
raw_path = output_dir / ('%s-run-%02d.json' % (name, repeat))
|
|
metrics, peaks = measure(
|
|
args.duration,
|
|
args.concurrency,
|
|
raw_path,
|
|
)
|
|
runs.append({'load': metrics, 'peak_resources': peaks, 'raw_file': raw_path.name})
|
|
return {
|
|
'name': name,
|
|
'api_replicas': replicas,
|
|
'api_cpu_limit_per_replica': cpu_per_replica,
|
|
'api_memory_limit_mib_per_replica': memory_per_replica,
|
|
'total_api_cpu_limit': round(replicas * cpu_per_replica, 3),
|
|
'total_api_memory_limit_mib': replicas * memory_per_replica,
|
|
'instances_observed': len(instances),
|
|
'runs': runs,
|
|
}
|
|
finally:
|
|
override.unlink(missing_ok=True)
|
|
|
|
|
|
def mean(values):
|
|
return sum(values) / len(values) if values else 0
|
|
|
|
|
|
def summarize(scenario):
|
|
runs = scenario['runs']
|
|
return {
|
|
'name': scenario['name'],
|
|
'api_replicas': scenario['api_replicas'],
|
|
'total_api_cpu_limit': scenario['total_api_cpu_limit'],
|
|
'total_api_memory_limit_mib': scenario['total_api_memory_limit_mib'],
|
|
'successful_rps_mean': round(
|
|
mean([run['load']['successful_rps'] for run in runs]), 3
|
|
),
|
|
'error_ratio_mean': round(
|
|
mean([run['load']['error_ratio'] or 0 for run in runs]), 6
|
|
),
|
|
'p95_latency_ms_mean': round(
|
|
mean([run['load']['successful_latency_ms']['p95'] or 0 for run in runs]),
|
|
3,
|
|
),
|
|
'api_cpu_peak_percent_mean': round(
|
|
mean([run['peak_resources'].get('api', {}).get('cpu_percent', 0) for run in runs]),
|
|
3,
|
|
),
|
|
'api_memory_peak_mib_mean': round(
|
|
mean(
|
|
[
|
|
run['peak_resources'].get('api', {}).get('memory_bytes', 0)
|
|
/ (1024 ** 2)
|
|
for run in runs
|
|
]
|
|
),
|
|
3,
|
|
),
|
|
'postgres_cpu_peak_percent_mean': round(
|
|
mean(
|
|
[
|
|
run['peak_resources'].get('postgres', {}).get('cpu_percent', 0)
|
|
for run in runs
|
|
]
|
|
),
|
|
3,
|
|
),
|
|
'postgres_memory_peak_mib_mean': round(
|
|
mean(
|
|
[
|
|
run['peak_resources'].get('postgres', {}).get('memory_bytes', 0)
|
|
/ (1024 ** 2)
|
|
for run in runs
|
|
]
|
|
),
|
|
3,
|
|
),
|
|
'nginx_cpu_peak_percent_mean': round(
|
|
mean([run['peak_resources'].get('nginx', {}).get('cpu_percent', 0) for run in runs]),
|
|
3,
|
|
),
|
|
'nginx_memory_peak_mib_mean': round(
|
|
mean(
|
|
[
|
|
run['peak_resources'].get('nginx', {}).get('memory_bytes', 0)
|
|
/ (1024 ** 2)
|
|
for run in runs
|
|
]
|
|
),
|
|
3,
|
|
),
|
|
}
|
|
|
|
|
|
def write_chart(path, summaries, value_key, title, unit):
|
|
width, height = 980, 430
|
|
left, top, chart_width, chart_height = 70, 55, 850, 275
|
|
maximum = max((item[value_key] for item in summaries), default=1) or 1
|
|
parts = [
|
|
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">'
|
|
% (width, height, width, height),
|
|
'<rect width="100%" height="100%" fill="white"/>',
|
|
'<text x="%d" y="30" font-family="sans-serif" font-size="20">%s</text>'
|
|
% (left, title),
|
|
]
|
|
for tick in range(5):
|
|
y = top + chart_height - chart_height * tick / 4
|
|
value = maximum * tick / 4
|
|
parts.append(
|
|
'<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="#d8dee8"/>'
|
|
% (left, y, left + chart_width, y)
|
|
)
|
|
parts.append(
|
|
'<text x="8" y="%.1f" font-family="sans-serif" font-size="12">%.1f</text>'
|
|
% (y + 4, value)
|
|
)
|
|
bar_width = 72
|
|
slot = chart_width / len(summaries)
|
|
for index, item in enumerate(summaries):
|
|
value = item[value_key]
|
|
bar_height = chart_height * value / maximum
|
|
x = left + slot * index + (slot - bar_width) / 2
|
|
y = top + chart_height - bar_height
|
|
color = '#2864a5' if item['name'].startswith('fixed') else '#d47a22'
|
|
label = item['name'].replace('fixed-total-', 'fixed ').replace('growing-total-', 'growth ')
|
|
parts.append(
|
|
'<rect x="%.1f" y="%.1f" width="%d" height="%.1f" fill="%s"/>'
|
|
% (x, y, bar_width, bar_height, color)
|
|
)
|
|
parts.append(
|
|
'<text x="%.1f" y="%.1f" text-anchor="middle" font-family="sans-serif" font-size="12">%.1f</text>'
|
|
% (x + bar_width / 2, max(top + 14, y - 6), value)
|
|
)
|
|
parts.append(
|
|
'<text x="%.1f" y="%d" text-anchor="middle" font-family="sans-serif" font-size="12">%s</text>'
|
|
% (x + bar_width / 2, top + chart_height + 22, label)
|
|
)
|
|
parts.extend(
|
|
[
|
|
'<text x="%d" y="%d" font-family="sans-serif" font-size="12">%s</text>'
|
|
% (left, height - 14, unit),
|
|
'<rect x="710" y="385" width="12" height="12" fill="#2864a5"/>',
|
|
'<text x="728" y="396" font-family="sans-serif" font-size="12">fixed API budget</text>',
|
|
'<rect x="850" y="385" width="12" height="12" fill="#d47a22"/>',
|
|
'<text x="868" y="396" font-family="sans-serif" font-size="12">growing API budget</text>',
|
|
'</svg>',
|
|
]
|
|
)
|
|
path.write_text('\n'.join(parts) + '\n', encoding='utf-8')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--duration', type=int, default=30)
|
|
parser.add_argument('--warmup', type=int, default=5)
|
|
parser.add_argument('--concurrency', type=int, default=16)
|
|
parser.add_argument('--repeats', type=int, default=3)
|
|
parser.add_argument('--fixed-total-cpus', type=float, default=4.0)
|
|
parser.add_argument('--fixed-total-memory-mib', type=int, default=2048)
|
|
parser.add_argument('--growth-cpus-per-replica', type=float, default=1.0)
|
|
parser.add_argument('--growth-memory-mib-per-replica', type=int, default=512)
|
|
parser.add_argument('--output', default='evidence/track1/benchmark.json')
|
|
args = parser.parse_args()
|
|
if (
|
|
args.duration < 1
|
|
or args.warmup < 1
|
|
or args.concurrency < 1
|
|
or args.repeats < 1
|
|
or args.fixed_total_cpus <= 0
|
|
or args.fixed_total_memory_mib < 1
|
|
or args.growth_cpus_per_replica <= 0
|
|
or args.growth_memory_mib_per_replica < 1
|
|
):
|
|
parser.error('duration, warmup, concurrency, repeats and resource limits must be positive')
|
|
if not (ROOT / '.env').is_file():
|
|
parser.error('Create .env and local Compose secrets first')
|
|
if not all((ROOT / 'secrets' / name).is_file() for name in ('postgres_password', 'database_url')):
|
|
parser.error('Run scripts/init-local-secrets.ps1 before benchmarking')
|
|
|
|
output = (ROOT / args.output).resolve()
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
command(['config', '--quiet'])
|
|
output_dir = output.parent
|
|
scenarios = []
|
|
try:
|
|
for replicas in (1, 2, 4):
|
|
fixed_cpu = round(args.fixed_total_cpus / replicas, 3)
|
|
fixed_memory = max(1, args.fixed_total_memory_mib // replicas)
|
|
scenarios.append(
|
|
run_scenario(
|
|
'fixed-total-%d' % replicas,
|
|
replicas,
|
|
fixed_cpu,
|
|
fixed_memory,
|
|
args,
|
|
output_dir,
|
|
)
|
|
)
|
|
for replicas in (1, 2, 4):
|
|
scenarios.append(
|
|
run_scenario(
|
|
'growing-total-%d' % replicas,
|
|
replicas,
|
|
args.growth_cpus_per_replica,
|
|
args.growth_memory_mib_per_replica,
|
|
args,
|
|
output_dir,
|
|
)
|
|
)
|
|
finally:
|
|
command(['up', '--detach', '--build', '--scale', 'api=2'], capture=True)
|
|
wait_ready()
|
|
|
|
docker_version = subprocess.run(
|
|
['docker', 'version', '--format', '{{.Server.Version}}'],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
docker_host = subprocess.run(
|
|
['docker', 'info', '--format', '{{.NCPU}}|{{.MemTotal}}|{{.OperatingSystem}}'],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
docker_host_values = docker_host.stdout.strip().split('|')
|
|
if docker_host.returncode or len(docker_host_values) != 3:
|
|
raise RuntimeError('Unable to capture Docker host CPU/RAM information')
|
|
report = {
|
|
'track': '1 — цена масштабирования',
|
|
'method': {
|
|
'workload': 'tests/load.py closed-loop POST messages through Nginx',
|
|
'duration_seconds_per_run': args.duration,
|
|
'warmup_seconds_per_scenario': args.warmup,
|
|
'concurrency': args.concurrency,
|
|
'repeats_per_scenario': args.repeats,
|
|
'fixed_total_api_limits': {
|
|
'cpus': args.fixed_total_cpus,
|
|
'memory_mib': args.fixed_total_memory_mib,
|
|
},
|
|
'growth_per_replica_limits': {
|
|
'cpus': args.growth_cpus_per_replica,
|
|
'memory_mib': args.growth_memory_mib_per_replica,
|
|
},
|
|
'host_cpu_count': os.cpu_count(),
|
|
'platform': platform.platform(),
|
|
'docker_server_version': docker_version.stdout.strip(),
|
|
'docker_host': {
|
|
'cpus': int(docker_host_values[0]),
|
|
'memory_mib': round(int(docker_host_values[1]) / (1024 ** 2), 3),
|
|
'operating_system': docker_host_values[2],
|
|
},
|
|
'measurement_note': (
|
|
'Successful HTTP throughput, not committed-write throughput. '
|
|
'Load generator is closed-loop. Docker stats are one-second samples '
|
|
'and can miss short CPU peaks.'
|
|
),
|
|
},
|
|
'scenarios': scenarios,
|
|
'summary': [summarize(scenario) for scenario in scenarios],
|
|
'raw_runs': True,
|
|
}
|
|
summary = [summarize(scenario) for scenario in scenarios]
|
|
rps_chart = output_dir / 'successful-rps.svg'
|
|
p95_chart = output_dir / 'p95-latency.svg'
|
|
write_chart(rps_chart, summary, 'successful_rps_mean', 'Successful HTTP throughput by scale', 'Successful requests per second')
|
|
write_chart(p95_chart, summary, 'p95_latency_ms_mean', 'Successful request p95 by scale', 'Milliseconds')
|
|
report['summary'] = summary
|
|
report['charts'] = [rps_chart.name, p95_chart.name]
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + '\n',
|
|
encoding='utf-8',
|
|
)
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
print('Saved repeatable benchmark evidence to ' + str(output))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
|
|
print('ERROR: ' + str(error), file=sys.stderr)
|
|
raise SystemExit(1)
|