Files
mtusi2026lab3/scripts/track1_benchmark.py
T
2026-11-04 16:11:13 +03:00

525 lines
18 KiB
Python

#!/usr/bin/env python3
"""Compare API replica counts under fixed and increasing total API resources."""
import argparse
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
import sys
import tempfile
import time
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
LOAD_SCRIPT = ROOT / 'tests' / 'load.py'
MEMORY_PATTERN = re.compile(r'([\d.]+)\s*(B|KiB|MiB|GiB|kB|MB|GB)', re.IGNORECASE)
os.environ.setdefault('PYTHONUTF8', '1')
def command(args, overlay=None, check=True, capture=False):
argv = COMPOSE + ['--env-file', str(ROOT / '.env')]
if overlay:
argv += ['--env-file', str(overlay)]
argv += ['-f', str(ROOT / 'compose.yaml')] + list(args)
result = subprocess.run(
argv,
cwd=ROOT,
check=False,
text=True,
capture_output=capture,
)
if check and result.returncode:
detail = (result.stderr or result.stdout or '').strip()
raise RuntimeError('Compose command failed: ' + ' '.join(args) + '\n' + detail)
return result
def wait_ready(timeout=90):
from urllib.error import URLError
from urllib.request import urlopen
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with urlopen(BASE_URL.rstrip('/') + '/health/ready', timeout=2) as response:
if response.status == 200:
return
except (OSError, URLError):
time.sleep(0.5)
raise TimeoutError('Load balancer did not become ready within %s seconds' % timeout)
def sample_instances(minimum, timeout=15):
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
client = Client(BASE_URL, timeout=5)
instances = set()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for _ in range(40):
try:
response = client.call(
'GET',
'/api/v1/users/me',
expected=401,
schema='Error',
)
instances.add(response['headers']['X-Instance-Id'])
except Failure:
time.sleep(0.1)
if len(instances) >= minimum:
return instances
time.sleep(1)
raise TimeoutError(
'Expected %d API instance IDs through Nginx; observed %d'
% (minimum, len(instances))
)
def parse_memory(value):
match = MEMORY_PATTERN.search(value)
if not match:
return 0
amount = float(match.group(1))
unit = match.group(2).lower()
factors = {
'b': 1,
'kib': 1024,
'mib': 1024 ** 2,
'gib': 1024 ** 3,
'kb': 1000,
'mb': 1000 ** 2,
'gb': 1000 ** 3,
}
return int(amount * factors[unit])
def collect_stats():
result = command(['ps', '-q'], check=True, capture=True)
container_ids = result.stdout.split()
if not container_ids:
return {}
stats = subprocess.run(
[
'docker',
'stats',
'--no-stream',
'--format',
'{{.Name}};{{.CPUPerc}};{{.MemUsage}}',
*container_ids,
],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
if stats.returncode:
raise RuntimeError('docker stats failed: ' + (stats.stderr or stats.stdout).strip())
values = {}
for line in stats.stdout.splitlines():
parts = line.split(';', 2)
if len(parts) != 3:
continue
name, cpu, memory = parts
service = next(
(item for item in ('api', 'postgres', 'nginx') if '-' + item + '-' in name),
None,
)
if service:
values.setdefault(service, []).append(
{
'cpu_percent': float(cpu.rstrip('%') or 0),
'memory_bytes': parse_memory(memory.split('/')[0]),
}
)
return values
def run_load(duration, concurrency, output=None):
args = [
sys.executable,
str(LOAD_SCRIPT),
'--base-url',
BASE_URL,
'--duration',
str(duration),
'--concurrency',
str(concurrency),
]
if output:
args += ['--output', str(output)]
return subprocess.run(
args,
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
def measure(duration, concurrency, output):
process = subprocess.Popen(
[
sys.executable,
str(LOAD_SCRIPT),
'--base-url',
BASE_URL,
'--duration',
str(duration),
'--concurrency',
str(concurrency),
'--output',
str(output),
],
cwd=ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
peaks = {}
try:
while process.poll() is None:
for service, entries in collect_stats().items():
values = peaks.setdefault(
service, {'cpu_percent': 0.0, 'memory_bytes': 0}
)
values['cpu_percent'] = max(
values['cpu_percent'],
sum(entry['cpu_percent'] for entry in entries),
)
values['memory_bytes'] = max(
values['memory_bytes'],
sum(entry['memory_bytes'] for entry in entries),
)
time.sleep(1)
stdout, stderr = process.communicate()
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=5)
if process.returncode:
raise RuntimeError('Load run failed:\n' + (stderr or stdout).strip())
return json.loads(output.read_text(encoding='utf-8')), peaks
def run_scenario(name, replicas, cpu_per_replica, memory_per_replica, args, output_dir):
with tempfile.NamedTemporaryFile(
mode='w',
encoding='ascii',
suffix='.env',
prefix='lab3-track1-',
delete=False,
) as handle:
handle.write('API_REPLICAS=%s\n' % replicas)
handle.write('API_CPU_LIMIT=%s\n' % cpu_per_replica)
handle.write('API_MEMORY_LIMIT=%sM\n' % memory_per_replica)
override = Path(handle.name)
try:
command(
['up', '--detach', '--build', '--scale', 'api=%d' % replicas],
overlay=override,
capture=True,
)
wait_ready()
instances = sample_instances(min(2, replicas))
running = command(['ps', '-q', 'api'], capture=True).stdout.split()
if len(running) != replicas:
raise RuntimeError(
'Expected %d running API replicas, found %d' % (replicas, len(running))
)
warmup = run_load(args.warmup, args.concurrency)
if warmup.returncode:
raise RuntimeError('Warm-up load failed: ' + warmup.stderr.strip())
runs = []
for repeat in range(1, args.repeats + 1):
raw_path = output_dir / ('%s-run-%02d.json' % (name, repeat))
metrics, peaks = measure(
args.duration,
args.concurrency,
raw_path,
)
runs.append({'load': metrics, 'peak_resources': peaks, 'raw_file': raw_path.name})
return {
'name': name,
'api_replicas': replicas,
'api_cpu_limit_per_replica': cpu_per_replica,
'api_memory_limit_mib_per_replica': memory_per_replica,
'total_api_cpu_limit': round(replicas * cpu_per_replica, 3),
'total_api_memory_limit_mib': replicas * memory_per_replica,
'instances_observed': len(instances),
'runs': runs,
}
finally:
override.unlink(missing_ok=True)
def mean(values):
return sum(values) / len(values) if values else 0
def summarize(scenario):
runs = scenario['runs']
return {
'name': scenario['name'],
'api_replicas': scenario['api_replicas'],
'total_api_cpu_limit': scenario['total_api_cpu_limit'],
'total_api_memory_limit_mib': scenario['total_api_memory_limit_mib'],
'successful_rps_mean': round(
mean([run['load']['successful_rps'] for run in runs]), 3
),
'error_ratio_mean': round(
mean([run['load']['error_ratio'] or 0 for run in runs]), 6
),
'p95_latency_ms_mean': round(
mean([run['load']['successful_latency_ms']['p95'] or 0 for run in runs]),
3,
),
'api_cpu_peak_percent_mean': round(
mean([run['peak_resources'].get('api', {}).get('cpu_percent', 0) for run in runs]),
3,
),
'api_memory_peak_mib_mean': round(
mean(
[
run['peak_resources'].get('api', {}).get('memory_bytes', 0)
/ (1024 ** 2)
for run in runs
]
),
3,
),
'postgres_cpu_peak_percent_mean': round(
mean(
[
run['peak_resources'].get('postgres', {}).get('cpu_percent', 0)
for run in runs
]
),
3,
),
'postgres_memory_peak_mib_mean': round(
mean(
[
run['peak_resources'].get('postgres', {}).get('memory_bytes', 0)
/ (1024 ** 2)
for run in runs
]
),
3,
),
'nginx_cpu_peak_percent_mean': round(
mean([run['peak_resources'].get('nginx', {}).get('cpu_percent', 0) for run in runs]),
3,
),
'nginx_memory_peak_mib_mean': round(
mean(
[
run['peak_resources'].get('nginx', {}).get('memory_bytes', 0)
/ (1024 ** 2)
for run in runs
]
),
3,
),
}
def write_chart(path, summaries, value_key, title, unit):
width, height = 980, 430
left, top, chart_width, chart_height = 70, 55, 850, 275
maximum = max((item[value_key] for item in summaries), default=1) or 1
parts = [
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">'
% (width, height, width, height),
'<rect width="100%" height="100%" fill="white"/>',
'<text x="%d" y="30" font-family="sans-serif" font-size="20">%s</text>'
% (left, title),
]
for tick in range(5):
y = top + chart_height - chart_height * tick / 4
value = maximum * tick / 4
parts.append(
'<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="#d8dee8"/>'
% (left, y, left + chart_width, y)
)
parts.append(
'<text x="8" y="%.1f" font-family="sans-serif" font-size="12">%.1f</text>'
% (y + 4, value)
)
bar_width = 72
slot = chart_width / len(summaries)
for index, item in enumerate(summaries):
value = item[value_key]
bar_height = chart_height * value / maximum
x = left + slot * index + (slot - bar_width) / 2
y = top + chart_height - bar_height
color = '#2864a5' if item['name'].startswith('fixed') else '#d47a22'
label = item['name'].replace('fixed-total-', 'fixed ').replace('growing-total-', 'growth ')
parts.append(
'<rect x="%.1f" y="%.1f" width="%d" height="%.1f" fill="%s"/>'
% (x, y, bar_width, bar_height, color)
)
parts.append(
'<text x="%.1f" y="%.1f" text-anchor="middle" font-family="sans-serif" font-size="12">%.1f</text>'
% (x + bar_width / 2, max(top + 14, y - 6), value)
)
parts.append(
'<text x="%.1f" y="%d" text-anchor="middle" font-family="sans-serif" font-size="12">%s</text>'
% (x + bar_width / 2, top + chart_height + 22, label)
)
parts.extend(
[
'<text x="%d" y="%d" font-family="sans-serif" font-size="12">%s</text>'
% (left, height - 14, unit),
'<rect x="710" y="385" width="12" height="12" fill="#2864a5"/>',
'<text x="728" y="396" font-family="sans-serif" font-size="12">fixed API budget</text>',
'<rect x="850" y="385" width="12" height="12" fill="#d47a22"/>',
'<text x="868" y="396" font-family="sans-serif" font-size="12">growing API budget</text>',
'</svg>',
]
)
path.write_text('\n'.join(parts) + '\n', encoding='utf-8')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--duration', type=int, default=30)
parser.add_argument('--warmup', type=int, default=5)
parser.add_argument('--concurrency', type=int, default=16)
parser.add_argument('--repeats', type=int, default=3)
parser.add_argument('--fixed-total-cpus', type=float, default=4.0)
parser.add_argument('--fixed-total-memory-mib', type=int, default=2048)
parser.add_argument('--growth-cpus-per-replica', type=float, default=1.0)
parser.add_argument('--growth-memory-mib-per-replica', type=int, default=512)
parser.add_argument('--output', default='evidence/track1/benchmark.json')
args = parser.parse_args()
if (
args.duration < 1
or args.warmup < 1
or args.concurrency < 1
or args.repeats < 1
or args.fixed_total_cpus <= 0
or args.fixed_total_memory_mib < 1
or args.growth_cpus_per_replica <= 0
or args.growth_memory_mib_per_replica < 1
):
parser.error('duration, warmup, concurrency, repeats and resource limits must be positive')
if not (ROOT / '.env').is_file():
parser.error('Create .env and local Compose secrets first')
if not all((ROOT / 'secrets' / name).is_file() for name in ('postgres_password', 'database_url')):
parser.error('Run scripts/init-local-secrets.ps1 before benchmarking')
output = (ROOT / args.output).resolve()
output.parent.mkdir(parents=True, exist_ok=True)
command(['config', '--quiet'])
output_dir = output.parent
scenarios = []
try:
for replicas in (1, 2, 4):
fixed_cpu = round(args.fixed_total_cpus / replicas, 3)
fixed_memory = max(1, args.fixed_total_memory_mib // replicas)
scenarios.append(
run_scenario(
'fixed-total-%d' % replicas,
replicas,
fixed_cpu,
fixed_memory,
args,
output_dir,
)
)
for replicas in (1, 2, 4):
scenarios.append(
run_scenario(
'growing-total-%d' % replicas,
replicas,
args.growth_cpus_per_replica,
args.growth_memory_mib_per_replica,
args,
output_dir,
)
)
finally:
command(['up', '--detach', '--build', '--scale', 'api=2'], capture=True)
wait_ready()
docker_version = subprocess.run(
['docker', 'version', '--format', '{{.Server.Version}}'],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
docker_host = subprocess.run(
['docker', 'info', '--format', '{{.NCPU}}|{{.MemTotal}}|{{.OperatingSystem}}'],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
docker_host_values = docker_host.stdout.strip().split('|')
if docker_host.returncode or len(docker_host_values) != 3:
raise RuntimeError('Unable to capture Docker host CPU/RAM information')
report = {
'track': '1 — цена масштабирования',
'method': {
'workload': 'tests/load.py closed-loop POST messages through Nginx',
'duration_seconds_per_run': args.duration,
'warmup_seconds_per_scenario': args.warmup,
'concurrency': args.concurrency,
'repeats_per_scenario': args.repeats,
'fixed_total_api_limits': {
'cpus': args.fixed_total_cpus,
'memory_mib': args.fixed_total_memory_mib,
},
'growth_per_replica_limits': {
'cpus': args.growth_cpus_per_replica,
'memory_mib': args.growth_memory_mib_per_replica,
},
'host_cpu_count': os.cpu_count(),
'platform': platform.platform(),
'docker_server_version': docker_version.stdout.strip(),
'docker_host': {
'cpus': int(docker_host_values[0]),
'memory_mib': round(int(docker_host_values[1]) / (1024 ** 2), 3),
'operating_system': docker_host_values[2],
},
'measurement_note': (
'Successful HTTP throughput, not committed-write throughput. '
'Load generator is closed-loop. Docker stats are one-second samples '
'and can miss short CPU peaks.'
),
},
'scenarios': scenarios,
'summary': [summarize(scenario) for scenario in scenarios],
'raw_runs': True,
}
summary = [summarize(scenario) for scenario in scenarios]
rps_chart = output_dir / 'successful-rps.svg'
p95_chart = output_dir / 'p95-latency.svg'
write_chart(rps_chart, summary, 'successful_rps_mean', 'Successful HTTP throughput by scale', 'Successful requests per second')
write_chart(p95_chart, summary, 'p95_latency_ms_mean', 'Successful request p95 by scale', 'Milliseconds')
report['summary'] = summary
report['charts'] = [rps_chart.name, p95_chart.name]
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + '\n',
encoding='utf-8',
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
print('Saved repeatable benchmark evidence to ' + str(output))
if __name__ == '__main__':
try:
main()
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
print('ERROR: ' + str(error), file=sys.stderr)
raise SystemExit(1)