forked from mtusi2026/mtusi2026lab3
lab3
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
$secrets = Join-Path $root 'secrets'
|
||||
$passwordPath = Join-Path $secrets 'postgres_password'
|
||||
$databaseUrlPath = Join-Path $secrets 'database_url'
|
||||
|
||||
if (-not (Test-Path (Join-Path $root '.env'))) {
|
||||
Copy-Item (Join-Path $root '.env.example') (Join-Path $root '.env')
|
||||
}
|
||||
|
||||
$hasPassword = Test-Path $passwordPath
|
||||
$hasDatabaseUrl = Test-Path $databaseUrlPath
|
||||
if ($hasPassword -or $hasDatabaseUrl) {
|
||||
if ($hasPassword -and $hasDatabaseUrl) {
|
||||
Write-Output 'Local secrets already exist; leaving them unchanged.'
|
||||
exit 0
|
||||
}
|
||||
throw 'The local secret set is incomplete. Restore both files from the same setup; do not rotate a password against an existing volume.'
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force $secrets | Out-Null
|
||||
$bytes = New-Object byte[] 32
|
||||
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try {
|
||||
$rng.GetBytes($bytes)
|
||||
}
|
||||
finally {
|
||||
$rng.Dispose()
|
||||
}
|
||||
$builder = New-Object System.Text.StringBuilder
|
||||
foreach ($byte in $bytes) {
|
||||
[void]$builder.Append($byte.ToString('x2'))
|
||||
}
|
||||
$password = $builder.ToString()
|
||||
[System.IO.File]::WriteAllText($passwordPath, $password, [System.Text.Encoding]::ASCII)
|
||||
$databaseUrl = 'postgresql://messenger:' + $password + '@postgres:5432/messenger'
|
||||
[System.IO.File]::WriteAllText($databaseUrlPath, $databaseUrl, [System.Text.Encoding]::ASCII)
|
||||
Write-Output 'Created local Compose secrets. Their values were not printed.'
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply database migrations once before API replicas start."""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from db import Store
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||
store = Store.connect()
|
||||
try:
|
||||
logging.info('database migrations are current')
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart api postgres redis
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart api postgres redis
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restart API replicas one at a time while the public load test is running."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
os.environ.setdefault('PYTHONUTF8', '1')
|
||||
|
||||
|
||||
def run(command, capture=False, timeout=None):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
check=False,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode:
|
||||
detail = (result.stderr or result.stdout or '').strip()
|
||||
raise RuntimeError('Command failed: ' + ' '.join(command) + '\n' + detail)
|
||||
return result.stdout.strip() if capture else ''
|
||||
|
||||
|
||||
def wait_healthy(container_id, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
status = run(
|
||||
[
|
||||
'docker',
|
||||
'inspect',
|
||||
'--format',
|
||||
'{{.State.Health.Status}}',
|
||||
container_id,
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
if status == 'healthy':
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError('API replica did not become healthy after restart')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--duration', type=int, default=40)
|
||||
parser.add_argument('--concurrency', type=int, default=16)
|
||||
parser.add_argument('--output', default='evidence/rolling-restart.json')
|
||||
args = parser.parse_args()
|
||||
if args.duration < 10 or args.concurrency < 1:
|
||||
parser.error('duration must be at least 10 seconds and concurrency must be positive')
|
||||
|
||||
containers = run(
|
||||
['docker', 'compose', 'ps', '-q', 'api'], capture=True
|
||||
).splitlines()
|
||||
if len(containers) < 2:
|
||||
parser.error('At least two running API replicas are required')
|
||||
|
||||
output = (ROOT / args.output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load_output = output.with_name(output.stem + '-load.json')
|
||||
load_process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tests' / 'load.py'),
|
||||
'--base-url',
|
||||
args.base_url,
|
||||
'--duration',
|
||||
str(args.duration),
|
||||
'--concurrency',
|
||||
str(args.concurrency),
|
||||
'--output',
|
||||
str(load_output),
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
restarts = []
|
||||
try:
|
||||
time.sleep(2)
|
||||
for container_id in containers:
|
||||
instance_id = run(
|
||||
[
|
||||
'docker',
|
||||
'inspect',
|
||||
'--format',
|
||||
'{{.Config.Hostname}}',
|
||||
container_id,
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
started = time.monotonic()
|
||||
run(['docker', 'restart', '--timeout', '15', container_id], timeout=30)
|
||||
wait_healthy(container_id)
|
||||
restarts.append(
|
||||
{
|
||||
'instance_id': instance_id,
|
||||
'ready_seconds': round(time.monotonic() - started, 3),
|
||||
}
|
||||
)
|
||||
stdout, stderr = load_process.communicate(timeout=max(90, args.duration + 30))
|
||||
except Exception:
|
||||
if load_process.poll() is None:
|
||||
load_process.terminate()
|
||||
load_process.wait(timeout=5)
|
||||
raise
|
||||
if load_process.returncode:
|
||||
raise RuntimeError('Load test failed:\n' + (stderr or stdout).strip())
|
||||
|
||||
report = {
|
||||
'operation': 'serial API replica restart under load; same image, no schema change',
|
||||
'duration_seconds': args.duration,
|
||||
'concurrency': args.concurrency,
|
||||
'restarted_replicas': restarts,
|
||||
'load': json.loads(load_output.read_text(encoding='utf-8')),
|
||||
'load_raw_file': load_output.name,
|
||||
'limitations': [
|
||||
'This verifies graceful process replacement, not a new-image rolling deployment.',
|
||||
'PostgreSQL and Redis are single instances; their failure is not covered.',
|
||||
],
|
||||
}
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps(report['load'], ensure_ascii=False, indent=2))
|
||||
print('Saved serial restart evidence to ' + str(output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
|
||||
print('ERROR: ' + str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a new API image and replace replicas one at a time under load."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
|
||||
os.environ.setdefault('PYTHONUTF8', '1')
|
||||
|
||||
|
||||
def run(args, capture=True, timeout=None):
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
check=False,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode:
|
||||
detail = (result.stderr or result.stdout or '').strip()
|
||||
raise RuntimeError('Operation failed: ' + ' '.join(args[:3]) + '\n' + detail)
|
||||
return result.stdout.strip() if capture else ''
|
||||
|
||||
|
||||
def compose(*args, capture=True, timeout=None):
|
||||
return run(COMPOSE + ['-f', str(ROOT / 'compose.yaml'), *args], capture, timeout)
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
return json.loads(run(['docker', 'inspect', container_id]))[0]
|
||||
|
||||
|
||||
def wait_healthy(container_id, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
state = inspect(container_id)['State']
|
||||
if state['Status'] == 'exited':
|
||||
raise RuntimeError('Replacement API container exited before becoming healthy')
|
||||
if state.get('Health', {}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError('Replacement API container did not become healthy')
|
||||
|
||||
|
||||
def wait_routed_to(instance_id, base_url, timeout=20):
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
client = Client(base_url, timeout=5)
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.call(
|
||||
'GET', '/api/v1/users/me', expected=401, schema='Error'
|
||||
)
|
||||
if response['headers'].get('X-Instance-Id') == instance_id:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError('Nginx did not route a request to the new API container')
|
||||
|
||||
|
||||
def candidate_command(old, image_id, name):
|
||||
config = old['Config']
|
||||
host_config = old['HostConfig']
|
||||
networks = old['NetworkSettings']['Networks']
|
||||
if len(networks) != 1:
|
||||
raise RuntimeError('Expected API to use exactly one private network')
|
||||
network = next(iter(networks))
|
||||
args = [
|
||||
'docker', 'run', '--detach',
|
||||
'--name', name,
|
||||
'--network', network,
|
||||
'--network-alias', 'api',
|
||||
'--restart', 'no',
|
||||
'--stop-timeout', '15',
|
||||
]
|
||||
if config.get('User'):
|
||||
args += ['--user', config['User']]
|
||||
if host_config.get('NanoCpus'):
|
||||
args += ['--cpus', str(host_config['NanoCpus'] / 1_000_000_000)]
|
||||
if host_config.get('Memory'):
|
||||
args += ['--memory', str(host_config['Memory'])]
|
||||
for env in config.get('Env', []):
|
||||
args += ['--env', env]
|
||||
for mount in old.get('Mounts', []):
|
||||
if mount['Destination'] != '/run/secrets/database_url':
|
||||
raise RuntimeError('Unexpected API mount; refusing to copy deployment state')
|
||||
args += [
|
||||
'--mount',
|
||||
'type=bind,source=%s,target=%s,readonly'
|
||||
% (mount['Source'], mount['Destination']),
|
||||
]
|
||||
args.append(image_id)
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=BASE_URL)
|
||||
parser.add_argument('--duration', type=int, default=90)
|
||||
parser.add_argument('--concurrency', type=int, default=16)
|
||||
parser.add_argument('--output', default='evidence/rolling-update.json')
|
||||
args = parser.parse_args()
|
||||
if args.duration < 30 or args.concurrency < 1:
|
||||
parser.error('duration must be at least 30 seconds and concurrency must be positive')
|
||||
|
||||
if not (ROOT / 'secrets' / 'database_url').is_file():
|
||||
parser.error('Run scripts/init-local-secrets.ps1 before the update')
|
||||
old_ids = compose('ps', '-q', 'api').splitlines()
|
||||
if len(old_ids) < 2:
|
||||
parser.error('At least two running API replicas are required')
|
||||
|
||||
old_image_ids = {inspect(container_id)['Image'] for container_id in old_ids}
|
||||
image = inspect(old_ids[0])['Config']['Image']
|
||||
revision = 'lab3-' + uuid.uuid4().hex[:12]
|
||||
run(
|
||||
[
|
||||
'docker',
|
||||
'build',
|
||||
'--build-arg',
|
||||
'BUILD_REVISION=' + revision,
|
||||
'--tag',
|
||||
image,
|
||||
'.',
|
||||
],
|
||||
capture=False,
|
||||
timeout=300,
|
||||
)
|
||||
image_id = run(['docker', 'image', 'inspect', '--format', '{{.Id}}', image])
|
||||
if image_id in old_image_ids:
|
||||
raise RuntimeError('The candidate API image is unchanged; refusing a no-op update')
|
||||
compose('run', '--rm', 'migrate', capture=False, timeout=120)
|
||||
|
||||
output = (ROOT / args.output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load_output = output.with_name(output.stem + '-load.json')
|
||||
load_process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tests' / 'load.py'),
|
||||
'--base-url',
|
||||
args.base_url,
|
||||
'--duration',
|
||||
str(args.duration),
|
||||
'--concurrency',
|
||||
str(args.concurrency),
|
||||
'--output',
|
||||
str(load_output),
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
candidates = []
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(2)
|
||||
for index, old_id in enumerate(old_ids, start=1):
|
||||
old = inspect(old_id)
|
||||
instance_id = old['Config']['Hostname']
|
||||
candidate_name = 'lab3-api-candidate-' + uuid.uuid4().hex[:10]
|
||||
started = time.monotonic()
|
||||
candidate_result = run(
|
||||
candidate_command(old, image_id, candidate_name),
|
||||
timeout=60,
|
||||
)
|
||||
candidate_id = candidate_result.splitlines()[0]
|
||||
candidates.append(candidate_id)
|
||||
new_instance_id = inspect(candidate_id)['Config']['Hostname']
|
||||
wait_healthy(candidate_id)
|
||||
wait_routed_to(new_instance_id, args.base_url)
|
||||
run(['docker', 'stop', '--timeout', '15', old_id], timeout=30)
|
||||
run(['docker', 'rm', old_id])
|
||||
replacements.append(
|
||||
{
|
||||
'old_instance_id': instance_id,
|
||||
'new_instance_id': new_instance_id,
|
||||
'ready_and_routed_seconds': round(time.monotonic() - started, 3),
|
||||
'sequence': index,
|
||||
}
|
||||
)
|
||||
|
||||
compose(
|
||||
'up', '--detach', '--scale', 'api=2', '--no-deps', 'api',
|
||||
capture=False, timeout=120,
|
||||
)
|
||||
managed_ids = compose('ps', '-q', 'api').splitlines()
|
||||
if len(managed_ids) != 2:
|
||||
raise RuntimeError('Compose did not restore exactly two managed API replicas')
|
||||
for container_id in managed_ids:
|
||||
wait_healthy(container_id)
|
||||
for candidate_id in candidates:
|
||||
run(['docker', 'rm', '--force', candidate_id])
|
||||
stdout, stderr = load_process.communicate(timeout=max(90, args.duration + 30))
|
||||
except Exception:
|
||||
if load_process.poll() is None:
|
||||
load_process.terminate()
|
||||
load_process.wait(timeout=5)
|
||||
raise
|
||||
if load_process.returncode:
|
||||
raise RuntimeError('Load test failed:\n' + (stderr or stdout).strip())
|
||||
|
||||
report = {
|
||||
'operation': 'new API image; serial container replacement with temporary capacity surge',
|
||||
'duration_seconds': args.duration,
|
||||
'concurrency': args.concurrency,
|
||||
'new_image_id': image_id,
|
||||
'build_revision': revision,
|
||||
'old_image_ids': sorted(old_image_ids),
|
||||
'replacements': replacements,
|
||||
'load': json.loads(load_output.read_text(encoding='utf-8')),
|
||||
'load_raw_file': load_output.name,
|
||||
'limitations': [
|
||||
'The schema migration in this exercise is additive and backward-compatible.',
|
||||
'PostgreSQL and Redis are single instances and remain SPOFs.',
|
||||
],
|
||||
}
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps(report['load'], ensure_ascii=False, indent=2))
|
||||
print('Saved rolling update evidence to ' + str(output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
|
||||
print('ERROR: ' + str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,21 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is required 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
set TARGET_STOPPED=
|
||||
for /f "delims=" %%i in ('docker compose ps -q api') do if not defined TARGET_STOPPED call :stop_if_target %%i
|
||||
if defined TARGET_STOPPED exit /b 0
|
||||
echo No running API replica matches TARGET_INSTANCE_ID 1>&2
|
||||
exit /b 1
|
||||
|
||||
:stop_if_target
|
||||
for /f "delims=" %%h in ('docker inspect --format "{{.Config.Hostname}}" %1') do if "%%h"=="%TARGET_INSTANCE_ID%" (
|
||||
docker stop --time 2 %1 >nul
|
||||
if errorlevel 1 exit /b 1
|
||||
set TARGET_STOPPED=1
|
||||
exit /b 0
|
||||
)
|
||||
exit /b 1
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
for container_id in $(docker compose ps -q api); do
|
||||
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 2 "$container_id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s\n' 'No running API replica matches TARGET_INSTANCE_ID' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,524 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare API replica counts under fixed and increasing total API resources."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
|
||||
LOAD_SCRIPT = ROOT / 'tests' / 'load.py'
|
||||
MEMORY_PATTERN = re.compile(r'([\d.]+)\s*(B|KiB|MiB|GiB|kB|MB|GB)', re.IGNORECASE)
|
||||
os.environ.setdefault('PYTHONUTF8', '1')
|
||||
|
||||
|
||||
def command(args, overlay=None, check=True, capture=False):
|
||||
argv = COMPOSE + ['--env-file', str(ROOT / '.env')]
|
||||
if overlay:
|
||||
argv += ['--env-file', str(overlay)]
|
||||
argv += ['-f', str(ROOT / 'compose.yaml')] + list(args)
|
||||
result = subprocess.run(
|
||||
argv,
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
)
|
||||
if check and result.returncode:
|
||||
detail = (result.stderr or result.stdout or '').strip()
|
||||
raise RuntimeError('Compose command failed: ' + ' '.join(args) + '\n' + detail)
|
||||
return result
|
||||
|
||||
|
||||
def wait_ready(timeout=90):
|
||||
from urllib.error import URLError
|
||||
from urllib.request import urlopen
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urlopen(BASE_URL.rstrip('/') + '/health/ready', timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except (OSError, URLError):
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError('Load balancer did not become ready within %s seconds' % timeout)
|
||||
|
||||
|
||||
def sample_instances(minimum, timeout=15):
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
client = Client(BASE_URL, timeout=5)
|
||||
instances = set()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for _ in range(40):
|
||||
try:
|
||||
response = client.call(
|
||||
'GET',
|
||||
'/api/v1/users/me',
|
||||
expected=401,
|
||||
schema='Error',
|
||||
)
|
||||
instances.add(response['headers']['X-Instance-Id'])
|
||||
except Failure:
|
||||
time.sleep(0.1)
|
||||
if len(instances) >= minimum:
|
||||
return instances
|
||||
time.sleep(1)
|
||||
raise TimeoutError(
|
||||
'Expected %d API instance IDs through Nginx; observed %d'
|
||||
% (minimum, len(instances))
|
||||
)
|
||||
|
||||
|
||||
def parse_memory(value):
|
||||
match = MEMORY_PATTERN.search(value)
|
||||
if not match:
|
||||
return 0
|
||||
amount = float(match.group(1))
|
||||
unit = match.group(2).lower()
|
||||
factors = {
|
||||
'b': 1,
|
||||
'kib': 1024,
|
||||
'mib': 1024 ** 2,
|
||||
'gib': 1024 ** 3,
|
||||
'kb': 1000,
|
||||
'mb': 1000 ** 2,
|
||||
'gb': 1000 ** 3,
|
||||
}
|
||||
return int(amount * factors[unit])
|
||||
|
||||
|
||||
def collect_stats():
|
||||
result = command(['ps', '-q'], check=True, capture=True)
|
||||
container_ids = result.stdout.split()
|
||||
if not container_ids:
|
||||
return {}
|
||||
stats = subprocess.run(
|
||||
[
|
||||
'docker',
|
||||
'stats',
|
||||
'--no-stream',
|
||||
'--format',
|
||||
'{{.Name}};{{.CPUPerc}};{{.MemUsage}}',
|
||||
*container_ids,
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if stats.returncode:
|
||||
raise RuntimeError('docker stats failed: ' + (stats.stderr or stats.stdout).strip())
|
||||
values = {}
|
||||
for line in stats.stdout.splitlines():
|
||||
parts = line.split(';', 2)
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
name, cpu, memory = parts
|
||||
service = next(
|
||||
(item for item in ('api', 'postgres', 'nginx') if '-' + item + '-' in name),
|
||||
None,
|
||||
)
|
||||
if service:
|
||||
values.setdefault(service, []).append(
|
||||
{
|
||||
'cpu_percent': float(cpu.rstrip('%') or 0),
|
||||
'memory_bytes': parse_memory(memory.split('/')[0]),
|
||||
}
|
||||
)
|
||||
return values
|
||||
|
||||
|
||||
def run_load(duration, concurrency, output=None):
|
||||
args = [
|
||||
sys.executable,
|
||||
str(LOAD_SCRIPT),
|
||||
'--base-url',
|
||||
BASE_URL,
|
||||
'--duration',
|
||||
str(duration),
|
||||
'--concurrency',
|
||||
str(concurrency),
|
||||
]
|
||||
if output:
|
||||
args += ['--output', str(output)]
|
||||
return subprocess.run(
|
||||
args,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def measure(duration, concurrency, output):
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(LOAD_SCRIPT),
|
||||
'--base-url',
|
||||
BASE_URL,
|
||||
'--duration',
|
||||
str(duration),
|
||||
'--concurrency',
|
||||
str(concurrency),
|
||||
'--output',
|
||||
str(output),
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
peaks = {}
|
||||
try:
|
||||
while process.poll() is None:
|
||||
for service, entries in collect_stats().items():
|
||||
values = peaks.setdefault(
|
||||
service, {'cpu_percent': 0.0, 'memory_bytes': 0}
|
||||
)
|
||||
values['cpu_percent'] = max(
|
||||
values['cpu_percent'],
|
||||
sum(entry['cpu_percent'] for entry in entries),
|
||||
)
|
||||
values['memory_bytes'] = max(
|
||||
values['memory_bytes'],
|
||||
sum(entry['memory_bytes'] for entry in entries),
|
||||
)
|
||||
time.sleep(1)
|
||||
stdout, stderr = process.communicate()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
if process.returncode:
|
||||
raise RuntimeError('Load run failed:\n' + (stderr or stdout).strip())
|
||||
return json.loads(output.read_text(encoding='utf-8')), peaks
|
||||
|
||||
|
||||
def run_scenario(name, replicas, cpu_per_replica, memory_per_replica, args, output_dir):
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode='w',
|
||||
encoding='ascii',
|
||||
suffix='.env',
|
||||
prefix='lab3-track1-',
|
||||
delete=False,
|
||||
) as handle:
|
||||
handle.write('API_REPLICAS=%s\n' % replicas)
|
||||
handle.write('API_CPU_LIMIT=%s\n' % cpu_per_replica)
|
||||
handle.write('API_MEMORY_LIMIT=%sM\n' % memory_per_replica)
|
||||
override = Path(handle.name)
|
||||
try:
|
||||
command(
|
||||
['up', '--detach', '--build', '--scale', 'api=%d' % replicas],
|
||||
overlay=override,
|
||||
capture=True,
|
||||
)
|
||||
wait_ready()
|
||||
instances = sample_instances(min(2, replicas))
|
||||
running = command(['ps', '-q', 'api'], capture=True).stdout.split()
|
||||
if len(running) != replicas:
|
||||
raise RuntimeError(
|
||||
'Expected %d running API replicas, found %d' % (replicas, len(running))
|
||||
)
|
||||
|
||||
warmup = run_load(args.warmup, args.concurrency)
|
||||
if warmup.returncode:
|
||||
raise RuntimeError('Warm-up load failed: ' + warmup.stderr.strip())
|
||||
|
||||
runs = []
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
raw_path = output_dir / ('%s-run-%02d.json' % (name, repeat))
|
||||
metrics, peaks = measure(
|
||||
args.duration,
|
||||
args.concurrency,
|
||||
raw_path,
|
||||
)
|
||||
runs.append({'load': metrics, 'peak_resources': peaks, 'raw_file': raw_path.name})
|
||||
return {
|
||||
'name': name,
|
||||
'api_replicas': replicas,
|
||||
'api_cpu_limit_per_replica': cpu_per_replica,
|
||||
'api_memory_limit_mib_per_replica': memory_per_replica,
|
||||
'total_api_cpu_limit': round(replicas * cpu_per_replica, 3),
|
||||
'total_api_memory_limit_mib': replicas * memory_per_replica,
|
||||
'instances_observed': len(instances),
|
||||
'runs': runs,
|
||||
}
|
||||
finally:
|
||||
override.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def mean(values):
|
||||
return sum(values) / len(values) if values else 0
|
||||
|
||||
|
||||
def summarize(scenario):
|
||||
runs = scenario['runs']
|
||||
return {
|
||||
'name': scenario['name'],
|
||||
'api_replicas': scenario['api_replicas'],
|
||||
'total_api_cpu_limit': scenario['total_api_cpu_limit'],
|
||||
'total_api_memory_limit_mib': scenario['total_api_memory_limit_mib'],
|
||||
'successful_rps_mean': round(
|
||||
mean([run['load']['successful_rps'] for run in runs]), 3
|
||||
),
|
||||
'error_ratio_mean': round(
|
||||
mean([run['load']['error_ratio'] or 0 for run in runs]), 6
|
||||
),
|
||||
'p95_latency_ms_mean': round(
|
||||
mean([run['load']['successful_latency_ms']['p95'] or 0 for run in runs]),
|
||||
3,
|
||||
),
|
||||
'api_cpu_peak_percent_mean': round(
|
||||
mean([run['peak_resources'].get('api', {}).get('cpu_percent', 0) for run in runs]),
|
||||
3,
|
||||
),
|
||||
'api_memory_peak_mib_mean': round(
|
||||
mean(
|
||||
[
|
||||
run['peak_resources'].get('api', {}).get('memory_bytes', 0)
|
||||
/ (1024 ** 2)
|
||||
for run in runs
|
||||
]
|
||||
),
|
||||
3,
|
||||
),
|
||||
'postgres_cpu_peak_percent_mean': round(
|
||||
mean(
|
||||
[
|
||||
run['peak_resources'].get('postgres', {}).get('cpu_percent', 0)
|
||||
for run in runs
|
||||
]
|
||||
),
|
||||
3,
|
||||
),
|
||||
'postgres_memory_peak_mib_mean': round(
|
||||
mean(
|
||||
[
|
||||
run['peak_resources'].get('postgres', {}).get('memory_bytes', 0)
|
||||
/ (1024 ** 2)
|
||||
for run in runs
|
||||
]
|
||||
),
|
||||
3,
|
||||
),
|
||||
'nginx_cpu_peak_percent_mean': round(
|
||||
mean([run['peak_resources'].get('nginx', {}).get('cpu_percent', 0) for run in runs]),
|
||||
3,
|
||||
),
|
||||
'nginx_memory_peak_mib_mean': round(
|
||||
mean(
|
||||
[
|
||||
run['peak_resources'].get('nginx', {}).get('memory_bytes', 0)
|
||||
/ (1024 ** 2)
|
||||
for run in runs
|
||||
]
|
||||
),
|
||||
3,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def write_chart(path, summaries, value_key, title, unit):
|
||||
width, height = 980, 430
|
||||
left, top, chart_width, chart_height = 70, 55, 850, 275
|
||||
maximum = max((item[value_key] for item in summaries), default=1) or 1
|
||||
parts = [
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">'
|
||||
% (width, height, width, height),
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
'<text x="%d" y="30" font-family="sans-serif" font-size="20">%s</text>'
|
||||
% (left, title),
|
||||
]
|
||||
for tick in range(5):
|
||||
y = top + chart_height - chart_height * tick / 4
|
||||
value = maximum * tick / 4
|
||||
parts.append(
|
||||
'<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="#d8dee8"/>'
|
||||
% (left, y, left + chart_width, y)
|
||||
)
|
||||
parts.append(
|
||||
'<text x="8" y="%.1f" font-family="sans-serif" font-size="12">%.1f</text>'
|
||||
% (y + 4, value)
|
||||
)
|
||||
bar_width = 72
|
||||
slot = chart_width / len(summaries)
|
||||
for index, item in enumerate(summaries):
|
||||
value = item[value_key]
|
||||
bar_height = chart_height * value / maximum
|
||||
x = left + slot * index + (slot - bar_width) / 2
|
||||
y = top + chart_height - bar_height
|
||||
color = '#2864a5' if item['name'].startswith('fixed') else '#d47a22'
|
||||
label = item['name'].replace('fixed-total-', 'fixed ').replace('growing-total-', 'growth ')
|
||||
parts.append(
|
||||
'<rect x="%.1f" y="%.1f" width="%d" height="%.1f" fill="%s"/>'
|
||||
% (x, y, bar_width, bar_height, color)
|
||||
)
|
||||
parts.append(
|
||||
'<text x="%.1f" y="%.1f" text-anchor="middle" font-family="sans-serif" font-size="12">%.1f</text>'
|
||||
% (x + bar_width / 2, max(top + 14, y - 6), value)
|
||||
)
|
||||
parts.append(
|
||||
'<text x="%.1f" y="%d" text-anchor="middle" font-family="sans-serif" font-size="12">%s</text>'
|
||||
% (x + bar_width / 2, top + chart_height + 22, label)
|
||||
)
|
||||
parts.extend(
|
||||
[
|
||||
'<text x="%d" y="%d" font-family="sans-serif" font-size="12">%s</text>'
|
||||
% (left, height - 14, unit),
|
||||
'<rect x="710" y="385" width="12" height="12" fill="#2864a5"/>',
|
||||
'<text x="728" y="396" font-family="sans-serif" font-size="12">fixed API budget</text>',
|
||||
'<rect x="850" y="385" width="12" height="12" fill="#d47a22"/>',
|
||||
'<text x="868" y="396" font-family="sans-serif" font-size="12">growing API budget</text>',
|
||||
'</svg>',
|
||||
]
|
||||
)
|
||||
path.write_text('\n'.join(parts) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--duration', type=int, default=30)
|
||||
parser.add_argument('--warmup', type=int, default=5)
|
||||
parser.add_argument('--concurrency', type=int, default=16)
|
||||
parser.add_argument('--repeats', type=int, default=3)
|
||||
parser.add_argument('--fixed-total-cpus', type=float, default=4.0)
|
||||
parser.add_argument('--fixed-total-memory-mib', type=int, default=2048)
|
||||
parser.add_argument('--growth-cpus-per-replica', type=float, default=1.0)
|
||||
parser.add_argument('--growth-memory-mib-per-replica', type=int, default=512)
|
||||
parser.add_argument('--output', default='evidence/track1/benchmark.json')
|
||||
args = parser.parse_args()
|
||||
if (
|
||||
args.duration < 1
|
||||
or args.warmup < 1
|
||||
or args.concurrency < 1
|
||||
or args.repeats < 1
|
||||
or args.fixed_total_cpus <= 0
|
||||
or args.fixed_total_memory_mib < 1
|
||||
or args.growth_cpus_per_replica <= 0
|
||||
or args.growth_memory_mib_per_replica < 1
|
||||
):
|
||||
parser.error('duration, warmup, concurrency, repeats and resource limits must be positive')
|
||||
if not (ROOT / '.env').is_file():
|
||||
parser.error('Create .env and local Compose secrets first')
|
||||
if not all((ROOT / 'secrets' / name).is_file() for name in ('postgres_password', 'database_url')):
|
||||
parser.error('Run scripts/init-local-secrets.ps1 before benchmarking')
|
||||
|
||||
output = (ROOT / args.output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
command(['config', '--quiet'])
|
||||
output_dir = output.parent
|
||||
scenarios = []
|
||||
try:
|
||||
for replicas in (1, 2, 4):
|
||||
fixed_cpu = round(args.fixed_total_cpus / replicas, 3)
|
||||
fixed_memory = max(1, args.fixed_total_memory_mib // replicas)
|
||||
scenarios.append(
|
||||
run_scenario(
|
||||
'fixed-total-%d' % replicas,
|
||||
replicas,
|
||||
fixed_cpu,
|
||||
fixed_memory,
|
||||
args,
|
||||
output_dir,
|
||||
)
|
||||
)
|
||||
for replicas in (1, 2, 4):
|
||||
scenarios.append(
|
||||
run_scenario(
|
||||
'growing-total-%d' % replicas,
|
||||
replicas,
|
||||
args.growth_cpus_per_replica,
|
||||
args.growth_memory_mib_per_replica,
|
||||
args,
|
||||
output_dir,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
command(['up', '--detach', '--build', '--scale', 'api=2'], capture=True)
|
||||
wait_ready()
|
||||
|
||||
docker_version = subprocess.run(
|
||||
['docker', 'version', '--format', '{{.Server.Version}}'],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
docker_host = subprocess.run(
|
||||
['docker', 'info', '--format', '{{.NCPU}}|{{.MemTotal}}|{{.OperatingSystem}}'],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
docker_host_values = docker_host.stdout.strip().split('|')
|
||||
if docker_host.returncode or len(docker_host_values) != 3:
|
||||
raise RuntimeError('Unable to capture Docker host CPU/RAM information')
|
||||
report = {
|
||||
'track': '1 — цена масштабирования',
|
||||
'method': {
|
||||
'workload': 'tests/load.py closed-loop POST messages through Nginx',
|
||||
'duration_seconds_per_run': args.duration,
|
||||
'warmup_seconds_per_scenario': args.warmup,
|
||||
'concurrency': args.concurrency,
|
||||
'repeats_per_scenario': args.repeats,
|
||||
'fixed_total_api_limits': {
|
||||
'cpus': args.fixed_total_cpus,
|
||||
'memory_mib': args.fixed_total_memory_mib,
|
||||
},
|
||||
'growth_per_replica_limits': {
|
||||
'cpus': args.growth_cpus_per_replica,
|
||||
'memory_mib': args.growth_memory_mib_per_replica,
|
||||
},
|
||||
'host_cpu_count': os.cpu_count(),
|
||||
'platform': platform.platform(),
|
||||
'docker_server_version': docker_version.stdout.strip(),
|
||||
'docker_host': {
|
||||
'cpus': int(docker_host_values[0]),
|
||||
'memory_mib': round(int(docker_host_values[1]) / (1024 ** 2), 3),
|
||||
'operating_system': docker_host_values[2],
|
||||
},
|
||||
'measurement_note': (
|
||||
'Successful HTTP throughput, not committed-write throughput. '
|
||||
'Load generator is closed-loop. Docker stats are one-second samples '
|
||||
'and can miss short CPU peaks.'
|
||||
),
|
||||
},
|
||||
'scenarios': scenarios,
|
||||
'summary': [summarize(scenario) for scenario in scenarios],
|
||||
'raw_runs': True,
|
||||
}
|
||||
summary = [summarize(scenario) for scenario in scenarios]
|
||||
rps_chart = output_dir / 'successful-rps.svg'
|
||||
p95_chart = output_dir / 'p95-latency.svg'
|
||||
write_chart(rps_chart, summary, 'successful_rps_mean', 'Successful HTTP throughput by scale', 'Successful requests per second')
|
||||
write_chart(p95_chart, summary, 'p95_latency_ms_mean', 'Successful request p95 by scale', 'Milliseconds')
|
||||
report['summary'] = summary
|
||||
report['charts'] = [rps_chart.name, p95_chart.name]
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + '\n',
|
||||
encoding='utf-8',
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
print('Saved repeatable benchmark evidence to ' + str(output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
|
||||
print('ERROR: ' + str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user