ЛР 3: балансировка, общая идемпотентность и исследование масштабирования
This commit is contained in:
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run exact make targets in Linux on Windows/Linux, retaining safe evidence."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
from run_experiments import compose, containers, docker, runner, snapshot, wait_instances
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def save(name, text):
|
||||
(ROOT / 'evidence' / name).write_text(text + '\n', encoding='utf-8')
|
||||
print('Saved ' + name if name.endswith('.json') else text, flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
save('checks.txt', runner('make', 'check', 'validate', 'test'))
|
||||
save('student-tests.txt', runner('python', 'student-tests/integration.py'))
|
||||
before = snapshot()
|
||||
save('restart-hook.txt', runner('python', 'scripts/stack_control.py', 'restart'))
|
||||
wait_instances(2)
|
||||
save('restart-processes.json', json.dumps({'before': before, 'after': snapshot()}, indent=2))
|
||||
save('persistence.txt', runner('make', 'test-persistence', 'ACTION_SCRIPT=scripts/restart.sh'))
|
||||
wait_instances(2)
|
||||
ids = {c['Id'] for c in containers('api')}
|
||||
stopped = set()
|
||||
rounds = []
|
||||
for i in range(1, 3):
|
||||
before = snapshot()
|
||||
text = runner('make', 'test-failover', 'ACTION_SCRIPT=scripts/stop-one.sh')
|
||||
after = snapshot()
|
||||
dead = [c for c in containers('api', True) if not c['State']['Running']]
|
||||
if len(dead) != 1:
|
||||
raise RuntimeError('Hook did not leave exactly one API stopped')
|
||||
stopped.add(dead[0]['Id'])
|
||||
rounds.append({'round': i, 'stopped_id': dead[0]['Id'][:12], 'before': before, 'after': after})
|
||||
save('failover-%d.txt' % i, text)
|
||||
docker('start', dead[0]['Id'])
|
||||
wait_instances(2)
|
||||
if stopped == ids:
|
||||
break
|
||||
save('failover-processes.json', json.dumps(rounds, indent=2))
|
||||
# The public scenario chooses its target via LB, so its two runs may select
|
||||
# the same process. The next tests deterministically cover EVERY replica.
|
||||
for index, container in enumerate(containers('api'), 1):
|
||||
before = snapshot()
|
||||
text = runner('python', 'student-tests/mixed_failover.py',
|
||||
'--target-hostname', container['Config']['Hostname'])
|
||||
save('mixed-failover-%d.json' % index, text)
|
||||
save('mixed-processes-%d.json' % index,
|
||||
json.dumps({'before': before, 'after': snapshot()}, indent=2))
|
||||
docker('start', container['Id'])
|
||||
wait_instances(2)
|
||||
save('final-smoke.txt', runner('make', 'test'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Persistent-connection async POST benchmark with raw per-request samples.
|
||||
|
||||
Closed loop, no retries; successful RPS counts HTTP acknowledgements, not commits.
|
||||
Bootstrap is outside measured time; credentials are never written to disk.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
from collections import Counter
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(len(values) * p) - 1)], 3) if values else None
|
||||
|
||||
|
||||
async def measure(base, actor, chat, duration, warmup, concurrency):
|
||||
samples = []
|
||||
limits = httpx.Limits(max_connections=concurrency, max_keepalive_connections=concurrency)
|
||||
async with httpx.AsyncClient(base_url=base, timeout=5, limits=limits, trust_env=False) as client:
|
||||
start = time.monotonic()
|
||||
measured_start, deadline = start + warmup, start + warmup + duration
|
||||
|
||||
async def worker(worker_id):
|
||||
index = 0
|
||||
while time.monotonic() < deadline:
|
||||
before = time.monotonic()
|
||||
try:
|
||||
response = await client.post('/api/v1/chats/' + chat['id'] + '/messages',
|
||||
headers={'Authorization': 'Bearer ' + actor['token'], 'Idempotency-Key': uuid.uuid4().hex},
|
||||
json={'text': 'load-%04d-%08d' % (worker_id, index)})
|
||||
status = response.status_code
|
||||
instance = response.headers.get('x-instance-id', '')
|
||||
except httpx.HTTPError:
|
||||
status, instance = 0, ''
|
||||
after = time.monotonic()
|
||||
if before >= measured_start:
|
||||
samples.append({'started_s': round(before - measured_start, 6),
|
||||
'latency_ms': round((after - before) * 1000, 3),
|
||||
'status': status, 'instance': instance})
|
||||
index += 1
|
||||
|
||||
await asyncio.gather(*(worker(i) for i in range(concurrency)))
|
||||
elapsed = time.monotonic() - measured_start
|
||||
return samples, elapsed
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--duration', type=float, default=20)
|
||||
parser.add_argument('--warmup', type=float, default=5)
|
||||
parser.add_argument('--concurrency', type=int, default=32)
|
||||
parser.add_argument('--output', required=True)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.concurrency <= 256 or not 1 <= args.duration <= 3600 or args.warmup < 0:
|
||||
parser.error('invalid load parameters')
|
||||
alice, _, _, chat = Client(args.base_url).bootstrap()
|
||||
samples, elapsed = asyncio.run(measure(args.base_url, alice, chat, args.duration, args.warmup, args.concurrency))
|
||||
success = [s['latency_ms'] for s in samples if s['status'] == 201]
|
||||
report = {'workload': 'closed-loop POST messages; unique keys, persistent connections, no retries',
|
||||
'duration_seconds': args.duration, 'warmup_seconds': args.warmup,
|
||||
'elapsed_seconds': round(elapsed, 3), 'concurrency': args.concurrency,
|
||||
'successful_requests': len(success), 'failed_requests': len(samples) - len(success),
|
||||
'successful_rps': round(len(success) / elapsed, 3),
|
||||
'error_ratio': round((len(samples) - len(success)) / len(samples), 6) if samples else None,
|
||||
'successful_latency_ms': {p: percentile(success, v) for p, v in [('p50', .5), ('p95', .95), ('p99', .99)]},
|
||||
'status_counts': dict(Counter(s['status'] for s in samples)),
|
||||
'instance_counts': dict(Counter(s['instance'] for s in samples)),
|
||||
'dataset': {'initial_messages_in_new_chat': 0, 'measured_attempts': len(samples),
|
||||
'text_template_codepoints': 18},
|
||||
'limits': 'Success-only latency; closed-loop coordinated omission. Client/host can bottleneck. '
|
||||
'Warmup inserts remain in history; requests started in warmup excluded from samples.'}
|
||||
dest = Path(args.output)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
raw = dest.with_suffix('.jsonl')
|
||||
raw.write_text(''.join(json.dumps(s) + '\n' for s in samples), encoding='utf-8')
|
||||
print(json.dumps(report, indent=2))
|
||||
if not success:
|
||||
raise SystemExit('No successful writes')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent persistent HTTP connection per worker; no shared HTTP client pool."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(len(values) * p) - 1)], 3) if values else None
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--duration', type=float, default=20)
|
||||
parser.add_argument('--warmup', type=float, default=5)
|
||||
parser.add_argument('--concurrency', type=int, default=32)
|
||||
parser.add_argument('--output', required=True)
|
||||
args = parser.parse_args()
|
||||
url = urlsplit(args.base_url)
|
||||
if url.scheme != 'http' or not url.hostname:
|
||||
parser.error('This local lab3 generator requires an HTTP base URL')
|
||||
if not 1 <= args.concurrency <= 256 or not 1 <= args.duration <= 3600 or args.warmup < 0:
|
||||
parser.error('invalid load parameters')
|
||||
alice, _, _, chat = Client(args.base_url).bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
measured_start = time.monotonic() + args.warmup
|
||||
deadline = measured_start + args.duration
|
||||
|
||||
def worker(worker_id):
|
||||
conn = http.client.HTTPConnection(url.hostname, url.port or 80, timeout=5)
|
||||
samples = []
|
||||
index = 0
|
||||
try:
|
||||
while time.monotonic() < deadline:
|
||||
data = json.dumps({'text': 'load-%04d-%08d' % (worker_id, index)}, separators=(',', ':')).encode()
|
||||
headers = {'Authorization': 'Bearer ' + alice['token'],
|
||||
'Content-Type': 'application/json', 'Idempotency-Key': uuid.uuid4().hex}
|
||||
before = time.monotonic()
|
||||
try:
|
||||
conn.request('POST', path, body=data, headers=headers)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
status = response.status
|
||||
instance = response.getheader('X-Instance-Id', '')
|
||||
except (OSError, http.client.HTTPException):
|
||||
status, instance = 0, ''
|
||||
conn.close()
|
||||
after = time.monotonic()
|
||||
if before >= measured_start:
|
||||
samples.append({'started_s': round(before - measured_start, 6),
|
||||
'latency_ms': round(1000 * (after - before), 3), 'status': status, 'instance': instance})
|
||||
index += 1
|
||||
finally:
|
||||
conn.close()
|
||||
return samples
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
batches = list(pool.map(worker, range(args.concurrency)))
|
||||
elapsed = time.monotonic() - measured_start
|
||||
samples = sorted((s for batch in batches for s in batch), key=lambda s: s['started_s'])
|
||||
success = [s['latency_ms'] for s in samples if s['status'] == 201]
|
||||
report = {'generator': 'stdlib http.client; independent persistent connection per thread',
|
||||
'workload': 'closed-loop POST messages; unique keys, persistent connections, no retries',
|
||||
'duration_seconds': args.duration, 'warmup_seconds': args.warmup,
|
||||
'elapsed_seconds': round(elapsed, 3), 'concurrency': args.concurrency,
|
||||
'successful_requests': len(success), 'failed_requests': len(samples) - len(success),
|
||||
'successful_rps': round(len(success) / elapsed, 3),
|
||||
'error_ratio': round((len(samples) - len(success)) / len(samples), 6) if samples else None,
|
||||
'successful_latency_ms': {label: percentile(success, p) for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]},
|
||||
'status_counts': dict(Counter(s['status'] for s in samples)),
|
||||
'instance_counts': dict(Counter(s['instance'] for s in samples)),
|
||||
'dataset': {'initial_messages_in_new_chat': 0, 'measured_attempts': len(samples), 'text_template_codepoints': 18},
|
||||
'limits': 'Success-only latency; closed-loop coordinated omission. Client/host can bottleneck. '
|
||||
'Warmup inserts remain in history, excluded from measured samples. HTTP acknowledgements, not COMMIT count.'}
|
||||
dest = Path(args.output)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
dest.with_suffix('.jsonl').write_text(''.join(json.dumps(s) + '\n' for s in samples), encoding='utf-8')
|
||||
print(json.dumps(report, indent=2))
|
||||
if not success:
|
||||
raise SystemExit('No successful writes')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local secrets once. Never overwrite a live installation's secrets."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
dest = ROOT / 'secrets'
|
||||
dest.mkdir(mode=0o700, exist_ok=True)
|
||||
password_path = dest / 'db_password'
|
||||
if not password_path.exists():
|
||||
password_path.write_text(secrets.token_hex(32), encoding='utf-8')
|
||||
password = password_path.read_text(encoding='utf-8').strip()
|
||||
values = {'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger',
|
||||
'cursor_key': secrets.token_hex(32)}
|
||||
for name, value in values.items():
|
||||
path = dest / name
|
||||
if not path.exists():
|
||||
path.write_text(value, encoding='utf-8')
|
||||
env = ROOT / '.env'
|
||||
if not env.exists():
|
||||
env.write_text((ROOT / '.env.example').read_text(encoding='utf-8'), encoding='utf-8')
|
||||
print('Local secrets and .env ready (existing files preserved; values not printed).')
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline consistency check: recorded summaries vs independent raw samples."""
|
||||
from collections import Counter
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def verify(record, folder, name):
|
||||
lines = (folder / (name + '.jsonl')).read_text(encoding='utf-8').splitlines()
|
||||
samples = [json.loads(line) for line in lines]
|
||||
successes = [s['latency_ms'] for s in samples if s['status'] == 201]
|
||||
assert len(successes) == record['successful_requests'], name + ': success count'
|
||||
assert all(s['instance'] for s in samples if s['status'] == 201), name + ': missing instance header'
|
||||
if 'replicas' in record:
|
||||
assert len({s['instance'] for s in samples if s['status'] == 201}) == record['replicas'], name + ': replica count'
|
||||
assert len(samples) - len(successes) == record['failed_requests'], name + ': error count'
|
||||
assert dict(Counter(str(s['status']) for s in samples)) == record['status_counts'], name + ': status counts'
|
||||
assert dict(Counter(s['instance'] for s in samples)) == record['instance_counts'], name + ': instance counts'
|
||||
assert abs(len(successes) / record['elapsed_seconds'] - record['successful_rps']) < .05, name + ': RPS'
|
||||
ordered = sorted(successes)
|
||||
for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]:
|
||||
value = ordered[max(0, math.ceil(len(ordered) * p) - 1)]
|
||||
assert abs(value - record['successful_latency_ms'][label]) < .0011, name + ': percentile ' + label
|
||||
resources = json.loads((folder / (name + '.resources.json')).read_text(encoding='utf-8'))
|
||||
assert resources['dataset_after']['messages'] >= resources['dataset_before']['messages'], name + ': dataset'
|
||||
assert any(s['containers'] for s in resources['resource_samples']), name + ': no resource samples'
|
||||
|
||||
|
||||
def main():
|
||||
immutable = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'contracts', 'tests', 'Makefile',
|
||||
'.github/workflows/contracts.yml']
|
||||
diff = subprocess.check_output(['git', 'diff', '0ab3a34', '--', *immutable], cwd=ROOT)
|
||||
assert not diff, 'Skeleton contract, rubric or public tests changed'
|
||||
records = json.loads((ROOT / 'evidence/scaling/summary.json').read_text(encoding='utf-8'))
|
||||
assert len({r['name'] for r in records}) == len(records), 'Duplicate run name'
|
||||
for record in records:
|
||||
verify(record, ROOT / 'evidence/scaling', record['name'])
|
||||
rolling = json.loads((ROOT / 'evidence/rolling.json').read_text(encoding='utf-8'))
|
||||
rolling_raw = ROOT / rolling.get('raw_prefix', 'evidence/scaling/rolling-update')
|
||||
verify(rolling['load'], rolling_raw.parent, rolling_raw.name)
|
||||
first, last = rolling['events'][0]['containers'], rolling['events'][-1]['containers']
|
||||
old = {c['id'] for c in first if c['service'] == 'api'}
|
||||
new = {c['id'] for c in last if c['service'] == 'api'}
|
||||
assert len(old) == len(new) == 2 and not old & new, 'Rolling did not replace both original APIs'
|
||||
for name in ('REPORT.md', 'RESEARCH.md', 'RUNBOOK.md', 'DEFENSE.md', 'PR_DESCRIPTION.md',
|
||||
'.github/pull_request_template.md'):
|
||||
assert 'TODO' not in (ROOT / name).read_text(encoding='utf-8'), name + ': unfilled placeholder'
|
||||
print('PASS: %d measured runs + rolling raw samples match counts, RPS and percentiles; '
|
||||
'resource evidence present; both API IDs replaced; contract/public tests unchanged.' % len(records))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small defense demonstration; outputs aliases and synthetic data, no credentials."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
c = Client()
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
instances = sorted({c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||||
for _ in range(20)})
|
||||
check(len(instances) >= 2, 'Expected >=2 replicas with the SAME session')
|
||||
message = c.message(alice, chat, 'Одно сообщение через общий LB')['json']
|
||||
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json']
|
||||
check(message in page['items'], 'Message not visible to another member')
|
||||
print(json.dumps({'same_session_served_by': instances, 'message': message,
|
||||
'visible_to_second_member': True}, ensure_ascii=False, indent=2))
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit short TTL and dependency failure checks; always restore baseline."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from run_experiments import compose, runner, wait_instances
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
env = dict(os.environ, SESSION_TTL_SECONDS='2')
|
||||
compose('up', '-d', '--no-deps', 'api', env=env)
|
||||
wait_instances(2)
|
||||
text = runner('python', 'student-tests/session_expiry.py')
|
||||
(ROOT / 'evidence/session-expiry.txt').write_text(text, encoding='utf-8')
|
||||
print(text)
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 'api')
|
||||
wait_instances(2)
|
||||
try:
|
||||
compose('stop', 'db')
|
||||
text = runner('python', 'student-tests/dependency_health.py')
|
||||
(ROOT / 'evidence/dependency-health.txt').write_text(text, encoding='utf-8')
|
||||
print(text)
|
||||
finally:
|
||||
compose('start', 'db')
|
||||
wait_instances(2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild static research figures and aggregate metrics from raw summaries."""
|
||||
from collections import defaultdict
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--directory', default='evidence/scaling')
|
||||
args = parser.parse_args()
|
||||
dest = ROOT / args.directory
|
||||
source_caption = 'Source: ' + args.directory + '/summary.json; 2026-11-06'
|
||||
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
|
||||
grouped = defaultdict(list)
|
||||
for record in records:
|
||||
grouped[(record['series'], record['replicas'], record['concurrency'])].append(record)
|
||||
aggregate = []
|
||||
for (series, replicas, concurrency), runs in grouped.items():
|
||||
cpu, ram = defaultdict(list), defaultdict(list)
|
||||
for run in runs:
|
||||
resource = json.loads((dest / (run['name'] + '.resources.json')).read_text(encoding='utf-8'))
|
||||
for sample in resource['resource_samples']:
|
||||
for c in sample['containers']:
|
||||
service = next((s for s in ('api', 'db', 'lb', 'tester') if '-' + s + '-' in c['Name']), None)
|
||||
if not service:
|
||||
continue
|
||||
cpu[service].append(float(c['CPUPerc'].rstrip('%')))
|
||||
value = c['MemUsage'].split('/')[0].strip()
|
||||
for unit, factor in [('KiB', 1 / 1024), ('MiB', 1), ('GiB', 1024), ('B', 1 / 1048576)]:
|
||||
if value.endswith(unit):
|
||||
ram[service].append(float(value[:-len(unit)]) * factor)
|
||||
break
|
||||
aggregate.append({'series': series, 'replicas': replicas, 'concurrency': concurrency,
|
||||
'runs': len(runs), 'api_cpu_each': runs[0]['api_cpu_each'],
|
||||
'api_memory_each': runs[0]['api_memory_each'], 'db_cpu': runs[0]['db_cpu'],
|
||||
'db_pool_size': runs[0].get('db_pool_size', 4),
|
||||
'median_rps': statistics.median(r['successful_rps'] for r in runs),
|
||||
'min_rps': min(r['successful_rps'] for r in runs),
|
||||
'max_rps': max(r['successful_rps'] for r in runs),
|
||||
'failed_requests_total': sum(r['failed_requests'] for r in runs),
|
||||
'attempts_total': sum(r['successful_requests'] + r['failed_requests'] for r in runs),
|
||||
'median_success_latency_ms': {p: statistics.median(r['successful_latency_ms'][p] for r in runs)
|
||||
for p in ('p50', 'p95', 'p99')},
|
||||
'sampled_cpu_percent_one_core': {s: {'mean': round(statistics.mean(v), 2), 'max': max(v)} for s, v in cpu.items()},
|
||||
'sampled_ram_mib_max_per_container': {s: round(max(v), 2) for s, v in ram.items() if v},
|
||||
'resource_note': 'Samples span bootstrap/warmup/measurement; CPU 100%=one core. API samples are per replica, not summed.'})
|
||||
order = {s: i for i, s in enumerate(('fixed', 'growing', 'db-improved',
|
||||
'saturation-baseline', 'saturation-improved', 'pool-four', 'pool-one', 'lb-unbounded', 'lb-bounded'))}
|
||||
aggregate.sort(key=lambda r: (order[r['series']], r['replicas'], r['concurrency']))
|
||||
(dest / 'aggregate.json').write_text(json.dumps(aggregate, indent=2) + '\n', encoding='utf-8')
|
||||
plt.rcParams.update({'figure.dpi': 150, 'axes.spines.top': False, 'axes.spines.right': False})
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), layout='constrained')
|
||||
for series, label, color in [('fixed', 'Fixed total API: 1 CPU / 512 MiB', '#2563eb'),
|
||||
('growing', 'Each API: 0.5 CPU / 256 MiB', '#b45309')]:
|
||||
rows = sorted((r for r in aggregate if r['series'] == series), key=lambda r: r['replicas'])
|
||||
x, y = [r['replicas'] for r in rows], [r['median_rps'] for r in rows]
|
||||
axes[0].errorbar(x, y, yerr=[[r['median_rps'] - r['min_rps'] for r in rows],
|
||||
[r['max_rps'] - r['median_rps'] for r in rows]],
|
||||
marker='o', capsize=4, label=label, color=color)
|
||||
axes[1].plot(x, [r['median_success_latency_ms']['p95'] for r in rows], marker='o', label=label, color=color)
|
||||
for ax in axes:
|
||||
ax.set_xlabel('API replicas (count)')
|
||||
ax.set_xticks([1, 2, 4])
|
||||
ax.grid(alpha=.2)
|
||||
ax.legend(fontsize=8)
|
||||
axes[0].set_ylabel('Successful HTTP 201 acknowledgements / second')
|
||||
axes[0].set_title('Scaling: median RPS; whiskers=min/max across repeats')
|
||||
axes[1].set_ylabel('Successful request p95 latency (ms)')
|
||||
axes[1].set_title('Scaling: median of run-level p95')
|
||||
fig.suptitle('POST messages via HAProxy; concurrency 32; DB 0.25 CPU\n' + source_caption, fontsize=11)
|
||||
fig.savefig(dest / 'scaling.png')
|
||||
plt.close(fig)
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), layout='constrained')
|
||||
for series, label, color in [('saturation-baseline', 'DB 0.25 CPU', '#b45309'),
|
||||
('saturation-improved', 'DB 1.0 CPU', '#2563eb')]:
|
||||
rows = sorted((r for r in aggregate if r['series'] == series), key=lambda r: r['concurrency'])
|
||||
x = [r['concurrency'] for r in rows]
|
||||
axes[0].errorbar(x, [r['median_rps'] for r in rows],
|
||||
yerr=[[r['median_rps'] - r['min_rps'] for r in rows], [r['max_rps'] - r['median_rps'] for r in rows]],
|
||||
marker='o', capsize=4, label=label, color=color)
|
||||
axes[1].plot(x, [r['median_success_latency_ms']['p95'] for r in rows], marker='o', label=label, color=color)
|
||||
for ax in axes:
|
||||
ax.set_xlabel('Concurrent closed-loop clients (count)')
|
||||
ax.set_xticks([1, 4, 8, 32, 64])
|
||||
ax.grid(alpha=.2)
|
||||
ax.legend()
|
||||
axes[0].set_ylabel('Successful HTTP 201 acknowledgements / second')
|
||||
axes[0].set_title('Shared dependency saturation: median and min/max')
|
||||
axes[1].set_ylabel('Successful request p95 latency (ms)')
|
||||
axes[1].set_title('Cost of concurrency: median of run-level p95')
|
||||
fig.suptitle('4 API x 0.5 CPU / 256 MiB; only DB CPU changes\n' + source_caption, fontsize=11)
|
||||
fig.savefig(dest / 'saturation.png')
|
||||
plt.close(fig)
|
||||
controls = [r for r in records if r['series'] == 'pool-four']
|
||||
candidates = [r for r in records if r['series'] == 'pool-one']
|
||||
if controls and candidates:
|
||||
controls.sort(key=lambda r: r['repeat'])
|
||||
candidates.sort(key=lambda r: r['repeat'])
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10, 4.5), layout='constrained')
|
||||
for series, label, offset, color in [(controls, 'Pool 4 per API', -.18, '#b45309'),
|
||||
(candidates, 'Pool 1 per API', .18, '#2563eb')]:
|
||||
x = [r['repeat'] + offset for r in series]
|
||||
axes[0].bar(x, [r['successful_rps'] for r in series], width=.35, label=label, color=color)
|
||||
axes[1].bar(x, [r['successful_latency_ms']['p95'] for r in series], width=.35, label=label, color=color)
|
||||
for ax in axes:
|
||||
ax.set_xlabel('Alternating experiment pair (number)')
|
||||
ax.set_xticks([r['repeat'] for r in controls])
|
||||
ax.legend()
|
||||
ax.grid(axis='y', alpha=.2)
|
||||
axes[0].set_ylabel('Successful HTTP 201 acknowledgements / second')
|
||||
axes[0].set_title('Connection pool control: observed RPS per run')
|
||||
axes[1].set_ylabel('Successful request p95 latency (ms)')
|
||||
axes[1].set_title('Connection pool control: observed p95 per run')
|
||||
fig.suptitle('Only pool limit changes: 4 API; DB 0.25 CPU; concurrency 32\n' + source_caption, fontsize=10)
|
||||
fig.savefig(dest / 'pool.png')
|
||||
plt.close(fig)
|
||||
print(json.dumps(aggregate, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stack_control.py restart
|
||||
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Host-side reproducible experiment controller. Requires Docker Compose v2.
|
||||
|
||||
Only this Compose project's API/DB quotas change. No data or volumes are deleted.
|
||||
The baseline two-replica limits are restored in finally, including after failure.
|
||||
"""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
from stack_control import containers, docker, snapshot
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, env=None):
|
||||
return subprocess.check_output(args, cwd=ROOT, env=env, text=True, encoding='utf-8',
|
||||
errors='replace', stderr=subprocess.PIPE)
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return command(['docker', 'compose', *args], env)
|
||||
|
||||
|
||||
def runner(*args):
|
||||
return compose('run', '--rm', '-T', 'tester', *args)
|
||||
|
||||
|
||||
def wait_instances(count):
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
apis = containers('api')
|
||||
if len(apis) == count and all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in apis):
|
||||
runner('python', 'scripts/wait_ready.py')
|
||||
# Let Docker DNS/HAProxy rise checks converge after a replacement.
|
||||
time.sleep(3)
|
||||
return
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('API containers did not become healthy')
|
||||
|
||||
|
||||
def configure(replicas, cpu, memory, db_cpu, pool=4):
|
||||
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu),
|
||||
API_MEMORY=memory, DB_POOL_SIZE=str(pool))
|
||||
compose('up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api', env=env)
|
||||
for c in containers('db'):
|
||||
docker('update', '--cpus', str(db_cpu), c['Id'])
|
||||
wait_instances(replicas)
|
||||
|
||||
|
||||
def db_counts():
|
||||
sql = "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats)," \
|
||||
"'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM sessions)," \
|
||||
"'commits',(SELECT xact_commit FROM pg_stat_database WHERE datname=current_database()))"
|
||||
result = json.loads(compose('exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql))
|
||||
counters = compose('exec', '-T', 'db', 'cat', '/sys/fs/cgroup/cpu.stat')
|
||||
result['db_cgroup_cpu_stat'] = {k: int(v) for k, v in (line.split() for line in counters.splitlines())}
|
||||
return result
|
||||
|
||||
|
||||
def monitored_benchmark(name, duration, warmup, concurrency, action=None):
|
||||
output = 'evidence/scaling/' + name + '.json'
|
||||
before = db_counts()
|
||||
samples = []
|
||||
proc = subprocess.Popen(['docker', 'compose', 'run', '--rm', '-T', 'tester',
|
||||
'python', os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py'),
|
||||
'--duration', str(duration), '--warmup', str(warmup),
|
||||
'--concurrency', str(concurrency), '--output', output], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='replace')
|
||||
action_error = []
|
||||
|
||||
def do_action():
|
||||
try:
|
||||
time.sleep(warmup + 3)
|
||||
action()
|
||||
except Exception as exc:
|
||||
action_error.append(type(exc).__name__ + ': ' + str(exc))
|
||||
|
||||
action_thread = threading.Thread(target=do_action) if action else None
|
||||
if action_thread:
|
||||
action_thread.start()
|
||||
while proc.poll() is None:
|
||||
try:
|
||||
ids = [c['Id'] for service in ('api', 'db', 'lb', 'tester') for c in containers(service)]
|
||||
if ids:
|
||||
lines = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids]).splitlines()
|
||||
samples.append({'utc': datetime.now(timezone.utc).isoformat(),
|
||||
'containers': [json.loads(line) for line in lines if line]})
|
||||
except subprocess.CalledProcessError:
|
||||
# --rm runner or a rolling-update target can disappear between ps,
|
||||
# inspect and stats. Missing telemetry must not abort a valid run.
|
||||
samples.append({'utc': datetime.now(timezone.utc).isoformat(), 'containers': [],
|
||||
'sampling_error': 'Container disappeared or Docker stats unavailable'})
|
||||
time.sleep(1)
|
||||
stdout, stderr = proc.communicate()
|
||||
if action_thread:
|
||||
action_thread.join()
|
||||
if proc.returncode or action_error:
|
||||
raise RuntimeError('Benchmark/action failed: ' + stdout[-1500:] + stderr[-1500:] + str(action_error))
|
||||
report = json.loads((ROOT / output).read_text(encoding='utf-8'))
|
||||
meta = {'limits_and_processes': snapshot(), 'dataset_before': before, 'dataset_after': db_counts(),
|
||||
'resource_samples': samples}
|
||||
(ROOT / output).with_suffix('.resources.json').write_text(json.dumps(meta, indent=2) + '\n', encoding='utf-8')
|
||||
print(name, report['successful_rps'], 'RPS; errors', report['failed_requests'], flush=True)
|
||||
return report
|
||||
|
||||
|
||||
def scaling(args):
|
||||
dest = ROOT / 'evidence/scaling'
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
hardware = {'utc': datetime.now(timezone.utc).isoformat(),
|
||||
'docker': json.loads(docker('info', '--format', '{{json .}}')),
|
||||
'note': 'Docker info sanitized to host capacity/version only below'}
|
||||
info = hardware['docker']
|
||||
hardware['docker'] = {k: info.get(k) for k in ('NCPU', 'MemTotal', 'Architecture', 'KernelVersion', 'OperatingSystem', 'ServerVersion')}
|
||||
hardware['images'] = [{ 'image': image, 'id': json.loads(docker('image', 'inspect', image))[0]['Id']}
|
||||
for image in ('mtusi-lab3-api:local', 'postgres:17.5-bookworm', 'haproxy:3.0.11-alpine')]
|
||||
hardware['benchmark_script'] = os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py')
|
||||
(dest / 'hardware.json').write_text(json.dumps(hardware, indent=2) + '\n', encoding='utf-8')
|
||||
summary = dest / 'summary.json'
|
||||
records = json.loads(summary.read_text(encoding='utf-8')) if args.resume and summary.exists() else []
|
||||
completed = {r['name'] for r in records}
|
||||
# All non-API quotas remain fixed. Fixed: TOTAL API 1CPU/512MiB.
|
||||
# Growing: EACH API .5CPU/256MiB. DB=.25CPU exposes a shared dependency limit.
|
||||
cases = [('fixed', n, 1 / n, str(512 // n) + 'm', .25) for n in (1, 2, 4)]
|
||||
cases += [('growing', n, .5, '256m', .25) for n in (1, 2, 4)]
|
||||
cases += [('db-improved', 4, .5, '256m', 1.0)]
|
||||
try:
|
||||
for series, n, cpu, memory, db_cpu in cases:
|
||||
if all('%s-n%d-r%d' % (series, n, r) in completed for r in range(1, args.repeats + 1)):
|
||||
continue
|
||||
pool = 1 if series == 'pool-one' else 4
|
||||
configure(n, cpu, memory, db_cpu, pool)
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
name = '%s-n%d-r%d' % (series, n, repeat)
|
||||
if name in completed:
|
||||
continue
|
||||
report = monitored_benchmark(name, args.duration, args.warmup, args.concurrency)
|
||||
records.append({'series': series, 'replicas': n, 'api_cpu_each': cpu,
|
||||
'api_memory_each': memory, 'db_cpu': db_cpu, 'db_pool_size': pool,
|
||||
'repeat': repeat, 'name': name, **report})
|
||||
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
||||
# Saturation sweep: identify the concurrency knee with all other limits fixed.
|
||||
for db_cpu, label in ((.25, 'baseline'), (1.0, 'improved')):
|
||||
configure(4, .5, '256m', db_cpu)
|
||||
for concurrency in ((1, 4, 8, 32, 64) if label == 'baseline' else (8, 32, 64)):
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
name = 'saturation-%s-c%d-r%d' % (label, concurrency, repeat)
|
||||
if name in completed:
|
||||
continue
|
||||
report = monitored_benchmark(name, args.duration, args.warmup, concurrency)
|
||||
records.append({'series': 'saturation-' + label, 'replicas': 4,
|
||||
'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': db_cpu,
|
||||
'repeat': repeat, 'name': name, **report})
|
||||
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 'lb')
|
||||
configure(2, .5, '256m', 1.0)
|
||||
|
||||
|
||||
def pool_control(args, records, dest):
|
||||
# Alternate 4/1/4/1/4/1 instead of comparing two distant best runs.
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
for pool, series in ((4, 'pool-four'), (1, 'pool-one')):
|
||||
configure(4, .5, '256m', .25, pool)
|
||||
name = '%s-n4-r%d' % (series, repeat)
|
||||
report = monitored_benchmark(name, args.duration, args.warmup, 32)
|
||||
records[:] = [r for r in records if r['name'] != name]
|
||||
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
|
||||
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool,
|
||||
'repeat': repeat, 'name': name, **report})
|
||||
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def refine(args):
|
||||
"""Continue an existing study with lower concurrency and a pool-size control.
|
||||
|
||||
Used after the first sweep located its maximum at the smallest tested C=8.
|
||||
These cases are also included in a fresh full `scaling` run.
|
||||
"""
|
||||
dest = ROOT / 'evidence/scaling'
|
||||
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
|
||||
cases = [('saturation-baseline', c, 4) for c in (1, 4)]
|
||||
try:
|
||||
for series, concurrency, pool in cases:
|
||||
configure(4, .5, '256m', .25, pool)
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
name = ('saturation-baseline-c%d-r%d' % (concurrency, repeat)
|
||||
if series == 'saturation-baseline' else 'pool-one-n4-r%d' % repeat)
|
||||
report = monitored_benchmark(name, args.duration, args.warmup, concurrency)
|
||||
records = [r for r in records if r['name'] != name]
|
||||
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
|
||||
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool,
|
||||
'repeat': repeat, 'name': name, **report})
|
||||
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
||||
pool_control(args, records, dest)
|
||||
finally:
|
||||
configure(2, .5, '256m', 1.0)
|
||||
|
||||
|
||||
def lb_control(args, records, dest):
|
||||
configure(4, .5, '256m', .25)
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
for bounded, series in ((False, 'lb-unbounded'), (True, 'lb-bounded')):
|
||||
files = ('-f', 'compose.yaml', '-f', 'infra/compose.lb-limit.yaml') if bounded else ()
|
||||
compose(*files, 'up', '-d', '--no-deps', 'lb')
|
||||
runner('python', 'scripts/wait_ready.py')
|
||||
name = '%s-n4-r%d' % (series, repeat)
|
||||
report = monitored_benchmark(name, args.duration, args.warmup, 32)
|
||||
config = ROOT / 'infra' / ('haproxy-bounded.cfg' if bounded else 'haproxy.cfg')
|
||||
records[:] = [r for r in records if r['name'] != name]
|
||||
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
|
||||
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': 4,
|
||||
'lb_maxconn_per_api': 1 if bounded else 0,
|
||||
'lb_config_sha256': hashlib.sha256(config.read_bytes()).hexdigest(),
|
||||
'repeat': repeat, 'name': name, **report})
|
||||
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
||||
text = runner('make', 'test')
|
||||
(ROOT / 'evidence/bounded-smoke.txt').write_text(text, encoding='utf-8')
|
||||
|
||||
|
||||
def backpressure(args):
|
||||
dest = ROOT / 'evidence/scaling'
|
||||
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
|
||||
try:
|
||||
lb_control(args, records, dest)
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 'lb')
|
||||
configure(2, .5, '256m', 1.0)
|
||||
|
||||
|
||||
def rolling(args):
|
||||
configure(2, .5, '256m', 1.0)
|
||||
original = [c['Id'] for c in containers('api')]
|
||||
events = [{'event': 'before', 'containers': snapshot()}]
|
||||
|
||||
def replace():
|
||||
# Surge +1, then replace originals one at a time, retaining >=2 live APIs.
|
||||
compose('up', '-d', '--no-deps', '--scale', 'api=3', 'api')
|
||||
wait_instances(3)
|
||||
for i, old in enumerate(original):
|
||||
docker('stop', '--time', '8', old)
|
||||
docker('rm', old)
|
||||
compose('up', '-d', '--no-deps', '--scale', 'api=' + str(3 if i == 0 else 2), 'api')
|
||||
wait_instances(3 if i == 0 else 2)
|
||||
events.append({'event': 'replaced-' + old[:12], 'utc': datetime.now(timezone.utc).isoformat(),
|
||||
'containers': snapshot()})
|
||||
if set(original) & {c['Id'] for c in containers('api')}:
|
||||
raise RuntimeError('An old API survived replacement')
|
||||
|
||||
try:
|
||||
result = monitored_benchmark('rolling-update', 75, 5, 16, replace)
|
||||
events.append({'event': 'after', 'containers': snapshot()})
|
||||
(ROOT / 'evidence/rolling.json').write_text(json.dumps({'events': events, 'load': result,
|
||||
'raw_prefix': 'evidence/scaling/rolling-update'}, indent=2) + '\n', encoding='utf-8')
|
||||
finally:
|
||||
configure(2, .5, '256m', 1.0)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode', choices=['scaling', 'refine', 'backpressure', 'rolling'])
|
||||
parser.add_argument('--duration', type=int, default=20)
|
||||
parser.add_argument('--warmup', type=int, default=5)
|
||||
parser.add_argument('--concurrency', type=int, default=32)
|
||||
parser.add_argument('--repeats', type=int, default=3)
|
||||
parser.add_argument('--resume', action='store_true', help='Keep completed summary entries and run only missing cases')
|
||||
args = parser.parse_args()
|
||||
os.chdir(ROOT)
|
||||
(ROOT / 'evidence/scaling').mkdir(parents=True, exist_ok=True)
|
||||
if args.mode == 'scaling':
|
||||
scaling(args)
|
||||
elif args.mode == 'refine':
|
||||
refine(args)
|
||||
elif args.mode == 'backpressure':
|
||||
backpressure(args)
|
||||
else:
|
||||
rolling(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operate ONLY containers bearing this lab's Compose project/service labels."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
PROJECT = os.getenv('COMPOSE_PROJECT_NAME', 'mtusi-lab3-chernenko')
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], text=True, encoding='utf-8',
|
||||
errors='replace', stderr=subprocess.PIPE).strip()
|
||||
|
||||
|
||||
def containers(service, all_states=False):
|
||||
ids = docker('ps', '-aq' if all_states else '-q',
|
||||
'--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=' + service).split()
|
||||
return json.loads(docker('inspect', *ids)) if ids else []
|
||||
|
||||
|
||||
def snapshot():
|
||||
return [{ 'service': c['Config']['Labels']['com.docker.compose.service'],
|
||||
'id': c['Id'][:12], 'hostname': c['Config']['Hostname'],
|
||||
'running': c['State']['Running'], 'started_at': c['State']['StartedAt'],
|
||||
'finished_at': c['State']['FinishedAt'],
|
||||
'nano_cpus': c['HostConfig']['NanoCpus'],
|
||||
'memory_bytes': c['HostConfig']['Memory'],
|
||||
'published_ports': c['NetworkSettings']['Ports']}
|
||||
for service in ('api', 'db', 'lb') for c in containers(service, True)]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action', choices=['stop-one', 'restart', 'snapshot'])
|
||||
args = parser.parse_args()
|
||||
if args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(), indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
target = os.environ.get('TARGET_INSTANCE_ID', '')
|
||||
matches = [c for c in containers('api') if target.startswith(c['Config']['Hostname'] + '-')]
|
||||
if len(matches) != 1 or len(containers('api')) < 2:
|
||||
raise SystemExit('Expected exactly one matching target and >=2 running API containers')
|
||||
docker('stop', '--time', '6', matches[0]['Id'])
|
||||
print('Stopped API container', matches[0]['Id'][:12], '; left stopped.')
|
||||
else:
|
||||
# Complete stop/start, including PostgreSQL/session store; preserve volume.
|
||||
print('Before:', json.dumps(snapshot()))
|
||||
apis = [c['Id'] for c in containers('api', True)]
|
||||
lbs = [c['Id'] for c in containers('lb', True)]
|
||||
dbs = [c['Id'] for c in containers('db', True)]
|
||||
if not apis or not lbs or len(dbs) != 1:
|
||||
raise SystemExit('Incomplete lab stack')
|
||||
docker('stop', '--time', '8', *lbs, *apis, *dbs)
|
||||
docker('start', *dbs)
|
||||
docker('start', *apis)
|
||||
docker('start', *lbs)
|
||||
print('After:', json.dumps(snapshot()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stack_control.py stop-one
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print('PASS: LB readiness')
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise SystemExit('Readiness timeout')
|
||||
time.sleep(0.5)
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the numeric research appendix from completed, inspected measurements."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
path = ROOT / 'evidence/scaling'
|
||||
rows = json.loads((path / 'aggregate.json').read_text(encoding='utf-8'))
|
||||
raw = json.loads((path / 'summary.json').read_text(encoding='utf-8'))
|
||||
labels = {'fixed': 'Фиксированный бюджет API', 'growing': 'Рост бюджета API',
|
||||
'pool-one': 'DB .25 CPU, pool=1 на API',
|
||||
'pool-four': 'DB .25 CPU, pool=4 (парный контроль)',
|
||||
'lb-unbounded': 'LB без ограничения активных запросов',
|
||||
'lb-bounded': 'LB maxconn=1 на API',
|
||||
'db-improved': 'Увеличен CPU БД', 'saturation-baseline': 'Насыщение: DB .25 CPU',
|
||||
'saturation-improved': 'Насыщение: DB 1 CPU'}
|
||||
lines = ['# Исследование: цена масштабирования', '',
|
||||
'Черненко, БПИ2301. Лабораторная 3, трек 1. Реальные локальные измерения 6 ноября 2026 года.', '',
|
||||
'Методика, запуск и ограничения: [REPORT.md](REPORT.md) и [RUNBOOK.md](RUNBOOK.md).',
|
||||
'Числа ниже автоматически рассчитаны из [summary.json](evidence/scaling/summary.json)',
|
||||
'и [aggregate.json](evidence/scaling/aggregate.json), а не введены вручную.', '',
|
||||
'## Гипотеза и условия', '',
|
||||
'Добавление API при неизменном общем ресурсе не обязано ускорять запись.',
|
||||
'Общая PostgreSQL ограничивает масштабирование; увеличение её CPU-квоты',
|
||||
'проверяется как отдельное улучшение при тех же API, LB и запросах.', '',
|
||||
'Каждый прогон: 5 секунд прогрева, затем 20 секунд измерения. Три повтора.',
|
||||
'POST текстового сообщения, уникальный ключ, постоянные HTTP-соединения, без retry.',
|
||||
'Генератор: стандартный http.client, по одному независимому постоянному',
|
||||
'соединению на поток. Closed loop; bootstrap вне измерений; новый чат на прогон.',
|
||||
'Размер текста 18 ASCII code points. Перцентили только успешных запросов.',
|
||||
'RPS включает время завершения последних измеренных запросов.', '',
|
||||
'Fixed: суммарно API 1 CPU / 512 MiB, делится на 1/2/4 процесса.',
|
||||
'Growing: каждой API 0.5 CPU / 256 MiB. В обеих сериях DB .25 CPU / 512 MiB,',
|
||||
'LB .5 CPU / 128 MiB, пул по 4 соединения на API.',
|
||||
'Суммарный серверный бюджет fixed: 1.75 CPU / 1152 MiB, плюс неизменный',
|
||||
'генератор 2 CPU / 512 MiB. Число возможных DB-соединений растёт с N.',
|
||||
'В improved меняется только DB CPU .25→1; API 4×.5 CPU / 256 MiB. Увеличение CPU БД —',
|
||||
'улучшение распределения ресурсов, а не оптимизация SQL при том же бюджете.', '',
|
||||
'## Исходные измерения и агрегирование', '',
|
||||
f'Завершённых прогонов в основной серии: **{len(raw)}**. '
|
||||
'Для каждой строки показана медиана трёх run-level результатов; диапазон — min/max RPS.',
|
||||
'Ошибки суммируются по всем повторам, поэтому не скрываются медианой.', '',
|
||||
'| Серия | API | Конкуренция | Медиана RPS [min; max] | Ошибки / попытки | p50 / p95 / p99, ms |',
|
||||
'| --- | ---: | ---: | ---: | ---: | ---: |']
|
||||
for r in rows:
|
||||
latency = r['median_success_latency_ms']
|
||||
lines.append('| %s | %d | %d | %.3f [%.3f; %.3f] | %d / %d | %.3f / %.3f / %.3f |' % (
|
||||
labels[r['series']], r['replicas'], r['concurrency'], r['median_rps'], r['min_rps'], r['max_rps'],
|
||||
r['failed_requests_total'], r['attempts_total'], latency['p50'], latency['p95'], latency['p99']))
|
||||
lines += ['', '', '',
|
||||
'', '',
|
||||
'## Ресурсы', '',
|
||||
'Docker stats CPU: 100% = один CPU. Сэмплы охватывают bootstrap, прогрев',
|
||||
'и измерение; среднее не взвешено по времени. API — значение одной реплики,',
|
||||
'не сумма всех API. RAM — максимальный наблюдавшийся Docker MemUsage',
|
||||
'одного контейнера. Это наблюдения, а не замена квот из inspect.', '',
|
||||
'| Серия / N / C | CPU API mean/max, % | CPU DB mean/max, % | CPU LB mean/max, % | RAM API/DB/LB max, MiB |',
|
||||
'| --- | ---: | ---: | ---: | ---: |']
|
||||
for r in rows:
|
||||
def cpu(service):
|
||||
v = r['sampled_cpu_percent_one_core'].get(service)
|
||||
return 'нет сэмпла' if not v else '%.2f / %.2f' % (v['mean'], v['max'])
|
||||
ram = r['sampled_ram_mib_max_per_container']
|
||||
lines.append('| %s / %d / %d | %s | %s | %s | %s |' % (
|
||||
labels[r['series']], r['replicas'], r['concurrency'], cpu('api'), cpu('db'), cpu('lb'),
|
||||
' / '.join(str(ram.get(s, 'нет сэмпла')) for s in ('api', 'db', 'lb'))))
|
||||
base = next(r for r in rows if r['series'] == 'growing' and r['replicas'] == 4)
|
||||
improved = next(r for r in rows if r['series'] == 'db-improved')
|
||||
ratio = improved['median_rps'] / base['median_rps']
|
||||
lines += ['', '## Проверка улучшения', '',
|
||||
'При тех же четырёх API, конкуренции 32 и пуле 4 увеличение только CPU БД',
|
||||
'с .25 до 1 изменило медианный successful throughput с **%.3f** до **%.3f RPS**: '
|
||||
'**%.3f× (%+.1f%%)**.' % (base['median_rps'], improved['median_rps'], ratio, (ratio - 1) * 100),
|
||||
'p95 успешных запросов: **%.3f → %.3f ms**. Ошибки по трём повторам: **%d → %d**.' % (
|
||||
base['median_success_latency_ms']['p95'], improved['median_success_latency_ms']['p95'],
|
||||
base['failed_requests_total'], improved['failed_requests_total']), '',
|
||||
'Это контролируемое сравнение одного параметра. Его нельзя переносить',
|
||||
'на произвольное оборудование или считать доказательством линейной масштабируемости.', '']
|
||||
pool_one = next((r for r in rows if r['series'] == 'pool-one'), None)
|
||||
if pool_one:
|
||||
control = next(r for r in rows if r['series'] == 'pool-four')
|
||||
lines += ['### Контроль размера пула', '',
|
||||
'При DB .25 CPU и тех же 4 API, бюджете CPU/RAM и конкуренции 32',
|
||||
'изменён только максимум соединений на API: 4→1. Условия чередовались',
|
||||
'4/1/4/1/4/1, по три повтора, чтобы уменьшить влияние временного дрейфа.',
|
||||
'Медиана throughput: **%.3f → %.3f RPS**, отношение **%.3f×**.' % (
|
||||
control['median_rps'], pool_one['median_rps'], pool_one['median_rps'] / control['median_rps']),
|
||||
'p95: **%.3f → %.3f ms**; ошибок **%d → %d**.' % (
|
||||
control['median_success_latency_ms']['p95'], pool_one['median_success_latency_ms']['p95'],
|
||||
control['failed_requests_total'], pool_one['failed_requests_total']),
|
||||
'Это проверка настройки пула, а не изменение API-контракта или надёжности COMMIT.', '']
|
||||
paired = []
|
||||
for repeat in range(1, 4):
|
||||
a = next(r for r in raw if r['series'] == 'pool-four' and r['repeat'] == repeat)
|
||||
b = next(r for r in raw if r['series'] == 'pool-one' and r['repeat'] == repeat)
|
||||
paired.append('%.3f×' % (b['successful_rps'] / a['successful_rps']))
|
||||
lines += ['Отношение RPS pool=1/pool=4 в трёх последовательных парах: ' + ', '.join(paired) + '.', '']
|
||||
lines += ['', '']
|
||||
back = next(r for r in rows if r['series'] == 'saturation-baseline' and r['concurrency'] == 32)
|
||||
lines += ['### Обратная проверка гипотезы о CPU', '',
|
||||
'После возврата DB к .25 CPU в отдельной серии C=32 наблюдалось **%.3f RPS**,' % back['median_rps'],
|
||||
'тогда как в серии DB 1 CPU — **%.3f RPS**. Первоначальное увеличение CPU' % improved['median_rps'],
|
||||
'проверяется этим возвратом, а не только одной лучшей попыткой.', '']
|
||||
if back['median_rps'] >= .9 * improved['median_rps']:
|
||||
lines += ['Возврат к меньшему CPU почти не снижает throughput: гипотеза о CPU',
|
||||
'как единственной причине не подтверждена устойчиво. Нужен другой',
|
||||
'контролируемый опыт; одна разница медиан не доказывает улучшение.', '']
|
||||
else:
|
||||
lines += ['После возврата меньшей CPU-квоты throughput снова снизился.',
|
||||
'Вместе с контролем остальных ресурсов это поддерживает влияние',
|
||||
'общей БД на наблюдаемую ёмкость. Это вывод для данного профиля,',
|
||||
'не утверждение, что DB CPU — единственное ограничение при любых настройках.', '']
|
||||
lines += ['### Наблюдение за общей зависимостью', '',
|
||||
'| Настройка БД | CPU time на вставленное сообщение, µs | Throttled periods / все periods |',
|
||||
'| --- | ---: | ---: |']
|
||||
for series, label in [('growing', '0.25 CPU, 4 API'), ('db-improved', '1 CPU, 4 API')]:
|
||||
usage, messages, periods, throttled = 0, 0, 0, 0
|
||||
for run in raw:
|
||||
if run['series'] != series or run['replicas'] != 4:
|
||||
continue
|
||||
resource = json.loads((path / (run['name'] + '.resources.json')).read_text(encoding='utf-8'))
|
||||
a, b = resource['dataset_before'], resource['dataset_after']
|
||||
if 'db_cgroup_cpu_stat' not in a:
|
||||
continue
|
||||
usage += b['db_cgroup_cpu_stat']['usage_usec'] - a['db_cgroup_cpu_stat']['usage_usec']
|
||||
periods += b['db_cgroup_cpu_stat']['nr_periods'] - a['db_cgroup_cpu_stat']['nr_periods']
|
||||
throttled += b['db_cgroup_cpu_stat']['nr_throttled'] - a['db_cgroup_cpu_stat']['nr_throttled']
|
||||
messages += b['messages'] - a['messages']
|
||||
if messages and periods:
|
||||
lines.append('| %s | %.3f | %d / %d (%.2f%%) |' % (
|
||||
label, usage / messages, throttled, periods, 100 * throttled / periods))
|
||||
lines += ['', 'Счётчики cgroup охватывают bootstrap/прогрев/измерение и служебные проверки;',
|
||||
'это диагностическое приближение, не чистая стоимость одной SQL-операции.', '']
|
||||
for series in ('saturation-baseline', 'saturation-improved'):
|
||||
sweep = sorted((r for r in rows if r['series'] == series), key=lambda r: r['concurrency'])
|
||||
maximum = max(r['median_rps'] for r in sweep)
|
||||
knee = next(r for r in sweep if r['median_rps'] >= .9 * maximum)
|
||||
lines += ['### ' + labels[series], '',
|
||||
'Рабочее определение наблюдаемого насыщения: наименьшая проверенная',
|
||||
'конкуренция, достигшая 90% наибольшей медианы RPS в данной сетке.',
|
||||
'По этому критерию: **C=%d**, медиана **%.3f RPS**, максимум в сетке **%.3f RPS**.' % (
|
||||
knee['concurrency'], knee['median_rps'], maximum),
|
||||
'Это оценка по дискретной сетке, а не точная физическая граница.', '']
|
||||
before = min(json.loads((path / (r['name'] + '.resources.json')).read_text(encoding='utf-8'))['dataset_before']['messages'] for r in raw)
|
||||
after = max(json.loads((path / (r['name'] + '.resources.json')).read_text(encoding='utf-8'))['dataset_after']['messages'] for r in raw)
|
||||
lines += ['## Данные, воспроизводимость и ограничения', '',
|
||||
f'Общий набор сообщений БД за серию вырос примерно с {before} до {after}.',
|
||||
'Точные users/chats/messages/sessions до и после каждого прогона — в',
|
||||
'`*.resources.json`; данные прогрева остаются в БД. Начальная история каждого',
|
||||
'нового чата пустая. Порядок серий последовательный, без рандомизации;',
|
||||
'рост индексов и прогрев cache могут влиять на сравнение.', '',
|
||||
'Серия возобновлялась после остановки Docker: 11 завершённых прогонов',
|
||||
'сохранены, недостающие выполнены с --resume. Перерыв между прогонами',
|
||||
'мог изменить состояние cache/температуру host; внутри измеренного прогона',
|
||||
'перезапуск Docker не выполнялся. Это дополнительное ограничение сравнения.', '',
|
||||
'Raw `*.jsonl` хранит старт запроса, latency, status и alias. Credentials,',
|
||||
'Bearer и body не записываются. `*.json` содержит итог отдельного прогона,',
|
||||
'`*.resources.json` — реальные квоты, container ID, CPU/RAM и объём данных.',
|
||||
'sampling_error означает пропущенный telemetry sample при исчезновении',
|
||||
'одноразового/заменяемого контейнера; это не нулевое потребление ресурса.', '',
|
||||
'CPU/RAM клиента также записаны в raw resources; клиент имеет лимит 2 CPU /',
|
||||
'512 MiB и может стать ограничением. Closed loop имеет coordinated omission.',
|
||||
'Неизолированный host, короткие прогоны и три повтора не дают доверительного',
|
||||
'интервала для production SLA. Ошибки и timeouts не входят в успешные',
|
||||
'latency-перцентили; successful RPS — подтверждения HTTP, не точный счётчик COMMIT.', '',
|
||||
'Увеличение реплик не устраняет единственные DB, LB, диск и Docker-host как SPOF.',
|
||||
'Исследование касается отказа API-процесса и локального масштабирования,',
|
||||
'а не отказоустойчивости к потере узла.', '',
|
||||
'## Почему предварительные опыты не смешаны с основными', '',
|
||||
'В предварительной серии использовался общий async HTTPX-клиент. Проверка',
|
||||
'другим генератором на том же стенде показала 270.793 вместо 61.529 RPS.',
|
||||
'Поэтому прежние низкие значения не приняты за saturation point сервера:',
|
||||
'вся основная серия полностью повторена текущим независимым генератором.',
|
||||
'Это диагностический контроль, не универсальное утверждение о HTTPX.',
|
||||
'Предварительные 57 опытов, включая отрицательные настройки пула и LB,',
|
||||
'сохранены отдельно: [архив и пояснения](evidence/exploratory-httpx/README.md).',
|
||||
'Причины внутри библиотеки не профилировались. Их нельзя приписывать БД.', '',
|
||||
'Перестроить числовое приложение и графики без повторной нагрузки:', '',
|
||||
'```sh',
|
||||
'docker compose run --rm -T tester python scripts/plot_results.py',
|
||||
'python3 scripts/write_research.py',
|
||||
'```', '']
|
||||
(ROOT / 'RESEARCH.md').write_text('\n'.join(lines), encoding='utf-8')
|
||||
print('Wrote RESEARCH.md from', len(raw), 'measured runs; DB CPU RPS ratio:', round(ratio, 3))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user