forked from mtusi2026/mtusi2026lab3
lab3
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run from tests/ in lab3. Uses the unchanged skeleton client and resilience helpers.
|
||||
Creates synthetic users/messages, stops only API containers, restores in finally.
|
||||
Metrics are a small closed-loop workload, not a capacity benchmark.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from client import Client, check
|
||||
from resilience import history
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / 'results' / ('level4-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
|
||||
|
||||
|
||||
def docker(*args):
|
||||
p = subprocess.run(['docker', *args], cwd=ROOT, text=True,
|
||||
capture_output=True, timeout=210)
|
||||
check(p.returncode == 0, 'Docker command failed: ' + ' '.join(args))
|
||||
return p.stdout.strip()
|
||||
|
||||
|
||||
def state(cid):
|
||||
return json.loads(docker('inspect', '--format', '{{json .State}}', cid))
|
||||
|
||||
|
||||
def ready(cid):
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
s = state(cid)
|
||||
if s.get('Running') and s.get('Health', {}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(1)
|
||||
raise RuntimeError('API did not become healthy: ' + cid[:12])
|
||||
|
||||
|
||||
def observe(c, actor, expected):
|
||||
deadline = time.monotonic() + 15
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
seen.add(r['headers']['X-Instance-Id'])
|
||||
if expected <= seen:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.1)
|
||||
raise RuntimeError('Both actual API hostnames must be observed through LB')
|
||||
|
||||
|
||||
def main():
|
||||
ids = docker('compose', 'ps', '--status', 'running', '-q', 'api').splitlines()
|
||||
check(len(ids) == 2, 'Start exactly two healthy API replicas first')
|
||||
names = {cid: docker('inspect', '--format', '{{.Config.Hostname}}', cid) for cid in ids}
|
||||
for cid in ids:
|
||||
ready(cid)
|
||||
OUT.mkdir(parents=True)
|
||||
summary = {'workload': '4 closed-loop workers, POST then GET, 50ms pause; HTTP timeout 2s',
|
||||
'replicas': names, 'experiments': []}
|
||||
c = Client(timeout=2)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||||
observe(c, alice, set(names.values()))
|
||||
|
||||
# One key, eight requests released together; a fresh chat isolates this test.
|
||||
barrier = threading.Barrier(8)
|
||||
key = uuid.uuid4().hex
|
||||
def replay(_):
|
||||
local = Client(timeout=10)
|
||||
barrier.wait(timeout=15)
|
||||
r = local.message(alice, chat, 'parallel-' + key, key=key)
|
||||
return {'instance': r['headers']['X-Instance-Id'], 'message': r['json']}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
replies = list(pool.map(replay, range(8)))
|
||||
check(all(r['message'] == replies[0]['message'] for r in replies), 'Concurrent replies differ')
|
||||
check(len({r['instance'] for r in replies}) == 2, 'Concurrency did not cover both replicas; rerun')
|
||||
check(history(c, bob, chat) == [replies[0]['message']], 'Concurrent replay created duplicates')
|
||||
summary['concurrency'] = {'pass': True, 'responses': replies}
|
||||
print('PASS: parallel replay, two replicas, one message', flush=True)
|
||||
|
||||
for cid in ids:
|
||||
observe(c, alice, set(names.values()))
|
||||
rows, confirmed = [], {}
|
||||
halt = threading.Event()
|
||||
lock = threading.Lock()
|
||||
start = time.monotonic()
|
||||
def worker():
|
||||
local = Client(timeout=2)
|
||||
key = uuid.uuid4().hex
|
||||
while not halt.is_set():
|
||||
# Preserve the key/body after any uncertain POST result.
|
||||
for method in ('POST', 'GET'):
|
||||
began = time.monotonic()
|
||||
row = {'method': method, 'start': began - start, 'ok': False, 'instance': '', 'error': ''}
|
||||
try:
|
||||
if method == 'POST':
|
||||
r = local.message(alice, chat, 'load-' + key, key=key)
|
||||
with lock:
|
||||
confirmed[r['json']['id']] = r['json']
|
||||
key = uuid.uuid4().hex
|
||||
else:
|
||||
r = local.call('GET', path, actor=alice, schema='MessagePage')
|
||||
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
|
||||
except Exception as exc:
|
||||
row['error'] = type(exc).__name__
|
||||
row['end'] = time.monotonic() - start
|
||||
with lock:
|
||||
rows.append(row)
|
||||
halt.wait(.05)
|
||||
pool = ThreadPoolExecutor(max_workers=4)
|
||||
jobs = [pool.submit(worker) for _ in range(4)]
|
||||
try:
|
||||
time.sleep(5)
|
||||
impact = time.monotonic() - start
|
||||
docker('stop', '--timeout', '20', cid)
|
||||
stopped = time.monotonic() - start
|
||||
stopped_state = state(cid)
|
||||
check(not stopped_state['Running'], 'Target is still running')
|
||||
print('Stopped ' + names[cid] + '; measuring for 15 seconds', flush=True)
|
||||
time.sleep(15)
|
||||
finally:
|
||||
halt.set()
|
||||
pool.shutdown(wait=True)
|
||||
# Restore even if HTTP validation or measurement fails.
|
||||
docker('start', cid)
|
||||
ready(cid)
|
||||
for job in jobs:
|
||||
job.result()
|
||||
observe(c, alice, set(names.values()))
|
||||
stored = history(c, bob, chat)
|
||||
by_id = {m['id']: m for m in stored}
|
||||
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed message lost')
|
||||
texts = [m['text'] for m in stored]
|
||||
check(len(texts) == len(set(texts)), 'Uncertain retry created duplicate')
|
||||
end = max(r['end'] for r in rows)
|
||||
def metrics(lo, hi):
|
||||
selected = [r for r in rows if lo <= r['start'] < hi]
|
||||
good = sum(r['ok'] for r in selected)
|
||||
return {'seconds': round(hi-lo, 3), 'attempts': len(selected), 'errors': len(selected)-good,
|
||||
'attempt_rps': round(len(selected)/(hi-lo), 2),
|
||||
'success_rps': round(good/(hi-lo), 2)}
|
||||
recovery = {}
|
||||
for method in ('POST', 'GET'):
|
||||
hits = [r['end']-impact for r in rows if r['method'] == method and r['ok']
|
||||
and r['start'] >= stopped and r['instance'] != names[cid]]
|
||||
recovery[method] = round(min(hits), 3) if hits else None
|
||||
result = {'target': names[cid], 'exit_code': stopped_state['ExitCode'],
|
||||
'stop_started_seconds': impact, 'stop_finished_seconds': stopped,
|
||||
'baseline': metrics(0, impact), 'during_and_after_stop': metrics(impact, end),
|
||||
'first_success_after_stop_seconds_from_action_start': recovery,
|
||||
'confirmed_messages_preserved': True}
|
||||
summary['experiments'].append(result)
|
||||
with (OUT / (names[cid] + '.csv')).open('w', newline='') as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
(OUT / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
|
||||
print(json.dumps(result, indent=2), flush=True)
|
||||
check(all(v is not None and v <= 10 for v in recovery.values()), 'Read/write recovery exceeded 10s')
|
||||
print('PASS: replica restored, session and data preserved', flush=True)
|
||||
print('Results:', OUT)
|
||||
print('This checks SIGTERM failover and concurrency, NOT rolling replacement.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
print('FAIL:', str(exc))
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential replacement of TWO API containers with the SAME image under load.
|
||||
Put beside level4_check.py, client.py and resilience.py in tests/.
|
||||
Does not build images, run migrations, modify dependencies or delete volumes.
|
||||
On failure it stops replacing containers and preserves the remaining replica.
|
||||
"""
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
from level4_check import docker, ready, observe
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def inspect(cid):
|
||||
# Only output safe fields; never persist container environment/secrets.
|
||||
return {'id': cid,
|
||||
'hostname': docker('inspect', '--format', '{{.Config.Hostname}}', cid),
|
||||
'image': docker('inspect', '--format', '{{.Image}}', cid)}
|
||||
|
||||
|
||||
def containers():
|
||||
return docker('compose', 'ps', '-a', '-q', 'api').splitlines()
|
||||
|
||||
|
||||
def main():
|
||||
original = containers()
|
||||
check(len(original) == 2, 'Expected exactly two API containers')
|
||||
for cid in original:
|
||||
ready(cid)
|
||||
info = {cid: inspect(cid) for cid in original}
|
||||
check(len({v['image'] for v in info.values()}) == 1, 'Both replicas must use the same image')
|
||||
original_image = info[original[0]]['image']
|
||||
# Verify the configured image tag still points at that same immutable image.
|
||||
config = json.loads(docker('compose', 'config', '--format', 'json'))
|
||||
api = config['services']['api']
|
||||
image_ref = api.get('image')
|
||||
check(bool(image_ref), 'API image must be explicitly configured')
|
||||
check(int(api.get('deploy', {}).get('replicas', 2)) == 2, 'Configure two replicas')
|
||||
check(docker('image', 'inspect', '--format', '{{.Id}}', image_ref) == original_image,
|
||||
'Image tag changed. This test expects replacement with the existing image')
|
||||
del config, api
|
||||
output = ROOT / 'results' / ('rolling-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
|
||||
output.mkdir(parents=True)
|
||||
report = {'script_version': 3, 'scope': 'Sequential replacement with the same image; no schema migration or version upgrade',
|
||||
'before': list(info.values()), 'steps': [], 'pass': False,
|
||||
'workload': '4 closed-loop workers; POST then GET; 100ms pause; HTTP timeout 2s',
|
||||
'migration_sha256': {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in sorted((ROOT / 'java/src/main/resources/db/migration').glob('*.sql'))}}
|
||||
c = Client(timeout=2)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||||
observe(c, alice, {v['hostname'] for v in info.values()})
|
||||
seed_key = uuid.uuid4().hex
|
||||
seed_text = 'rolling-seed-' + seed_key
|
||||
seed = c.message(alice, chat, seed_text, key=seed_key)['json']
|
||||
confirmed = {seed['id']: seed}
|
||||
rows = []
|
||||
halt = threading.Event()
|
||||
lock = threading.Lock()
|
||||
beginning = time.monotonic()
|
||||
def worker():
|
||||
local = Client(timeout=2)
|
||||
key = uuid.uuid4().hex
|
||||
while not halt.is_set():
|
||||
for method in ('POST', 'GET'):
|
||||
row = {'method': method, 'start': time.monotonic()-beginning,
|
||||
'ok': False, 'instance': '', 'error': ''}
|
||||
try:
|
||||
if method == 'POST':
|
||||
r = local.message(alice, chat, 'rolling-' + key, key=key)
|
||||
with lock:
|
||||
confirmed[r['json']['id']] = r['json']
|
||||
key = uuid.uuid4().hex
|
||||
else:
|
||||
r = local.call('GET', path, actor=alice, schema='MessagePage')
|
||||
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
|
||||
except Exception as exc:
|
||||
row['error'] = str(exc) if isinstance(exc, AssertionError) else type(exc).__name__
|
||||
row['end'] = time.monotonic()-beginning
|
||||
with lock:
|
||||
rows.append(row)
|
||||
halt.wait(.1)
|
||||
pool = ThreadPoolExecutor(max_workers=4)
|
||||
jobs = [pool.submit(worker) for _ in range(4)]
|
||||
try:
|
||||
time.sleep(5)
|
||||
for old in original:
|
||||
current = containers()
|
||||
check(len(current) == 2 and old in current, 'Unexpected API container set')
|
||||
survivor = next(cid for cid in current if cid != old)
|
||||
ready(survivor)
|
||||
step = {'old': info[old], 'survivor': inspect(survivor),
|
||||
'start': time.monotonic()-beginning}
|
||||
report['steps'].append(step)
|
||||
print('Replacing ' + info[old]['hostname'], flush=True)
|
||||
docker('stop', '--timeout', '20', old)
|
||||
check(not json.loads(docker('inspect', '--format', '{{json .State.Running}}', old)),
|
||||
'Old container is still running')
|
||||
docker('rm', old)
|
||||
# --no-recreate preserves the survivor and creates only the missing replica.
|
||||
# --no-deps prevents restarting migrate, DB, Redis or LB.
|
||||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build',
|
||||
'--pull', 'never', '--scale', 'api=2', 'api')
|
||||
current = containers()
|
||||
added = set(current)-set([old, survivor])
|
||||
check(len(current) == 2 and survivor in current and len(added) == 1,
|
||||
'Replacement affected an unexpected container')
|
||||
new = added.pop()
|
||||
step['new'] = inspect(new)
|
||||
check(step['new']['image'] == original_image, 'Replacement image differs')
|
||||
ready(new)
|
||||
ready(survivor)
|
||||
# Wait until the replacement actually receives traffic through LB.
|
||||
observe(c, alice, {step['new']['hostname'], step['survivor']['hostname']})
|
||||
step['ready_through_lb'] = time.monotonic()-beginning
|
||||
# A control POST can time out just like a background POST.
|
||||
# Retry the SAME key/body, record all attempts, and keep a hard deadline.
|
||||
replay_started = time.monotonic()
|
||||
replay_deadline = replay_started + 10
|
||||
step['control_replay'] = {'attempts': 0, 'errors': [], 'pass': False}
|
||||
while True:
|
||||
remaining = replay_deadline - time.monotonic()
|
||||
check(remaining > 0, 'Control replay did not recover within 10s')
|
||||
step['control_replay']['attempts'] += 1
|
||||
try:
|
||||
control = Client(timeout=min(2, remaining))
|
||||
replay = control.message(alice, chat, seed_text, key=seed_key)
|
||||
except Failure as exc:
|
||||
detail = str(exc)
|
||||
step['control_replay']['errors'].append(detail)
|
||||
# Contract errors, authorization errors and conflicts are not retried.
|
||||
if not ('connection/TLS failure' in detail
|
||||
or any('received ' + str(code) in detail for code in (502, 503, 504))):
|
||||
raise
|
||||
time.sleep(min(.1, max(0, replay_deadline-time.monotonic())))
|
||||
continue
|
||||
check(replay['json'] == seed,
|
||||
'Idempotency replay changed after replacement')
|
||||
step['control_replay']['pass'] = True
|
||||
step['control_replay']['elapsed_seconds'] = round(time.monotonic()-replay_started, 3)
|
||||
break
|
||||
# Readiness is not proof of timely application traffic under load.
|
||||
# Keep the original 2s client timeout; retain every timeout/error.
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
with lock:
|
||||
counts = {method: sum(1 for r in rows
|
||||
if r['ok'] and r['method'] == method
|
||||
and r['instance'] == step['new']['hostname']
|
||||
and r['start'] >= step['ready_through_lb'])
|
||||
for method in ('POST', 'GET')}
|
||||
if (min(counts.values()) >= 3
|
||||
and time.monotonic()-beginning >= step['ready_through_lb']+5):
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(.1)
|
||||
step['new_replica_successes_after_ready'] = counts
|
||||
step['end'] = time.monotonic()-beginning
|
||||
with lock:
|
||||
selected = [r.copy() for r in rows if step['start'] <= r['start']
|
||||
and r['end'] <= step['end']]
|
||||
duration = step['end']-step['start']
|
||||
good = [r for r in selected if r['ok']]
|
||||
step['completed_requests'] = len(selected)
|
||||
step['errors'] = len(selected)-len(good)
|
||||
step['success_rps'] = round(len(good)/duration, 2)
|
||||
step['replacement_seconds'] = round(step['ready_through_lb']-step['start'], 3)
|
||||
for method in ('POST', 'GET'):
|
||||
check(any(r['method'] == method and r['instance'] == step['survivor']['hostname']
|
||||
and r['end'] < step['ready_through_lb'] for r in good),
|
||||
'No successful ' + method + ' on survivor during replacement')
|
||||
check(any(r['method'] == method and r['instance'] == step['new']['hostname']
|
||||
for r in good), 'New replica did not serve ' + method)
|
||||
check(min(counts.values()) >= 3,
|
||||
'New replica did not complete 3 POST and 3 GET within 30s after readiness')
|
||||
print(json.dumps(step, indent=2), flush=True)
|
||||
halt.set()
|
||||
pool.shutdown(wait=True)
|
||||
for job in jobs:
|
||||
job.result()
|
||||
messages = history(c, bob, chat)
|
||||
by_id = {m['id']: m for m in messages}
|
||||
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed data lost')
|
||||
texts = [m['text'] for m in messages]
|
||||
check(len(texts) == len(set(texts)), 'Retry created duplicate messages')
|
||||
report['confirmed_messages_preserved'] = len(confirmed)
|
||||
report['after'] = [inspect(cid) for cid in containers()]
|
||||
check(not set(original).intersection(v['id'] for v in report['after']),
|
||||
'An original container remains')
|
||||
report['pass'] = True
|
||||
except Exception as exc:
|
||||
report['error'] = str(exc)
|
||||
raise
|
||||
finally:
|
||||
halt.set()
|
||||
pool.shutdown(wait=True)
|
||||
if rows:
|
||||
with (output / 'requests.csv').open('w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(sorted(rows, key=lambda r: r['start']))
|
||||
(output / 'summary.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
|
||||
print('Results:', output, flush=True)
|
||||
print('PASS: both containers replaced, survivor served reads/writes, session and data preserved')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
print('FAIL:', str(exc))
|
||||
print('Stop the experiment. Inspect docker compose ps -a and logs before continuing.')
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab3 scaling pilot. Put in tests/ beside client.py. Python standard library.
|
||||
Open-loop arrivals; bounded client concurrency; overload drops are counted.
|
||||
Creates synthetic users/chats/messages. Does not change Docker configuration.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from client import Client
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PG_SQL = """SELECT json_build_object(
|
||||
'connections',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database()),
|
||||
'active',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND pid<>pg_backend_pid()),
|
||||
'lock_waits',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock'),
|
||||
'waits',(SELECT COALESCE(json_agg(t),'[]'::json) FROM
|
||||
(SELECT wait_event_type,wait_event,count(*) AS n FROM pg_stat_activity
|
||||
WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active'
|
||||
GROUP BY wait_event_type,wait_event) t),
|
||||
'db',(SELECT row_to_json(d) FROM
|
||||
(SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_returned,tup_fetched,
|
||||
tup_inserted,deadlocks,temp_bytes FROM pg_stat_database WHERE datname=current_database()) d));"""
|
||||
|
||||
|
||||
def command(args, timeout=20):
|
||||
p = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=timeout)
|
||||
if p.returncode:
|
||||
raise RuntimeError('Command failed: ' + ' '.join(args[:4]))
|
||||
return p.stdout.strip()
|
||||
|
||||
|
||||
def percentile(values, q):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(len(values)*q)-1)], 2)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--label', required=True)
|
||||
p.add_argument('--rates', default='10,25,50,100,200')
|
||||
p.add_argument('--seconds', type=int, default=30)
|
||||
p.add_argument('--warmup', type=int, default=15)
|
||||
p.add_argument('--workers', type=int, default=128)
|
||||
p.add_argument('--timeout', type=float, default=3)
|
||||
args = p.parse_args()
|
||||
rates = [float(x) for x in args.rates.split(',')]
|
||||
if any(r <= 0 for r in rates) or args.seconds < 5 or args.workers < 1 or args.warmup < 0:
|
||||
p.error('Positive rates/workers, duration >=5s, warmup >=0 required')
|
||||
label = ''.join(x for x in args.label if x.isalnum() or x in '-_')
|
||||
output = ROOT / 'results' / ('scale-' + label + '-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
|
||||
output.mkdir(parents=True)
|
||||
c = Client(timeout=args.timeout)
|
||||
alice = c.user('scale')
|
||||
metadata = {'args': vars(args), 'base_url': c.base,
|
||||
'profile': '80% GET chat history limit=20, 20% POST unique message; one session; deterministic schedule',
|
||||
'limits': [], 'summary': [], 'utc_started': datetime.now(timezone.utc).isoformat()}
|
||||
ids = command(['docker','compose','ps','-q']).splitlines()
|
||||
for cid in ids:
|
||||
raw = json.loads(command(['docker','inspect',cid]))[0]
|
||||
if raw['Config']['Labels'].get('com.docker.compose.service') == 'migrate':
|
||||
continue
|
||||
env = dict(x.split('=',1) for x in raw['Config'].get('Env',[]) if '=' in x)
|
||||
metadata['limits'].append({'id':cid,'name':raw['Name'],
|
||||
'service':raw['Config']['Labels'].get('com.docker.compose.service'),
|
||||
'image':raw['Image'],'cpu':raw['HostConfig'].get('NanoCpus',0)/1e9,
|
||||
'memory_bytes':raw['HostConfig'].get('Memory'),
|
||||
'db_pool_size':env.get('DB_POOL_SIZE')})
|
||||
del raw, env
|
||||
metadata['docker_resources'] = json.loads(command(['docker','info','--format',
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}']))
|
||||
api_names = {x['id'][:12] for x in metadata['limits'] if x['service']=='api'}
|
||||
seen = set()
|
||||
for _ in range(40):
|
||||
seen.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
|
||||
if not api_names <= seen:
|
||||
raise RuntimeError('Not all running API containers were observed through LB')
|
||||
metadata['observed_instances'] = sorted(seen)
|
||||
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
|
||||
|
||||
stop_sample = threading.Event()
|
||||
phase = {'name': 'setup', 'rate': 0}
|
||||
origin = time.monotonic()
|
||||
def sampler():
|
||||
with (output/'resources.jsonl').open('w',encoding='utf-8') as f:
|
||||
while not stop_sample.is_set():
|
||||
record = {'seconds':time.monotonic()-origin, **phase.copy()}
|
||||
try:
|
||||
stats = command(['docker','stats','--no-stream','--format','{{json .}}',*ids],timeout=10)
|
||||
record['docker'] = [json.loads(line) for line in stats.splitlines() if line.strip()]
|
||||
except Exception as exc:
|
||||
record['docker_error'] = type(exc).__name__
|
||||
try:
|
||||
# Variables are expanded in the container, never printed or persisted.
|
||||
pg = command(['docker','compose','exec','-T','postgres','sh','-c',
|
||||
'exec psql -X -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"',
|
||||
'sh',PG_SQL],timeout=10)
|
||||
record['postgres'] = json.loads(pg)
|
||||
except Exception as exc:
|
||||
record['postgres_error'] = type(exc).__name__
|
||||
record['sample_end_seconds'] = time.monotonic()-origin
|
||||
f.write(json.dumps(record)+'\n'); f.flush()
|
||||
stop_sample.wait(2)
|
||||
monitor = threading.Thread(target=sampler,daemon=True)
|
||||
monitor.start()
|
||||
local = threading.local()
|
||||
lock = threading.Lock()
|
||||
slots = threading.BoundedSemaphore(args.workers)
|
||||
fields = ['stage','rate','index','method','scheduled','started','finished','latency_ms',
|
||||
'schedule_lag_ms','status','instance','ok','error']
|
||||
try:
|
||||
with (output/'requests.csv').open('w',newline='',encoding='utf-8') as file:
|
||||
writer = csv.DictWriter(file,fieldnames=fields);writer.writeheader()
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
for rate in rates:
|
||||
# Each rate starts with a fresh chat and identical 20-message seed.
|
||||
chat = c.call('POST','/api/v1/chats',201,actor=alice,
|
||||
body={'title':'Scaling benchmark','member_ids':[alice['id']]},schema='Chat')['json']
|
||||
for i in range(20):
|
||||
c.message(alice,chat,'seed-'+str(i))
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
for stage, duration in [('warmup',args.warmup),('measure',args.seconds)]:
|
||||
if not duration:continue
|
||||
phase.update(name=stage,rate=rate)
|
||||
rows = []
|
||||
start = time.monotonic()
|
||||
def save(row):
|
||||
with lock:
|
||||
rows.append(row);writer.writerow(row)
|
||||
def request(index, due, stage=stage, rate=rate, path=path, rows=rows):
|
||||
started = time.monotonic()
|
||||
row = dict(stage=stage,rate=rate,index=index,method='POST' if index%5==0 else 'GET',
|
||||
scheduled=due-origin,started=started-origin,finished='',latency_ms='',
|
||||
schedule_lag_ms=round((started-due)*1000,3),status=0,instance='',ok=False,error='')
|
||||
try:
|
||||
if not hasattr(local,'client'):local.client=Client(timeout=args.timeout)
|
||||
method=row['method']
|
||||
# Benchmark does NOT retry: every original attempt stays visible.
|
||||
key=uuid.uuid4().hex
|
||||
r=local.client.call(method,path if method=='POST' else path+'?limit=20',
|
||||
expected=[200,201,400,401,403,404,409,429,500,502,503,504],actor=alice,
|
||||
body={'text':'scaling-'+key} if method=='POST' else None,
|
||||
headers={'Idempotency-Key':key} if method=='POST' else None,
|
||||
schema='Message' if method=='POST' else 'MessagePage')
|
||||
row.update(status=r['status'],instance=r['headers'].get('X-Instance-Id',''),
|
||||
ok=r['status']==(201 if method=='POST' else 200))
|
||||
if not row['ok']:row['error']='HTTP '+str(r['status'])
|
||||
except Exception as exc:
|
||||
row['error']=str(exc) if isinstance(exc,AssertionError) else type(exc).__name__
|
||||
finally:
|
||||
finish=time.monotonic()
|
||||
row.update(finished=finish-origin,latency_ms=round((finish-started)*1000,3))
|
||||
save(row);slots.release()
|
||||
futures=[]
|
||||
for i in range(math.ceil(rate*duration)):
|
||||
due=start+i/rate
|
||||
delay=due-time.monotonic()
|
||||
if delay>0:time.sleep(delay)
|
||||
now=time.monotonic()
|
||||
# Do not create catch-up bursts or silently queue unlimited work.
|
||||
if now-due>.1 or not slots.acquire(blocking=False):
|
||||
save(dict(stage=stage,rate=rate,index=i,method='POST' if i%5==0 else 'GET',
|
||||
scheduled=due-origin,started='',finished=now-origin,latency_ms='',
|
||||
schedule_lag_ms=round((now-due)*1000,3),status=0,instance='',ok=False,
|
||||
error='generator_drop'))
|
||||
else:
|
||||
futures.append(pool.submit(request,i,due))
|
||||
remainder=start+duration-time.monotonic()
|
||||
if remainder>0:time.sleep(remainder)
|
||||
for f in futures:f.result()
|
||||
ended=time.monotonic()
|
||||
file.flush()
|
||||
if stage=='measure':
|
||||
good=[r for r in rows if r['ok']]
|
||||
latency=[r['latency_ms'] for r in good]
|
||||
drops=sum(r['error']=='generator_drop' for r in rows)
|
||||
summary={'target_rps':rate,'scheduled_seconds':duration,
|
||||
'elapsed_with_drain_seconds':round(ended-start,3),'scheduled_requests':len(rows),
|
||||
'sent':len(rows)-drops,'generator_drops':drops,'successes':len(good),
|
||||
'errors':len(rows)-len(good)-drops,
|
||||
'success_rps_with_drain':round(len(good)/(ended-start),3),
|
||||
'sent_rps':round((len(rows)-drops)/duration,3),
|
||||
'failure_pct_including_drops':round(100*(len(rows)-len(good))/len(rows),3),
|
||||
'p50_ms_success':percentile(latency,.5),'p95_ms_success':percentile(latency,.95),
|
||||
'p99_ms_success':percentile(latency,.99),
|
||||
'p95_schedule_lag_ms':percentile([r['schedule_lag_ms'] for r in rows],.95),
|
||||
'instances':sorted({r['instance'] for r in good})}
|
||||
metadata['summary'].append(summary)
|
||||
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
|
||||
print(json.dumps(summary),flush=True)
|
||||
phase.update(name='cooldown',rate=rate)
|
||||
time.sleep(5)
|
||||
finally:
|
||||
stop_sample.set();monitor.join(timeout=25)
|
||||
print('Results:',output,flush=True)
|
||||
print('DONE: send summary.json and resources.jsonl for saturation analysis')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as exc:
|
||||
print('FAIL:',str(exc));raise SystemExit(1)
|
||||
Reference in New Issue
Block a user