This commit is contained in:
2026-10-26 21:32:36 +03:00
parent 0ab3a341aa
commit 551139d628
125 changed files with 1516665 additions and 22 deletions
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Run from tests/ in lab3. Uses the unchanged skeleton client and resilience helpers.
Creates synthetic users/messages, stops only API containers, restores in finally.
Metrics are a small closed-loop workload, not a capacity benchmark.
"""
import csv
import json
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from client import Client, check
from resilience import history
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'results' / ('level4-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
def docker(*args):
p = subprocess.run(['docker', *args], cwd=ROOT, text=True,
capture_output=True, timeout=210)
check(p.returncode == 0, 'Docker command failed: ' + ' '.join(args))
return p.stdout.strip()
def state(cid):
return json.loads(docker('inspect', '--format', '{{json .State}}', cid))
def ready(cid):
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
s = state(cid)
if s.get('Running') and s.get('Health', {}).get('Status') == 'healthy':
return
time.sleep(1)
raise RuntimeError('API did not become healthy: ' + cid[:12])
def observe(c, actor, expected):
deadline = time.monotonic() + 15
seen = set()
while time.monotonic() < deadline:
try:
r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
seen.add(r['headers']['X-Instance-Id'])
if expected <= seen:
return
except Exception:
pass
time.sleep(.1)
raise RuntimeError('Both actual API hostnames must be observed through LB')
def main():
ids = docker('compose', 'ps', '--status', 'running', '-q', 'api').splitlines()
check(len(ids) == 2, 'Start exactly two healthy API replicas first')
names = {cid: docker('inspect', '--format', '{{.Config.Hostname}}', cid) for cid in ids}
for cid in ids:
ready(cid)
OUT.mkdir(parents=True)
summary = {'workload': '4 closed-loop workers, POST then GET, 50ms pause; HTTP timeout 2s',
'replicas': names, 'experiments': []}
c = Client(timeout=2)
alice, bob, _, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
observe(c, alice, set(names.values()))
# One key, eight requests released together; a fresh chat isolates this test.
barrier = threading.Barrier(8)
key = uuid.uuid4().hex
def replay(_):
local = Client(timeout=10)
barrier.wait(timeout=15)
r = local.message(alice, chat, 'parallel-' + key, key=key)
return {'instance': r['headers']['X-Instance-Id'], 'message': r['json']}
with ThreadPoolExecutor(max_workers=8) as pool:
replies = list(pool.map(replay, range(8)))
check(all(r['message'] == replies[0]['message'] for r in replies), 'Concurrent replies differ')
check(len({r['instance'] for r in replies}) == 2, 'Concurrency did not cover both replicas; rerun')
check(history(c, bob, chat) == [replies[0]['message']], 'Concurrent replay created duplicates')
summary['concurrency'] = {'pass': True, 'responses': replies}
print('PASS: parallel replay, two replicas, one message', flush=True)
for cid in ids:
observe(c, alice, set(names.values()))
rows, confirmed = [], {}
halt = threading.Event()
lock = threading.Lock()
start = time.monotonic()
def worker():
local = Client(timeout=2)
key = uuid.uuid4().hex
while not halt.is_set():
# Preserve the key/body after any uncertain POST result.
for method in ('POST', 'GET'):
began = time.monotonic()
row = {'method': method, 'start': began - start, 'ok': False, 'instance': '', 'error': ''}
try:
if method == 'POST':
r = local.message(alice, chat, 'load-' + key, key=key)
with lock:
confirmed[r['json']['id']] = r['json']
key = uuid.uuid4().hex
else:
r = local.call('GET', path, actor=alice, schema='MessagePage')
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
except Exception as exc:
row['error'] = type(exc).__name__
row['end'] = time.monotonic() - start
with lock:
rows.append(row)
halt.wait(.05)
pool = ThreadPoolExecutor(max_workers=4)
jobs = [pool.submit(worker) for _ in range(4)]
try:
time.sleep(5)
impact = time.monotonic() - start
docker('stop', '--timeout', '20', cid)
stopped = time.monotonic() - start
stopped_state = state(cid)
check(not stopped_state['Running'], 'Target is still running')
print('Stopped ' + names[cid] + '; measuring for 15 seconds', flush=True)
time.sleep(15)
finally:
halt.set()
pool.shutdown(wait=True)
# Restore even if HTTP validation or measurement fails.
docker('start', cid)
ready(cid)
for job in jobs:
job.result()
observe(c, alice, set(names.values()))
stored = history(c, bob, chat)
by_id = {m['id']: m for m in stored}
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed message lost')
texts = [m['text'] for m in stored]
check(len(texts) == len(set(texts)), 'Uncertain retry created duplicate')
end = max(r['end'] for r in rows)
def metrics(lo, hi):
selected = [r for r in rows if lo <= r['start'] < hi]
good = sum(r['ok'] for r in selected)
return {'seconds': round(hi-lo, 3), 'attempts': len(selected), 'errors': len(selected)-good,
'attempt_rps': round(len(selected)/(hi-lo), 2),
'success_rps': round(good/(hi-lo), 2)}
recovery = {}
for method in ('POST', 'GET'):
hits = [r['end']-impact for r in rows if r['method'] == method and r['ok']
and r['start'] >= stopped and r['instance'] != names[cid]]
recovery[method] = round(min(hits), 3) if hits else None
result = {'target': names[cid], 'exit_code': stopped_state['ExitCode'],
'stop_started_seconds': impact, 'stop_finished_seconds': stopped,
'baseline': metrics(0, impact), 'during_and_after_stop': metrics(impact, end),
'first_success_after_stop_seconds_from_action_start': recovery,
'confirmed_messages_preserved': True}
summary['experiments'].append(result)
with (OUT / (names[cid] + '.csv')).open('w', newline='') as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
(OUT / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
print(json.dumps(result, indent=2), flush=True)
check(all(v is not None and v <= 10 for v in recovery.values()), 'Read/write recovery exceeded 10s')
print('PASS: replica restored, session and data preserved', flush=True)
print('Results:', OUT)
print('This checks SIGTERM failover and concurrency, NOT rolling replacement.')
if __name__ == '__main__':
try:
main()
except Exception as exc:
print('FAIL:', str(exc))
raise SystemExit(1)
+224
View File
@@ -0,0 +1,224 @@
#!/usr/bin/env python3
"""Sequential replacement of TWO API containers with the SAME image under load.
Put beside level4_check.py, client.py and resilience.py in tests/.
Does not build images, run migrations, modify dependencies or delete volumes.
On failure it stops replacing containers and preserves the remaining replica.
"""
import csv
import hashlib
import json
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from client import Client, Failure, check
from resilience import history
from level4_check import docker, ready, observe
ROOT = Path(__file__).resolve().parents[1]
def inspect(cid):
# Only output safe fields; never persist container environment/secrets.
return {'id': cid,
'hostname': docker('inspect', '--format', '{{.Config.Hostname}}', cid),
'image': docker('inspect', '--format', '{{.Image}}', cid)}
def containers():
return docker('compose', 'ps', '-a', '-q', 'api').splitlines()
def main():
original = containers()
check(len(original) == 2, 'Expected exactly two API containers')
for cid in original:
ready(cid)
info = {cid: inspect(cid) for cid in original}
check(len({v['image'] for v in info.values()}) == 1, 'Both replicas must use the same image')
original_image = info[original[0]]['image']
# Verify the configured image tag still points at that same immutable image.
config = json.loads(docker('compose', 'config', '--format', 'json'))
api = config['services']['api']
image_ref = api.get('image')
check(bool(image_ref), 'API image must be explicitly configured')
check(int(api.get('deploy', {}).get('replicas', 2)) == 2, 'Configure two replicas')
check(docker('image', 'inspect', '--format', '{{.Id}}', image_ref) == original_image,
'Image tag changed. This test expects replacement with the existing image')
del config, api
output = ROOT / 'results' / ('rolling-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
output.mkdir(parents=True)
report = {'script_version': 3, 'scope': 'Sequential replacement with the same image; no schema migration or version upgrade',
'before': list(info.values()), 'steps': [], 'pass': False,
'workload': '4 closed-loop workers; POST then GET; 100ms pause; HTTP timeout 2s',
'migration_sha256': {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted((ROOT / 'java/src/main/resources/db/migration').glob('*.sql'))}}
c = Client(timeout=2)
alice, bob, _, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
observe(c, alice, {v['hostname'] for v in info.values()})
seed_key = uuid.uuid4().hex
seed_text = 'rolling-seed-' + seed_key
seed = c.message(alice, chat, seed_text, key=seed_key)['json']
confirmed = {seed['id']: seed}
rows = []
halt = threading.Event()
lock = threading.Lock()
beginning = time.monotonic()
def worker():
local = Client(timeout=2)
key = uuid.uuid4().hex
while not halt.is_set():
for method in ('POST', 'GET'):
row = {'method': method, 'start': time.monotonic()-beginning,
'ok': False, 'instance': '', 'error': ''}
try:
if method == 'POST':
r = local.message(alice, chat, 'rolling-' + key, key=key)
with lock:
confirmed[r['json']['id']] = r['json']
key = uuid.uuid4().hex
else:
r = local.call('GET', path, actor=alice, schema='MessagePage')
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
except Exception as exc:
row['error'] = str(exc) if isinstance(exc, AssertionError) else type(exc).__name__
row['end'] = time.monotonic()-beginning
with lock:
rows.append(row)
halt.wait(.1)
pool = ThreadPoolExecutor(max_workers=4)
jobs = [pool.submit(worker) for _ in range(4)]
try:
time.sleep(5)
for old in original:
current = containers()
check(len(current) == 2 and old in current, 'Unexpected API container set')
survivor = next(cid for cid in current if cid != old)
ready(survivor)
step = {'old': info[old], 'survivor': inspect(survivor),
'start': time.monotonic()-beginning}
report['steps'].append(step)
print('Replacing ' + info[old]['hostname'], flush=True)
docker('stop', '--timeout', '20', old)
check(not json.loads(docker('inspect', '--format', '{{json .State.Running}}', old)),
'Old container is still running')
docker('rm', old)
# --no-recreate preserves the survivor and creates only the missing replica.
# --no-deps prevents restarting migrate, DB, Redis or LB.
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build',
'--pull', 'never', '--scale', 'api=2', 'api')
current = containers()
added = set(current)-set([old, survivor])
check(len(current) == 2 and survivor in current and len(added) == 1,
'Replacement affected an unexpected container')
new = added.pop()
step['new'] = inspect(new)
check(step['new']['image'] == original_image, 'Replacement image differs')
ready(new)
ready(survivor)
# Wait until the replacement actually receives traffic through LB.
observe(c, alice, {step['new']['hostname'], step['survivor']['hostname']})
step['ready_through_lb'] = time.monotonic()-beginning
# A control POST can time out just like a background POST.
# Retry the SAME key/body, record all attempts, and keep a hard deadline.
replay_started = time.monotonic()
replay_deadline = replay_started + 10
step['control_replay'] = {'attempts': 0, 'errors': [], 'pass': False}
while True:
remaining = replay_deadline - time.monotonic()
check(remaining > 0, 'Control replay did not recover within 10s')
step['control_replay']['attempts'] += 1
try:
control = Client(timeout=min(2, remaining))
replay = control.message(alice, chat, seed_text, key=seed_key)
except Failure as exc:
detail = str(exc)
step['control_replay']['errors'].append(detail)
# Contract errors, authorization errors and conflicts are not retried.
if not ('connection/TLS failure' in detail
or any('received ' + str(code) in detail for code in (502, 503, 504))):
raise
time.sleep(min(.1, max(0, replay_deadline-time.monotonic())))
continue
check(replay['json'] == seed,
'Idempotency replay changed after replacement')
step['control_replay']['pass'] = True
step['control_replay']['elapsed_seconds'] = round(time.monotonic()-replay_started, 3)
break
# Readiness is not proof of timely application traffic under load.
# Keep the original 2s client timeout; retain every timeout/error.
deadline = time.monotonic() + 30
while True:
with lock:
counts = {method: sum(1 for r in rows
if r['ok'] and r['method'] == method
and r['instance'] == step['new']['hostname']
and r['start'] >= step['ready_through_lb'])
for method in ('POST', 'GET')}
if (min(counts.values()) >= 3
and time.monotonic()-beginning >= step['ready_through_lb']+5):
break
if time.monotonic() >= deadline:
break
time.sleep(.1)
step['new_replica_successes_after_ready'] = counts
step['end'] = time.monotonic()-beginning
with lock:
selected = [r.copy() for r in rows if step['start'] <= r['start']
and r['end'] <= step['end']]
duration = step['end']-step['start']
good = [r for r in selected if r['ok']]
step['completed_requests'] = len(selected)
step['errors'] = len(selected)-len(good)
step['success_rps'] = round(len(good)/duration, 2)
step['replacement_seconds'] = round(step['ready_through_lb']-step['start'], 3)
for method in ('POST', 'GET'):
check(any(r['method'] == method and r['instance'] == step['survivor']['hostname']
and r['end'] < step['ready_through_lb'] for r in good),
'No successful ' + method + ' on survivor during replacement')
check(any(r['method'] == method and r['instance'] == step['new']['hostname']
for r in good), 'New replica did not serve ' + method)
check(min(counts.values()) >= 3,
'New replica did not complete 3 POST and 3 GET within 30s after readiness')
print(json.dumps(step, indent=2), flush=True)
halt.set()
pool.shutdown(wait=True)
for job in jobs:
job.result()
messages = history(c, bob, chat)
by_id = {m['id']: m for m in messages}
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed data lost')
texts = [m['text'] for m in messages]
check(len(texts) == len(set(texts)), 'Retry created duplicate messages')
report['confirmed_messages_preserved'] = len(confirmed)
report['after'] = [inspect(cid) for cid in containers()]
check(not set(original).intersection(v['id'] for v in report['after']),
'An original container remains')
report['pass'] = True
except Exception as exc:
report['error'] = str(exc)
raise
finally:
halt.set()
pool.shutdown(wait=True)
if rows:
with (output / 'requests.csv').open('w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(sorted(rows, key=lambda r: r['start']))
(output / 'summary.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
print('Results:', output, flush=True)
print('PASS: both containers replaced, survivor served reads/writes, session and data preserved')
if __name__ == '__main__':
try:
main()
except Exception as exc:
print('FAIL:', str(exc))
print('Stop the experiment. Inspect docker compose ps -a and logs before continuing.')
raise SystemExit(1)
+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""Lab3 scaling pilot. Put in tests/ beside client.py. Python standard library.
Open-loop arrivals; bounded client concurrency; overload drops are counted.
Creates synthetic users/chats/messages. Does not change Docker configuration.
"""
import argparse
import csv
import json
import math
import os
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from client import Client
ROOT = Path(__file__).resolve().parents[1]
PG_SQL = """SELECT json_build_object(
'connections',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database()),
'active',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND pid<>pg_backend_pid()),
'lock_waits',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock'),
'waits',(SELECT COALESCE(json_agg(t),'[]'::json) FROM
(SELECT wait_event_type,wait_event,count(*) AS n FROM pg_stat_activity
WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active'
GROUP BY wait_event_type,wait_event) t),
'db',(SELECT row_to_json(d) FROM
(SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_returned,tup_fetched,
tup_inserted,deadlocks,temp_bytes FROM pg_stat_database WHERE datname=current_database()) d));"""
def command(args, timeout=20):
p = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=timeout)
if p.returncode:
raise RuntimeError('Command failed: ' + ' '.join(args[:4]))
return p.stdout.strip()
def percentile(values, q):
if not values:
return None
values = sorted(values)
return round(values[max(0, math.ceil(len(values)*q)-1)], 2)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--label', required=True)
p.add_argument('--rates', default='10,25,50,100,200')
p.add_argument('--seconds', type=int, default=30)
p.add_argument('--warmup', type=int, default=15)
p.add_argument('--workers', type=int, default=128)
p.add_argument('--timeout', type=float, default=3)
args = p.parse_args()
rates = [float(x) for x in args.rates.split(',')]
if any(r <= 0 for r in rates) or args.seconds < 5 or args.workers < 1 or args.warmup < 0:
p.error('Positive rates/workers, duration >=5s, warmup >=0 required')
label = ''.join(x for x in args.label if x.isalnum() or x in '-_')
output = ROOT / 'results' / ('scale-' + label + '-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
output.mkdir(parents=True)
c = Client(timeout=args.timeout)
alice = c.user('scale')
metadata = {'args': vars(args), 'base_url': c.base,
'profile': '80% GET chat history limit=20, 20% POST unique message; one session; deterministic schedule',
'limits': [], 'summary': [], 'utc_started': datetime.now(timezone.utc).isoformat()}
ids = command(['docker','compose','ps','-q']).splitlines()
for cid in ids:
raw = json.loads(command(['docker','inspect',cid]))[0]
if raw['Config']['Labels'].get('com.docker.compose.service') == 'migrate':
continue
env = dict(x.split('=',1) for x in raw['Config'].get('Env',[]) if '=' in x)
metadata['limits'].append({'id':cid,'name':raw['Name'],
'service':raw['Config']['Labels'].get('com.docker.compose.service'),
'image':raw['Image'],'cpu':raw['HostConfig'].get('NanoCpus',0)/1e9,
'memory_bytes':raw['HostConfig'].get('Memory'),
'db_pool_size':env.get('DB_POOL_SIZE')})
del raw, env
metadata['docker_resources'] = json.loads(command(['docker','info','--format',
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}']))
api_names = {x['id'][:12] for x in metadata['limits'] if x['service']=='api'}
seen = set()
for _ in range(40):
seen.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
if not api_names <= seen:
raise RuntimeError('Not all running API containers were observed through LB')
metadata['observed_instances'] = sorted(seen)
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
stop_sample = threading.Event()
phase = {'name': 'setup', 'rate': 0}
origin = time.monotonic()
def sampler():
with (output/'resources.jsonl').open('w',encoding='utf-8') as f:
while not stop_sample.is_set():
record = {'seconds':time.monotonic()-origin, **phase.copy()}
try:
stats = command(['docker','stats','--no-stream','--format','{{json .}}',*ids],timeout=10)
record['docker'] = [json.loads(line) for line in stats.splitlines() if line.strip()]
except Exception as exc:
record['docker_error'] = type(exc).__name__
try:
# Variables are expanded in the container, never printed or persisted.
pg = command(['docker','compose','exec','-T','postgres','sh','-c',
'exec psql -X -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"',
'sh',PG_SQL],timeout=10)
record['postgres'] = json.loads(pg)
except Exception as exc:
record['postgres_error'] = type(exc).__name__
record['sample_end_seconds'] = time.monotonic()-origin
f.write(json.dumps(record)+'\n'); f.flush()
stop_sample.wait(2)
monitor = threading.Thread(target=sampler,daemon=True)
monitor.start()
local = threading.local()
lock = threading.Lock()
slots = threading.BoundedSemaphore(args.workers)
fields = ['stage','rate','index','method','scheduled','started','finished','latency_ms',
'schedule_lag_ms','status','instance','ok','error']
try:
with (output/'requests.csv').open('w',newline='',encoding='utf-8') as file:
writer = csv.DictWriter(file,fieldnames=fields);writer.writeheader()
with ThreadPoolExecutor(max_workers=args.workers) as pool:
for rate in rates:
# Each rate starts with a fresh chat and identical 20-message seed.
chat = c.call('POST','/api/v1/chats',201,actor=alice,
body={'title':'Scaling benchmark','member_ids':[alice['id']]},schema='Chat')['json']
for i in range(20):
c.message(alice,chat,'seed-'+str(i))
path = '/api/v1/chats/'+chat['id']+'/messages'
for stage, duration in [('warmup',args.warmup),('measure',args.seconds)]:
if not duration:continue
phase.update(name=stage,rate=rate)
rows = []
start = time.monotonic()
def save(row):
with lock:
rows.append(row);writer.writerow(row)
def request(index, due, stage=stage, rate=rate, path=path, rows=rows):
started = time.monotonic()
row = dict(stage=stage,rate=rate,index=index,method='POST' if index%5==0 else 'GET',
scheduled=due-origin,started=started-origin,finished='',latency_ms='',
schedule_lag_ms=round((started-due)*1000,3),status=0,instance='',ok=False,error='')
try:
if not hasattr(local,'client'):local.client=Client(timeout=args.timeout)
method=row['method']
# Benchmark does NOT retry: every original attempt stays visible.
key=uuid.uuid4().hex
r=local.client.call(method,path if method=='POST' else path+'?limit=20',
expected=[200,201,400,401,403,404,409,429,500,502,503,504],actor=alice,
body={'text':'scaling-'+key} if method=='POST' else None,
headers={'Idempotency-Key':key} if method=='POST' else None,
schema='Message' if method=='POST' else 'MessagePage')
row.update(status=r['status'],instance=r['headers'].get('X-Instance-Id',''),
ok=r['status']==(201 if method=='POST' else 200))
if not row['ok']:row['error']='HTTP '+str(r['status'])
except Exception as exc:
row['error']=str(exc) if isinstance(exc,AssertionError) else type(exc).__name__
finally:
finish=time.monotonic()
row.update(finished=finish-origin,latency_ms=round((finish-started)*1000,3))
save(row);slots.release()
futures=[]
for i in range(math.ceil(rate*duration)):
due=start+i/rate
delay=due-time.monotonic()
if delay>0:time.sleep(delay)
now=time.monotonic()
# Do not create catch-up bursts or silently queue unlimited work.
if now-due>.1 or not slots.acquire(blocking=False):
save(dict(stage=stage,rate=rate,index=i,method='POST' if i%5==0 else 'GET',
scheduled=due-origin,started='',finished=now-origin,latency_ms='',
schedule_lag_ms=round((now-due)*1000,3),status=0,instance='',ok=False,
error='generator_drop'))
else:
futures.append(pool.submit(request,i,due))
remainder=start+duration-time.monotonic()
if remainder>0:time.sleep(remainder)
for f in futures:f.result()
ended=time.monotonic()
file.flush()
if stage=='measure':
good=[r for r in rows if r['ok']]
latency=[r['latency_ms'] for r in good]
drops=sum(r['error']=='generator_drop' for r in rows)
summary={'target_rps':rate,'scheduled_seconds':duration,
'elapsed_with_drain_seconds':round(ended-start,3),'scheduled_requests':len(rows),
'sent':len(rows)-drops,'generator_drops':drops,'successes':len(good),
'errors':len(rows)-len(good)-drops,
'success_rps_with_drain':round(len(good)/(ended-start),3),
'sent_rps':round((len(rows)-drops)/duration,3),
'failure_pct_including_drops':round(100*(len(rows)-len(good))/len(rows),3),
'p50_ms_success':percentile(latency,.5),'p95_ms_success':percentile(latency,.95),
'p99_ms_success':percentile(latency,.99),
'p95_schedule_lag_ms':percentile([r['schedule_lag_ms'] for r in rows],.95),
'instances':sorted({r['instance'] for r in good})}
metadata['summary'].append(summary)
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
print(json.dumps(summary),flush=True)
phase.update(name='cooldown',rate=rate)
time.sleep(5)
finally:
stop_sample.set();monitor.join(timeout=25)
print('Results:',output,flush=True)
print('DONE: send summary.json and resources.jsonl for saturation analysis')
if __name__=='__main__':
try:main()
except Exception as exc:
print('FAIL:',str(exc));raise SystemExit(1)