Files
mtusi2026lab3/scripts/experiment.py
T

256 lines
9.9 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import json
import math
import os
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
def docker(*args):
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
def api_containers():
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
def describe(cid):
return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid)
def wait_healthy(cid, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy':
return
time.sleep(0.2)
raise SystemExit(cid + ' did not become healthy')
class Load:
def __init__(self, client, actor, chat, workers):
self.client, self.actor, self.chat = client, actor, chat
self.workers = workers
self.lock = threading.Lock()
self.events = []
self.confirmed = []
self.stop_flag = threading.Event()
self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)]
self.started = None
def worker(self, index):
local = Client(BASE_URL, timeout=2)
key = uuid.uuid4().hex
while not self.stop_flag.is_set():
try:
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
with self.lock:
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'], response['elapsed']))
self.confirmed.append(response['json'])
key = uuid.uuid4().hex
except Failure:
with self.lock:
self.events.append((time.monotonic(), False, None, None))
time.sleep(0.05)
def __enter__(self):
self.started = time.monotonic()
for thread in self.threads:
thread.start()
return self
def __exit__(self, *exc):
self.stop_flag.set()
for thread in self.threads:
thread.join()
def window(self, start, end):
with self.lock:
events = [e for e in self.events if start <= e[0] < end]
ok = sorted(e[0] for e in events if e[1])
latencies = sorted(e[3] for e in events if e[1])
points = [start, *ok, end]
def percentile(p):
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 1) if latencies else None
return {
'seconds': round(end - start, 3),
'successful_writes': len(ok),
'failed_attempts': len(events) - len(ok),
'error_ratio': round((len(events) - len(ok)) / len(events), 4) if events else None,
'successful_rps': round(len(ok) / (end - start), 1),
'latency_ms_successful': {'p50': percentile(0.5), 'p95': percentile(0.95), 'p99': percentile(0.99)},
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
'instances': sorted({e[2] for e in events if e[1]}),
}
def last_seen(self, instance, start, end):
with self.lock:
times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end]
return max(times) if times else None
def first_seen_after(self, instance, after):
with self.lock:
times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after]
return min(times) if times else None
def verify_history(client, actor, chat, confirmed):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
items = []
while True:
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
items.extend(page['items'])
if page['next_cursor'] is None:
break
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
texts = [m['text'] for m in items]
stored = {m['id']: m for m in items}
return {
'confirmed': len(confirmed),
'stored': len(items),
'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m),
'duplicates': len(texts) - len(set(texts)),
}
def resources():
ids = docker('compose', 'ps', '-q').split()
return docker('stats', '--no-stream', '--format', '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}', *ids).splitlines()
def messages_in_db():
return int(docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT count(*) FROM messages'))
def failover(load, args):
results = []
with load:
time.sleep(args.warmup)
usage = resources()
for cid in api_containers():
t0 = time.monotonic()
docker('kill' if args.kill else 'stop', cid)
stopped = time.monotonic()
time.sleep(args.outage)
outage = load.window(t0, time.monotonic())
restore = time.monotonic()
docker('start', cid)
wait_healthy(cid)
healthy = time.monotonic()
time.sleep(args.warmup)
rejoined = load.first_seen_after(cid, restore)
last = load.last_seen(cid, t0, restore)
results.append({
'container': describe(cid),
'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)',
'before': load.window(t0 - args.warmup, t0),
'stop_command_seconds': round(stopped - t0, 3),
'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None,
'outage_window': outage,
'restore_until_healthy_seconds': round(healthy - restore, 3),
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
'after_restore': load.window(healthy, time.monotonic()),
})
return {'scenario': 'sequential failover of each api replica', 'resources_under_load': usage, 'replicas': results}
def rolling(load, args):
old = api_containers()
before = [describe(cid) for cid in old]
steps = []
with load:
time.sleep(args.warmup)
usage = resources()
t_start = time.monotonic()
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
for cid in old:
t0 = time.monotonic()
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api')
added = time.monotonic()
docker('stop', cid)
docker('rm', cid)
steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)})
time.sleep(args.warmup)
total = load.window(t_start, time.monotonic())
return {
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
'resources_under_load': usage,
'before': before,
'after': [describe(cid) for cid in api_containers()],
'steps': steps,
'during_update': total,
}
def idempotency(client, actor, chat, args):
key = uuid.uuid4().hex
text = 'parallel ' + key
def send(_):
local = Client(BASE_URL, timeout=10)
response = local.message(actor, chat, text, key=key)
return response['json']['id'], response['headers']['X-Instance-Id']
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
results = list(pool.map(send, range(args.parallel)))
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text]
return {
'scenario': 'parallel POST with one Idempotency-Key through LB',
'requests': args.parallel,
'all_201': True,
'distinct_message_ids': len({r[0] for r in results}),
'instances': sorted({r[1] for r in results}),
'messages_in_history': len(stored),
}
def main():
parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB')
parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency'])
parser.add_argument('--workers', type=int, default=4)
parser.add_argument('--warmup', type=float, default=5)
parser.add_argument('--outage', type=float, default=10)
parser.add_argument('--parallel', type=int, default=20)
parser.add_argument('--kill', action='store_true')
parser.add_argument('--output')
args = parser.parse_args()
client = Client(BASE_URL)
alice, bob, _, chat = client.bootstrap()
scenarios = {'failover': failover, 'rolling': rolling}
if args.scenario in scenarios:
load = Load(client, alice, chat, args.workers)
dataset = messages_in_db()
report = scenarios[args.scenario](load, args)
report['dataset'] = {'messages_in_db_before': dataset, 'messages_in_db_after': messages_in_db()}
report['history'] = verify_history(client, bob, chat, load.confirmed)
report['load'] = {'workers': args.workers, 'model': 'closed loop, POST message', 'warmup_seconds': args.warmup, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
else:
report = idempotency(client, alice, chat, args)
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(text + '\n')
print(text)
if __name__ == '__main__':
main()