forked from mtusi2026/mtusi2026lab3
256 lines
9.9 KiB
Python
Executable File
256 lines
9.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure
|
|
|
|
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def api_containers():
|
|
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
|
|
|
|
|
|
def describe(cid):
|
|
return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid)
|
|
|
|
|
|
def wait_healthy(cid, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy':
|
|
return
|
|
time.sleep(0.2)
|
|
raise SystemExit(cid + ' did not become healthy')
|
|
|
|
|
|
class Load:
|
|
def __init__(self, client, actor, chat, workers):
|
|
self.client, self.actor, self.chat = client, actor, chat
|
|
self.workers = workers
|
|
self.lock = threading.Lock()
|
|
self.events = []
|
|
self.confirmed = []
|
|
self.stop_flag = threading.Event()
|
|
self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)]
|
|
self.started = None
|
|
|
|
def worker(self, index):
|
|
local = Client(BASE_URL, timeout=2)
|
|
key = uuid.uuid4().hex
|
|
while not self.stop_flag.is_set():
|
|
try:
|
|
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
|
|
with self.lock:
|
|
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'], response['elapsed']))
|
|
self.confirmed.append(response['json'])
|
|
key = uuid.uuid4().hex
|
|
except Failure:
|
|
with self.lock:
|
|
self.events.append((time.monotonic(), False, None, None))
|
|
time.sleep(0.05)
|
|
|
|
def __enter__(self):
|
|
self.started = time.monotonic()
|
|
for thread in self.threads:
|
|
thread.start()
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
self.stop_flag.set()
|
|
for thread in self.threads:
|
|
thread.join()
|
|
|
|
def window(self, start, end):
|
|
with self.lock:
|
|
events = [e for e in self.events if start <= e[0] < end]
|
|
ok = sorted(e[0] for e in events if e[1])
|
|
latencies = sorted(e[3] for e in events if e[1])
|
|
points = [start, *ok, end]
|
|
|
|
def percentile(p):
|
|
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 1) if latencies else None
|
|
|
|
return {
|
|
'seconds': round(end - start, 3),
|
|
'successful_writes': len(ok),
|
|
'failed_attempts': len(events) - len(ok),
|
|
'error_ratio': round((len(events) - len(ok)) / len(events), 4) if events else None,
|
|
'successful_rps': round(len(ok) / (end - start), 1),
|
|
'latency_ms_successful': {'p50': percentile(0.5), 'p95': percentile(0.95), 'p99': percentile(0.99)},
|
|
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
|
|
'instances': sorted({e[2] for e in events if e[1]}),
|
|
}
|
|
|
|
def last_seen(self, instance, start, end):
|
|
with self.lock:
|
|
times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end]
|
|
return max(times) if times else None
|
|
|
|
def first_seen_after(self, instance, after):
|
|
with self.lock:
|
|
times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after]
|
|
return min(times) if times else None
|
|
|
|
|
|
def verify_history(client, actor, chat, confirmed):
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
|
items = []
|
|
while True:
|
|
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
|
items.extend(page['items'])
|
|
if page['next_cursor'] is None:
|
|
break
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
|
|
texts = [m['text'] for m in items]
|
|
stored = {m['id']: m for m in items}
|
|
return {
|
|
'confirmed': len(confirmed),
|
|
'stored': len(items),
|
|
'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m),
|
|
'duplicates': len(texts) - len(set(texts)),
|
|
}
|
|
|
|
|
|
def resources():
|
|
ids = docker('compose', 'ps', '-q').split()
|
|
return docker('stats', '--no-stream', '--format', '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}', *ids).splitlines()
|
|
|
|
|
|
def messages_in_db():
|
|
return int(docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT count(*) FROM messages'))
|
|
|
|
|
|
def failover(load, args):
|
|
results = []
|
|
with load:
|
|
time.sleep(args.warmup)
|
|
usage = resources()
|
|
for cid in api_containers():
|
|
t0 = time.monotonic()
|
|
docker('kill' if args.kill else 'stop', cid)
|
|
stopped = time.monotonic()
|
|
time.sleep(args.outage)
|
|
outage = load.window(t0, time.monotonic())
|
|
restore = time.monotonic()
|
|
docker('start', cid)
|
|
wait_healthy(cid)
|
|
healthy = time.monotonic()
|
|
time.sleep(args.warmup)
|
|
rejoined = load.first_seen_after(cid, restore)
|
|
last = load.last_seen(cid, t0, restore)
|
|
results.append({
|
|
'container': describe(cid),
|
|
'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)',
|
|
'before': load.window(t0 - args.warmup, t0),
|
|
'stop_command_seconds': round(stopped - t0, 3),
|
|
'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None,
|
|
'outage_window': outage,
|
|
'restore_until_healthy_seconds': round(healthy - restore, 3),
|
|
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
|
|
'after_restore': load.window(healthy, time.monotonic()),
|
|
})
|
|
return {'scenario': 'sequential failover of each api replica', 'resources_under_load': usage, 'replicas': results}
|
|
|
|
|
|
def rolling(load, args):
|
|
old = api_containers()
|
|
before = [describe(cid) for cid in old]
|
|
steps = []
|
|
with load:
|
|
time.sleep(args.warmup)
|
|
usage = resources()
|
|
t_start = time.monotonic()
|
|
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
|
|
for cid in old:
|
|
t0 = time.monotonic()
|
|
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api')
|
|
added = time.monotonic()
|
|
docker('stop', cid)
|
|
docker('rm', cid)
|
|
steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)})
|
|
time.sleep(args.warmup)
|
|
total = load.window(t_start, time.monotonic())
|
|
return {
|
|
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
|
|
'resources_under_load': usage,
|
|
'before': before,
|
|
'after': [describe(cid) for cid in api_containers()],
|
|
'steps': steps,
|
|
'during_update': total,
|
|
}
|
|
|
|
|
|
def idempotency(client, actor, chat, args):
|
|
key = uuid.uuid4().hex
|
|
text = 'parallel ' + key
|
|
|
|
def send(_):
|
|
local = Client(BASE_URL, timeout=10)
|
|
response = local.message(actor, chat, text, key=key)
|
|
return response['json']['id'], response['headers']['X-Instance-Id']
|
|
|
|
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
|
|
results = list(pool.map(send, range(args.parallel)))
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
|
stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text]
|
|
return {
|
|
'scenario': 'parallel POST with one Idempotency-Key through LB',
|
|
'requests': args.parallel,
|
|
'all_201': True,
|
|
'distinct_message_ids': len({r[0] for r in results}),
|
|
'instances': sorted({r[1] for r in results}),
|
|
'messages_in_history': len(stored),
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB')
|
|
parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency'])
|
|
parser.add_argument('--workers', type=int, default=4)
|
|
parser.add_argument('--warmup', type=float, default=5)
|
|
parser.add_argument('--outage', type=float, default=10)
|
|
parser.add_argument('--parallel', type=int, default=20)
|
|
parser.add_argument('--kill', action='store_true')
|
|
parser.add_argument('--output')
|
|
args = parser.parse_args()
|
|
|
|
client = Client(BASE_URL)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
scenarios = {'failover': failover, 'rolling': rolling}
|
|
if args.scenario in scenarios:
|
|
load = Load(client, alice, chat, args.workers)
|
|
dataset = messages_in_db()
|
|
report = scenarios[args.scenario](load, args)
|
|
report['dataset'] = {'messages_in_db_before': dataset, 'messages_in_db_after': messages_in_db()}
|
|
report['history'] = verify_history(client, bob, chat, load.confirmed)
|
|
report['load'] = {'workers': args.workers, 'model': 'closed loop, POST message', 'warmup_seconds': args.warmup, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
|
|
else:
|
|
report = idempotency(client, alice, chat, args)
|
|
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.output).write_text(text + '\n')
|
|
print(text)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|