304 lines
13 KiB
Python
Executable File
304 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs).
|
|
|
|
failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe ->
|
|
docker start the same container -> wait until it serves again -> settle. Restores before the next one.
|
|
action: baseline -> run --action script (e.g. rolling-update) under load -> observe.
|
|
|
|
Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body)
|
|
and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded.
|
|
At the end the whole chat history is read back: every confirmed message must be present exactly once.
|
|
Session tokens stay in memory and are never printed.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check, run # noqa: E402
|
|
from resilience import history # noqa: E402
|
|
|
|
PROJECT = 'messenger-lab3'
|
|
|
|
|
|
def docker(*args, timeout=60):
|
|
return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip()
|
|
|
|
|
|
def api_containers():
|
|
"""(name, short id) of running api containers of this Compose project, sorted by name."""
|
|
out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
|
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}')
|
|
return sorted(tuple(line.split()) for line in out.splitlines() if line.strip())
|
|
|
|
|
|
class Load:
|
|
def __init__(self, base_url, concurrency):
|
|
self.base_url = base_url
|
|
self.concurrency = concurrency
|
|
self.events = [] # (t, op, ok, instance, version, latency)
|
|
self.confirmed = {} # message id -> text
|
|
self.lock = threading.Lock()
|
|
self.stop = threading.Event()
|
|
self.t0 = None
|
|
|
|
def now(self):
|
|
return time.monotonic() - self.t0
|
|
|
|
def record(self, op, ok, response=None, latency=0.0):
|
|
headers = response['headers'] if response else {}
|
|
event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None,
|
|
headers.get('X-App-Version') if response else None, latency)
|
|
with self.lock:
|
|
self.events.append(event)
|
|
|
|
def worker(self, worker_id, alice, chat):
|
|
c = Client(self.base_url, timeout=5)
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20'
|
|
index = 0
|
|
key = text = None
|
|
while not self.stop.is_set():
|
|
if key is None:
|
|
key = uuid.uuid4().hex
|
|
text = 'series-%d-%d-%s' % (worker_id, index, key[:8])
|
|
start = time.monotonic()
|
|
try:
|
|
response = c.message(alice, chat, text, key=key)
|
|
self.record('write', True, response, time.monotonic() - start)
|
|
with self.lock:
|
|
self.confirmed[response['json']['id']] = text
|
|
key = None
|
|
index += 1
|
|
except Failure:
|
|
self.record('write', False, latency=time.monotonic() - start)
|
|
time.sleep(0.05)
|
|
continue # same key and text: the first attempt may have committed
|
|
start = time.monotonic()
|
|
try:
|
|
response = c.call('GET', path, actor=alice, schema='MessagePage')
|
|
self.record('read', True, response, time.monotonic() - start)
|
|
except Failure:
|
|
self.record('read', False, latency=time.monotonic() - start)
|
|
time.sleep(0.05)
|
|
|
|
def start(self, alice, chat):
|
|
self.t0 = time.monotonic()
|
|
self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True)
|
|
for i in range(self.concurrency)]
|
|
for t in self.threads:
|
|
t.start()
|
|
|
|
def finish(self):
|
|
self.stop.set()
|
|
for t in self.threads:
|
|
t.join(timeout=10)
|
|
|
|
def window(self, start, end):
|
|
return [e for e in self.events if start <= e[0] < end]
|
|
|
|
|
|
def summarize(events, start, end):
|
|
ok = [e for e in events if e[2]]
|
|
duration = max(end - start, 1e-9)
|
|
latencies = sorted(e[5] for e in ok)
|
|
p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None
|
|
return {
|
|
'seconds': round(duration, 2),
|
|
'rps_ok': round(len(ok) / duration, 1),
|
|
'writes_ok': sum(1 for e in ok if e[1] == 'write'),
|
|
'reads_ok': sum(1 for e in ok if e[1] == 'read'),
|
|
'errors': sum(1 for e in events if not e[2]),
|
|
'p95_ms': p95,
|
|
'instances': dict(Counter(e[3] for e in ok)),
|
|
}
|
|
|
|
|
|
def timeline(load, end):
|
|
buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()})
|
|
for t, _, ok, instance, version, _ in load.events:
|
|
if t >= end:
|
|
continue
|
|
b = buckets[int(t)]
|
|
if ok:
|
|
b['ok'] += 1
|
|
b['instances'][instance] += 1
|
|
b['versions'][version] += 1
|
|
else:
|
|
b['err'] += 1
|
|
return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'],
|
|
'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])}
|
|
for s in range(int(end) + 1)]
|
|
|
|
|
|
def max_gap(events, start, end):
|
|
"""Longest interval without any successful response inside [start, end)."""
|
|
times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end]
|
|
return round(max(b - a for a, b in zip(times, times[1:])), 3)
|
|
|
|
|
|
def wait_serving(load, instance, since, limit=30):
|
|
deadline = time.monotonic() + limit
|
|
while time.monotonic() < deadline:
|
|
with load.lock:
|
|
hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None)
|
|
if hit is not None:
|
|
return hit
|
|
time.sleep(0.2)
|
|
return None
|
|
|
|
|
|
def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill):
|
|
time.sleep(baseline)
|
|
t_stop = load.now()
|
|
if kill:
|
|
docker('kill', cid)
|
|
else:
|
|
docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30)
|
|
t_stopped = load.now()
|
|
state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid)
|
|
time.sleep(observe)
|
|
t_start = load.now()
|
|
docker('start', cid)
|
|
rejoined = wait_serving(load, cid, t_start)
|
|
time.sleep(settle)
|
|
t_end = load.now()
|
|
|
|
with load.lock:
|
|
events = list(load.events)
|
|
base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop)
|
|
outage = [e for e in events if t_stop <= e[0] < t_start]
|
|
last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None)
|
|
first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None)
|
|
# Recovery: first full second after the container stopped whose throughput is >= 90% of baseline.
|
|
recovered = None
|
|
for s in range(int(t_stopped), int(t_start)):
|
|
n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1)
|
|
if n >= 0.9 * base['rps_ok']:
|
|
recovered = s + 1 - t_stopped
|
|
break
|
|
return {
|
|
'replica': name,
|
|
'instance_id': cid,
|
|
'state_after_stop': state,
|
|
'stop_command_seconds': round(t_stopped - t_stop, 2),
|
|
'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None,
|
|
'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None,
|
|
'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None,
|
|
'max_gap_without_success_s': max_gap(events, t_stop, t_start),
|
|
'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None,
|
|
'baseline': base,
|
|
'during_stop_and_outage': summarize(outage, t_stop, t_start),
|
|
'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end),
|
|
'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)},
|
|
}
|
|
|
|
|
|
def action_run(load, action, baseline, observe):
|
|
time.sleep(baseline)
|
|
t_action = load.now()
|
|
proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600)
|
|
t_done = load.now()
|
|
time.sleep(observe)
|
|
t_end = load.now()
|
|
with load.lock:
|
|
events = list(load.events)
|
|
during = [e for e in events if t_action <= e[0] < t_done]
|
|
return {
|
|
'action': Path(action).name,
|
|
'action_exit_code': proc.returncode,
|
|
'action_output': proc.stdout.strip().splitlines()[-30:],
|
|
'action_seconds': round(t_done - t_action, 2),
|
|
'max_gap_without_success_s': max_gap(events, t_action, t_done),
|
|
'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action),
|
|
'during_action': summarize(during, t_action, t_done),
|
|
'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end),
|
|
'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)),
|
|
'versions_during': dict(Counter(e[4] for e in during if e[2])),
|
|
'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)),
|
|
'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)},
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument('--base-url', default=None)
|
|
p.add_argument('--mode', choices=['failover', 'action'], default='failover')
|
|
p.add_argument('--action', help='script to run under load in --mode action')
|
|
p.add_argument('--concurrency', type=int, default=4)
|
|
p.add_argument('--baseline', type=float, default=10)
|
|
p.add_argument('--observe', type=float, default=15)
|
|
p.add_argument('--settle', type=float, default=5)
|
|
p.add_argument('--stop-timeout', type=int, default=9)
|
|
p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop')
|
|
p.add_argument('--output', help='JSON result file (no tokens)')
|
|
args = p.parse_args()
|
|
check(args.mode != 'action' or args.action, '--action is required in action mode')
|
|
|
|
c = Client(args.base_url)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
replicas = api_containers()
|
|
if args.mode == 'failover':
|
|
check(len(replicas) >= 2, 'need at least two running api containers')
|
|
load = Load(args.base_url, args.concurrency)
|
|
load.start(alice, chat)
|
|
runs = []
|
|
try:
|
|
if args.mode == 'failover':
|
|
for name, cid in replicas:
|
|
print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True)
|
|
runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle,
|
|
args.stop_timeout, args.kill))
|
|
r = runs[-1]
|
|
print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % (
|
|
r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'],
|
|
r['first_success_after_container_exit_s'], r['baseline']['rps_ok'],
|
|
r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'],
|
|
r['rejoin_after_docker_start_s']), flush=True)
|
|
else:
|
|
print('running %s under load...' % args.action, flush=True)
|
|
runs.append(action_run(load, args.action, args.baseline, args.observe))
|
|
finally:
|
|
load.finish()
|
|
end = load.now()
|
|
|
|
confirmed = dict(load.confirmed)
|
|
stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')]
|
|
counts = Counter(m['text'] for m in stored)
|
|
duplicates = sum(n - 1 for n in counts.values() if n > 1)
|
|
stored_ids = {m['id'] for m in stored}
|
|
missing = [mid for mid in confirmed if mid not in stored_ids]
|
|
integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored),
|
|
'duplicates': duplicates, 'missing_confirmed': len(missing)}
|
|
report = {
|
|
'mode': args.mode,
|
|
'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout,
|
|
'concurrency': args.concurrency,
|
|
'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest',
|
|
'replicas_before': [name for name, _ in replicas],
|
|
'runs': runs,
|
|
'total': summarize(load.events, 0, end),
|
|
'integrity': integrity,
|
|
'timeline_per_second': timeline(load, end),
|
|
}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
out = Path(args.output)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(text + '\n', encoding='utf-8')
|
|
print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False))
|
|
check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity)
|
|
if args.mode == 'action':
|
|
check(runs[0]['action_exit_code'] == 0, 'action script failed')
|
|
print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|