Files

304 lines
13 KiB
Python
Executable File

#!/usr/bin/env python3
"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs).
failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe ->
docker start the same container -> wait until it serves again -> settle. Restores before the next one.
action: baseline -> run --action script (e.g. rolling-update) under load -> observe.
Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body)
and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded.
At the end the whole chat history is read back: every confirmed message must be present exactly once.
Session tokens stay in memory and are never printed.
"""
import argparse
import json
import subprocess
import sys
import threading
import time
import uuid
from collections import Counter, defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, run # noqa: E402
from resilience import history # noqa: E402
PROJECT = 'messenger-lab3'
def docker(*args, timeout=60):
return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip()
def api_containers():
"""(name, short id) of running api containers of this Compose project, sorted by name."""
out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}')
return sorted(tuple(line.split()) for line in out.splitlines() if line.strip())
class Load:
def __init__(self, base_url, concurrency):
self.base_url = base_url
self.concurrency = concurrency
self.events = [] # (t, op, ok, instance, version, latency)
self.confirmed = {} # message id -> text
self.lock = threading.Lock()
self.stop = threading.Event()
self.t0 = None
def now(self):
return time.monotonic() - self.t0
def record(self, op, ok, response=None, latency=0.0):
headers = response['headers'] if response else {}
event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None,
headers.get('X-App-Version') if response else None, latency)
with self.lock:
self.events.append(event)
def worker(self, worker_id, alice, chat):
c = Client(self.base_url, timeout=5)
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20'
index = 0
key = text = None
while not self.stop.is_set():
if key is None:
key = uuid.uuid4().hex
text = 'series-%d-%d-%s' % (worker_id, index, key[:8])
start = time.monotonic()
try:
response = c.message(alice, chat, text, key=key)
self.record('write', True, response, time.monotonic() - start)
with self.lock:
self.confirmed[response['json']['id']] = text
key = None
index += 1
except Failure:
self.record('write', False, latency=time.monotonic() - start)
time.sleep(0.05)
continue # same key and text: the first attempt may have committed
start = time.monotonic()
try:
response = c.call('GET', path, actor=alice, schema='MessagePage')
self.record('read', True, response, time.monotonic() - start)
except Failure:
self.record('read', False, latency=time.monotonic() - start)
time.sleep(0.05)
def start(self, alice, chat):
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True)
for i in range(self.concurrency)]
for t in self.threads:
t.start()
def finish(self):
self.stop.set()
for t in self.threads:
t.join(timeout=10)
def window(self, start, end):
return [e for e in self.events if start <= e[0] < end]
def summarize(events, start, end):
ok = [e for e in events if e[2]]
duration = max(end - start, 1e-9)
latencies = sorted(e[5] for e in ok)
p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None
return {
'seconds': round(duration, 2),
'rps_ok': round(len(ok) / duration, 1),
'writes_ok': sum(1 for e in ok if e[1] == 'write'),
'reads_ok': sum(1 for e in ok if e[1] == 'read'),
'errors': sum(1 for e in events if not e[2]),
'p95_ms': p95,
'instances': dict(Counter(e[3] for e in ok)),
}
def timeline(load, end):
buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()})
for t, _, ok, instance, version, _ in load.events:
if t >= end:
continue
b = buckets[int(t)]
if ok:
b['ok'] += 1
b['instances'][instance] += 1
b['versions'][version] += 1
else:
b['err'] += 1
return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'],
'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])}
for s in range(int(end) + 1)]
def max_gap(events, start, end):
"""Longest interval without any successful response inside [start, end)."""
times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end]
return round(max(b - a for a, b in zip(times, times[1:])), 3)
def wait_serving(load, instance, since, limit=30):
deadline = time.monotonic() + limit
while time.monotonic() < deadline:
with load.lock:
hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None)
if hit is not None:
return hit
time.sleep(0.2)
return None
def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill):
time.sleep(baseline)
t_stop = load.now()
if kill:
docker('kill', cid)
else:
docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30)
t_stopped = load.now()
state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid)
time.sleep(observe)
t_start = load.now()
docker('start', cid)
rejoined = wait_serving(load, cid, t_start)
time.sleep(settle)
t_end = load.now()
with load.lock:
events = list(load.events)
base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop)
outage = [e for e in events if t_stop <= e[0] < t_start]
last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None)
first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None)
# Recovery: first full second after the container stopped whose throughput is >= 90% of baseline.
recovered = None
for s in range(int(t_stopped), int(t_start)):
n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1)
if n >= 0.9 * base['rps_ok']:
recovered = s + 1 - t_stopped
break
return {
'replica': name,
'instance_id': cid,
'state_after_stop': state,
'stop_command_seconds': round(t_stopped - t_stop, 2),
'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None,
'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None,
'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None,
'max_gap_without_success_s': max_gap(events, t_stop, t_start),
'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None,
'baseline': base,
'during_stop_and_outage': summarize(outage, t_stop, t_start),
'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end),
'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)},
}
def action_run(load, action, baseline, observe):
time.sleep(baseline)
t_action = load.now()
proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600)
t_done = load.now()
time.sleep(observe)
t_end = load.now()
with load.lock:
events = list(load.events)
during = [e for e in events if t_action <= e[0] < t_done]
return {
'action': Path(action).name,
'action_exit_code': proc.returncode,
'action_output': proc.stdout.strip().splitlines()[-30:],
'action_seconds': round(t_done - t_action, 2),
'max_gap_without_success_s': max_gap(events, t_action, t_done),
'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action),
'during_action': summarize(during, t_action, t_done),
'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end),
'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)),
'versions_during': dict(Counter(e[4] for e in during if e[2])),
'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)),
'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)},
}
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('--base-url', default=None)
p.add_argument('--mode', choices=['failover', 'action'], default='failover')
p.add_argument('--action', help='script to run under load in --mode action')
p.add_argument('--concurrency', type=int, default=4)
p.add_argument('--baseline', type=float, default=10)
p.add_argument('--observe', type=float, default=15)
p.add_argument('--settle', type=float, default=5)
p.add_argument('--stop-timeout', type=int, default=9)
p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop')
p.add_argument('--output', help='JSON result file (no tokens)')
args = p.parse_args()
check(args.mode != 'action' or args.action, '--action is required in action mode')
c = Client(args.base_url)
alice, bob, _, chat = c.bootstrap()
replicas = api_containers()
if args.mode == 'failover':
check(len(replicas) >= 2, 'need at least two running api containers')
load = Load(args.base_url, args.concurrency)
load.start(alice, chat)
runs = []
try:
if args.mode == 'failover':
for name, cid in replicas:
print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True)
runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle,
args.stop_timeout, args.kill))
r = runs[-1]
print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % (
r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'],
r['first_success_after_container_exit_s'], r['baseline']['rps_ok'],
r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'],
r['rejoin_after_docker_start_s']), flush=True)
else:
print('running %s under load...' % args.action, flush=True)
runs.append(action_run(load, args.action, args.baseline, args.observe))
finally:
load.finish()
end = load.now()
confirmed = dict(load.confirmed)
stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')]
counts = Counter(m['text'] for m in stored)
duplicates = sum(n - 1 for n in counts.values() if n > 1)
stored_ids = {m['id'] for m in stored}
missing = [mid for mid in confirmed if mid not in stored_ids]
integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored),
'duplicates': duplicates, 'missing_confirmed': len(missing)}
report = {
'mode': args.mode,
'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout,
'concurrency': args.concurrency,
'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest',
'replicas_before': [name for name, _ in replicas],
'runs': runs,
'total': summarize(load.events, 0, end),
'integrity': integrity,
'timeline_per_second': timeline(load, end),
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
out = Path(args.output)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(text + '\n', encoding='utf-8')
print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False))
check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity)
if args.mode == 'action':
check(runs[0]['action_exit_code'] == 0, 'action script failed')
print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed)))
if __name__ == '__main__':
run(main)