level 4, changed to HAProxy
This commit is contained in:
Executable
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs).
|
||||
|
||||
failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe ->
|
||||
docker start the same container -> wait until it serves again -> settle. Restores before the next one.
|
||||
action: baseline -> run --action script (e.g. rolling-update) under load -> observe.
|
||||
|
||||
Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body)
|
||||
and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded.
|
||||
At the end the whole chat history is read back: every confirmed message must be present exactly once.
|
||||
Session tokens stay in memory and are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
PROJECT = 'messenger-lab3'
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
"""(name, short id) of running api containers of this Compose project, sorted by name."""
|
||||
out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}')
|
||||
return sorted(tuple(line.split()) for line in out.splitlines() if line.strip())
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base_url, concurrency):
|
||||
self.base_url = base_url
|
||||
self.concurrency = concurrency
|
||||
self.events = [] # (t, op, ok, instance, version, latency)
|
||||
self.confirmed = {} # message id -> text
|
||||
self.lock = threading.Lock()
|
||||
self.stop = threading.Event()
|
||||
self.t0 = None
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def record(self, op, ok, response=None, latency=0.0):
|
||||
headers = response['headers'] if response else {}
|
||||
event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None,
|
||||
headers.get('X-App-Version') if response else None, latency)
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def worker(self, worker_id, alice, chat):
|
||||
c = Client(self.base_url, timeout=5)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20'
|
||||
index = 0
|
||||
key = text = None
|
||||
while not self.stop.is_set():
|
||||
if key is None:
|
||||
key = uuid.uuid4().hex
|
||||
text = 'series-%d-%d-%s' % (worker_id, index, key[:8])
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = c.message(alice, chat, text, key=key)
|
||||
self.record('write', True, response, time.monotonic() - start)
|
||||
with self.lock:
|
||||
self.confirmed[response['json']['id']] = text
|
||||
key = None
|
||||
index += 1
|
||||
except Failure:
|
||||
self.record('write', False, latency=time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
continue # same key and text: the first attempt may have committed
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = c.call('GET', path, actor=alice, schema='MessagePage')
|
||||
self.record('read', True, response, time.monotonic() - start)
|
||||
except Failure:
|
||||
self.record('read', False, latency=time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self, alice, chat):
|
||||
self.t0 = time.monotonic()
|
||||
self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True)
|
||||
for i in range(self.concurrency)]
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=10)
|
||||
|
||||
def window(self, start, end):
|
||||
return [e for e in self.events if start <= e[0] < end]
|
||||
|
||||
|
||||
def summarize(events, start, end):
|
||||
ok = [e for e in events if e[2]]
|
||||
duration = max(end - start, 1e-9)
|
||||
latencies = sorted(e[5] for e in ok)
|
||||
p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None
|
||||
return {
|
||||
'seconds': round(duration, 2),
|
||||
'rps_ok': round(len(ok) / duration, 1),
|
||||
'writes_ok': sum(1 for e in ok if e[1] == 'write'),
|
||||
'reads_ok': sum(1 for e in ok if e[1] == 'read'),
|
||||
'errors': sum(1 for e in events if not e[2]),
|
||||
'p95_ms': p95,
|
||||
'instances': dict(Counter(e[3] for e in ok)),
|
||||
}
|
||||
|
||||
|
||||
def timeline(load, end):
|
||||
buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()})
|
||||
for t, _, ok, instance, version, _ in load.events:
|
||||
if t >= end:
|
||||
continue
|
||||
b = buckets[int(t)]
|
||||
if ok:
|
||||
b['ok'] += 1
|
||||
b['instances'][instance] += 1
|
||||
b['versions'][version] += 1
|
||||
else:
|
||||
b['err'] += 1
|
||||
return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'],
|
||||
'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])}
|
||||
for s in range(int(end) + 1)]
|
||||
|
||||
|
||||
def max_gap(events, start, end):
|
||||
"""Longest interval without any successful response inside [start, end)."""
|
||||
times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end]
|
||||
return round(max(b - a for a, b in zip(times, times[1:])), 3)
|
||||
|
||||
|
||||
def wait_serving(load, instance, since, limit=30):
|
||||
deadline = time.monotonic() + limit
|
||||
while time.monotonic() < deadline:
|
||||
with load.lock:
|
||||
hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None)
|
||||
if hit is not None:
|
||||
return hit
|
||||
time.sleep(0.2)
|
||||
return None
|
||||
|
||||
|
||||
def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill):
|
||||
time.sleep(baseline)
|
||||
t_stop = load.now()
|
||||
if kill:
|
||||
docker('kill', cid)
|
||||
else:
|
||||
docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30)
|
||||
t_stopped = load.now()
|
||||
state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid)
|
||||
time.sleep(observe)
|
||||
t_start = load.now()
|
||||
docker('start', cid)
|
||||
rejoined = wait_serving(load, cid, t_start)
|
||||
time.sleep(settle)
|
||||
t_end = load.now()
|
||||
|
||||
with load.lock:
|
||||
events = list(load.events)
|
||||
base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop)
|
||||
outage = [e for e in events if t_stop <= e[0] < t_start]
|
||||
last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None)
|
||||
first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None)
|
||||
# Recovery: first full second after the container stopped whose throughput is >= 90% of baseline.
|
||||
recovered = None
|
||||
for s in range(int(t_stopped), int(t_start)):
|
||||
n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1)
|
||||
if n >= 0.9 * base['rps_ok']:
|
||||
recovered = s + 1 - t_stopped
|
||||
break
|
||||
return {
|
||||
'replica': name,
|
||||
'instance_id': cid,
|
||||
'state_after_stop': state,
|
||||
'stop_command_seconds': round(t_stopped - t_stop, 2),
|
||||
'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None,
|
||||
'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None,
|
||||
'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None,
|
||||
'max_gap_without_success_s': max_gap(events, t_stop, t_start),
|
||||
'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None,
|
||||
'baseline': base,
|
||||
'during_stop_and_outage': summarize(outage, t_stop, t_start),
|
||||
'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end),
|
||||
'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)},
|
||||
}
|
||||
|
||||
|
||||
def action_run(load, action, baseline, observe):
|
||||
time.sleep(baseline)
|
||||
t_action = load.now()
|
||||
proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600)
|
||||
t_done = load.now()
|
||||
time.sleep(observe)
|
||||
t_end = load.now()
|
||||
with load.lock:
|
||||
events = list(load.events)
|
||||
during = [e for e in events if t_action <= e[0] < t_done]
|
||||
return {
|
||||
'action': Path(action).name,
|
||||
'action_exit_code': proc.returncode,
|
||||
'action_output': proc.stdout.strip().splitlines()[-30:],
|
||||
'action_seconds': round(t_done - t_action, 2),
|
||||
'max_gap_without_success_s': max_gap(events, t_action, t_done),
|
||||
'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action),
|
||||
'during_action': summarize(during, t_action, t_done),
|
||||
'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end),
|
||||
'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)),
|
||||
'versions_during': dict(Counter(e[4] for e in during if e[2])),
|
||||
'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)),
|
||||
'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--mode', choices=['failover', 'action'], default='failover')
|
||||
p.add_argument('--action', help='script to run under load in --mode action')
|
||||
p.add_argument('--concurrency', type=int, default=4)
|
||||
p.add_argument('--baseline', type=float, default=10)
|
||||
p.add_argument('--observe', type=float, default=15)
|
||||
p.add_argument('--settle', type=float, default=5)
|
||||
p.add_argument('--stop-timeout', type=int, default=9)
|
||||
p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop')
|
||||
p.add_argument('--output', help='JSON result file (no tokens)')
|
||||
args = p.parse_args()
|
||||
check(args.mode != 'action' or args.action, '--action is required in action mode')
|
||||
|
||||
c = Client(args.base_url)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
replicas = api_containers()
|
||||
if args.mode == 'failover':
|
||||
check(len(replicas) >= 2, 'need at least two running api containers')
|
||||
load = Load(args.base_url, args.concurrency)
|
||||
load.start(alice, chat)
|
||||
runs = []
|
||||
try:
|
||||
if args.mode == 'failover':
|
||||
for name, cid in replicas:
|
||||
print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True)
|
||||
runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle,
|
||||
args.stop_timeout, args.kill))
|
||||
r = runs[-1]
|
||||
print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % (
|
||||
r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'],
|
||||
r['first_success_after_container_exit_s'], r['baseline']['rps_ok'],
|
||||
r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'],
|
||||
r['rejoin_after_docker_start_s']), flush=True)
|
||||
else:
|
||||
print('running %s under load...' % args.action, flush=True)
|
||||
runs.append(action_run(load, args.action, args.baseline, args.observe))
|
||||
finally:
|
||||
load.finish()
|
||||
end = load.now()
|
||||
|
||||
confirmed = dict(load.confirmed)
|
||||
stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')]
|
||||
counts = Counter(m['text'] for m in stored)
|
||||
duplicates = sum(n - 1 for n in counts.values() if n > 1)
|
||||
stored_ids = {m['id'] for m in stored}
|
||||
missing = [mid for mid in confirmed if mid not in stored_ids]
|
||||
integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored),
|
||||
'duplicates': duplicates, 'missing_confirmed': len(missing)}
|
||||
report = {
|
||||
'mode': args.mode,
|
||||
'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout,
|
||||
'concurrency': args.concurrency,
|
||||
'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest',
|
||||
'replicas_before': [name for name, _ in replicas],
|
||||
'runs': runs,
|
||||
'total': summarize(load.events, 0, end),
|
||||
'integrity': integrity,
|
||||
'timeline_per_second': timeline(load, end),
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
out = Path(args.output)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(text + '\n', encoding='utf-8')
|
||||
print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False))
|
||||
check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity)
|
||||
if args.mode == 'action':
|
||||
check(runs[0]['action_exit_code'] == 0, 'action script failed')
|
||||
print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point: runs lb-status.sh with Git Bash.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0lb-status.sh"
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# HAProxy view of the api pool: slot, replica IP, state (2=UP, 0=DOWN), check result. Empty template slots are hidden.
|
||||
docker compose --env-file .env exec -T lb sh -c \
|
||||
"echo 'show servers state api' | nc 127.0.0.1 9999 | awk '\$1 !~ /^#/ && NF > 10 && \$5 != \"-\" {print \$4, \$5, (\$6 == 2 ? \"UP\" : \"DOWN\")}'"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point: runs rolling-update.sh with Git Bash. Set APP_VERSION first.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0rolling-update.sh"
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Replace API replicas one at a time with a new build while the LB keeps serving.
|
||||
# APP_VERSION=v2 scripts/rolling-update.sh
|
||||
# 1. Build messenger-lab3-app:$APP_VERSION.
|
||||
# 2. Run migrations ONCE (expand-only, compatible with replicas still on the old build).
|
||||
# 3. For each old replica: start one new replica (N+1), wait until Docker reports it healthy AND HAProxy
|
||||
# marks it UP, then docker stop the old one (SIGTERM -> drain -> exit) and remove it.
|
||||
# Capacity never drops below N. Afterwards put APP_VERSION=$APP_VERSION into .env, otherwise a later
|
||||
# `docker compose up` recreates replicas from the version written there.
|
||||
: "${APP_VERSION:?set APP_VERSION of the new build}"
|
||||
export APP_VERSION
|
||||
project="${COMPOSE_PROJECT_NAME:-messenger-lab3}"
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
api_ids() {
|
||||
docker ps -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.service=api"
|
||||
}
|
||||
lb_up() { sh scripts/lb-status.sh | grep -c ' UP$' || true; }
|
||||
|
||||
old=$(api_ids)
|
||||
count=$(printf '%s\n' "$old" | grep -c . || true)
|
||||
[ "$count" -ge 1 ] || { echo "no running api replicas" >&2; exit 1; }
|
||||
echo "$(date +%T) rolling $count replica(s) to messenger-lab3-app:$APP_VERSION"
|
||||
|
||||
compose build --quiet api migrate
|
||||
compose run --rm --no-deps -T migrate
|
||||
echo "$(date +%T) migrations applied once"
|
||||
|
||||
for id in $old; do
|
||||
before=$(lb_up)
|
||||
compose up -d --no-deps --no-recreate --wait --scale "api=$((count + 1))" api >/dev/null 2>&1
|
||||
new=$(api_ids | grep -v -F "$old" || true)
|
||||
i=0
|
||||
until [ "$(lb_up)" -gt "$before" ]; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -le 60 ] || { echo "new replica did not become UP in HAProxy" >&2; exit 1; }
|
||||
sleep 0.5
|
||||
done
|
||||
echo "$(date +%T) new replica(s) UP in LB: $(printf '%s ' $new); stopping old $id"
|
||||
docker stop --time 9 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
echo "$(date +%T) old $id drained and removed"
|
||||
done
|
||||
|
||||
compose ps api --format '{{.Name}} {{.Image}} {{.Status}}'
|
||||
echo "$(date +%T) done; set APP_VERSION=$APP_VERSION in .env to keep this version on the next compose up"
|
||||
+3
-2
@@ -23,6 +23,7 @@ if [ "$count" -ne 1 ]; then
|
||||
exit 1
|
||||
fi
|
||||
name=$(docker inspect --format '{{.Name}}' "$matches")
|
||||
# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit.
|
||||
docker stop --time 8 "$matches" >/dev/null
|
||||
# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish.
|
||||
# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit.
|
||||
docker stop --time 9 "$matches" >/dev/null
|
||||
echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)"
|
||||
|
||||
Reference in New Issue
Block a user