level 4, changed to HAProxy

This commit is contained in:
2026-11-07 18:18:43 +03:00
parent 124c83e450
commit eb4e00e412
33 changed files with 3932 additions and 231 deletions
+303
View File
@@ -0,0 +1,303 @@
#!/usr/bin/env python3
"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs).
failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe ->
docker start the same container -> wait until it serves again -> settle. Restores before the next one.
action: baseline -> run --action script (e.g. rolling-update) under load -> observe.
Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body)
and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded.
At the end the whole chat history is read back: every confirmed message must be present exactly once.
Session tokens stay in memory and are never printed.
"""
import argparse
import json
import subprocess
import sys
import threading
import time
import uuid
from collections import Counter, defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, run # noqa: E402
from resilience import history # noqa: E402
PROJECT = 'messenger-lab3'
def docker(*args, timeout=60):
return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip()
def api_containers():
"""(name, short id) of running api containers of this Compose project, sorted by name."""
out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}')
return sorted(tuple(line.split()) for line in out.splitlines() if line.strip())
class Load:
def __init__(self, base_url, concurrency):
self.base_url = base_url
self.concurrency = concurrency
self.events = [] # (t, op, ok, instance, version, latency)
self.confirmed = {} # message id -> text
self.lock = threading.Lock()
self.stop = threading.Event()
self.t0 = None
def now(self):
return time.monotonic() - self.t0
def record(self, op, ok, response=None, latency=0.0):
headers = response['headers'] if response else {}
event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None,
headers.get('X-App-Version') if response else None, latency)
with self.lock:
self.events.append(event)
def worker(self, worker_id, alice, chat):
c = Client(self.base_url, timeout=5)
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20'
index = 0
key = text = None
while not self.stop.is_set():
if key is None:
key = uuid.uuid4().hex
text = 'series-%d-%d-%s' % (worker_id, index, key[:8])
start = time.monotonic()
try:
response = c.message(alice, chat, text, key=key)
self.record('write', True, response, time.monotonic() - start)
with self.lock:
self.confirmed[response['json']['id']] = text
key = None
index += 1
except Failure:
self.record('write', False, latency=time.monotonic() - start)
time.sleep(0.05)
continue # same key and text: the first attempt may have committed
start = time.monotonic()
try:
response = c.call('GET', path, actor=alice, schema='MessagePage')
self.record('read', True, response, time.monotonic() - start)
except Failure:
self.record('read', False, latency=time.monotonic() - start)
time.sleep(0.05)
def start(self, alice, chat):
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True)
for i in range(self.concurrency)]
for t in self.threads:
t.start()
def finish(self):
self.stop.set()
for t in self.threads:
t.join(timeout=10)
def window(self, start, end):
return [e for e in self.events if start <= e[0] < end]
def summarize(events, start, end):
ok = [e for e in events if e[2]]
duration = max(end - start, 1e-9)
latencies = sorted(e[5] for e in ok)
p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None
return {
'seconds': round(duration, 2),
'rps_ok': round(len(ok) / duration, 1),
'writes_ok': sum(1 for e in ok if e[1] == 'write'),
'reads_ok': sum(1 for e in ok if e[1] == 'read'),
'errors': sum(1 for e in events if not e[2]),
'p95_ms': p95,
'instances': dict(Counter(e[3] for e in ok)),
}
def timeline(load, end):
buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()})
for t, _, ok, instance, version, _ in load.events:
if t >= end:
continue
b = buckets[int(t)]
if ok:
b['ok'] += 1
b['instances'][instance] += 1
b['versions'][version] += 1
else:
b['err'] += 1
return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'],
'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])}
for s in range(int(end) + 1)]
def max_gap(events, start, end):
"""Longest interval without any successful response inside [start, end)."""
times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end]
return round(max(b - a for a, b in zip(times, times[1:])), 3)
def wait_serving(load, instance, since, limit=30):
deadline = time.monotonic() + limit
while time.monotonic() < deadline:
with load.lock:
hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None)
if hit is not None:
return hit
time.sleep(0.2)
return None
def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill):
time.sleep(baseline)
t_stop = load.now()
if kill:
docker('kill', cid)
else:
docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30)
t_stopped = load.now()
state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid)
time.sleep(observe)
t_start = load.now()
docker('start', cid)
rejoined = wait_serving(load, cid, t_start)
time.sleep(settle)
t_end = load.now()
with load.lock:
events = list(load.events)
base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop)
outage = [e for e in events if t_stop <= e[0] < t_start]
last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None)
first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None)
# Recovery: first full second after the container stopped whose throughput is >= 90% of baseline.
recovered = None
for s in range(int(t_stopped), int(t_start)):
n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1)
if n >= 0.9 * base['rps_ok']:
recovered = s + 1 - t_stopped
break
return {
'replica': name,
'instance_id': cid,
'state_after_stop': state,
'stop_command_seconds': round(t_stopped - t_stop, 2),
'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None,
'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None,
'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None,
'max_gap_without_success_s': max_gap(events, t_stop, t_start),
'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None,
'baseline': base,
'during_stop_and_outage': summarize(outage, t_stop, t_start),
'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end),
'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)},
}
def action_run(load, action, baseline, observe):
time.sleep(baseline)
t_action = load.now()
proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600)
t_done = load.now()
time.sleep(observe)
t_end = load.now()
with load.lock:
events = list(load.events)
during = [e for e in events if t_action <= e[0] < t_done]
return {
'action': Path(action).name,
'action_exit_code': proc.returncode,
'action_output': proc.stdout.strip().splitlines()[-30:],
'action_seconds': round(t_done - t_action, 2),
'max_gap_without_success_s': max_gap(events, t_action, t_done),
'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action),
'during_action': summarize(during, t_action, t_done),
'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end),
'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)),
'versions_during': dict(Counter(e[4] for e in during if e[2])),
'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)),
'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)},
}
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('--base-url', default=None)
p.add_argument('--mode', choices=['failover', 'action'], default='failover')
p.add_argument('--action', help='script to run under load in --mode action')
p.add_argument('--concurrency', type=int, default=4)
p.add_argument('--baseline', type=float, default=10)
p.add_argument('--observe', type=float, default=15)
p.add_argument('--settle', type=float, default=5)
p.add_argument('--stop-timeout', type=int, default=9)
p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop')
p.add_argument('--output', help='JSON result file (no tokens)')
args = p.parse_args()
check(args.mode != 'action' or args.action, '--action is required in action mode')
c = Client(args.base_url)
alice, bob, _, chat = c.bootstrap()
replicas = api_containers()
if args.mode == 'failover':
check(len(replicas) >= 2, 'need at least two running api containers')
load = Load(args.base_url, args.concurrency)
load.start(alice, chat)
runs = []
try:
if args.mode == 'failover':
for name, cid in replicas:
print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True)
runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle,
args.stop_timeout, args.kill))
r = runs[-1]
print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % (
r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'],
r['first_success_after_container_exit_s'], r['baseline']['rps_ok'],
r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'],
r['rejoin_after_docker_start_s']), flush=True)
else:
print('running %s under load...' % args.action, flush=True)
runs.append(action_run(load, args.action, args.baseline, args.observe))
finally:
load.finish()
end = load.now()
confirmed = dict(load.confirmed)
stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')]
counts = Counter(m['text'] for m in stored)
duplicates = sum(n - 1 for n in counts.values() if n > 1)
stored_ids = {m['id'] for m in stored}
missing = [mid for mid in confirmed if mid not in stored_ids]
integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored),
'duplicates': duplicates, 'missing_confirmed': len(missing)}
report = {
'mode': args.mode,
'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout,
'concurrency': args.concurrency,
'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest',
'replicas_before': [name for name, _ in replicas],
'runs': runs,
'total': summarize(load.events, 0, end),
'integrity': integrity,
'timeline_per_second': timeline(load, end),
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
out = Path(args.output)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(text + '\n', encoding='utf-8')
print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False))
check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity)
if args.mode == 'action':
check(runs[0]['action_exit_code'] == 0, 'action script failed')
print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed)))
if __name__ == '__main__':
run(main)
+3
View File
@@ -0,0 +1,3 @@
@echo off
rem Windows entry point: runs lb-status.sh with Git Bash.
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0lb-status.sh"
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# HAProxy view of the api pool: slot, replica IP, state (2=UP, 0=DOWN), check result. Empty template slots are hidden.
docker compose --env-file .env exec -T lb sh -c \
"echo 'show servers state api' | nc 127.0.0.1 9999 | awk '\$1 !~ /^#/ && NF > 10 && \$5 != \"-\" {print \$4, \$5, (\$6 == 2 ? \"UP\" : \"DOWN\")}'"
+3
View File
@@ -0,0 +1,3 @@
@echo off
rem Windows entry point: runs rolling-update.sh with Git Bash. Set APP_VERSION first.
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0rolling-update.sh"
+47
View File
@@ -0,0 +1,47 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Replace API replicas one at a time with a new build while the LB keeps serving.
# APP_VERSION=v2 scripts/rolling-update.sh
# 1. Build messenger-lab3-app:$APP_VERSION.
# 2. Run migrations ONCE (expand-only, compatible with replicas still on the old build).
# 3. For each old replica: start one new replica (N+1), wait until Docker reports it healthy AND HAProxy
# marks it UP, then docker stop the old one (SIGTERM -> drain -> exit) and remove it.
# Capacity never drops below N. Afterwards put APP_VERSION=$APP_VERSION into .env, otherwise a later
# `docker compose up` recreates replicas from the version written there.
: "${APP_VERSION:?set APP_VERSION of the new build}"
export APP_VERSION
project="${COMPOSE_PROJECT_NAME:-messenger-lab3}"
compose() { docker compose --env-file .env "$@"; }
api_ids() {
docker ps -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.service=api"
}
lb_up() { sh scripts/lb-status.sh | grep -c ' UP$' || true; }
old=$(api_ids)
count=$(printf '%s\n' "$old" | grep -c . || true)
[ "$count" -ge 1 ] || { echo "no running api replicas" >&2; exit 1; }
echo "$(date +%T) rolling $count replica(s) to messenger-lab3-app:$APP_VERSION"
compose build --quiet api migrate
compose run --rm --no-deps -T migrate
echo "$(date +%T) migrations applied once"
for id in $old; do
before=$(lb_up)
compose up -d --no-deps --no-recreate --wait --scale "api=$((count + 1))" api >/dev/null 2>&1
new=$(api_ids | grep -v -F "$old" || true)
i=0
until [ "$(lb_up)" -gt "$before" ]; do
i=$((i + 1))
[ "$i" -le 60 ] || { echo "new replica did not become UP in HAProxy" >&2; exit 1; }
sleep 0.5
done
echo "$(date +%T) new replica(s) UP in LB: $(printf '%s ' $new); stopping old $id"
docker stop --time 9 "$id" >/dev/null
docker rm "$id" >/dev/null
echo "$(date +%T) old $id drained and removed"
done
compose ps api --format '{{.Name}} {{.Image}} {{.Status}}'
echo "$(date +%T) done; set APP_VERSION=$APP_VERSION in .env to keep this version on the next compose up"
+3 -2
View File
@@ -23,6 +23,7 @@ if [ "$count" -ne 1 ]; then
exit 1
fi
name=$(docker inspect --format '{{.Name}}' "$matches")
# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit.
docker stop --time 8 "$matches" >/dev/null
# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish.
# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit.
docker stop --time 9 "$matches" >/dev/null
echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)"