feat: add resource limits, per-service secrets and failover/rolling experiments

This commit is contained in:
2026-11-01 15:13:01 +03:00
parent 53035d475f
commit 210ff67c01
3 changed files with 269 additions and 9 deletions
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
def docker(*args):
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
def api_containers():
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
def describe(cid):
return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid)
def wait_healthy(cid, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy':
return
time.sleep(0.2)
raise SystemExit(cid + ' did not become healthy')
class Load:
def __init__(self, client, actor, chat, workers):
self.client, self.actor, self.chat = client, actor, chat
self.workers = workers
self.lock = threading.Lock()
self.events = []
self.confirmed = []
self.stop_flag = threading.Event()
self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)]
self.started = None
def worker(self, index):
local = Client(BASE_URL, timeout=2)
key = uuid.uuid4().hex
while not self.stop_flag.is_set():
try:
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
with self.lock:
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
self.confirmed.append(response['json'])
key = uuid.uuid4().hex
except Failure:
with self.lock:
self.events.append((time.monotonic(), False, None))
time.sleep(0.05)
def __enter__(self):
self.started = time.monotonic()
for thread in self.threads:
thread.start()
return self
def __exit__(self, *exc):
self.stop_flag.set()
for thread in self.threads:
thread.join()
def window(self, start, end):
with self.lock:
events = [e for e in self.events if start <= e[0] < end]
ok = sorted(e[0] for e in events if e[1])
points = [start, *ok, end]
return {
'seconds': round(end - start, 3),
'successful_writes': len(ok),
'failed_attempts': sum(1 for e in events if not e[1]),
'successful_rps': round(len(ok) / (end - start), 1),
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
'instances': sorted({e[2] for e in events if e[1]}),
}
def last_seen(self, instance, start, end):
with self.lock:
times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end]
return max(times) if times else None
def first_seen_after(self, instance, after):
with self.lock:
times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after]
return min(times) if times else None
def verify_history(client, actor, chat, confirmed):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
items = []
while True:
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
items.extend(page['items'])
if page['next_cursor'] is None:
break
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
texts = [m['text'] for m in items]
stored = {m['id']: m for m in items}
return {
'confirmed': len(confirmed),
'stored': len(items),
'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m),
'duplicates': len(texts) - len(set(texts)),
}
def failover(load, args):
results = []
with load:
time.sleep(args.warmup)
for cid in api_containers():
t0 = time.monotonic()
docker('kill' if args.kill else 'stop', cid)
stopped = time.monotonic()
time.sleep(args.outage)
outage = load.window(t0, time.monotonic())
restore = time.monotonic()
docker('start', cid)
wait_healthy(cid)
healthy = time.monotonic()
time.sleep(args.warmup)
rejoined = load.first_seen_after(cid, restore)
last = load.last_seen(cid, t0, restore)
results.append({
'container': describe(cid),
'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)',
'before': load.window(t0 - args.warmup, t0),
'stop_command_seconds': round(stopped - t0, 3),
'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None,
'outage_window': outage,
'restore_until_healthy_seconds': round(healthy - restore, 3),
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
'after_restore': load.window(healthy, time.monotonic()),
})
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
def rolling(load, args):
old = api_containers()
before = [describe(cid) for cid in old]
steps = []
with load:
time.sleep(args.warmup)
t_start = time.monotonic()
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
for cid in old:
t0 = time.monotonic()
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api')
added = time.monotonic()
docker('stop', cid)
docker('rm', cid)
steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)})
time.sleep(args.warmup)
total = load.window(t_start, time.monotonic())
return {
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
'before': before,
'after': [describe(cid) for cid in api_containers()],
'steps': steps,
'during_update': total,
}
def idempotency(client, actor, chat, args):
key = uuid.uuid4().hex
text = 'parallel ' + key
def send(_):
local = Client(BASE_URL, timeout=10)
response = local.message(actor, chat, text, key=key)
return response['json']['id'], response['headers']['X-Instance-Id']
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
results = list(pool.map(send, range(args.parallel)))
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text]
return {
'scenario': 'parallel POST with one Idempotency-Key through LB',
'requests': args.parallel,
'all_201': True,
'distinct_message_ids': len({r[0] for r in results}),
'instances': sorted({r[1] for r in results}),
'messages_in_history': len(stored),
}
def main():
parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB')
parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency'])
parser.add_argument('--workers', type=int, default=4)
parser.add_argument('--warmup', type=float, default=5)
parser.add_argument('--outage', type=float, default=10)
parser.add_argument('--parallel', type=int, default=20)
parser.add_argument('--kill', action='store_true')
parser.add_argument('--output')
args = parser.parse_args()
client = Client(BASE_URL)
alice, bob, _, chat = client.bootstrap()
scenarios = {'failover': failover, 'rolling': rolling}
if args.scenario in scenarios:
load = Load(client, alice, chat, args.workers)
report = scenarios[args.scenario](load, args)
report['history'] = verify_history(client, bob, chat, load.confirmed)
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
else:
report = idempotency(client, alice, chat, args)
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(text + '\n')
print(text)
if __name__ == '__main__':
main()