Добавить проверки идемпотентности, отказов и rolling update
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import uuid
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Concurrent Idempotency-Key check through the load balancer')
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
parallel = 32
|
||||
key = 'idem-' + uuid.uuid4().hex
|
||||
barrier = threading.Barrier(parallel)
|
||||
|
||||
def send(_):
|
||||
local = Client(args.base_url, args.ca_file)
|
||||
barrier.wait()
|
||||
return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')
|
||||
|
||||
with ThreadPoolExecutor(max_workers=parallel) as pool:
|
||||
results = list(pool.map(send, range(parallel)))
|
||||
ids = {r['json']['id'] for r in results}
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids))
|
||||
check(len(instances) >= 2, 'requests were served by one instance only')
|
||||
check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays')
|
||||
|
||||
listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items']
|
||||
check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history')
|
||||
|
||||
c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'x'})
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'})
|
||||
other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json']
|
||||
check(other['id'] not in ids, 'key must be scoped per user')
|
||||
print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def sh(*cmd):
|
||||
return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api')
|
||||
return [x[:12] for x in out.split()]
|
||||
|
||||
|
||||
def wait_healthy(count, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}')
|
||||
if out.count('(healthy)') >= count:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure('replicas did not become healthy')
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base, ca, concurrency):
|
||||
c = Client(base, ca)
|
||||
self.alice, _, _, self.chat = c.bootstrap()
|
||||
self.base, self.ca, self.n = base, ca, concurrency
|
||||
self.events = []
|
||||
self.lock = threading.Lock()
|
||||
self.stop = threading.Event()
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def worker(self):
|
||||
c = Client(self.base, self.ca, timeout=3)
|
||||
while not self.stop.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
first_error = None
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
r = c.message(self.alice, self.chat, 'k-' + key, key=key)
|
||||
t = time.monotonic() - self.t0
|
||||
with self.lock:
|
||||
self.events.append((t, True, r['headers']['X-Instance-Id'], first_error))
|
||||
break
|
||||
except Failure:
|
||||
t = time.monotonic() - self.t0
|
||||
first_error = first_error or t
|
||||
with self.lock:
|
||||
self.events.append((t, False, None, None))
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self):
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.n)
|
||||
self.futures = [self.pool.submit(self.worker) for _ in range(self.n)]
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
def summary(self, begin, end):
|
||||
ev = [e for e in self.events if begin <= e[0] < end]
|
||||
ok = [e for e in ev if e[1]]
|
||||
bad = [e for e in ev if not e[1]]
|
||||
span = max(end - begin, 1e-9)
|
||||
return {
|
||||
'window_seconds': round(span, 2),
|
||||
'successful_requests': len(ok),
|
||||
'failed_attempts': len(bad),
|
||||
'successful_rps': round(len(ok) / span, 1),
|
||||
'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None,
|
||||
'instances': sorted({e[2] for e in ok}),
|
||||
}
|
||||
|
||||
def instances_between(self, begin, end=float('inf')):
|
||||
return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end})
|
||||
|
||||
|
||||
def failover_each(args):
|
||||
ids = api_containers()
|
||||
check(len(ids) >= 2, 'need at least two replicas')
|
||||
load = Load(args.base_url, args.ca_file, args.concurrency)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
results = []
|
||||
base = load.summary(2, args.warmup)
|
||||
for target in ids:
|
||||
before = time.monotonic() - load.t0
|
||||
sh('docker', 'stop', '-t', '10', target)
|
||||
stopped = time.monotonic() - load.t0
|
||||
time.sleep(args.hold)
|
||||
after_stop = time.monotonic() - load.t0
|
||||
window = load.summary(before, after_stop)
|
||||
sh('docker', 'start', target)
|
||||
wait_healthy(len(ids))
|
||||
restored = time.monotonic() - load.t0
|
||||
time.sleep(args.warmup)
|
||||
results.append({
|
||||
'stopped_replica': target,
|
||||
'during_failure': window,
|
||||
'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop),
|
||||
'stop_command_seconds': round(stopped - before, 2),
|
||||
'restore_seconds': round(restored - after_stop, 2),
|
||||
'after_restore': load.summary(restored + 1, restored + args.warmup),
|
||||
})
|
||||
load.finish()
|
||||
return {'baseline': base, 'replicas': results}
|
||||
|
||||
|
||||
def rolling(args):
|
||||
old = api_containers()
|
||||
check(len(old) >= 2, 'need at least two replicas')
|
||||
n = len(old)
|
||||
load = Load(args.base_url, args.ca_file, args.concurrency)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
begin = time.monotonic() - load.t0
|
||||
sh('scripts/rolling-update.sh')
|
||||
end = time.monotonic() - load.t0
|
||||
time.sleep(args.warmup)
|
||||
load.finish()
|
||||
return {
|
||||
'old_replicas': old,
|
||||
'new_replicas': api_containers(),
|
||||
'update_window': load.summary(begin, end),
|
||||
'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True)
|
||||
p.add_argument('--concurrency', type=int, default=8)
|
||||
p.add_argument('--warmup', type=float, default=8)
|
||||
p.add_argument('--hold', type=float, default=12)
|
||||
p.add_argument('--output')
|
||||
args = arguments('Failure and rolling update under load', configure)
|
||||
report = failover_each(args) if args.mode == 'failover-each' else rolling(args)
|
||||
text = json.dumps(report, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + '\n')
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user