From cdd20c667f19d04df61ab93ea6ee074e69348360 Mon Sep 17 00:00:00 2001 From: gera Date: Fri, 23 Oct 2026 01:00:01 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=B8=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=BD=D0=BE=D1=81=D1=82=D0=B8,=20=D0=BE=D1=82=D0=BA=D0=B0?= =?UTF-8?q?=D0=B7=D0=BE=D0=B2=20=D0=B8=20rolling=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/idempotency.py | 42 ++++++++++++ tests/scenario.py | 155 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 tests/idempotency.py create mode 100644 tests/scenario.py diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 0000000..951d438 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import threading +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Concurrent Idempotency-Key check through the load balancer') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + parallel = 32 + key = 'idem-' + uuid.uuid4().hex + barrier = threading.Barrier(parallel) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(send, range(parallel))) + ids = {r['json']['id'] for r in results} + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids)) + check(len(instances) >= 2, 'requests were served by one instance only') + check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays') + + listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] + check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history') + + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'}) + other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + check(other['id'] not in ids, 'key must be scoped per user') + print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/tests/scenario.py b/tests/scenario.py new file mode 100644 index 0000000..f7ddd70 --- /dev/null +++ b/tests/scenario.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import threading +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent.parent + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') + return [x[:12] for x in out.split()] + + +def wait_healthy(count, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') + if out.count('(healthy)') >= count: + return + time.sleep(1) + raise Failure('replicas did not become healthy') + + +class Load: + def __init__(self, base, ca, concurrency): + c = Client(base, ca) + self.alice, _, _, self.chat = c.bootstrap() + self.base, self.ca, self.n = base, ca, concurrency + self.events = [] + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = time.monotonic() + + def worker(self): + c = Client(self.base, self.ca, timeout=3) + while not self.stop.is_set(): + key = uuid.uuid4().hex + first_error = None + while not self.stop.is_set(): + try: + r = c.message(self.alice, self.chat, 'k-' + key, key=key) + t = time.monotonic() - self.t0 + with self.lock: + self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) + break + except Failure: + t = time.monotonic() - self.t0 + first_error = first_error or t + with self.lock: + self.events.append((t, False, None, None)) + time.sleep(0.05) + + def start(self): + self.pool = ThreadPoolExecutor(max_workers=self.n) + self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] + + def finish(self): + self.stop.set() + self.pool.shutdown(wait=True) + + def summary(self, begin, end): + ev = [e for e in self.events if begin <= e[0] < end] + ok = [e for e in ev if e[1]] + bad = [e for e in ev if not e[1]] + span = max(end - begin, 1e-9) + return { + 'window_seconds': round(span, 2), + 'successful_requests': len(ok), + 'failed_attempts': len(bad), + 'successful_rps': round(len(ok) / span, 1), + 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, + 'instances': sorted({e[2] for e in ok}), + } + + def instances_between(self, begin, end=float('inf')): + return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) + + +def failover_each(args): + ids = api_containers() + check(len(ids) >= 2, 'need at least two replicas') + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + results = [] + base = load.summary(2, args.warmup) + for target in ids: + before = time.monotonic() - load.t0 + sh('docker', 'stop', '-t', '10', target) + stopped = time.monotonic() - load.t0 + time.sleep(args.hold) + after_stop = time.monotonic() - load.t0 + window = load.summary(before, after_stop) + sh('docker', 'start', target) + wait_healthy(len(ids)) + restored = time.monotonic() - load.t0 + time.sleep(args.warmup) + results.append({ + 'stopped_replica': target, + 'during_failure': window, + 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), + 'stop_command_seconds': round(stopped - before, 2), + 'restore_seconds': round(restored - after_stop, 2), + 'after_restore': load.summary(restored + 1, restored + args.warmup), + }) + load.finish() + return {'baseline': base, 'replicas': results} + + +def rolling(args): + old = api_containers() + check(len(old) >= 2, 'need at least two replicas') + n = len(old) + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + begin = time.monotonic() - load.t0 + sh('scripts/rolling-update.sh') + end = time.monotonic() - load.t0 + time.sleep(args.warmup) + load.finish() + return { + 'old_replicas': old, + 'new_replicas': api_containers(), + 'update_window': load.summary(begin, end), + 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], + } + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) + p.add_argument('--concurrency', type=int, default=8) + p.add_argument('--warmup', type=float, default=8) + p.add_argument('--hold', type=float, default=12) + p.add_argument('--output') + args = arguments('Failure and rolling update under load', configure) + report = failover_each(args) if args.mode == 'failover-each' else rolling(args) + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + run(main)