#!/usr/bin/env python3 """Эксперименты уровня 4 lab3. Стенд (docker compose up -d --build --wait) должен быть уже запущен. failover останавливает каждую API-реплику по очереди под нагрузкой, восстанавливает её и только потом берёт следующую; считает RPS, ошибки и время восстановления idempotency параллельно шлёт один Idempotency-Key через LB (разные реплики): должно быть ровно 1 сообщение rolling поочерёдная замена всех реплик на новую сборку под нагрузкой (migrate запускается один раз) Результаты (без токенов) пишутся в evidence/*.json. Данные создаются синтетические и не удаляются. Использует публичный tests/client.py и docker CLI из каталога проекта. """ import argparse import json import math import os from pathlib import Path import subprocess import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check # noqa: E402 PROJECT = 'mtusi-lab3' # ---------- docker ---------- def docker(*args, env=None, timeout=180, ok=True, with_stderr=False): proc = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, env=env) if ok and proc.returncode != 0: raise Failure('docker ' + ' '.join(args[:3]) + ' failed: ' + proc.stderr.strip()[:300]) return (proc.stdout + proc.stderr).strip() if with_stderr else proc.stdout.strip() def api_containers(all_states=False): """Короткие ID контейнеров сервиса api. Совпадают с X-Instance-Id (hostname контейнера).""" args = ['ps', '-q', '--filter', 'label=com.docker.compose.project=' + PROJECT, '--filter', 'label=com.docker.compose.service=api'] if all_states: args.insert(1, '-a') return sorted(x for x in docker(*args).split() if x) def wait_healthy(ids, timeout=90): deadline = time.monotonic() + timeout while True: states = {i: docker('inspect', '-f', '{{.State.Health.Status}}', i) for i in ids} if all(s == 'healthy' for s in states.values()): return check(time.monotonic() < deadline, 'containers did not become healthy: ' + str(states)) time.sleep(1) def wait_instances(c, actor, expected, timeout=60): """Ждать, пока через LB ответят все ожидаемые реплики (т.е. LB включил их в ротацию).""" deadline = time.monotonic() + timeout seen = set() while True: for _ in range(20): try: seen.add(c.call('GET', '/api/v1/users/me', actor=actor)['headers'].get('X-Instance-Id')) except Failure: pass if set(expected) <= seen: return check(time.monotonic() < deadline, 'LB did not route to all replicas: missing ' + str(set(expected) - seen)) time.sleep(0.5) def resource_snapshot(): out = docker('stats', '--no-stream', '--format', '{{.Name}}: cpu {{.CPUPerc}}, mem {{.MemUsage}}', ok=False) return [line for line in out.splitlines() if line] # ---------- нагрузка ---------- class Load: """Замкнутая нагрузка: запись сообщения (с ретраями ТОГО ЖЕ ключа при ошибке) + чтение страницы истории.""" def __init__(self, base_url, ca_file, actor, chat, workers): self.base_url, self.ca_file, self.actor, self.chat, self.workers = base_url, ca_file, actor, chat, workers self.events, self.acked = [], [] # events: (t_start, t_end, kind, ok, instance, error_reason) self.stop_event = threading.Event() self.lock = threading.Lock() self.threads = [] self.t0 = time.monotonic() def now(self): return time.monotonic() - self.t0 def record(self, start, kind, ok, instance, error=None): # Причина без пути (в нём id чата): "expected [201], received 503" или "connection/TLS failure (...)". reason = str(error).split(': ', 1)[-1] if error else None with self.lock: self.events.append((start, self.now(), kind, ok, instance, reason)) def worker(self, wid): c = Client(self.base_url, self.ca_file, timeout=2) path = '/api/v1/chats/' + self.chat['id'] + '/messages' i = 0 while not self.stop_event.is_set(): key = uuid.uuid4().hex text = 'exp-%d-%d-%s' % (wid, i, key) i += 1 for _ in range(10): start = self.now() try: r = c.message(self.actor, self.chat, text, key=key) self.record(start, 'write', True, r['headers'].get('X-Instance-Id')) with self.lock: self.acked.append(r['json']) break except Failure as exc: self.record(start, 'write', False, None, exc) # неопределённый результат: повтор с тем же ключом if self.stop_event.wait(0.1): break start = self.now() try: r = c.call('GET', path + '?limit=1', actor=self.actor, schema='MessagePage') self.record(start, 'read', True, r['headers'].get('X-Instance-Id')) except Failure as exc: self.record(start, 'read', False, None, exc) def start(self): self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)] for t in self.threads: t.start() def stop(self): self.stop_event.set() for t in self.threads: t.join(timeout=15) def percentile(sorted_values, p): if not sorted_values: return None return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1) def window(events, lo, hi): """Метрики по запросам, завершившимся в [lo, hi).""" sel = [e for e in events if lo <= e[1] < hi] ok = [e for e in sel if e[3]] errors = [e for e in sel if not e[3]] lat = sorted(e[1] - e[0] for e in ok) span = max(hi - lo, 1e-9) reasons = {} for e in errors: key = e[2] + ': ' + (e[5] or 'unknown') reasons[key] = reasons.get(key, 0) + 1 return { 'seconds': round(span, 2), 'successful_requests': len(ok), 'failed_requests': len(errors), 'error_reasons': reasons, 'error_ratio': round(len(errors) / len(sel), 4) if sel else None, 'successful_rps': round(len(ok) / span, 1), 'latency_ms': {'p50': percentile(lat, .5), 'p95': percentile(lat, .95), 'p99': percentile(lat, .99)}, } def timeline(events, end): rows = [] for s in range(int(end) + 1): sel = [e for e in events if s <= e[1] < s + 1] rows.append({'t': s, 'ok': sum(1 for e in sel if e[3]), 'errors': sum(1 for e in sel if not e[3])}) return rows def last_error_after(events, moment): late = [e[1] for e in events if not e[3] and e[1] >= moment] return round(max(late) - moment, 2) if late else 0.0 def history(c, actor, chat): path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' result, seen = [], set() for _ in range(100000): page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] result.extend(page['items']) cursor = page['next_cursor'] if cursor is None: return result check(cursor not in seen, 'cursor cycle') seen.add(cursor) path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(cursor, safe='') raise Failure('history pagination exceeded safety bound') def verify(c, actor, chat, acked): """Подтверждённые сообщения не потеряны, а повторы с тем же ключом не создали дублей.""" messages = history(c, actor, chat) by_id = {m['id']: m for m in messages} lost = sum(1 for a in acked if by_id.get(a['id']) != a) texts = [m['text'] for m in messages if m['text'].startswith('exp-')] duplicates = len(texts) - len(set(texts)) return {'acked_messages': len(acked), 'messages_in_history': len(messages), 'lost_acked_messages': lost, 'duplicate_messages': duplicates} def write_evidence(out_dir, name, data): out_dir.mkdir(parents=True, exist_ok=True) path = out_dir / name path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print('written', path) def setup(args): c = Client(args.base_url, args.ca_file) alice, bob, eve, chat = c.bootstrap() return c, alice, bob, chat # ---------- эксперименты ---------- def run_failover(args): c, alice, bob, chat = setup(args) containers = api_containers() check(len(containers) >= 2, 'need at least 2 running API replicas, found %d' % len(containers)) results = [] for target in containers: # Предыдущая реплика уже восстановлена; убеждаемся, что в ротации все. docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300) wait_instances(c, alice, containers) load = Load(args.base_url, args.ca_file, alice, chat, args.workers) load.start() time.sleep(args.warmup) resources = resource_snapshot() t_stop = load.now() docker('stop', '-t', '8', target) stop_command_seconds = round(load.now() - t_stop, 2) time.sleep(args.after) t_restore = load.now() docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300) wait_instances(c, alice, containers) t_back = load.now() time.sleep(args.settle) load.stop() end = load.now() stats = verify(c, bob, chat, load.acked) after_stop = [e for e in load.events if e[1] >= t_stop and e[3]] first_other = min((e[1] for e in after_stop if e[4] != target and e[2] == 'write'), default=None) # Сколько запросов остановленная реплика обслужила после возврата `docker stop` и до восстановления: должно быть 0. served_by_target_after_stop = sum(1 for e in load.events if t_stop + stop_command_seconds <= e[0] < t_restore and e[3] and e[4] == target) results.append({ 'stopped_replica': target, 'baseline_before_stop': window(load.events, 0, t_stop), 'stop_to_restore': window(load.events, t_stop, t_restore), 'after_restore': window(load.events, t_restore, end), 'docker_stop_seconds': stop_command_seconds, 'last_error_seconds_after_stop': last_error_after(load.events, t_stop), 'first_success_on_other_replica_seconds': round(first_other - t_stop, 2) if first_other else None, 'requests_served_by_stopped_replica_after_stop_returned': served_by_target_after_stop, # Включает `docker compose up --wait` (повторный migrate, start_period healthcheck) и rise 2 в LB. 'seconds_until_replica_back_in_rotation': round(t_back - t_restore, 2), 'instances_seen': sorted({e[4] for e in load.events if e[4]}), 'resource_snapshot_under_load': resources, 'timeline_per_second': timeline(load.events, end), **stats, }) print('replica %s: errors=%d, last error %.2fs after stop, lost=%d, dup=%d' % ( target, results[-1]['stop_to_restore']['failed_requests'], results[-1]['last_error_seconds_after_stop'], stats['lost_acked_messages'], stats['duplicate_messages'])) write_evidence(Path(args.output), 'failover-sequential.json', { 'method': 'closed loop %d workers: POST message (retry same Idempotency-Key on error) + GET page; ' 'client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next' % args.workers, 'replicas': containers, 'warmup_seconds': args.warmup, 'seconds_down': args.after, 'runs': results}) for r in results: check(r['lost_acked_messages'] == 0 and r['duplicate_messages'] == 0, 'data integrity violated') def run_idempotency(args): c, alice, bob, chat = setup(args) path = '/api/v1/chats/' + chat['id'] + '/messages' instances, rounds = set(), [] for _ in range(args.rounds): key = uuid.uuid4().hex text = 'idem-' + key barrier = threading.Barrier(args.parallel) def send(_): local = Client(args.base_url, args.ca_file, timeout=10) barrier.wait() try: r = local.message(alice, chat, text, key=key) return r['status'], r['json']['id'], r['headers'].get('X-Instance-Id') except Failure as exc: return None, str(exc), None with ThreadPoolExecutor(max_workers=args.parallel) as pool: answers = list(pool.map(send, range(args.parallel))) ok = [a for a in answers if a[0] == 201] instances.update(a[2] for a in ok) rounds.append({'responses_201': len(ok), 'failures': len(answers) - len(ok), 'distinct_message_ids': len({a[1] for a in ok}), 'distinct_instances': len({a[2] for a in ok})}) messages = history(c, bob, chat) texts = [m['text'] for m in messages] created = [texts.count(m['text']) for m in messages if m['text'].startswith('idem-')] # Тот же ключ с другим телом -> 409. key = uuid.uuid4().hex c.message(alice, chat, 'idem-first-' + key, key=key) other_body = c.call('POST', path, 409, actor=alice, body={'text': 'idem-other-' + key}, headers={'Idempotency-Key': key}) report = { 'method': '%d rounds; each round %d simultaneous identical POSTs with one Idempotency-Key through the LB ' '(separate connections, released by a barrier)' % (args.rounds, args.parallel), 'rounds': args.rounds, 'parallel_requests_per_round': args.parallel, 'instances_seen': sorted(i for i in instances if i), 'all_rounds_single_message_id': all(r['distinct_message_ids'] == 1 for r in rounds), 'failed_requests': sum(r['failures'] for r in rounds), 'rounds_with_requests_on_2plus_replicas': sum(1 for r in rounds if r['distinct_instances'] >= 2), 'messages_created_per_key_max': max(created) if created else 0, 'same_key_other_body_status': other_body['status'], 'per_round': rounds, } write_evidence(Path(args.output), 'idempotency-parallel.json', report) check(report['all_rounds_single_message_id'], 'a key produced several message ids') check(report['messages_created_per_key_max'] == 1, 'a key created several messages') check(report['failed_requests'] == 0, 'some parallel requests failed') def run_rolling(args): c, alice, bob, chat = setup(args) old = api_containers() n = len(old) check(n >= 2, 'need at least 2 running API replicas') old_image = {i: docker('inspect', '-f', '{{.Image}}', i) for i in old} # Migrate запускается один раз, до замены; миграции только добавляют (expand), старые реплики их переживают. # Alembic пишет журнал в stderr. migrate_log = docker('compose', 'run', '--rm', 'migrate', timeout=300, with_stderr=True) load = Load(args.base_url, args.ca_file, alice, chat, args.workers) load.start() time.sleep(args.warmup) t_start = load.now() env = dict(os.environ, APP_VERSION='rolling-' + time.strftime('%H%M%S')) docker('compose', 'build', 'api', env=env, timeout=600) docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (2 * n), 'api', env=env, timeout=300) new = [i for i in api_containers() if i not in old] check(len(new) == n, 'expected %d new replicas, found %d' % (n, len(new))) wait_healthy(new) wait_instances(c, alice, new) steps = [] for victim in old: t = load.now() docker('stop', '-t', '15', victim) docker('rm', victim) steps.append({'replaced': victim, 'seconds_into_update': round(t - t_start, 2)}) time.sleep(args.step_pause) docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % n, 'api', env=env, timeout=300) t_done = load.now() time.sleep(args.settle) load.stop() end = load.now() stats = verify(c, bob, chat, load.acked) seen_old = [e for e in load.events if e[3] and e[4] in old] seen_new = [e for e in load.events if e[3] and e[4] in new] report = { 'method': 'closed loop %d workers under rolling replacement: start N new replicas of the new build, ' 'wait until healthy and routed by LB, then stop+remove old replicas one at a time' % args.workers, 'replicas_before': old, 'replicas_after': api_containers(), 'new_replicas': new, 'old_image_ids': old_image, 'new_image_ids': {i: docker('inspect', '-f', '{{.Image}}', i) for i in new}, 'app_version_build_arg': env['APP_VERSION'], 'update_seconds': round(t_done - t_start, 2), 'steps': steps, 'before_update': window(load.events, 0, t_start), 'during_update': window(load.events, t_start, t_done), 'after_update': window(load.events, t_done, end), # Все моменты — секунды от начала обновления, как и steps[].seconds_into_update. 'first_new_replica_response_seconds_into_update': round(min((e[1] for e in seen_new), default=t_start) - t_start, 2), 'last_old_replica_response_seconds_into_update': round(max((e[1] for e in seen_old), default=t_start) - t_start, 2), 'migrate_output_tail': migrate_log.splitlines()[-3:], 'timeline_per_second': timeline(load.events, end), **stats, } write_evidence(Path(args.output), 'rolling-update.json', report) print('rolling: errors during update=%d, lost=%d, dup=%d' % ( report['during_update']['failed_requests'], stats['lost_acked_messages'], stats['duplicate_messages'])) check(stats['lost_acked_messages'] == 0 and stats['duplicate_messages'] == 0, 'data integrity violated') def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('experiment', choices=['failover', 'idempotency', 'rolling']) p.add_argument('--base-url', default=None) p.add_argument('--ca-file', default=None) p.add_argument('--workers', type=int, default=8, help='closed-loop workers (failover, rolling)') p.add_argument('--warmup', type=float, default=5, help='seconds of load before the action') p.add_argument('--after', type=float, default=15, help='failover: seconds the replica stays down under load') p.add_argument('--settle', type=float, default=8, help='seconds of load after the action') p.add_argument('--step-pause', type=float, default=3, help='rolling: pause between replacing replicas') p.add_argument('--rounds', type=int, default=30, help='idempotency: number of keys') p.add_argument('--parallel', type=int, default=16, help='idempotency: simultaneous requests per key') p.add_argument('--output', default=str(ROOT / 'evidence')) args = p.parse_args() try: {'failover': run_failover, 'idempotency': run_idempotency, 'rolling': run_rolling}[args.experiment](args) except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc: print('FAIL:', exc) raise SystemExit(1) print('PASS:', args.experiment) if __name__ == '__main__': main()