ЛР3: балансировка HAProxy, 2 реплики API, идемпотентность, отказоустойчивость (на 4)
- X-Instance-Id в каждом ответе; обязательный Idempotency-Key, таблица idempotency_keys (миграция 0002, одна транзакция, конкурентные повторы через уникальный индекс) - graceful shutdown: SIGTERM -> ready=503 -> drain 3 с -> закрытие порта - compose: lb (HAProxy) + api x2 без портов + разовый migrate + postgres, лимиты CPU/RAM - хуки stop-one/restart/restore-replicas, scripts/experiments.py (failover, idempotency, rolling) - свои тесты: 44 (13 новых); evidence: smoke, pytest, persistence, failover, эксперименты
This commit is contained in:
@@ -0,0 +1,413 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Эксперименты уровня 4 lab3. Стенд (docker compose up -d --build --wait) должен быть уже запущен.
|
||||
|
||||
failover останавливает каждую API-реплику по очереди под нагрузкой, восстанавливает её и
|
||||
только потом берёт следующую; считает RPS, ошибки и время восстановления
|
||||
idempotency параллельно шлёт один Idempotency-Key через LB (разные реплики): должно быть ровно 1 сообщение
|
||||
rolling поочерёдная замена всех реплик на новую сборку под нагрузкой (migrate запускается один раз)
|
||||
|
||||
Результаты (без токенов) пишутся в evidence/*.json. Данные создаются синтетические и не удаляются.
|
||||
Использует публичный tests/client.py и docker CLI из каталога проекта.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
PROJECT = 'mtusi-lab3'
|
||||
|
||||
|
||||
# ---------- docker ----------
|
||||
|
||||
def docker(*args, env=None, timeout=180, ok=True, with_stderr=False):
|
||||
proc = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8',
|
||||
errors='replace', timeout=timeout, env=env)
|
||||
if ok and proc.returncode != 0:
|
||||
raise Failure('docker ' + ' '.join(args[:3]) + ' failed: ' + proc.stderr.strip()[:300])
|
||||
return (proc.stdout + proc.stderr).strip() if with_stderr else proc.stdout.strip()
|
||||
|
||||
|
||||
def api_containers(all_states=False):
|
||||
"""Короткие ID контейнеров сервиса api. Совпадают с X-Instance-Id (hostname контейнера)."""
|
||||
args = ['ps', '-q', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api']
|
||||
if all_states:
|
||||
args.insert(1, '-a')
|
||||
return sorted(x for x in docker(*args).split() if x)
|
||||
|
||||
|
||||
def wait_healthy(ids, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
states = {i: docker('inspect', '-f', '{{.State.Health.Status}}', i) for i in ids}
|
||||
if all(s == 'healthy' for s in states.values()):
|
||||
return
|
||||
check(time.monotonic() < deadline, 'containers did not become healthy: ' + str(states))
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def wait_instances(c, actor, expected, timeout=60):
|
||||
"""Ждать, пока через LB ответят все ожидаемые реплики (т.е. LB включил их в ротацию)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
seen = set()
|
||||
while True:
|
||||
for _ in range(20):
|
||||
try:
|
||||
seen.add(c.call('GET', '/api/v1/users/me', actor=actor)['headers'].get('X-Instance-Id'))
|
||||
except Failure:
|
||||
pass
|
||||
if set(expected) <= seen:
|
||||
return
|
||||
check(time.monotonic() < deadline, 'LB did not route to all replicas: missing ' + str(set(expected) - seen))
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def resource_snapshot():
|
||||
out = docker('stats', '--no-stream', '--format', '{{.Name}}: cpu {{.CPUPerc}}, mem {{.MemUsage}}', ok=False)
|
||||
return [line for line in out.splitlines() if line]
|
||||
|
||||
|
||||
# ---------- нагрузка ----------
|
||||
|
||||
class Load:
|
||||
"""Замкнутая нагрузка: запись сообщения (с ретраями ТОГО ЖЕ ключа при ошибке) + чтение страницы истории."""
|
||||
|
||||
def __init__(self, base_url, ca_file, actor, chat, workers):
|
||||
self.base_url, self.ca_file, self.actor, self.chat, self.workers = base_url, ca_file, actor, chat, workers
|
||||
self.events, self.acked = [], [] # events: (t_start, t_end, kind, ok, instance, error_reason)
|
||||
self.stop_event = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.threads = []
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def record(self, start, kind, ok, instance, error=None):
|
||||
# Причина без пути (в нём id чата): "expected [201], received 503" или "connection/TLS failure (...)".
|
||||
reason = str(error).split(': ', 1)[-1] if error else None
|
||||
with self.lock:
|
||||
self.events.append((start, self.now(), kind, ok, instance, reason))
|
||||
|
||||
def worker(self, wid):
|
||||
c = Client(self.base_url, self.ca_file, timeout=2)
|
||||
path = '/api/v1/chats/' + self.chat['id'] + '/messages'
|
||||
i = 0
|
||||
while not self.stop_event.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
text = 'exp-%d-%d-%s' % (wid, i, key)
|
||||
i += 1
|
||||
for _ in range(10):
|
||||
start = self.now()
|
||||
try:
|
||||
r = c.message(self.actor, self.chat, text, key=key)
|
||||
self.record(start, 'write', True, r['headers'].get('X-Instance-Id'))
|
||||
with self.lock:
|
||||
self.acked.append(r['json'])
|
||||
break
|
||||
except Failure as exc:
|
||||
self.record(start, 'write', False, None, exc) # неопределённый результат: повтор с тем же ключом
|
||||
if self.stop_event.wait(0.1):
|
||||
break
|
||||
start = self.now()
|
||||
try:
|
||||
r = c.call('GET', path + '?limit=1', actor=self.actor, schema='MessagePage')
|
||||
self.record(start, 'read', True, r['headers'].get('X-Instance-Id'))
|
||||
except Failure as exc:
|
||||
self.record(start, 'read', False, None, exc)
|
||||
|
||||
def start(self):
|
||||
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
|
||||
def percentile(sorted_values, p):
|
||||
if not sorted_values:
|
||||
return None
|
||||
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
|
||||
|
||||
|
||||
def window(events, lo, hi):
|
||||
"""Метрики по запросам, завершившимся в [lo, hi)."""
|
||||
sel = [e for e in events if lo <= e[1] < hi]
|
||||
ok = [e for e in sel if e[3]]
|
||||
errors = [e for e in sel if not e[3]]
|
||||
lat = sorted(e[1] - e[0] for e in ok)
|
||||
span = max(hi - lo, 1e-9)
|
||||
reasons = {}
|
||||
for e in errors:
|
||||
key = e[2] + ': ' + (e[5] or 'unknown')
|
||||
reasons[key] = reasons.get(key, 0) + 1
|
||||
return {
|
||||
'seconds': round(span, 2),
|
||||
'successful_requests': len(ok), 'failed_requests': len(errors), 'error_reasons': reasons,
|
||||
'error_ratio': round(len(errors) / len(sel), 4) if sel else None,
|
||||
'successful_rps': round(len(ok) / span, 1),
|
||||
'latency_ms': {'p50': percentile(lat, .5), 'p95': percentile(lat, .95), 'p99': percentile(lat, .99)},
|
||||
}
|
||||
|
||||
|
||||
def timeline(events, end):
|
||||
rows = []
|
||||
for s in range(int(end) + 1):
|
||||
sel = [e for e in events if s <= e[1] < s + 1]
|
||||
rows.append({'t': s, 'ok': sum(1 for e in sel if e[3]), 'errors': sum(1 for e in sel if not e[3])})
|
||||
return rows
|
||||
|
||||
|
||||
def last_error_after(events, moment):
|
||||
late = [e[1] for e in events if not e[3] and e[1] >= moment]
|
||||
return round(max(late) - moment, 2) if late else 0.0
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||||
result, seen = [], set()
|
||||
for _ in range(100000):
|
||||
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return result
|
||||
check(cursor not in seen, 'cursor cycle')
|
||||
seen.add(cursor)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(cursor, safe='')
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def verify(c, actor, chat, acked):
|
||||
"""Подтверждённые сообщения не потеряны, а повторы с тем же ключом не создали дублей."""
|
||||
messages = history(c, actor, chat)
|
||||
by_id = {m['id']: m for m in messages}
|
||||
lost = sum(1 for a in acked if by_id.get(a['id']) != a)
|
||||
texts = [m['text'] for m in messages if m['text'].startswith('exp-')]
|
||||
duplicates = len(texts) - len(set(texts))
|
||||
return {'acked_messages': len(acked), 'messages_in_history': len(messages),
|
||||
'lost_acked_messages': lost, 'duplicate_messages': duplicates}
|
||||
|
||||
|
||||
def write_evidence(out_dir, name, data):
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / name
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('written', path)
|
||||
|
||||
|
||||
def setup(args):
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
return c, alice, bob, chat
|
||||
|
||||
|
||||
# ---------- эксперименты ----------
|
||||
|
||||
def run_failover(args):
|
||||
c, alice, bob, chat = setup(args)
|
||||
containers = api_containers()
|
||||
check(len(containers) >= 2, 'need at least 2 running API replicas, found %d' % len(containers))
|
||||
results = []
|
||||
for target in containers:
|
||||
# Предыдущая реплика уже восстановлена; убеждаемся, что в ротации все.
|
||||
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
|
||||
wait_instances(c, alice, containers)
|
||||
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
resources = resource_snapshot()
|
||||
t_stop = load.now()
|
||||
docker('stop', '-t', '8', target)
|
||||
stop_command_seconds = round(load.now() - t_stop, 2)
|
||||
time.sleep(args.after)
|
||||
t_restore = load.now()
|
||||
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
|
||||
wait_instances(c, alice, containers)
|
||||
t_back = load.now()
|
||||
time.sleep(args.settle)
|
||||
load.stop()
|
||||
end = load.now()
|
||||
stats = verify(c, bob, chat, load.acked)
|
||||
after_stop = [e for e in load.events if e[1] >= t_stop and e[3]]
|
||||
first_other = min((e[1] for e in after_stop if e[4] != target and e[2] == 'write'), default=None)
|
||||
# Сколько запросов остановленная реплика обслужила после возврата `docker stop` и до восстановления: должно быть 0.
|
||||
served_by_target_after_stop = sum(1 for e in load.events
|
||||
if t_stop + stop_command_seconds <= e[0] < t_restore and e[3] and e[4] == target)
|
||||
results.append({
|
||||
'stopped_replica': target,
|
||||
'baseline_before_stop': window(load.events, 0, t_stop),
|
||||
'stop_to_restore': window(load.events, t_stop, t_restore),
|
||||
'after_restore': window(load.events, t_restore, end),
|
||||
'docker_stop_seconds': stop_command_seconds,
|
||||
'last_error_seconds_after_stop': last_error_after(load.events, t_stop),
|
||||
'first_success_on_other_replica_seconds': round(first_other - t_stop, 2) if first_other else None,
|
||||
'requests_served_by_stopped_replica_after_stop_returned': served_by_target_after_stop,
|
||||
# Включает `docker compose up --wait` (повторный migrate, start_period healthcheck) и rise 2 в LB.
|
||||
'seconds_until_replica_back_in_rotation': round(t_back - t_restore, 2),
|
||||
'instances_seen': sorted({e[4] for e in load.events if e[4]}),
|
||||
'resource_snapshot_under_load': resources,
|
||||
'timeline_per_second': timeline(load.events, end),
|
||||
**stats,
|
||||
})
|
||||
print('replica %s: errors=%d, last error %.2fs after stop, lost=%d, dup=%d' % (
|
||||
target, results[-1]['stop_to_restore']['failed_requests'],
|
||||
results[-1]['last_error_seconds_after_stop'], stats['lost_acked_messages'], stats['duplicate_messages']))
|
||||
write_evidence(Path(args.output), 'failover-sequential.json', {
|
||||
'method': 'closed loop %d workers: POST message (retry same Idempotency-Key on error) + GET page; '
|
||||
'client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next' % args.workers,
|
||||
'replicas': containers, 'warmup_seconds': args.warmup, 'seconds_down': args.after, 'runs': results})
|
||||
for r in results:
|
||||
check(r['lost_acked_messages'] == 0 and r['duplicate_messages'] == 0, 'data integrity violated')
|
||||
|
||||
|
||||
def run_idempotency(args):
|
||||
c, alice, bob, chat = setup(args)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
instances, rounds = set(), []
|
||||
for _ in range(args.rounds):
|
||||
key = uuid.uuid4().hex
|
||||
text = 'idem-' + key
|
||||
barrier = threading.Barrier(args.parallel)
|
||||
|
||||
def send(_):
|
||||
local = Client(args.base_url, args.ca_file, timeout=10)
|
||||
barrier.wait()
|
||||
try:
|
||||
r = local.message(alice, chat, text, key=key)
|
||||
return r['status'], r['json']['id'], r['headers'].get('X-Instance-Id')
|
||||
except Failure as exc:
|
||||
return None, str(exc), None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
|
||||
answers = list(pool.map(send, range(args.parallel)))
|
||||
ok = [a for a in answers if a[0] == 201]
|
||||
instances.update(a[2] for a in ok)
|
||||
rounds.append({'responses_201': len(ok), 'failures': len(answers) - len(ok),
|
||||
'distinct_message_ids': len({a[1] for a in ok}),
|
||||
'distinct_instances': len({a[2] for a in ok})})
|
||||
messages = history(c, bob, chat)
|
||||
texts = [m['text'] for m in messages]
|
||||
created = [texts.count(m['text']) for m in messages if m['text'].startswith('idem-')]
|
||||
# Тот же ключ с другим телом -> 409.
|
||||
key = uuid.uuid4().hex
|
||||
c.message(alice, chat, 'idem-first-' + key, key=key)
|
||||
other_body = c.call('POST', path, 409, actor=alice, body={'text': 'idem-other-' + key},
|
||||
headers={'Idempotency-Key': key})
|
||||
report = {
|
||||
'method': '%d rounds; each round %d simultaneous identical POSTs with one Idempotency-Key through the LB '
|
||||
'(separate connections, released by a barrier)' % (args.rounds, args.parallel),
|
||||
'rounds': args.rounds, 'parallel_requests_per_round': args.parallel,
|
||||
'instances_seen': sorted(i for i in instances if i),
|
||||
'all_rounds_single_message_id': all(r['distinct_message_ids'] == 1 for r in rounds),
|
||||
'failed_requests': sum(r['failures'] for r in rounds),
|
||||
'rounds_with_requests_on_2plus_replicas': sum(1 for r in rounds if r['distinct_instances'] >= 2),
|
||||
'messages_created_per_key_max': max(created) if created else 0,
|
||||
'same_key_other_body_status': other_body['status'],
|
||||
'per_round': rounds,
|
||||
}
|
||||
write_evidence(Path(args.output), 'idempotency-parallel.json', report)
|
||||
check(report['all_rounds_single_message_id'], 'a key produced several message ids')
|
||||
check(report['messages_created_per_key_max'] == 1, 'a key created several messages')
|
||||
check(report['failed_requests'] == 0, 'some parallel requests failed')
|
||||
|
||||
|
||||
def run_rolling(args):
|
||||
c, alice, bob, chat = setup(args)
|
||||
old = api_containers()
|
||||
n = len(old)
|
||||
check(n >= 2, 'need at least 2 running API replicas')
|
||||
old_image = {i: docker('inspect', '-f', '{{.Image}}', i) for i in old}
|
||||
# Migrate запускается один раз, до замены; миграции только добавляют (expand), старые реплики их переживают.
|
||||
# Alembic пишет журнал в stderr.
|
||||
migrate_log = docker('compose', 'run', '--rm', 'migrate', timeout=300, with_stderr=True)
|
||||
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
t_start = load.now()
|
||||
env = dict(os.environ, APP_VERSION='rolling-' + time.strftime('%H%M%S'))
|
||||
docker('compose', 'build', 'api', env=env, timeout=600)
|
||||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (2 * n), 'api',
|
||||
env=env, timeout=300)
|
||||
new = [i for i in api_containers() if i not in old]
|
||||
check(len(new) == n, 'expected %d new replicas, found %d' % (n, len(new)))
|
||||
wait_healthy(new)
|
||||
wait_instances(c, alice, new)
|
||||
steps = []
|
||||
for victim in old:
|
||||
t = load.now()
|
||||
docker('stop', '-t', '15', victim)
|
||||
docker('rm', victim)
|
||||
steps.append({'replaced': victim, 'seconds_into_update': round(t - t_start, 2)})
|
||||
time.sleep(args.step_pause)
|
||||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % n, 'api', env=env, timeout=300)
|
||||
t_done = load.now()
|
||||
time.sleep(args.settle)
|
||||
load.stop()
|
||||
end = load.now()
|
||||
stats = verify(c, bob, chat, load.acked)
|
||||
seen_old = [e for e in load.events if e[3] and e[4] in old]
|
||||
seen_new = [e for e in load.events if e[3] and e[4] in new]
|
||||
report = {
|
||||
'method': 'closed loop %d workers under rolling replacement: start N new replicas of the new build, '
|
||||
'wait until healthy and routed by LB, then stop+remove old replicas one at a time' % args.workers,
|
||||
'replicas_before': old, 'replicas_after': api_containers(), 'new_replicas': new,
|
||||
'old_image_ids': old_image, 'new_image_ids': {i: docker('inspect', '-f', '{{.Image}}', i) for i in new},
|
||||
'app_version_build_arg': env['APP_VERSION'],
|
||||
'update_seconds': round(t_done - t_start, 2),
|
||||
'steps': steps,
|
||||
'before_update': window(load.events, 0, t_start),
|
||||
'during_update': window(load.events, t_start, t_done),
|
||||
'after_update': window(load.events, t_done, end),
|
||||
# Все моменты — секунды от начала обновления, как и steps[].seconds_into_update.
|
||||
'first_new_replica_response_seconds_into_update': round(min((e[1] for e in seen_new), default=t_start) - t_start, 2),
|
||||
'last_old_replica_response_seconds_into_update': round(max((e[1] for e in seen_old), default=t_start) - t_start, 2),
|
||||
'migrate_output_tail': migrate_log.splitlines()[-3:],
|
||||
'timeline_per_second': timeline(load.events, end),
|
||||
**stats,
|
||||
}
|
||||
write_evidence(Path(args.output), 'rolling-update.json', report)
|
||||
print('rolling: errors during update=%d, lost=%d, dup=%d' % (
|
||||
report['during_update']['failed_requests'], stats['lost_acked_messages'], stats['duplicate_messages']))
|
||||
check(stats['lost_acked_messages'] == 0 and stats['duplicate_messages'] == 0, 'data integrity violated')
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('experiment', choices=['failover', 'idempotency', 'rolling'])
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--ca-file', default=None)
|
||||
p.add_argument('--workers', type=int, default=8, help='closed-loop workers (failover, rolling)')
|
||||
p.add_argument('--warmup', type=float, default=5, help='seconds of load before the action')
|
||||
p.add_argument('--after', type=float, default=15, help='failover: seconds the replica stays down under load')
|
||||
p.add_argument('--settle', type=float, default=8, help='seconds of load after the action')
|
||||
p.add_argument('--step-pause', type=float, default=3, help='rolling: pause between replacing replicas')
|
||||
p.add_argument('--rounds', type=int, default=30, help='idempotency: number of keys')
|
||||
p.add_argument('--parallel', type=int, default=16, help='idempotency: simultaneous requests per key')
|
||||
p.add_argument('--output', default=str(ROOT / 'evidence'))
|
||||
args = p.parse_args()
|
||||
try:
|
||||
{'failover': run_failover, 'idempotency': run_idempotency, 'rolling': run_rolling}[args.experiment](args)
|
||||
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
|
||||
print('FAIL:', exc)
|
||||
raise SystemExit(1)
|
||||
print('PASS:', args.experiment)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user