diff --git a/compose.yaml b/compose.yaml index d7b3c185..1707b02b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -10,11 +10,20 @@ services: - -ec - | cd /secrets - [ -s pg_password ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > pg_password - printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat pg_password)" > database_url - [ -s redis_password ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > redis_password - printf 'redis://:%s@redis:6379/0' "$$(cat redis_password)" > cache_url - chmod 0444 pg_password database_url redis_password cache_url + mkdir -p postgres redis api migrate + for name in pg_password:postgres redis_password:redis; do + file=$${name%%:*}; dir=$${name##*:} + [ ! -s "$$file" ] || mv "$$file" "$$dir/$$file" + [ -s "$$dir/$$file" ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file" + done + rm -f database_url cache_url + printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat postgres/pg_password)" > api/database_url + printf 'redis://:%s@redis:6379/0' "$$(cat redis/redis_password)" > api/cache_url + printf 'postgresql://messenger:%s@postgres:5432/messenger' "$$(cat postgres/pg_password)" > migrate/database_url + chmod 0444 postgres/* redis/* api/* migrate/* + deploy: + resources: + limits: { cpus: "0.1", memory: 32M } postgres: image: postgres:17.2-alpine @@ -24,7 +33,10 @@ services: POSTGRES_PASSWORD_FILE: /run/secrets/pg_password volumes: - pgdata:/var/lib/postgresql/data - - secrets:/run/secrets:ro + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: postgres } } + deploy: + resources: + limits: { cpus: "1.0", memory: 512M } healthcheck: test: ["CMD", "pg_isready", "-U", "messenger", "-d", "messenger"] interval: 5s @@ -43,7 +55,10 @@ services: - -c - exec redis-server --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$(cat /run/secrets/redis_password)" volumes: - - secrets:/run/secrets:ro + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: redis } } + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } healthcheck: test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep -q PONG"] interval: 5s @@ -62,7 +77,10 @@ services: environment: DATABASE_URL_FILE: /run/secrets/database_url volumes: - - secrets:/run/secrets:ro + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: migrate } } + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } depends_on: postgres: condition: service_healthy @@ -82,9 +100,11 @@ services: DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url volumes: - - secrets:/run/secrets:ro + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } } deploy: replicas: ${API_REPLICAS:-2} + resources: + limits: { cpus: "0.5", memory: 256M } stop_grace_period: 10s depends_on: migrate: @@ -99,6 +119,9 @@ services: - "127.0.0.1:${APP_PORT:-8080}:8080" volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + deploy: + resources: + limits: { cpus: "0.25", memory: 64M } depends_on: api: condition: service_healthy @@ -117,6 +140,9 @@ services: volumes: - .:/work - /var/run/docker.sock:/var/run/docker.sock + deploy: + resources: + limits: { cpus: "1.0", memory: 256M } volumes: pgdata: diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index d8c9285d..ae335b5b 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -1,5 +1,6 @@ global log stdout format raw local0 info + maxconn 2000 defaults mode http diff --git a/scripts/experiment.py b/scripts/experiment.py new file mode 100644 index 00000000..02f811dd --- /dev/null +++ b/scripts/experiment.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + +BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080') + + +def docker(*args): + return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()] + + +def describe(cid): + return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid) + + +def wait_healthy(cid, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy': + return + time.sleep(0.2) + raise SystemExit(cid + ' did not become healthy') + + +class Load: + def __init__(self, client, actor, chat, workers): + self.client, self.actor, self.chat = client, actor, chat + self.workers = workers + self.lock = threading.Lock() + self.events = [] + self.confirmed = [] + self.stop_flag = threading.Event() + self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)] + self.started = None + + def worker(self, index): + local = Client(BASE_URL, timeout=2) + key = uuid.uuid4().hex + while not self.stop_flag.is_set(): + try: + response = local.message(self.actor, self.chat, 'load ' + key, key=key) + with self.lock: + self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'])) + self.confirmed.append(response['json']) + key = uuid.uuid4().hex + except Failure: + with self.lock: + self.events.append((time.monotonic(), False, None)) + time.sleep(0.05) + + def __enter__(self): + self.started = time.monotonic() + for thread in self.threads: + thread.start() + return self + + def __exit__(self, *exc): + self.stop_flag.set() + for thread in self.threads: + thread.join() + + def window(self, start, end): + with self.lock: + events = [e for e in self.events if start <= e[0] < end] + ok = sorted(e[0] for e in events if e[1]) + points = [start, *ok, end] + return { + 'seconds': round(end - start, 3), + 'successful_writes': len(ok), + 'failed_attempts': sum(1 for e in events if not e[1]), + 'successful_rps': round(len(ok) / (end - start), 1), + 'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3), + 'instances': sorted({e[2] for e in events if e[1]}), + } + + def last_seen(self, instance, start, end): + with self.lock: + times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end] + return max(times) if times else None + + def first_seen_after(self, instance, after): + with self.lock: + times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after] + return min(times) if times else None + + +def verify_history(client, actor, chat, confirmed): + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' + items = [] + while True: + page = client.call('GET', path, actor=actor, schema='MessagePage')['json'] + items.extend(page['items']) + if page['next_cursor'] is None: + break + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') + texts = [m['text'] for m in items] + stored = {m['id']: m for m in items} + return { + 'confirmed': len(confirmed), + 'stored': len(items), + 'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m), + 'duplicates': len(texts) - len(set(texts)), + } + + +def failover(load, args): + results = [] + with load: + time.sleep(args.warmup) + for cid in api_containers(): + t0 = time.monotonic() + docker('kill' if args.kill else 'stop', cid) + stopped = time.monotonic() + time.sleep(args.outage) + outage = load.window(t0, time.monotonic()) + restore = time.monotonic() + docker('start', cid) + wait_healthy(cid) + healthy = time.monotonic() + time.sleep(args.warmup) + rejoined = load.first_seen_after(cid, restore) + last = load.last_seen(cid, t0, restore) + results.append({ + 'container': describe(cid), + 'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)', + 'before': load.window(t0 - args.warmup, t0), + 'stop_command_seconds': round(stopped - t0, 3), + 'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None, + 'outage_window': outage, + 'restore_until_healthy_seconds': round(healthy - restore, 3), + 'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None, + 'after_restore': load.window(healthy, time.monotonic()), + }) + return {'scenario': 'sequential failover of each api replica', 'replicas': results} + + +def rolling(load, args): + old = api_containers() + before = [describe(cid) for cid in old] + steps = [] + with load: + time.sleep(args.warmup) + t_start = time.monotonic() + docker('compose', 'run', '--rm', '--no-deps', 'migrate') + for cid in old: + t0 = time.monotonic() + docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api') + added = time.monotonic() + docker('stop', cid) + docker('rm', cid) + steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)}) + time.sleep(args.warmup) + total = load.window(t_start, time.monotonic()) + return { + 'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one', + 'before': before, + 'after': [describe(cid) for cid in api_containers()], + 'steps': steps, + 'during_update': total, + } + + +def idempotency(client, actor, chat, args): + key = uuid.uuid4().hex + text = 'parallel ' + key + + def send(_): + local = Client(BASE_URL, timeout=10) + response = local.message(actor, chat, text, key=key) + return response['json']['id'], response['headers']['X-Instance-Id'] + + with ThreadPoolExecutor(max_workers=args.parallel) as pool: + results = list(pool.map(send, range(args.parallel))) + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' + stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text] + return { + 'scenario': 'parallel POST with one Idempotency-Key through LB', + 'requests': args.parallel, + 'all_201': True, + 'distinct_message_ids': len({r[0] for r in results}), + 'instances': sorted({r[1] for r in results}), + 'messages_in_history': len(stored), + } + + +def main(): + parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB') + parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency']) + parser.add_argument('--workers', type=int, default=4) + parser.add_argument('--warmup', type=float, default=5) + parser.add_argument('--outage', type=float, default=10) + parser.add_argument('--parallel', type=int, default=20) + parser.add_argument('--kill', action='store_true') + parser.add_argument('--output') + args = parser.parse_args() + + client = Client(BASE_URL) + alice, bob, _, chat = client.bootstrap() + scenarios = {'failover': failover, 'rolling': rolling} + if args.scenario in scenarios: + load = Load(client, alice, chat, args.workers) + report = scenarios[args.scenario](load, args) + report['history'] = verify_history(client, bob, chat, load.confirmed) + report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'} + else: + report = idempotency(client, alice, chat, args) + report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + main()