forked from mtusi2026/mtusi2026lab3
feat: add resource limits, per-service secrets and failover/rolling experiments
This commit is contained in:
+35
-9
@@ -10,11 +10,20 @@ services:
|
||||
- -ec
|
||||
- |
|
||||
cd /secrets
|
||||
[ -s pg_password ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > pg_password
|
||||
printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat pg_password)" > database_url
|
||||
[ -s redis_password ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > redis_password
|
||||
printf 'redis://:%s@redis:6379/0' "$$(cat redis_password)" > cache_url
|
||||
chmod 0444 pg_password database_url redis_password cache_url
|
||||
mkdir -p postgres redis api migrate
|
||||
for name in pg_password:postgres redis_password:redis; do
|
||||
file=$${name%%:*}; dir=$${name##*:}
|
||||
[ ! -s "$$file" ] || mv "$$file" "$$dir/$$file"
|
||||
[ -s "$$dir/$$file" ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file"
|
||||
done
|
||||
rm -f database_url cache_url
|
||||
printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat postgres/pg_password)" > api/database_url
|
||||
printf 'redis://:%s@redis:6379/0' "$$(cat redis/redis_password)" > api/cache_url
|
||||
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$$(cat postgres/pg_password)" > migrate/database_url
|
||||
chmod 0444 postgres/* redis/* api/* migrate/*
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.1", memory: 32M }
|
||||
|
||||
postgres:
|
||||
image: postgres:17.2-alpine
|
||||
@@ -24,7 +33,10 @@ services:
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
- secrets:/run/secrets:ro
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: postgres } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 512M }
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "messenger", "-d", "messenger"]
|
||||
interval: 5s
|
||||
@@ -43,7 +55,10 @@ services:
|
||||
- -c
|
||||
- exec redis-server --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$(cat /run/secrets/redis_password)"
|
||||
volumes:
|
||||
- secrets:/run/secrets:ro
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: redis } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep -q PONG"]
|
||||
interval: 5s
|
||||
@@ -62,7 +77,10 @@ services:
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
volumes:
|
||||
- secrets:/run/secrets:ro
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: migrate } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -82,9 +100,11 @@ services:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
volumes:
|
||||
- secrets:/run/secrets:ro
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } }
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
stop_grace_period: 10s
|
||||
depends_on:
|
||||
migrate:
|
||||
@@ -99,6 +119,9 @@ services:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 64M }
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
@@ -117,6 +140,9 @@ services:
|
||||
volumes:
|
||||
- .:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 256M }
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2000
|
||||
|
||||
defaults
|
||||
mode http
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
|
||||
|
||||
|
||||
def describe(cid):
|
||||
return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid)
|
||||
|
||||
|
||||
def wait_healthy(cid, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy':
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise SystemExit(cid + ' did not become healthy')
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, client, actor, chat, workers):
|
||||
self.client, self.actor, self.chat = client, actor, chat
|
||||
self.workers = workers
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.confirmed = []
|
||||
self.stop_flag = threading.Event()
|
||||
self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)]
|
||||
self.started = None
|
||||
|
||||
def worker(self, index):
|
||||
local = Client(BASE_URL, timeout=2)
|
||||
key = uuid.uuid4().hex
|
||||
while not self.stop_flag.is_set():
|
||||
try:
|
||||
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
|
||||
self.confirmed.append(response['json'])
|
||||
key = uuid.uuid4().hex
|
||||
except Failure:
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), False, None))
|
||||
time.sleep(0.05)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.stop_flag.set()
|
||||
for thread in self.threads:
|
||||
thread.join()
|
||||
|
||||
def window(self, start, end):
|
||||
with self.lock:
|
||||
events = [e for e in self.events if start <= e[0] < end]
|
||||
ok = sorted(e[0] for e in events if e[1])
|
||||
points = [start, *ok, end]
|
||||
return {
|
||||
'seconds': round(end - start, 3),
|
||||
'successful_writes': len(ok),
|
||||
'failed_attempts': sum(1 for e in events if not e[1]),
|
||||
'successful_rps': round(len(ok) / (end - start), 1),
|
||||
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
|
||||
'instances': sorted({e[2] for e in events if e[1]}),
|
||||
}
|
||||
|
||||
def last_seen(self, instance, start, end):
|
||||
with self.lock:
|
||||
times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end]
|
||||
return max(times) if times else None
|
||||
|
||||
def first_seen_after(self, instance, after):
|
||||
with self.lock:
|
||||
times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after]
|
||||
return min(times) if times else None
|
||||
|
||||
|
||||
def verify_history(client, actor, chat, confirmed):
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||||
items = []
|
||||
while True:
|
||||
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
items.extend(page['items'])
|
||||
if page['next_cursor'] is None:
|
||||
break
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
|
||||
texts = [m['text'] for m in items]
|
||||
stored = {m['id']: m for m in items}
|
||||
return {
|
||||
'confirmed': len(confirmed),
|
||||
'stored': len(items),
|
||||
'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m),
|
||||
'duplicates': len(texts) - len(set(texts)),
|
||||
}
|
||||
|
||||
|
||||
def failover(load, args):
|
||||
results = []
|
||||
with load:
|
||||
time.sleep(args.warmup)
|
||||
for cid in api_containers():
|
||||
t0 = time.monotonic()
|
||||
docker('kill' if args.kill else 'stop', cid)
|
||||
stopped = time.monotonic()
|
||||
time.sleep(args.outage)
|
||||
outage = load.window(t0, time.monotonic())
|
||||
restore = time.monotonic()
|
||||
docker('start', cid)
|
||||
wait_healthy(cid)
|
||||
healthy = time.monotonic()
|
||||
time.sleep(args.warmup)
|
||||
rejoined = load.first_seen_after(cid, restore)
|
||||
last = load.last_seen(cid, t0, restore)
|
||||
results.append({
|
||||
'container': describe(cid),
|
||||
'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)',
|
||||
'before': load.window(t0 - args.warmup, t0),
|
||||
'stop_command_seconds': round(stopped - t0, 3),
|
||||
'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None,
|
||||
'outage_window': outage,
|
||||
'restore_until_healthy_seconds': round(healthy - restore, 3),
|
||||
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
|
||||
'after_restore': load.window(healthy, time.monotonic()),
|
||||
})
|
||||
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
|
||||
|
||||
|
||||
def rolling(load, args):
|
||||
old = api_containers()
|
||||
before = [describe(cid) for cid in old]
|
||||
steps = []
|
||||
with load:
|
||||
time.sleep(args.warmup)
|
||||
t_start = time.monotonic()
|
||||
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
|
||||
for cid in old:
|
||||
t0 = time.monotonic()
|
||||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api')
|
||||
added = time.monotonic()
|
||||
docker('stop', cid)
|
||||
docker('rm', cid)
|
||||
steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)})
|
||||
time.sleep(args.warmup)
|
||||
total = load.window(t_start, time.monotonic())
|
||||
return {
|
||||
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
|
||||
'before': before,
|
||||
'after': [describe(cid) for cid in api_containers()],
|
||||
'steps': steps,
|
||||
'during_update': total,
|
||||
}
|
||||
|
||||
|
||||
def idempotency(client, actor, chat, args):
|
||||
key = uuid.uuid4().hex
|
||||
text = 'parallel ' + key
|
||||
|
||||
def send(_):
|
||||
local = Client(BASE_URL, timeout=10)
|
||||
response = local.message(actor, chat, text, key=key)
|
||||
return response['json']['id'], response['headers']['X-Instance-Id']
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
|
||||
results = list(pool.map(send, range(args.parallel)))
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||||
stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text]
|
||||
return {
|
||||
'scenario': 'parallel POST with one Idempotency-Key through LB',
|
||||
'requests': args.parallel,
|
||||
'all_201': True,
|
||||
'distinct_message_ids': len({r[0] for r in results}),
|
||||
'instances': sorted({r[1] for r in results}),
|
||||
'messages_in_history': len(stored),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB')
|
||||
parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency'])
|
||||
parser.add_argument('--workers', type=int, default=4)
|
||||
parser.add_argument('--warmup', type=float, default=5)
|
||||
parser.add_argument('--outage', type=float, default=10)
|
||||
parser.add_argument('--parallel', type=int, default=20)
|
||||
parser.add_argument('--kill', action='store_true')
|
||||
parser.add_argument('--output')
|
||||
args = parser.parse_args()
|
||||
|
||||
client = Client(BASE_URL)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
scenarios = {'failover': failover, 'rolling': rolling}
|
||||
if args.scenario in scenarios:
|
||||
load = Load(client, alice, chat, args.workers)
|
||||
report = scenarios[args.scenario](load, args)
|
||||
report['history'] = verify_history(client, bob, chat, load.confirmed)
|
||||
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
|
||||
else:
|
||||
report = idempotency(client, alice, chat, args)
|
||||
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(text + '\n')
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user