feat: add resource limits, per-service secrets and failover/rolling experiments

This commit is contained in:
2026-11-01 15:13:01 +03:00
parent 53035d475f
commit 210ff67c01
3 changed files with 269 additions and 9 deletions
+35 -9
View File
@@ -10,11 +10,20 @@ services:
- -ec
- |
cd /secrets
[ -s pg_password ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > pg_password
printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat pg_password)" > database_url
[ -s redis_password ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > redis_password
printf 'redis://:%s@redis:6379/0' "$$(cat redis_password)" > cache_url
chmod 0444 pg_password database_url redis_password cache_url
mkdir -p postgres redis api migrate
for name in pg_password:postgres redis_password:redis; do
file=$${name%%:*}; dir=$${name##*:}
[ ! -s "$$file" ] || mv "$$file" "$$dir/$$file"
[ -s "$$dir/$$file" ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file"
done
rm -f database_url cache_url
printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat postgres/pg_password)" > api/database_url
printf 'redis://:%s@redis:6379/0' "$$(cat redis/redis_password)" > api/cache_url
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$$(cat postgres/pg_password)" > migrate/database_url
chmod 0444 postgres/* redis/* api/* migrate/*
deploy:
resources:
limits: { cpus: "0.1", memory: 32M }
postgres:
image: postgres:17.2-alpine
@@ -24,7 +33,10 @@ services:
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
volumes:
- pgdata:/var/lib/postgresql/data
- secrets:/run/secrets:ro
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: postgres } }
deploy:
resources:
limits: { cpus: "1.0", memory: 512M }
healthcheck:
test: ["CMD", "pg_isready", "-U", "messenger", "-d", "messenger"]
interval: 5s
@@ -43,7 +55,10 @@ services:
- -c
- exec redis-server --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$(cat /run/secrets/redis_password)"
volumes:
- secrets:/run/secrets:ro
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: redis } }
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
healthcheck:
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep -q PONG"]
interval: 5s
@@ -62,7 +77,10 @@ services:
environment:
DATABASE_URL_FILE: /run/secrets/database_url
volumes:
- secrets:/run/secrets:ro
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: migrate } }
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
depends_on:
postgres:
condition: service_healthy
@@ -82,9 +100,11 @@ services:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
volumes:
- secrets:/run/secrets:ro
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } }
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "0.5", memory: 256M }
stop_grace_period: 10s
depends_on:
migrate:
@@ -99,6 +119,9 @@ services:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
deploy:
resources:
limits: { cpus: "0.25", memory: 64M }
depends_on:
api:
condition: service_healthy
@@ -117,6 +140,9 @@ services:
volumes:
- .:/work
- /var/run/docker.sock:/var/run/docker.sock
deploy:
resources:
limits: { cpus: "1.0", memory: 256M }
volumes:
pgdata:
+1
View File
@@ -1,5 +1,6 @@
global
log stdout format raw local0 info
maxconn 2000
defaults
mode http
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
def docker(*args):
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
def api_containers():
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
def describe(cid):
return docker('inspect', '-f', '{{.Name}} {{.State.Status}} image={{slice .Image 7 19}}', cid)
def wait_healthy(cid, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if docker('inspect', '-f', '{{.State.Health.Status}}', cid) == 'healthy':
return
time.sleep(0.2)
raise SystemExit(cid + ' did not become healthy')
class Load:
def __init__(self, client, actor, chat, workers):
self.client, self.actor, self.chat = client, actor, chat
self.workers = workers
self.lock = threading.Lock()
self.events = []
self.confirmed = []
self.stop_flag = threading.Event()
self.threads = [threading.Thread(target=self.worker, args=(i,)) for i in range(workers)]
self.started = None
def worker(self, index):
local = Client(BASE_URL, timeout=2)
key = uuid.uuid4().hex
while not self.stop_flag.is_set():
try:
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
with self.lock:
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
self.confirmed.append(response['json'])
key = uuid.uuid4().hex
except Failure:
with self.lock:
self.events.append((time.monotonic(), False, None))
time.sleep(0.05)
def __enter__(self):
self.started = time.monotonic()
for thread in self.threads:
thread.start()
return self
def __exit__(self, *exc):
self.stop_flag.set()
for thread in self.threads:
thread.join()
def window(self, start, end):
with self.lock:
events = [e for e in self.events if start <= e[0] < end]
ok = sorted(e[0] for e in events if e[1])
points = [start, *ok, end]
return {
'seconds': round(end - start, 3),
'successful_writes': len(ok),
'failed_attempts': sum(1 for e in events if not e[1]),
'successful_rps': round(len(ok) / (end - start), 1),
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
'instances': sorted({e[2] for e in events if e[1]}),
}
def last_seen(self, instance, start, end):
with self.lock:
times = [e[0] for e in self.events if e[1] and e[2] == instance and start <= e[0] < end]
return max(times) if times else None
def first_seen_after(self, instance, after):
with self.lock:
times = [e[0] for e in self.events if e[1] and e[2] == instance and e[0] >= after]
return min(times) if times else None
def verify_history(client, actor, chat, confirmed):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
items = []
while True:
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
items.extend(page['items'])
if page['next_cursor'] is None:
break
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
texts = [m['text'] for m in items]
stored = {m['id']: m for m in items}
return {
'confirmed': len(confirmed),
'stored': len(items),
'lost_confirmed': sum(1 for m in confirmed if stored.get(m['id']) != m),
'duplicates': len(texts) - len(set(texts)),
}
def failover(load, args):
results = []
with load:
time.sleep(args.warmup)
for cid in api_containers():
t0 = time.monotonic()
docker('kill' if args.kill else 'stop', cid)
stopped = time.monotonic()
time.sleep(args.outage)
outage = load.window(t0, time.monotonic())
restore = time.monotonic()
docker('start', cid)
wait_healthy(cid)
healthy = time.monotonic()
time.sleep(args.warmup)
rejoined = load.first_seen_after(cid, restore)
last = load.last_seen(cid, t0, restore)
results.append({
'container': describe(cid),
'action': 'docker kill (SIGKILL)' if args.kill else 'docker stop (SIGTERM, graceful)',
'before': load.window(t0 - args.warmup, t0),
'stop_command_seconds': round(stopped - t0, 3),
'target_last_success_after_stop_seconds': round(last - t0, 3) if last else None,
'outage_window': outage,
'restore_until_healthy_seconds': round(healthy - restore, 3),
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
'after_restore': load.window(healthy, time.monotonic()),
})
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
def rolling(load, args):
old = api_containers()
before = [describe(cid) for cid in old]
steps = []
with load:
time.sleep(args.warmup)
t_start = time.monotonic()
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
for cid in old:
t0 = time.monotonic()
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(old) + 1), '--wait', 'api')
added = time.monotonic()
docker('stop', cid)
docker('rm', cid)
steps.append({'replaced': cid, 'new_replica_ready_seconds': round(added - t0, 3), 'step_seconds': round(time.monotonic() - t0, 3)})
time.sleep(args.warmup)
total = load.window(t_start, time.monotonic())
return {
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
'before': before,
'after': [describe(cid) for cid in api_containers()],
'steps': steps,
'during_update': total,
}
def idempotency(client, actor, chat, args):
key = uuid.uuid4().hex
text = 'parallel ' + key
def send(_):
local = Client(BASE_URL, timeout=10)
response = local.message(actor, chat, text, key=key)
return response['json']['id'], response['headers']['X-Instance-Id']
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
results = list(pool.map(send, range(args.parallel)))
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
stored = [m for m in client.call('GET', path, actor=actor, schema='MessagePage')['json']['items'] if m['text'] == text]
return {
'scenario': 'parallel POST with one Idempotency-Key through LB',
'requests': args.parallel,
'all_201': True,
'distinct_message_ids': len({r[0] for r in results}),
'instances': sorted({r[1] for r in results}),
'messages_in_history': len(stored),
}
def main():
parser = argparse.ArgumentParser(description='Lab3 level-4 experiments under load through LB')
parser.add_argument('scenario', choices=['failover', 'rolling', 'idempotency'])
parser.add_argument('--workers', type=int, default=4)
parser.add_argument('--warmup', type=float, default=5)
parser.add_argument('--outage', type=float, default=10)
parser.add_argument('--parallel', type=int, default=20)
parser.add_argument('--kill', action='store_true')
parser.add_argument('--output')
args = parser.parse_args()
client = Client(BASE_URL)
alice, bob, _, chat = client.bootstrap()
scenarios = {'failover': failover, 'rolling': rolling}
if args.scenario in scenarios:
load = Load(client, alice, chat, args.workers)
report = scenarios[args.scenario](load, args)
report['history'] = verify_history(client, bob, chat, load.confirmed)
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
else:
report = idempotency(client, alice, chat, args)
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(text + '\n')
print(text)
if __name__ == '__main__':
main()