forked from mtusi2026/mtusi2026lab3
feat: add evidence collection, experiment results and report
This commit is contained in:
@@ -38,11 +38,12 @@ echo 'container.txt'
|
||||
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
|
||||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)"
|
||||
echo '## container_name не задан (одно имя сервиса масштабируется)'
|
||||
run sh -c "docker compose config | grep -c container_name"
|
||||
run sh -c "docker compose config | grep -c container_name || true"
|
||||
echo '## Секреты: каждый сервис видит только свои файлы'
|
||||
for service in postgres redis api lb; do
|
||||
for service in postgres redis api; do
|
||||
run docker compose exec -T "$service" ls /run/secrets
|
||||
done
|
||||
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
|
||||
run docker compose exec -T lb haproxy -v
|
||||
run docker compose exec -T postgres postgres --version
|
||||
run docker compose exec -T redis redis-server --version
|
||||
@@ -120,7 +121,7 @@ echo 'failures.txt'
|
||||
sleep 3
|
||||
echo '## 2. Redis снова доступен: logout отзывает токен на всех репликах'
|
||||
run python3 scripts/probe.py logout
|
||||
echo '## 3. PostgreSQL остановлен: readiness реплик 503 → LB исключает их и сам отвечает 503 без внутренних деталей'
|
||||
echo '## 3. PostgreSQL остановлен: live 200, запрос с токеном — 503 без внутренних деталей, аноним — 401'
|
||||
run docker compose stop postgres
|
||||
run python3 scripts/probe.py status GET /health/live
|
||||
run python3 scripts/probe.py status GET /api/v1/chats 'Bearer unknown-token'
|
||||
@@ -132,7 +133,25 @@ echo 'failures.txt'
|
||||
run python3 scripts/probe.py status GET /api/v1/users/me
|
||||
run docker compose start api
|
||||
run python3 scripts/probe.py ready
|
||||
echo '## 5. Expiry: реплики пересозданы с SESSION_TTL_SECONDS=2'
|
||||
echo '## 5. Живая, но неготовая реплика: PostgreSQL отклоняет подключения только от одной API (pg_hba reject)'
|
||||
target=$(docker compose ps -q api | head -1 | cut -c1-12)
|
||||
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$target")
|
||||
hba=/var/lib/postgresql/data/pg_hba.conf
|
||||
echo "target=$target ip=$ip"
|
||||
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba"
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null
|
||||
sleep 4
|
||||
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')"
|
||||
run python3 scripts/probe.py instances 10
|
||||
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]'
|
||||
echo
|
||||
docker compose exec -T postgres sh -c "cp /tmp/pg_hba.bak $hba && chown postgres:postgres $hba && chmod 600 $hba"
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf()" >/dev/null
|
||||
sleep 4
|
||||
echo '### доступ возвращён: реплика снова готова и получает запросы'
|
||||
run python3 scripts/probe.py instances 6
|
||||
echo '## 6. Expiry: реплики пересозданы с SESSION_TTL_SECONDS=2'
|
||||
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --wait api >/dev/null 2>&1
|
||||
run sh -c "docker exec \$(docker compose ps -q api | head -1) printenv SESSION_TTL_SECONDS"
|
||||
run python3 scripts/probe.py expiry 2
|
||||
|
||||
+27
-5
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -57,12 +58,12 @@ class Load:
|
||||
try:
|
||||
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
|
||||
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'], response['elapsed']))
|
||||
self.confirmed.append(response['json'])
|
||||
key = uuid.uuid4().hex
|
||||
except Failure:
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), False, None))
|
||||
self.events.append((time.monotonic(), False, None, None))
|
||||
time.sleep(0.05)
|
||||
|
||||
def __enter__(self):
|
||||
@@ -80,12 +81,19 @@ class Load:
|
||||
with self.lock:
|
||||
events = [e for e in self.events if start <= e[0] < end]
|
||||
ok = sorted(e[0] for e in events if e[1])
|
||||
latencies = sorted(e[3] for e in events if e[1])
|
||||
points = [start, *ok, end]
|
||||
|
||||
def percentile(p):
|
||||
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 1) if latencies else None
|
||||
|
||||
return {
|
||||
'seconds': round(end - start, 3),
|
||||
'successful_writes': len(ok),
|
||||
'failed_attempts': sum(1 for e in events if not e[1]),
|
||||
'failed_attempts': len(events) - len(ok),
|
||||
'error_ratio': round((len(events) - len(ok)) / len(events), 4) if events else None,
|
||||
'successful_rps': round(len(ok) / (end - start), 1),
|
||||
'latency_ms_successful': {'p50': percentile(0.5), 'p95': percentile(0.95), 'p99': percentile(0.99)},
|
||||
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
|
||||
'instances': sorted({e[2] for e in events if e[1]}),
|
||||
}
|
||||
@@ -120,10 +128,20 @@ def verify_history(client, actor, chat, confirmed):
|
||||
}
|
||||
|
||||
|
||||
def resources():
|
||||
ids = docker('compose', 'ps', '-q').split()
|
||||
return docker('stats', '--no-stream', '--format', '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}', *ids).splitlines()
|
||||
|
||||
|
||||
def messages_in_db():
|
||||
return int(docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT count(*) FROM messages'))
|
||||
|
||||
|
||||
def failover(load, args):
|
||||
results = []
|
||||
with load:
|
||||
time.sleep(args.warmup)
|
||||
usage = resources()
|
||||
for cid in api_containers():
|
||||
t0 = time.monotonic()
|
||||
docker('kill' if args.kill else 'stop', cid)
|
||||
@@ -148,7 +166,7 @@ def failover(load, args):
|
||||
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
|
||||
'after_restore': load.window(healthy, time.monotonic()),
|
||||
})
|
||||
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
|
||||
return {'scenario': 'sequential failover of each api replica', 'resources_under_load': usage, 'replicas': results}
|
||||
|
||||
|
||||
def rolling(load, args):
|
||||
@@ -157,6 +175,7 @@ def rolling(load, args):
|
||||
steps = []
|
||||
with load:
|
||||
time.sleep(args.warmup)
|
||||
usage = resources()
|
||||
t_start = time.monotonic()
|
||||
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
|
||||
for cid in old:
|
||||
@@ -170,6 +189,7 @@ def rolling(load, args):
|
||||
total = load.window(t_start, time.monotonic())
|
||||
return {
|
||||
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
|
||||
'resources_under_load': usage,
|
||||
'before': before,
|
||||
'after': [describe(cid) for cid in api_containers()],
|
||||
'steps': steps,
|
||||
@@ -216,9 +236,11 @@ def main():
|
||||
scenarios = {'failover': failover, 'rolling': rolling}
|
||||
if args.scenario in scenarios:
|
||||
load = Load(client, alice, chat, args.workers)
|
||||
dataset = messages_in_db()
|
||||
report = scenarios[args.scenario](load, args)
|
||||
report['dataset'] = {'messages_in_db_before': dataset, 'messages_in_db_after': messages_in_db()}
|
||||
report['history'] = verify_history(client, bob, chat, load.confirmed)
|
||||
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
|
||||
report['load'] = {'workers': args.workers, 'model': 'closed loop, POST message', 'warmup_seconds': args.warmup, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
|
||||
else:
|
||||
report = idempotency(client, alice, chat, args)
|
||||
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
+37
-13
@@ -11,17 +11,45 @@ import uuid
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None):
|
||||
def call(method, path, body=None, headers=None, with_headers=False):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
content_type = {'Content-Type': 'application/json'} if data else {}
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method, headers={**content_type, **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status, json.loads(response.read() or b'null')
|
||||
result = response.status, json.loads(response.read() or b'null'), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, json.loads(error.read() or b'null')
|
||||
result = error.code, json.loads(error.read() or b'null'), error.headers
|
||||
except OSError as error:
|
||||
return 0, str(error)
|
||||
result = 0, str(error), {}
|
||||
return result if with_headers else result[:2]
|
||||
|
||||
|
||||
def login():
|
||||
username = 'probe_' + uuid.uuid4().hex[:8]
|
||||
password = 'Probe-check-' + uuid.uuid4().hex
|
||||
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Probe', 'password': password})[0])
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
if code != 201:
|
||||
sys.exit('login failed: %s %s' % (code, session))
|
||||
print('login ->', code, 'expires_at:', session['expires_at'])
|
||||
return {'Authorization': 'Bearer ' + session['token']}
|
||||
|
||||
|
||||
def instances(count):
|
||||
auth = login()
|
||||
for _ in range(count):
|
||||
code, _, headers = call('GET', '/api/v1/users/me', headers=auth, with_headers=True)
|
||||
print('GET /api/v1/users/me ->', code, 'X-Instance-Id:', headers.get('X-Instance-Id'))
|
||||
|
||||
|
||||
def logout():
|
||||
auth = login()
|
||||
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
code, body = call('DELETE', '/api/v1/auth/sessions/current', headers=auth)
|
||||
print('DELETE /api/v1/auth/sessions/current ->', code, json.dumps(body, ensure_ascii=False) if body else '')
|
||||
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
|
||||
|
||||
def ready(timeout=90):
|
||||
@@ -40,15 +68,7 @@ def status(method, path, authorization=None):
|
||||
|
||||
|
||||
def expiry(ttl):
|
||||
username = 'expiry_' + uuid.uuid4().hex[:8]
|
||||
password = 'Expiry-check-' + uuid.uuid4().hex
|
||||
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Expiry', 'password': password})[0])
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
if code != 201:
|
||||
sys.exit('login failed: %s %s' % (code, session))
|
||||
print('login ->', code, 'expires_at:', session['expires_at'])
|
||||
auth = {'Authorization': 'Bearer ' + session['token']}
|
||||
auth = login()
|
||||
print('GET /api/v1/users/me (fresh) ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
time.sleep(ttl + 1)
|
||||
print('GET /api/v1/users/me (after %ds) ->' % (ttl + 1), call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
@@ -62,3 +82,7 @@ if __name__ == '__main__':
|
||||
status(*sys.argv[2:5])
|
||||
elif command == 'expiry':
|
||||
expiry(int(sys.argv[2]))
|
||||
elif command == 'instances':
|
||||
instances(int(sys.argv[2]))
|
||||
elif command == 'logout':
|
||||
logout()
|
||||
|
||||
Reference in New Issue
Block a user