feat: add evidence collection, experiment results and report

This commit is contained in:
2026-11-01 17:51:45 +03:00
parent 8f8c89cb19
commit 252ebd017a
14 changed files with 1109 additions and 36 deletions
+23 -4
View File
@@ -38,11 +38,12 @@ echo 'container.txt'
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -q)"
echo '## container_name не задан (одно имя сервиса масштабируется)'
run sh -c "docker compose config | grep -c container_name"
run sh -c "docker compose config | grep -c container_name || true"
echo '## Секреты: каждый сервис видит только свои файлы'
for service in postgres redis api lb; do
for service in postgres redis api; do
run docker compose exec -T "$service" ls /run/secrets
done
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
run docker compose exec -T lb haproxy -v
run docker compose exec -T postgres postgres --version
run docker compose exec -T redis redis-server --version
@@ -120,7 +121,7 @@ echo 'failures.txt'
sleep 3
echo '## 2. Redis снова доступен: logout отзывает токен на всех репликах'
run python3 scripts/probe.py logout
echo '## 3. PostgreSQL остановлен: readiness реплик 503 → LB исключает их и сам отвечает 503 без внутренних деталей'
echo '## 3. PostgreSQL остановлен: live 200, запрос с токеном — 503 без внутренних деталей, аноним — 401'
run docker compose stop postgres
run python3 scripts/probe.py status GET /health/live
run python3 scripts/probe.py status GET /api/v1/chats 'Bearer unknown-token'
@@ -132,7 +133,25 @@ echo 'failures.txt'
run python3 scripts/probe.py status GET /api/v1/users/me
run docker compose start api
run python3 scripts/probe.py ready
echo '## 5. Expiry: реплики пересозданы с SESSION_TTL_SECONDS=2'
echo '## 5. Живая, но неготовая реплика: PostgreSQL отклоняет подключения только от одной API (pg_hba reject)'
target=$(docker compose ps -q api | head -1 | cut -c1-12)
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$target")
hba=/var/lib/postgresql/data/pg_hba.conf
echo "target=$target ip=$ip"
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
docker compose exec -T postgres sh -c "cp $hba /tmp/pg_hba.bak && { echo 'host all all $ip/32 reject'; cat /tmp/pg_hba.bak; } > $hba && chown postgres:postgres $hba && chmod 600 $hba"
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '$ip'" >/dev/null
sleep 4
run docker exec "$target" node -e "let n=0;const g=p=>require('http').get('http://127.0.0.1:8080'+p,r=>{console.log(p,r.statusCode);r.resume();if(++n===2)process.exit(0)});g('/health/live');g('/health/ready')"
run python3 scripts/probe.py instances 10
docker compose logs --no-log-prefix --since "$since" lb 2>&1 | grep -E 'is DOWN|is UP' | grep -v '^\[WARNING\]'
echo
docker compose exec -T postgres sh -c "cp /tmp/pg_hba.bak $hba && chown postgres:postgres $hba && chmod 600 $hba"
docker compose exec -T postgres psql -U messenger -d messenger -tAc "SELECT pg_reload_conf()" >/dev/null
sleep 4
echo '### доступ возвращён: реплика снова готова и получает запросы'
run python3 scripts/probe.py instances 6
echo '## 6. Expiry: реплики пересозданы с SESSION_TTL_SECONDS=2'
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --wait api >/dev/null 2>&1
run sh -c "docker exec \$(docker compose ps -q api | head -1) printenv SESSION_TTL_SECONDS"
run python3 scripts/probe.py expiry 2
+27 -5
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
import json
import math
import os
import subprocess
import sys
@@ -57,12 +58,12 @@ class Load:
try:
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
with self.lock:
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'], response['elapsed']))
self.confirmed.append(response['json'])
key = uuid.uuid4().hex
except Failure:
with self.lock:
self.events.append((time.monotonic(), False, None))
self.events.append((time.monotonic(), False, None, None))
time.sleep(0.05)
def __enter__(self):
@@ -80,12 +81,19 @@ class Load:
with self.lock:
events = [e for e in self.events if start <= e[0] < end]
ok = sorted(e[0] for e in events if e[1])
latencies = sorted(e[3] for e in events if e[1])
points = [start, *ok, end]
def percentile(p):
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 1) if latencies else None
return {
'seconds': round(end - start, 3),
'successful_writes': len(ok),
'failed_attempts': sum(1 for e in events if not e[1]),
'failed_attempts': len(events) - len(ok),
'error_ratio': round((len(events) - len(ok)) / len(events), 4) if events else None,
'successful_rps': round(len(ok) / (end - start), 1),
'latency_ms_successful': {'p50': percentile(0.5), 'p95': percentile(0.95), 'p99': percentile(0.99)},
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
'instances': sorted({e[2] for e in events if e[1]}),
}
@@ -120,10 +128,20 @@ def verify_history(client, actor, chat, confirmed):
}
def resources():
ids = docker('compose', 'ps', '-q').split()
return docker('stats', '--no-stream', '--format', '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}', *ids).splitlines()
def messages_in_db():
return int(docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT count(*) FROM messages'))
def failover(load, args):
results = []
with load:
time.sleep(args.warmup)
usage = resources()
for cid in api_containers():
t0 = time.monotonic()
docker('kill' if args.kill else 'stop', cid)
@@ -148,7 +166,7 @@ def failover(load, args):
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
'after_restore': load.window(healthy, time.monotonic()),
})
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
return {'scenario': 'sequential failover of each api replica', 'resources_under_load': usage, 'replicas': results}
def rolling(load, args):
@@ -157,6 +175,7 @@ def rolling(load, args):
steps = []
with load:
time.sleep(args.warmup)
usage = resources()
t_start = time.monotonic()
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
for cid in old:
@@ -170,6 +189,7 @@ def rolling(load, args):
total = load.window(t_start, time.monotonic())
return {
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
'resources_under_load': usage,
'before': before,
'after': [describe(cid) for cid in api_containers()],
'steps': steps,
@@ -216,9 +236,11 @@ def main():
scenarios = {'failover': failover, 'rolling': rolling}
if args.scenario in scenarios:
load = Load(client, alice, chat, args.workers)
dataset = messages_in_db()
report = scenarios[args.scenario](load, args)
report['dataset'] = {'messages_in_db_before': dataset, 'messages_in_db_after': messages_in_db()}
report['history'] = verify_history(client, bob, chat, load.confirmed)
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
report['load'] = {'workers': args.workers, 'model': 'closed loop, POST message', 'warmup_seconds': args.warmup, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
else:
report = idempotency(client, alice, chat, args)
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+37 -13
View File
@@ -11,17 +11,45 @@ import uuid
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
def call(method, path, body=None, headers=None):
def call(method, path, body=None, headers=None, with_headers=False):
data = json.dumps(body).encode() if body is not None else None
content_type = {'Content-Type': 'application/json'} if data else {}
request = urllib.request.Request(BASE + path, data=data, method=method, headers={**content_type, **(headers or {})})
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status, json.loads(response.read() or b'null')
result = response.status, json.loads(response.read() or b'null'), response.headers
except urllib.error.HTTPError as error:
return error.code, json.loads(error.read() or b'null')
result = error.code, json.loads(error.read() or b'null'), error.headers
except OSError as error:
return 0, str(error)
result = 0, str(error), {}
return result if with_headers else result[:2]
def login():
username = 'probe_' + uuid.uuid4().hex[:8]
password = 'Probe-check-' + uuid.uuid4().hex
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Probe', 'password': password})[0])
credential = base64.b64encode((username + ':' + password).encode()).decode()
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
if code != 201:
sys.exit('login failed: %s %s' % (code, session))
print('login ->', code, 'expires_at:', session['expires_at'])
return {'Authorization': 'Bearer ' + session['token']}
def instances(count):
auth = login()
for _ in range(count):
code, _, headers = call('GET', '/api/v1/users/me', headers=auth, with_headers=True)
print('GET /api/v1/users/me ->', code, 'X-Instance-Id:', headers.get('X-Instance-Id'))
def logout():
auth = login()
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
code, body = call('DELETE', '/api/v1/auth/sessions/current', headers=auth)
print('DELETE /api/v1/auth/sessions/current ->', code, json.dumps(body, ensure_ascii=False) if body else '')
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
def ready(timeout=90):
@@ -40,15 +68,7 @@ def status(method, path, authorization=None):
def expiry(ttl):
username = 'expiry_' + uuid.uuid4().hex[:8]
password = 'Expiry-check-' + uuid.uuid4().hex
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Expiry', 'password': password})[0])
credential = base64.b64encode((username + ':' + password).encode()).decode()
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
if code != 201:
sys.exit('login failed: %s %s' % (code, session))
print('login ->', code, 'expires_at:', session['expires_at'])
auth = {'Authorization': 'Bearer ' + session['token']}
auth = login()
print('GET /api/v1/users/me (fresh) ->', call('GET', '/api/v1/users/me', headers=auth)[0])
time.sleep(ttl + 1)
print('GET /api/v1/users/me (after %ds) ->' % (ttl + 1), call('GET', '/api/v1/users/me', headers=auth)[0])
@@ -62,3 +82,7 @@ if __name__ == '__main__':
status(*sys.argv[2:5])
elif command == 'expiry':
expiry(int(sys.argv[2]))
elif command == 'instances':
instances(int(sys.argv[2]))
elif command == 'logout':
logout()