feat: add evidence collection, experiment results and report

This commit is contained in:
2026-11-01 17:51:45 +03:00
parent 8f8c89cb19
commit 252ebd017a
14 changed files with 1109 additions and 36 deletions
+27 -5
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
import json
import math
import os
import subprocess
import sys
@@ -57,12 +58,12 @@ class Load:
try:
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
with self.lock:
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'], response['elapsed']))
self.confirmed.append(response['json'])
key = uuid.uuid4().hex
except Failure:
with self.lock:
self.events.append((time.monotonic(), False, None))
self.events.append((time.monotonic(), False, None, None))
time.sleep(0.05)
def __enter__(self):
@@ -80,12 +81,19 @@ class Load:
with self.lock:
events = [e for e in self.events if start <= e[0] < end]
ok = sorted(e[0] for e in events if e[1])
latencies = sorted(e[3] for e in events if e[1])
points = [start, *ok, end]
def percentile(p):
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 1) if latencies else None
return {
'seconds': round(end - start, 3),
'successful_writes': len(ok),
'failed_attempts': sum(1 for e in events if not e[1]),
'failed_attempts': len(events) - len(ok),
'error_ratio': round((len(events) - len(ok)) / len(events), 4) if events else None,
'successful_rps': round(len(ok) / (end - start), 1),
'latency_ms_successful': {'p50': percentile(0.5), 'p95': percentile(0.95), 'p99': percentile(0.99)},
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
'instances': sorted({e[2] for e in events if e[1]}),
}
@@ -120,10 +128,20 @@ def verify_history(client, actor, chat, confirmed):
}
def resources():
ids = docker('compose', 'ps', '-q').split()
return docker('stats', '--no-stream', '--format', '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}', *ids).splitlines()
def messages_in_db():
return int(docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT count(*) FROM messages'))
def failover(load, args):
results = []
with load:
time.sleep(args.warmup)
usage = resources()
for cid in api_containers():
t0 = time.monotonic()
docker('kill' if args.kill else 'stop', cid)
@@ -148,7 +166,7 @@ def failover(load, args):
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
'after_restore': load.window(healthy, time.monotonic()),
})
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
return {'scenario': 'sequential failover of each api replica', 'resources_under_load': usage, 'replicas': results}
def rolling(load, args):
@@ -157,6 +175,7 @@ def rolling(load, args):
steps = []
with load:
time.sleep(args.warmup)
usage = resources()
t_start = time.monotonic()
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
for cid in old:
@@ -170,6 +189,7 @@ def rolling(load, args):
total = load.window(t_start, time.monotonic())
return {
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
'resources_under_load': usage,
'before': before,
'after': [describe(cid) for cid in api_containers()],
'steps': steps,
@@ -216,9 +236,11 @@ def main():
scenarios = {'failover': failover, 'rolling': rolling}
if args.scenario in scenarios:
load = Load(client, alice, chat, args.workers)
dataset = messages_in_db()
report = scenarios[args.scenario](load, args)
report['dataset'] = {'messages_in_db_before': dataset, 'messages_in_db_after': messages_in_db()}
report['history'] = verify_history(client, bob, chat, load.confirmed)
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
report['load'] = {'workers': args.workers, 'model': 'closed loop, POST message', 'warmup_seconds': args.warmup, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
else:
report = idempotency(client, alice, chat, args)
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())