forked from mtusi2026/mtusi2026lab3
feat: add evidence collection, experiment results and report
This commit is contained in:
+27
-5
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -57,12 +58,12 @@ class Load:
|
||||
try:
|
||||
response = local.message(self.actor, self.chat, 'load ' + key, key=key)
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id']))
|
||||
self.events.append((time.monotonic(), True, response['headers']['X-Instance-Id'], response['elapsed']))
|
||||
self.confirmed.append(response['json'])
|
||||
key = uuid.uuid4().hex
|
||||
except Failure:
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), False, None))
|
||||
self.events.append((time.monotonic(), False, None, None))
|
||||
time.sleep(0.05)
|
||||
|
||||
def __enter__(self):
|
||||
@@ -80,12 +81,19 @@ class Load:
|
||||
with self.lock:
|
||||
events = [e for e in self.events if start <= e[0] < end]
|
||||
ok = sorted(e[0] for e in events if e[1])
|
||||
latencies = sorted(e[3] for e in events if e[1])
|
||||
points = [start, *ok, end]
|
||||
|
||||
def percentile(p):
|
||||
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 1) if latencies else None
|
||||
|
||||
return {
|
||||
'seconds': round(end - start, 3),
|
||||
'successful_writes': len(ok),
|
||||
'failed_attempts': sum(1 for e in events if not e[1]),
|
||||
'failed_attempts': len(events) - len(ok),
|
||||
'error_ratio': round((len(events) - len(ok)) / len(events), 4) if events else None,
|
||||
'successful_rps': round(len(ok) / (end - start), 1),
|
||||
'latency_ms_successful': {'p50': percentile(0.5), 'p95': percentile(0.95), 'p99': percentile(0.99)},
|
||||
'max_write_gap_seconds': round(max(b - a for a, b in zip(points, points[1:])), 3),
|
||||
'instances': sorted({e[2] for e in events if e[1]}),
|
||||
}
|
||||
@@ -120,10 +128,20 @@ def verify_history(client, actor, chat, confirmed):
|
||||
}
|
||||
|
||||
|
||||
def resources():
|
||||
ids = docker('compose', 'ps', '-q').split()
|
||||
return docker('stats', '--no-stream', '--format', '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}', *ids).splitlines()
|
||||
|
||||
|
||||
def messages_in_db():
|
||||
return int(docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT count(*) FROM messages'))
|
||||
|
||||
|
||||
def failover(load, args):
|
||||
results = []
|
||||
with load:
|
||||
time.sleep(args.warmup)
|
||||
usage = resources()
|
||||
for cid in api_containers():
|
||||
t0 = time.monotonic()
|
||||
docker('kill' if args.kill else 'stop', cid)
|
||||
@@ -148,7 +166,7 @@ def failover(load, args):
|
||||
'restore_until_serving_seconds': round(rejoined - restore, 3) if rejoined else None,
|
||||
'after_restore': load.window(healthy, time.monotonic()),
|
||||
})
|
||||
return {'scenario': 'sequential failover of each api replica', 'replicas': results}
|
||||
return {'scenario': 'sequential failover of each api replica', 'resources_under_load': usage, 'replicas': results}
|
||||
|
||||
|
||||
def rolling(load, args):
|
||||
@@ -157,6 +175,7 @@ def rolling(load, args):
|
||||
steps = []
|
||||
with load:
|
||||
time.sleep(args.warmup)
|
||||
usage = resources()
|
||||
t_start = time.monotonic()
|
||||
docker('compose', 'run', '--rm', '--no-deps', 'migrate')
|
||||
for cid in old:
|
||||
@@ -170,6 +189,7 @@ def rolling(load, args):
|
||||
total = load.window(t_start, time.monotonic())
|
||||
return {
|
||||
'scenario': 'rolling replacement: migrate once, then add new replica, wait healthy, stop old one',
|
||||
'resources_under_load': usage,
|
||||
'before': before,
|
||||
'after': [describe(cid) for cid in api_containers()],
|
||||
'steps': steps,
|
||||
@@ -216,9 +236,11 @@ def main():
|
||||
scenarios = {'failover': failover, 'rolling': rolling}
|
||||
if args.scenario in scenarios:
|
||||
load = Load(client, alice, chat, args.workers)
|
||||
dataset = messages_in_db()
|
||||
report = scenarios[args.scenario](load, args)
|
||||
report['dataset'] = {'messages_in_db_before': dataset, 'messages_in_db_after': messages_in_db()}
|
||||
report['history'] = verify_history(client, bob, chat, load.confirmed)
|
||||
report['load'] = {'workers': args.workers, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
|
||||
report['load'] = {'workers': args.workers, 'model': 'closed loop, POST message', 'warmup_seconds': args.warmup, 'client_timeout_seconds': 2, 'retry': 'same Idempotency-Key until success'}
|
||||
else:
|
||||
report = idempotency(client, alice, chat, args)
|
||||
report['date'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
Reference in New Issue
Block a user