forked from mtusi2026/mtusi2026lab3
208 lines
10 KiB
Python
208 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable read/write load with explicit stop/update and sanitized evidence."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from client import Client, Failure, arguments, check, run
|
|
import manage
|
|
|
|
|
|
def whole_history(client, actor, chat):
|
|
items, cursor = [], None
|
|
while True:
|
|
path = '/api/v1/chats/%s/messages?limit=100' % chat['id']
|
|
if cursor:
|
|
path += '&cursor=' + quote(cursor, safe='')
|
|
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
|
items.extend(page['items'])
|
|
cursor = page['next_cursor']
|
|
if cursor is None:
|
|
return items
|
|
|
|
|
|
def scenario(args, service=None):
|
|
c = Client(args.base_url, args.ca_file, timeout=2)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
seed_key = uuid.uuid4().hex
|
|
seed = c.message(alice, chat, 'До воздействия', key=seed_key)['json']
|
|
events, samples, confirmations = [], [], {}
|
|
lock = threading.Lock()
|
|
started = time.monotonic()
|
|
end = threading.Event()
|
|
workload_started = threading.Barrier(5)
|
|
|
|
def load(worker_id):
|
|
local = Client(args.base_url, args.ca_file, timeout=2)
|
|
key = uuid.uuid4().hex
|
|
workload_started.wait()
|
|
while not end.is_set():
|
|
text = 'experiment-%d-%s' % (worker_id, key)
|
|
start = time.monotonic()
|
|
try:
|
|
reply = local.message(alice, chat, text, key=key)
|
|
with lock:
|
|
confirmations[text] = reply['json']
|
|
event = {'kind': 'write', 'ok': True, 'instance': reply['headers']['X-Instance-Id']}
|
|
key = uuid.uuid4().hex
|
|
except Failure:
|
|
event = {'kind': 'write', 'ok': False}
|
|
# Keep the same body/key after uncertain response.
|
|
event.update(at=time.monotonic() - started, elapsed=time.monotonic() - start)
|
|
with lock:
|
|
events.append(event)
|
|
start = time.monotonic()
|
|
try:
|
|
local.call('GET', '/api/v1/chats/%s/messages?limit=1' % chat['id'], actor=bob, schema='MessagePage')
|
|
ok = True
|
|
except Failure:
|
|
ok = False
|
|
with lock:
|
|
events.append({'kind': 'read', 'ok': ok, 'at': time.monotonic()-started,
|
|
'elapsed': time.monotonic()-start})
|
|
|
|
def stats():
|
|
ids = [x for name in (*manage.SERVICES, 'nginx', 'postgres', 'redis') for x in manage.containers(name)]
|
|
while not end.is_set():
|
|
proc = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids],
|
|
text=True, capture_output=True, timeout=10)
|
|
with lock:
|
|
samples.append({'at': round(time.monotonic()-started, 3),
|
|
'containers': [json.loads(x) for x in proc.stdout.splitlines() if x]})
|
|
end.wait(1)
|
|
|
|
def status():
|
|
return [{'service': s, 'id': data['Id'][:12], 'instance': data['Config']['Hostname'],
|
|
'running': data['State']['Running'], 'image': data['Image']}
|
|
for s in manage.SERVICES for data in map(manage.inspect, manage.containers(s))]
|
|
|
|
before = status()
|
|
monitor = threading.Thread(target=stats, daemon=True)
|
|
action_at = action_done = None
|
|
hook_output = ''
|
|
action_error = None
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
jobs = [pool.submit(load, i) for i in range(4)]
|
|
workload_started.wait()
|
|
monitor.start()
|
|
try:
|
|
time.sleep(3) # Same warm-up for each scenario.
|
|
action_at = time.monotonic()-started
|
|
if service:
|
|
container = manage.containers(service)[0]
|
|
target = manage.inspect(container)['Config']['Hostname']
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
|
proc = subprocess.run([str(ROOT/'scripts/stop-one.sh')], cwd=ROOT, env=env,
|
|
capture_output=True, text=True, timeout=10)
|
|
else:
|
|
proc = subprocess.run([str(ROOT/'scripts/rolling-update.sh')], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=180)
|
|
action_done = time.monotonic()-started
|
|
hook_output = proc.stdout + proc.stderr
|
|
check(proc.returncode == 0, 'action failed; inspect sanitized action log')
|
|
time.sleep(12 if service else 5)
|
|
except Exception as exc:
|
|
action_error = exc
|
|
finally:
|
|
end.set()
|
|
for job in jobs:
|
|
job.result()
|
|
monitor.join(timeout=15)
|
|
after = status()
|
|
history = whole_history(c, bob, chat)
|
|
by_text = {}
|
|
for item in history:
|
|
by_text.setdefault(item['text'], []).append(item)
|
|
missing = [text for text, msg in confirmations.items() if by_text.get(text) != [msg]]
|
|
duplicate_texts = sum(len(items)-1 for items in by_text.values())
|
|
check(by_text.get(seed['text']) == [seed], 'pre-action confirmed message lost')
|
|
check(c.message(alice, chat, seed['text'], key=seed_key)['json'] == seed, 'replay across action differs')
|
|
|
|
def summary(selected, elapsed):
|
|
successes = [e for e in selected if e['ok']]
|
|
latencies = sorted(e['elapsed'] * 1000 for e in successes)
|
|
def percentile(p):
|
|
return round(latencies[max(0, math.ceil(len(latencies)*p)-1)], 3) if latencies else None
|
|
return {'successful_requests': len(successes), 'failed_requests': len(selected)-len(successes),
|
|
'successful_rps': round(len(successes)/elapsed, 3) if elapsed else None,
|
|
'error_ratio': round(1-len(successes)/len(selected), 6) if selected else None,
|
|
'successful_latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)}}
|
|
|
|
elapsed = time.monotonic()-started
|
|
recovery = {}
|
|
for kind in ('read', 'write'):
|
|
times = [e['at']-action_at for e in events if e['ok'] and e['kind']==kind and e['at']>=action_done]
|
|
recovery[kind] = round(min(times), 3) if times else None
|
|
success_gaps = {}
|
|
for kind in ('read', 'write'):
|
|
timeline = sorted(e['at'] for e in events if e['ok'] and e['kind']==kind)
|
|
# Include boundaries of the action, so silence during the entire action
|
|
# cannot be hidden by success only before or after it.
|
|
timeline = [action_at] + [t for t in timeline if action_at<=t<=(action_done or elapsed)] + [action_done or elapsed]
|
|
success_gaps[kind] = round(max(b-a for a,b in zip(timeline,timeline[1:])), 3)
|
|
report = {'scenario': 'stop-'+service if service else 'rolling-update', 'concurrency': 4,
|
|
'warmup_seconds': 3, 'elapsed_seconds': round(elapsed, 3),
|
|
'action_start_seconds': round(action_at, 3), 'action_done_seconds': round(action_done, 3) if action_done else None,
|
|
'first_success_after_action_from_action_start_seconds': recovery,
|
|
'before': before, 'after': after,
|
|
'http': summary(events, elapsed),
|
|
'read': summary([e for e in events if e['kind']=='read'], elapsed),
|
|
'write': summary([e for e in events if e['kind']=='write'], elapsed),
|
|
'during_action': summary([e for e in events if action_at<=e['at']<=(action_done or elapsed)], (action_done or elapsed)-action_at),
|
|
'confirmed_messages': len(confirmations), 'history_messages': len(history),
|
|
'missing_or_changed_confirmations': len(missing), 'duplicate_texts': duplicate_texts,
|
|
'resources': samples, 'maximum_gap_between_successes_during_action_seconds': success_gaps,
|
|
'note': 'Closed-loop, four writers alternate POST and GET. Percentiles cover successful HTTP requests only; retries preserve key/body. Recovery is first success after hook completion measured from hook start; not an outage duration. One Docker host; no node/DB/LB HA claim.'}
|
|
name = report['scenario']
|
|
(ROOT/'evidence'/('%s.json' % name)).write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
|
(ROOT/'evidence'/('%s-action.txt' % name)).write_text(hook_output)
|
|
check(action_error is None, 'action failed: %s' % action_error)
|
|
check(not missing and not duplicate_texts, 'lost/duplicated messages')
|
|
if service:
|
|
check(all(x is not None and x<=10 for x in recovery.values()), 'read/write recovery exceeded 10s')
|
|
target_entry = next(e for e in after if e['service']==service)
|
|
check(not target_entry['running'], 'target replica is still running')
|
|
recent_instances = {e.get('instance') for e in events if e['kind']=='write' and e['ok'] and e['at']>action_done+3}
|
|
check(target_entry['instance'] not in recent_instances, 'stopped target still serves writes')
|
|
else:
|
|
check(all(x<=3 for x in success_gaps.values()), 'rolling read/write availability gap exceeded 3 seconds')
|
|
for kind in ('read', 'write'):
|
|
for left in range(int(action_at), int(action_done), 3):
|
|
right = min(left+3, action_done)
|
|
check(any(e['ok'] and e['kind']==kind and left<=e['at']<=right for e in events),
|
|
'no successful %s during rolling window' % kind)
|
|
check(all(a['id']!=b['id'] for a,b in zip(before, after)), 'not every replica was replaced')
|
|
check(all(a['image']!=b['image'] for a,b in zip(before, after)), 'new image was not deployed')
|
|
print(json.dumps({k:v for k,v in report.items() if k not in ('resources','before','after','note')}, ensure_ascii=False))
|
|
print('PASS:', name)
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--scenario', choices=('failover', 'rolling'), required=True)
|
|
args = arguments(__doc__, configure)
|
|
try:
|
|
if args.scenario == 'failover':
|
|
for service in manage.SERVICES:
|
|
manage.restore()
|
|
scenario(args, service)
|
|
else:
|
|
scenario(args)
|
|
finally:
|
|
manage.restore()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|