Files
mtusi2026lab3-fork/student-tests/experiments.py
T

208 lines
10 KiB
Python

#!/usr/bin/env python3
"""Repeatable read/write load with explicit stop/update and sanitized evidence."""
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, arguments, check, run
import manage
def whole_history(client, actor, chat):
items, cursor = [], None
while True:
path = '/api/v1/chats/%s/messages?limit=100' % chat['id']
if cursor:
path += '&cursor=' + quote(cursor, safe='')
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
items.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return items
def scenario(args, service=None):
c = Client(args.base_url, args.ca_file, timeout=2)
alice, bob, eve, chat = c.bootstrap()
seed_key = uuid.uuid4().hex
seed = c.message(alice, chat, 'До воздействия', key=seed_key)['json']
events, samples, confirmations = [], [], {}
lock = threading.Lock()
started = time.monotonic()
end = threading.Event()
workload_started = threading.Barrier(5)
def load(worker_id):
local = Client(args.base_url, args.ca_file, timeout=2)
key = uuid.uuid4().hex
workload_started.wait()
while not end.is_set():
text = 'experiment-%d-%s' % (worker_id, key)
start = time.monotonic()
try:
reply = local.message(alice, chat, text, key=key)
with lock:
confirmations[text] = reply['json']
event = {'kind': 'write', 'ok': True, 'instance': reply['headers']['X-Instance-Id']}
key = uuid.uuid4().hex
except Failure:
event = {'kind': 'write', 'ok': False}
# Keep the same body/key after uncertain response.
event.update(at=time.monotonic() - started, elapsed=time.monotonic() - start)
with lock:
events.append(event)
start = time.monotonic()
try:
local.call('GET', '/api/v1/chats/%s/messages?limit=1' % chat['id'], actor=bob, schema='MessagePage')
ok = True
except Failure:
ok = False
with lock:
events.append({'kind': 'read', 'ok': ok, 'at': time.monotonic()-started,
'elapsed': time.monotonic()-start})
def stats():
ids = [x for name in (*manage.SERVICES, 'nginx', 'postgres', 'redis') for x in manage.containers(name)]
while not end.is_set():
proc = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids],
text=True, capture_output=True, timeout=10)
with lock:
samples.append({'at': round(time.monotonic()-started, 3),
'containers': [json.loads(x) for x in proc.stdout.splitlines() if x]})
end.wait(1)
def status():
return [{'service': s, 'id': data['Id'][:12], 'instance': data['Config']['Hostname'],
'running': data['State']['Running'], 'image': data['Image']}
for s in manage.SERVICES for data in map(manage.inspect, manage.containers(s))]
before = status()
monitor = threading.Thread(target=stats, daemon=True)
action_at = action_done = None
hook_output = ''
action_error = None
with ThreadPoolExecutor(max_workers=4) as pool:
jobs = [pool.submit(load, i) for i in range(4)]
workload_started.wait()
monitor.start()
try:
time.sleep(3) # Same warm-up for each scenario.
action_at = time.monotonic()-started
if service:
container = manage.containers(service)[0]
target = manage.inspect(container)['Config']['Hostname']
env = dict(os.environ, TARGET_INSTANCE_ID=target)
proc = subprocess.run([str(ROOT/'scripts/stop-one.sh')], cwd=ROOT, env=env,
capture_output=True, text=True, timeout=10)
else:
proc = subprocess.run([str(ROOT/'scripts/rolling-update.sh')], cwd=ROOT,
capture_output=True, text=True, timeout=180)
action_done = time.monotonic()-started
hook_output = proc.stdout + proc.stderr
check(proc.returncode == 0, 'action failed; inspect sanitized action log')
time.sleep(12 if service else 5)
except Exception as exc:
action_error = exc
finally:
end.set()
for job in jobs:
job.result()
monitor.join(timeout=15)
after = status()
history = whole_history(c, bob, chat)
by_text = {}
for item in history:
by_text.setdefault(item['text'], []).append(item)
missing = [text for text, msg in confirmations.items() if by_text.get(text) != [msg]]
duplicate_texts = sum(len(items)-1 for items in by_text.values())
check(by_text.get(seed['text']) == [seed], 'pre-action confirmed message lost')
check(c.message(alice, chat, seed['text'], key=seed_key)['json'] == seed, 'replay across action differs')
def summary(selected, elapsed):
successes = [e for e in selected if e['ok']]
latencies = sorted(e['elapsed'] * 1000 for e in successes)
def percentile(p):
return round(latencies[max(0, math.ceil(len(latencies)*p)-1)], 3) if latencies else None
return {'successful_requests': len(successes), 'failed_requests': len(selected)-len(successes),
'successful_rps': round(len(successes)/elapsed, 3) if elapsed else None,
'error_ratio': round(1-len(successes)/len(selected), 6) if selected else None,
'successful_latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)}}
elapsed = time.monotonic()-started
recovery = {}
for kind in ('read', 'write'):
times = [e['at']-action_at for e in events if e['ok'] and e['kind']==kind and e['at']>=action_done]
recovery[kind] = round(min(times), 3) if times else None
success_gaps = {}
for kind in ('read', 'write'):
timeline = sorted(e['at'] for e in events if e['ok'] and e['kind']==kind)
# Include boundaries of the action, so silence during the entire action
# cannot be hidden by success only before or after it.
timeline = [action_at] + [t for t in timeline if action_at<=t<=(action_done or elapsed)] + [action_done or elapsed]
success_gaps[kind] = round(max(b-a for a,b in zip(timeline,timeline[1:])), 3)
report = {'scenario': 'stop-'+service if service else 'rolling-update', 'concurrency': 4,
'warmup_seconds': 3, 'elapsed_seconds': round(elapsed, 3),
'action_start_seconds': round(action_at, 3), 'action_done_seconds': round(action_done, 3) if action_done else None,
'first_success_after_action_from_action_start_seconds': recovery,
'before': before, 'after': after,
'http': summary(events, elapsed),
'read': summary([e for e in events if e['kind']=='read'], elapsed),
'write': summary([e for e in events if e['kind']=='write'], elapsed),
'during_action': summary([e for e in events if action_at<=e['at']<=(action_done or elapsed)], (action_done or elapsed)-action_at),
'confirmed_messages': len(confirmations), 'history_messages': len(history),
'missing_or_changed_confirmations': len(missing), 'duplicate_texts': duplicate_texts,
'resources': samples, 'maximum_gap_between_successes_during_action_seconds': success_gaps,
'note': 'Closed-loop, four writers alternate POST and GET. Percentiles cover successful HTTP requests only; retries preserve key/body. Recovery is first success after hook completion measured from hook start; not an outage duration. One Docker host; no node/DB/LB HA claim.'}
name = report['scenario']
(ROOT/'evidence'/('%s.json' % name)).write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
(ROOT/'evidence'/('%s-action.txt' % name)).write_text(hook_output)
check(action_error is None, 'action failed: %s' % action_error)
check(not missing and not duplicate_texts, 'lost/duplicated messages')
if service:
check(all(x is not None and x<=10 for x in recovery.values()), 'read/write recovery exceeded 10s')
target_entry = next(e for e in after if e['service']==service)
check(not target_entry['running'], 'target replica is still running')
recent_instances = {e.get('instance') for e in events if e['kind']=='write' and e['ok'] and e['at']>action_done+3}
check(target_entry['instance'] not in recent_instances, 'stopped target still serves writes')
else:
check(all(x<=3 for x in success_gaps.values()), 'rolling read/write availability gap exceeded 3 seconds')
for kind in ('read', 'write'):
for left in range(int(action_at), int(action_done), 3):
right = min(left+3, action_done)
check(any(e['ok'] and e['kind']==kind and left<=e['at']<=right for e in events),
'no successful %s during rolling window' % kind)
check(all(a['id']!=b['id'] for a,b in zip(before, after)), 'not every replica was replaced')
check(all(a['image']!=b['image'] for a,b in zip(before, after)), 'new image was not deployed')
print(json.dumps({k:v for k,v in report.items() if k not in ('resources','before','after','note')}, ensure_ascii=False))
print('PASS:', name)
def main():
def configure(p):
p.add_argument('--scenario', choices=('failover', 'rolling'), required=True)
args = arguments(__doc__, configure)
try:
if args.scenario == 'failover':
for service in manage.SERVICES:
manage.restore()
scenario(args, service)
else:
scenario(args)
finally:
manage.restore()
if __name__ == '__main__':
run(main)