forked from mtusi2026/mtusi2026lab3
135 lines
6.3 KiB
Python
135 lines
6.3 KiB
Python
"""Real container failures/replacements while readers and writers use only the LB."""
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from client import Client, Failure
|
|
from resilience import history
|
|
from stop_one import instance
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(['docker', *args], text=True).strip()
|
|
|
|
|
|
def wait_instances(expected):
|
|
deadline = time.monotonic() + 30
|
|
observed = set()
|
|
c = Client(timeout=1)
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
observed.add(c.call('GET', '/health/ready', schema='Health')['headers']['X-Instance-Id'])
|
|
if len(observed) >= expected:
|
|
return observed
|
|
except Failure:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise AssertionError('Not enough ready instances through LB')
|
|
|
|
|
|
def phase(actor, reader, chat, action, duration=12):
|
|
started = time.monotonic()
|
|
stop = threading.Event()
|
|
replies, failures, confirmed = [], [], {}
|
|
action_time = [None]
|
|
def worker(index):
|
|
c = Client(timeout=2)
|
|
key = uuid.uuid4().hex
|
|
content = 'fault-' + key
|
|
while not stop.is_set():
|
|
before = time.monotonic()
|
|
try:
|
|
if index < 2:
|
|
response = c.message(actor, chat, content, key=key)
|
|
confirmed[response['json']['id']] = response['json']
|
|
key = uuid.uuid4().hex
|
|
content = 'fault-' + key
|
|
else:
|
|
response = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=50', actor=reader, schema='MessagePage')
|
|
replies.append({'t': time.monotonic() - started, 'latency_ms': (time.monotonic() - before) * 1000, 'write': index < 2, 'instance': response['headers']['X-Instance-Id']})
|
|
except Failure:
|
|
failures.append({'t': time.monotonic() - started, 'write': index < 2})
|
|
# An uncertain POST keeps its original body/key until confirmation.
|
|
stop.wait(0.02)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
|
futures = [executor.submit(worker, i) for i in range(4)]
|
|
time.sleep(2)
|
|
action_time[0] = time.monotonic() - started
|
|
try:
|
|
action_record = action()
|
|
action_completed = time.monotonic() - started
|
|
except BaseException:
|
|
stop.set()
|
|
raise
|
|
remaining = duration - (time.monotonic() - started)
|
|
if remaining > 0:
|
|
stop.wait(remaining)
|
|
stop.set()
|
|
for future in futures:
|
|
future.result()
|
|
measurement_elapsed = time.monotonic() - started
|
|
found = history(Client(), reader, chat)
|
|
by_id = {v['id']: v for v in found}
|
|
assert all(by_id.get(k) == v for k, v in confirmed.items()), 'acknowledged message lost'
|
|
contents = [v['text'] for v in found]
|
|
assert len(contents) == len(set(contents)), 'uncertain retry duplicated a message'
|
|
target = action_record.get('instance', action_record.get('old_instance'))
|
|
recovered = [v['t'] - action_time[0] for v in replies if v['write'] and v['t'] >= action_completed and v['instance'] != target]
|
|
assert all(v['instance'] != target for v in replies[-5:]), 'removed instance still serves final replies'
|
|
assert recovered and min(recovered) <= 10, 'write recovery exceeded 10 seconds'
|
|
times = sorted(v['latency_ms'] for v in replies)
|
|
return {'duration': round(measurement_elapsed, 3), 'action_at': action_time[0], 'action_completed_at': action_completed, 'action': action_record, 'successful_rps': len(replies) / measurement_elapsed, 'successful_requests': len(replies), 'errors': len(failures), 'write_recovery_seconds': min(recovered), 'p50_ms': statistics.median(times), 'p95_ms': times[int(.95 * (len(times)-1))], 'p99_ms': times[int(.99 * (len(times)-1))], 'confirmed_messages_verified': len(confirmed), 'samples': replies, 'failed_samples': failures}
|
|
|
|
|
|
def main():
|
|
c = Client()
|
|
actor, reader, _, chat = c.bootstrap()
|
|
originals = docker('compose', 'ps', '-q', 'api').splitlines()
|
|
assert len(originals) == 2
|
|
wait_instances(2)
|
|
results = []
|
|
for container in originals:
|
|
alias = instance(container)
|
|
def stop_one():
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=alias)
|
|
subprocess.run([str(ROOT / 'scripts/stop-one.sh')], env=env, check=True, stdout=subprocess.DEVNULL)
|
|
assert docker('inspect', '--format', '{{.State.Running}}', container) == 'false'
|
|
return {'container': container, 'instance': alias, 'running_after': False}
|
|
result = phase(actor, reader, chat, stop_one)
|
|
result['kind'] = 'stop'
|
|
results.append(result)
|
|
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', 'api'], check=True, stdout=subprocess.DEVNULL)
|
|
wait_instances(2)
|
|
print('PASS: stopped actual replica', container[:12], 'under read/write load; existing session/data preserved.', flush=True)
|
|
for container in originals:
|
|
old_alias = instance(container)
|
|
def replace():
|
|
docker('stop', '--time', '8', container)
|
|
docker('rm', container)
|
|
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', '--no-recreate', 'api'], check=True, stdout=subprocess.DEVNULL)
|
|
new_ids = docker('compose', 'ps', '-q', 'api').splitlines()
|
|
assert container not in new_ids and len(new_ids) == 2
|
|
new_aliases = wait_instances(2)
|
|
assert old_alias not in new_aliases
|
|
return {'old_container': container, 'old_instance': old_alias, 'new_containers': new_ids, 'observed_instances': sorted(new_aliases)}
|
|
result = phase(actor, reader, chat, replace, duration=18)
|
|
result['kind'] = 'rolling_replace'
|
|
results.append(result)
|
|
print('PASS: replaced actual replica', container[:12], 'under load; DNS rediscovery verified.', flush=True)
|
|
(ROOT / 'evidence/faults.json').write_text(json.dumps(results, indent=2))
|
|
print('PASS: each API stopped and then replaced sequentially; all confirmations and original sessions verified.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|