Files
mtusi2026lab3/student-tests/faults.py

135 lines
6.3 KiB
Python

"""Real container failures/replacements while readers and writers use only the LB."""
import concurrent.futures
import json
import os
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure
from resilience import history
from stop_one import instance
def docker(*args):
return subprocess.check_output(['docker', *args], text=True).strip()
def wait_instances(expected):
deadline = time.monotonic() + 30
observed = set()
c = Client(timeout=1)
while time.monotonic() < deadline:
try:
observed.add(c.call('GET', '/health/ready', schema='Health')['headers']['X-Instance-Id'])
if len(observed) >= expected:
return observed
except Failure:
pass
time.sleep(0.1)
raise AssertionError('Not enough ready instances through LB')
def phase(actor, reader, chat, action, duration=12):
started = time.monotonic()
stop = threading.Event()
replies, failures, confirmed = [], [], {}
action_time = [None]
def worker(index):
c = Client(timeout=2)
key = uuid.uuid4().hex
content = 'fault-' + key
while not stop.is_set():
before = time.monotonic()
try:
if index < 2:
response = c.message(actor, chat, content, key=key)
confirmed[response['json']['id']] = response['json']
key = uuid.uuid4().hex
content = 'fault-' + key
else:
response = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=50', actor=reader, schema='MessagePage')
replies.append({'t': time.monotonic() - started, 'latency_ms': (time.monotonic() - before) * 1000, 'write': index < 2, 'instance': response['headers']['X-Instance-Id']})
except Failure:
failures.append({'t': time.monotonic() - started, 'write': index < 2})
# An uncertain POST keeps its original body/key until confirmation.
stop.wait(0.02)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
futures = [executor.submit(worker, i) for i in range(4)]
time.sleep(2)
action_time[0] = time.monotonic() - started
try:
action_record = action()
action_completed = time.monotonic() - started
except BaseException:
stop.set()
raise
remaining = duration - (time.monotonic() - started)
if remaining > 0:
stop.wait(remaining)
stop.set()
for future in futures:
future.result()
measurement_elapsed = time.monotonic() - started
found = history(Client(), reader, chat)
by_id = {v['id']: v for v in found}
assert all(by_id.get(k) == v for k, v in confirmed.items()), 'acknowledged message lost'
contents = [v['text'] for v in found]
assert len(contents) == len(set(contents)), 'uncertain retry duplicated a message'
target = action_record.get('instance', action_record.get('old_instance'))
recovered = [v['t'] - action_time[0] for v in replies if v['write'] and v['t'] >= action_completed and v['instance'] != target]
assert all(v['instance'] != target for v in replies[-5:]), 'removed instance still serves final replies'
assert recovered and min(recovered) <= 10, 'write recovery exceeded 10 seconds'
times = sorted(v['latency_ms'] for v in replies)
return {'duration': round(measurement_elapsed, 3), 'action_at': action_time[0], 'action_completed_at': action_completed, 'action': action_record, 'successful_rps': len(replies) / measurement_elapsed, 'successful_requests': len(replies), 'errors': len(failures), 'write_recovery_seconds': min(recovered), 'p50_ms': statistics.median(times), 'p95_ms': times[int(.95 * (len(times)-1))], 'p99_ms': times[int(.99 * (len(times)-1))], 'confirmed_messages_verified': len(confirmed), 'samples': replies, 'failed_samples': failures}
def main():
c = Client()
actor, reader, _, chat = c.bootstrap()
originals = docker('compose', 'ps', '-q', 'api').splitlines()
assert len(originals) == 2
wait_instances(2)
results = []
for container in originals:
alias = instance(container)
def stop_one():
env = dict(os.environ, TARGET_INSTANCE_ID=alias)
subprocess.run([str(ROOT / 'scripts/stop-one.sh')], env=env, check=True, stdout=subprocess.DEVNULL)
assert docker('inspect', '--format', '{{.State.Running}}', container) == 'false'
return {'container': container, 'instance': alias, 'running_after': False}
result = phase(actor, reader, chat, stop_one)
result['kind'] = 'stop'
results.append(result)
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', 'api'], check=True, stdout=subprocess.DEVNULL)
wait_instances(2)
print('PASS: stopped actual replica', container[:12], 'under read/write load; existing session/data preserved.', flush=True)
for container in originals:
old_alias = instance(container)
def replace():
docker('stop', '--time', '8', container)
docker('rm', container)
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', '--no-recreate', 'api'], check=True, stdout=subprocess.DEVNULL)
new_ids = docker('compose', 'ps', '-q', 'api').splitlines()
assert container not in new_ids and len(new_ids) == 2
new_aliases = wait_instances(2)
assert old_alias not in new_aliases
return {'old_container': container, 'old_instance': old_alias, 'new_containers': new_ids, 'observed_instances': sorted(new_aliases)}
result = phase(actor, reader, chat, replace, duration=18)
result['kind'] = 'rolling_replace'
results.append(result)
print('PASS: replaced actual replica', container[:12], 'under load; DNS rediscovery verified.', flush=True)
(ROOT / 'evidence/faults.json').write_text(json.dumps(results, indent=2))
print('PASS: each API stopped and then replaced sequentially; all confirmations and original sessions verified.')
if __name__ == '__main__':
main()