forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
134 lines
7.3 KiB
Python
134 lines
7.3 KiB
Python
"""Run unchanged public resilience tests and record which Compose processes changed."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
|
|
text=True, check=True, timeout=120).stdout
|
|
|
|
|
|
def snapshot():
|
|
ids = docker('compose', 'ps', '--all', '-q').split()
|
|
states = json.loads(docker('inspect', *ids))
|
|
return {state['Name'].lstrip('/'): {
|
|
'service': state['Config']['Labels']['com.docker.compose.service'],
|
|
'instance': state['Config']['Hostname'], 'startedAt': state['State']['StartedAt'],
|
|
'status': state['State']['Status'], 'restartCount': state['RestartCount'],
|
|
} for state in states if state['Config']['Labels']['com.docker.compose.service']
|
|
in {'api', 'nginx', 'postgres', 'redis', 's3', 'reconciler'}}
|
|
|
|
|
|
def public_test(mode):
|
|
extension = 'cmd' if os.name == 'nt' else 'sh'
|
|
hook = ROOT / 'scripts' / (('restart.' if mode == 'persistence' else 'stop-one.') + extension)
|
|
result = subprocess.run([sys.executable, '-X', 'utf8', '-u', str(ROOT / 'tests/resilience.py'),
|
|
'--mode', mode, '--action-script', str(hook), '--base-url', 'http://127.0.0.1:8080'],
|
|
cwd=ROOT, capture_output=True, text=True, timeout=240)
|
|
print(result.stdout, end='', flush=True)
|
|
if result.returncode:
|
|
raise RuntimeError(f'Public {mode} test failed: {result.stderr.strip() or result.stdout.strip()}')
|
|
return result.stdout.strip()
|
|
|
|
|
|
def warm_writes():
|
|
# Exercise writes and attachment I/O before measuring a steady-state API failure.
|
|
client = Client('http://127.0.0.1:8080', timeout=10)
|
|
alice, _, _, chat = client.bootstrap()
|
|
timings = {}
|
|
for index in range(30):
|
|
result = client.message(alice, chat, f'Failover warm-up {index}')
|
|
timings.setdefault(result['headers']['X-Instance-Id'], []).append(result['elapsed'])
|
|
if len(timings) == 2 and min(map(len, timings.values())) >= 3:
|
|
break
|
|
assert len(timings) == 2 and min(map(len, timings.values())) >= 3, 'Warm-up did not reach both API write paths'
|
|
result = {instance: {'writes': len(values), 'minSeconds': min(values), 'maxSeconds': max(values),
|
|
'uploadSeconds': [], 'downloadSeconds': []}
|
|
for instance, values in timings.items()}
|
|
image = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
for index in range(30):
|
|
uploaded = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=warmup.png',
|
|
201, actor=alice, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
result[uploaded['headers']['X-Instance-Id']]['uploadSeconds'].append(uploaded['elapsed'])
|
|
path = '/api/v1/attachments/' + uploaded['json']['id']
|
|
client.call('GET', path, actor=alice, schema='Attachment')
|
|
downloaded = client.call('GET', path + '/content', actor=alice)
|
|
assert downloaded['bytes'] == image, 'Warm-up attachment checksum mismatch'
|
|
result[downloaded['headers']['X-Instance-Id']]['downloadSeconds'].append(downloaded['elapsed'])
|
|
if all(entry['uploadSeconds'] and entry['downloadSeconds'] for entry in result.values()):
|
|
break
|
|
assert all(entry['uploadSeconds'] and entry['downloadSeconds'] for entry in result.values()), 'Attachment warm-up did not reach both replicas'
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--mode', choices=['all', 'failover'], default='all')
|
|
args = parser.parse_args()
|
|
report = {'status': 'failed', 'failover': []}
|
|
report['writeWarmups'] = []
|
|
try:
|
|
before = snapshot()
|
|
assert sum(state['service'] == 'api' for state in before.values()) == 2, 'Expected two API replicas'
|
|
assert all(state['status'] == 'running' for state in before.values()), 'Start the complete stack first'
|
|
if args.mode == 'all':
|
|
output = public_test('persistence')
|
|
after = snapshot()
|
|
assert before.keys() == after.keys(), 'Restart hook recreated runtime containers'
|
|
assert all(state['startedAt'] != before[name]['startedAt'] and state['status'] == 'running'
|
|
for name, state in after.items()), 'Hook did not restart every runtime service'
|
|
report['persistence'] = {'before': before, 'after': after, 'output': output}
|
|
else:
|
|
previous = json.loads((ROOT / 'evidence/hooks.json').read_text(encoding='utf-8'))
|
|
assert previous.get('persistence'), 'Run the full suite first to record persistence evidence'
|
|
report['persistence'] = previous['persistence']
|
|
report['persistenceEvidenceReused'] = True
|
|
after = before
|
|
expected = {state['instance'] for state in after.values() if state['service'] == 'api'}
|
|
stopped_instances = set()
|
|
# The public test selects the replica that served its POST, so repeat independent
|
|
# successful scenarios until each replica has been targeted. No failing run is retried.
|
|
for _ in range(6):
|
|
if stopped_instances == expected:
|
|
break
|
|
before = snapshot()
|
|
try:
|
|
report['writeWarmups'].append(warm_writes())
|
|
output = public_test('failover')
|
|
after = snapshot()
|
|
stopped = [state for state in after.values() if state['service'] == 'api' and state['status'] == 'exited']
|
|
assert len(stopped) == 1, 'Stop-one hook did not leave exactly one API stopped'
|
|
assert all(state['startedAt'] == before[name]['startedAt'] and state['status'] == 'running'
|
|
for name, state in after.items() if state['service'] != 'api'), 'Failover touched non-API services'
|
|
target = stopped[0]['instance']
|
|
stopped_instances.add(target)
|
|
report['failover'].append({'stoppedInstance': target, 'before': before, 'after': after, 'output': output})
|
|
finally:
|
|
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build', '--wait', '--wait-timeout', '90', 'api')
|
|
time.sleep(5) # Allow the Nginx readiness poll to observe the restored API.
|
|
assert stopped_instances == expected, 'Public scenarios did not cover both API replicas'
|
|
report['status'] = 'passed'
|
|
print('PASS: public persistence/failover hooks; all stores restarted; each API stopped separately; replicas restored')
|
|
except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error:
|
|
report['error'] = str(error) if isinstance(error, (AssertionError, RuntimeError)) else type(error).__name__
|
|
raise
|
|
finally:
|
|
(ROOT / 'evidence/hooks.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error:
|
|
print('FAIL: ' + (str(error) if isinstance(error, (AssertionError, RuntimeError)) else type(error).__name__), file=sys.stderr)
|
|
sys.exit(1)
|