Files
mtusi2026lab4/test/hooks.py
T
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

134 lines
7.3 KiB
Python

"""Run unchanged public resilience tests and record which Compose processes changed."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
def docker(*args):
return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
text=True, check=True, timeout=120).stdout
def snapshot():
ids = docker('compose', 'ps', '--all', '-q').split()
states = json.loads(docker('inspect', *ids))
return {state['Name'].lstrip('/'): {
'service': state['Config']['Labels']['com.docker.compose.service'],
'instance': state['Config']['Hostname'], 'startedAt': state['State']['StartedAt'],
'status': state['State']['Status'], 'restartCount': state['RestartCount'],
} for state in states if state['Config']['Labels']['com.docker.compose.service']
in {'api', 'nginx', 'postgres', 'redis', 's3', 'reconciler'}}
def public_test(mode):
extension = 'cmd' if os.name == 'nt' else 'sh'
hook = ROOT / 'scripts' / (('restart.' if mode == 'persistence' else 'stop-one.') + extension)
result = subprocess.run([sys.executable, '-X', 'utf8', '-u', str(ROOT / 'tests/resilience.py'),
'--mode', mode, '--action-script', str(hook), '--base-url', 'http://127.0.0.1:8080'],
cwd=ROOT, capture_output=True, text=True, timeout=240)
print(result.stdout, end='', flush=True)
if result.returncode:
raise RuntimeError(f'Public {mode} test failed: {result.stderr.strip() or result.stdout.strip()}')
return result.stdout.strip()
def warm_writes():
# Exercise writes and attachment I/O before measuring a steady-state API failure.
client = Client('http://127.0.0.1:8080', timeout=10)
alice, _, _, chat = client.bootstrap()
timings = {}
for index in range(30):
result = client.message(alice, chat, f'Failover warm-up {index}')
timings.setdefault(result['headers']['X-Instance-Id'], []).append(result['elapsed'])
if len(timings) == 2 and min(map(len, timings.values())) >= 3:
break
assert len(timings) == 2 and min(map(len, timings.values())) >= 3, 'Warm-up did not reach both API write paths'
result = {instance: {'writes': len(values), 'minSeconds': min(values), 'maxSeconds': max(values),
'uploadSeconds': [], 'downloadSeconds': []}
for instance, values in timings.items()}
image = (ROOT / 'tests/fixtures/sample.png').read_bytes()
for index in range(30):
uploaded = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=warmup.png',
201, actor=alice, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment')
result[uploaded['headers']['X-Instance-Id']]['uploadSeconds'].append(uploaded['elapsed'])
path = '/api/v1/attachments/' + uploaded['json']['id']
client.call('GET', path, actor=alice, schema='Attachment')
downloaded = client.call('GET', path + '/content', actor=alice)
assert downloaded['bytes'] == image, 'Warm-up attachment checksum mismatch'
result[downloaded['headers']['X-Instance-Id']]['downloadSeconds'].append(downloaded['elapsed'])
if all(entry['uploadSeconds'] and entry['downloadSeconds'] for entry in result.values()):
break
assert all(entry['uploadSeconds'] and entry['downloadSeconds'] for entry in result.values()), 'Attachment warm-up did not reach both replicas'
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--mode', choices=['all', 'failover'], default='all')
args = parser.parse_args()
report = {'status': 'failed', 'failover': []}
report['writeWarmups'] = []
try:
before = snapshot()
assert sum(state['service'] == 'api' for state in before.values()) == 2, 'Expected two API replicas'
assert all(state['status'] == 'running' for state in before.values()), 'Start the complete stack first'
if args.mode == 'all':
output = public_test('persistence')
after = snapshot()
assert before.keys() == after.keys(), 'Restart hook recreated runtime containers'
assert all(state['startedAt'] != before[name]['startedAt'] and state['status'] == 'running'
for name, state in after.items()), 'Hook did not restart every runtime service'
report['persistence'] = {'before': before, 'after': after, 'output': output}
else:
previous = json.loads((ROOT / 'evidence/hooks.json').read_text(encoding='utf-8'))
assert previous.get('persistence'), 'Run the full suite first to record persistence evidence'
report['persistence'] = previous['persistence']
report['persistenceEvidenceReused'] = True
after = before
expected = {state['instance'] for state in after.values() if state['service'] == 'api'}
stopped_instances = set()
# The public test selects the replica that served its POST, so repeat independent
# successful scenarios until each replica has been targeted. No failing run is retried.
for _ in range(6):
if stopped_instances == expected:
break
before = snapshot()
try:
report['writeWarmups'].append(warm_writes())
output = public_test('failover')
after = snapshot()
stopped = [state for state in after.values() if state['service'] == 'api' and state['status'] == 'exited']
assert len(stopped) == 1, 'Stop-one hook did not leave exactly one API stopped'
assert all(state['startedAt'] == before[name]['startedAt'] and state['status'] == 'running'
for name, state in after.items() if state['service'] != 'api'), 'Failover touched non-API services'
target = stopped[0]['instance']
stopped_instances.add(target)
report['failover'].append({'stoppedInstance': target, 'before': before, 'after': after, 'output': output})
finally:
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--no-build', '--wait', '--wait-timeout', '90', 'api')
time.sleep(5) # Allow the Nginx readiness poll to observe the restored API.
assert stopped_instances == expected, 'Public scenarios did not cover both API replicas'
report['status'] = 'passed'
print('PASS: public persistence/failover hooks; all stores restarted; each API stopped separately; replicas restored')
except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error:
report['error'] = str(error) if isinstance(error, (AssertionError, RuntimeError)) else type(error).__name__
raise
finally:
(ROOT / 'evidence/hooks.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
if __name__ == '__main__':
try:
main()
except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error:
print('FAIL: ' + (str(error) if isinstance(error, (AssertionError, RuntimeError)) else type(error).__name__), file=sys.stderr)
sys.exit(1)