96 lines
4.2 KiB
Python
96 lines
4.2 KiB
Python
"""Stop each actual uploader replica and verify its bytes through the survivor."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def instances():
|
|
text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT,
|
|
encoding='utf-8')
|
|
result = []
|
|
for line in text.splitlines():
|
|
container, replica, state, alias, image = line.split()
|
|
result.append({'container': container, 'replica': replica,
|
|
'state': state, 'alias': alias, 'image': image})
|
|
return result
|
|
|
|
|
|
def restore(c):
|
|
subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT,
|
|
check=True, stdout=subprocess.DEVNULL)
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id']
|
|
for _ in range(6)}
|
|
if len(observed) >= 2:
|
|
return
|
|
time.sleep(0.3)
|
|
raise AssertionError('two replicas did not return after restore')
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json')
|
|
args = p.parse_args()
|
|
c = Client(args.base_url, timeout=10)
|
|
actor, reader, _, chat = c.bootstrap()
|
|
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
wanted = instances()
|
|
check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted),
|
|
'this experiment requires two running API replicas')
|
|
report = {'scenario': 'stop actual upload-serving replica; download via survivor',
|
|
'cases': [], 'ok': False}
|
|
try:
|
|
for target in wanted:
|
|
upload = None
|
|
for _ in range(10):
|
|
attempt = c.call('POST', '/api/v1/chats/' + chat['id'] +
|
|
'/attachments?filename=sample.png', 201,
|
|
actor=actor, raw=data,
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
if attempt['headers']['X-Instance-Id'] == target['alias']:
|
|
upload = attempt
|
|
break
|
|
check(upload is not None, 'target replica did not serve an upload')
|
|
before = instances()
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=target['alias'])
|
|
started = time.monotonic()
|
|
subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env,
|
|
check=True, stdout=subprocess.DEVNULL, timeout=15)
|
|
stop_seconds = time.monotonic() - started
|
|
downloaded = c.call('GET', '/api/v1/attachments/' +
|
|
upload['json']['id'] + '/content', actor=reader)
|
|
survivor = downloaded['headers']['X-Instance-Id']
|
|
check(survivor != target['alias'], 'stopped uploader still served download')
|
|
check(downloaded['bytes'] == data, 'bytes changed after uploader stop')
|
|
after = instances()
|
|
check(any(x['container'] == target['container'] and x['state'] != 'running'
|
|
for x in after), 'uploader was not left stopped')
|
|
report['cases'].append({'uploader': target, 'download_instance': survivor,
|
|
'stop_seconds': round(stop_seconds, 3),
|
|
'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(),
|
|
'same_bytes': True, 'before': before, 'after': after})
|
|
restore(c)
|
|
report['ok'] = True
|
|
finally:
|
|
# This script only restores the API containers that it deliberately stopped.
|
|
restore(c)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']),
|
|
'output': str(args.output)}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|