"""Stop exactly the API that accepted an upload, for each of the two APIs.""" import hashlib import os import subprocess import time from client import Client, arguments, check, run def main(): args = arguments('Upload/download failover on both APIs') c = Client(args.base_url) alice,bob,eve,chat = c.bootstrap() content = open('tests/fixtures/sample.png','rb').read() path = '/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png' for service in ('api1','api2'): cid = subprocess.check_output(['docker','compose','ps','-q',service],text=True).strip() target = subprocess.check_output(['docker','inspect','--format','{{.Config.Hostname}}',cid],text=True).strip() for _ in range(20): uploaded = c.call('POST',path,201,actor=alice,raw=content,headers={'Content-Type':'image/png'},schema='Attachment') if uploaded['headers']['X-Instance-Id'] == target: break else: raise AssertionError('Target API not observed through LB') ident = uploaded['json']['id'] c.message(alice,chat,'',attachments=[ident]) started = time.monotonic() try: subprocess.run(['scripts/stop-one.sh'],env=dict(os.environ,TARGET_INSTANCE_ID=target),check=True,stdout=subprocess.DEVNULL) while time.monotonic()-started < 10: try: downloaded = c.call('GET','/api/v1/attachments/'+ident+'/content',actor=bob) except Exception: time.sleep(.2) continue check(downloaded['headers']['X-Instance-Id'] != target,'stopped API still responds') check(downloaded['bytes']==content,'original changed') print('PASS:',service,'uploaded; stopped; other API returned SHA-256',hashlib.sha256(content).hexdigest(),'after',round(time.monotonic()-started,3),'seconds',flush=True) break else: raise AssertionError('Download did not recover in 10 seconds') finally: subprocess.run(['docker','compose','start',service],check=True) for _ in range(30): health = subprocess.check_output(['docker','inspect','--format','{{.State.Health.Status}}',cid],text=True).strip() if health == 'healthy': time.sleep(2) # allow HAProxy rise check break time.sleep(1) else: raise AssertionError('API did not recover') if __name__=='__main__': run(main)