"""Check readiness and upload when S3 is unavailable; always restore S3.""" import subprocess import time from client import Client, arguments, run def main(): args = arguments('Storage failure') c = Client(args.base_url) alice,bob,eve,chat = c.bootstrap() data = open('tests/fixtures/sample.png','rb').read() # Call an API inside its container because LB correctly removes unready APIs. code = '''import urllib.request, urllib.error for path, data, headers in CASES: try: r = urllib.request.urlopen(urllib.request.Request('http://localhost:8000'+path,data=data,headers=headers),timeout=10) except urllib.error.HTTPError as e: r = e assert r.code == 503, r.code print('PASS: S3 down returns 503 for readiness and upload') ''' cases = [('/health/ready',None,{}),('/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',data,{'Content-Type':'image/png','Authorization':'Bearer '+alice['token']})] try: subprocess.run(['docker','compose','stop','-t','2','s3'],check=True) subprocess.run(['docker','compose','exec','-T','api1','python','-'],input='CASES='+repr(cases)+'\n'+code,text=True,check=True) finally: subprocess.run(['docker','compose','start','s3'],check=True) for _ in range(60): try: c.call('GET','/health/ready') print('PASS: S3 restored') break except Exception: time.sleep(1) else: raise AssertionError('S3 did not recover') if __name__=='__main__': run(main)