40 lines
1.5 KiB
Python
40 lines
1.5 KiB
Python
"""Check readiness and upload when S3 is unavailable; always restore S3."""
|
|
import subprocess
|
|
import time
|
|
from client import Client, arguments, run
|
|
|
|
|
|
def main():
|
|
args = arguments('Storage failure')
|
|
c = Client(args.base_url)
|
|
alice,bob,eve,chat = c.bootstrap()
|
|
data = open('tests/fixtures/sample.png','rb').read()
|
|
# Call an API inside its container because LB correctly removes unready APIs.
|
|
code = '''import urllib.request, urllib.error
|
|
for path, data, headers in CASES:
|
|
try:
|
|
r = urllib.request.urlopen(urllib.request.Request('http://localhost:8000'+path,data=data,headers=headers),timeout=10)
|
|
except urllib.error.HTTPError as e:
|
|
r = e
|
|
assert r.code == 503, r.code
|
|
print('PASS: S3 down returns 503 for readiness and upload')
|
|
'''
|
|
cases = [('/health/ready',None,{}),('/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',data,{'Content-Type':'image/png','Authorization':'Bearer '+alice['token']})]
|
|
try:
|
|
subprocess.run(['docker','compose','stop','-t','2','s3'],check=True)
|
|
subprocess.run(['docker','compose','exec','-T','api1','python','-'],input='CASES='+repr(cases)+'\n'+code,text=True,check=True)
|
|
finally:
|
|
subprocess.run(['docker','compose','start','s3'],check=True)
|
|
for _ in range(60):
|
|
try:
|
|
c.call('GET','/health/ready')
|
|
print('PASS: S3 restored')
|
|
break
|
|
except Exception:
|
|
time.sleep(1)
|
|
else:
|
|
raise AssertionError('S3 did not recover')
|
|
|
|
if __name__=='__main__':
|
|
run(main)
|