102 lines
5.9 KiB
Python
102 lines
5.9 KiB
Python
"""Sequential infrastructure faults; restores every stopped service in finally."""
|
|
import http.client
|
|
import json
|
|
import time
|
|
import urllib.parse
|
|
from common import ROOT, Client, check, command, compose, mc, sql, wait_ready, save
|
|
|
|
|
|
def inside(program, data=None):
|
|
return command(['docker','run','--rm','-i','--network','mtusi-lab4_default',
|
|
'--mount','type=bind,source='+str(ROOT)+',target=/workspace',
|
|
'mtusi-messenger:lab4-tests','python','-X','utf8','-c',program],
|
|
input=json.dumps(data) if data is not None else '')
|
|
|
|
|
|
def main():
|
|
c=Client(timeout=40)
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
att,content=c.attachment(alice,chat)
|
|
results=[]
|
|
key='original/'+att['id']
|
|
status=inside("import urllib.request,urllib.error;\ntry:\n r=urllib.request.urlopen('http://s3:9000/messenger-attachments/"+key+"');print(r.status)\nexcept urllib.error.HTTPError as e: print(e.code)")
|
|
check(status=='403','anonymous S3 access was not denied')
|
|
results.append('private bucket anonymous GET: 403')
|
|
guard=inside("""import urllib.request,urllib.error
|
|
cases=[('/messenger-attachments/original/guard-test',{'X-Amz-Content-Sha256':'STREAMING-UNSIGNED-PAYLOAD-TRAILER'},'PUT'),
|
|
('/minio/storage/v63/rmpl',{},'POST'),('/messenger-attachments/x?select&select-type=2',{},'POST'),
|
|
('/messenger-attachments/x',{'X-Minio-Source-Replication-Request':'true'},'PUT')]
|
|
for path,headers,method in cases:
|
|
try:
|
|
response=urllib.request.urlopen(urllib.request.Request('http://s3:9000'+path,data=b'',headers=headers,method=method))
|
|
assert response.status==403
|
|
except urllib.error.HTTPError as e: assert e.code==403
|
|
print('PASS')""")
|
|
check(guard=='PASS','MinIO network guard failed')
|
|
results.append('unsigned trailers, storage REST, S3 Select and replication headers blocked at private proxy')
|
|
# API identity can use only this bucket/prefix, never administrative APIs.
|
|
denied=mc('mc alias set app http://s3:9000 "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null\n'
|
|
'if mc admin user list app >/dev/null 2>&1; then exit 10; fi\n'
|
|
'if mc mb app/forbidden-bucket >/dev/null 2>&1; then exit 11; fi\n'
|
|
'if printf x | mc pipe app/messenger-attachments/outside-prefix >/dev/null 2>&1; then exit 12; fi\n'
|
|
'echo denied')
|
|
check(denied=='denied','S3 API policy too broad')
|
|
results.append('API account cannot administer, create bucket or write outside original/')
|
|
# Check bootstrap twice without replacing secrets or losing bytes.
|
|
compose('run','--rm','--no-deps','-T','s3-init')
|
|
compose('run','--rm','--no-deps','-T','s3-init')
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==content,'bootstrap lost original')
|
|
results.append('idempotent bootstrap preserves data and credentials')
|
|
try:
|
|
compose('stop','s3')
|
|
# Readiness removes both replicas from LB, so query internal API to
|
|
# distinguish the application JSON error from the LB transport error.
|
|
program='''import sys,json
|
|
sys.path.insert(0,'tests')
|
|
from client import Client
|
|
x=json.load(sys.stdin);c=Client('http://api:8080',timeout=40)
|
|
c.call('GET','/health/live',schema='Health')
|
|
c.call('GET','/health/ready',503)
|
|
c.call('POST','/api/v1/chats/'+x['chat']+'/attachments?filename=a.png',503,actor=x['actor'],raw=open('tests/fixtures/sample.png','rb').read(),headers={'Content-Type':'image/png'})
|
|
c.call('GET','/api/v1/attachments/'+x['attachment']+'/content',503,actor=x['actor'])
|
|
print('PASS')'''
|
|
check(inside(program,{'actor':alice,'chat':chat['id'],'attachment':att['id']})=='PASS','S3 outage error mapping')
|
|
finally:
|
|
compose('start','s3');wait_ready()
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==content,'S3 restart lost data')
|
|
results.append('S3 outage: live 200, ready/upload/content 503, restart retains bytes')
|
|
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=uploader.png',201,
|
|
actor=alice,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
uploader=response['headers']['X-Instance-Id'];uploaded=response['json']
|
|
ids=compose('ps','-q','api').split()
|
|
selected=[]
|
|
for container in ids:
|
|
hostname=command(['docker','inspect','--format','{{.Config.Hostname}}',container])
|
|
if hostname==uploader:selected.append(container)
|
|
check(len(selected)==1,'uploading replica did not map to exactly one container')
|
|
try:
|
|
command(['docker','stop','--time','9',selected[0]])
|
|
wait_ready()
|
|
download=c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',actor=bob)
|
|
check(download['bytes']==content,'replica failover changed original')
|
|
check(download['headers']['X-Instance-Id']!=uploader,'download still used stopped uploader')
|
|
finally:
|
|
compose('up','-d','--no-deps','--scale','api=2','api');wait_ready()
|
|
results.append('exact uploading container stopped; another replica returned identical original with old session')
|
|
# An interrupted body must not reach PUT or metadata insertion.
|
|
before=sql('SELECT count(*) FROM attachments')
|
|
url=urllib.parse.urlsplit(c.base)
|
|
conn=http.client.HTTPConnection(url.hostname,url.port or 80,timeout=5)
|
|
conn.putrequest('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=abort.png')
|
|
conn.putheader('Authorization','Bearer '+alice['token'])
|
|
conn.putheader('Content-Type','image/png');conn.putheader('Content-Length',str(len(content)+1000));conn.endheaders()
|
|
conn.send(content[:30]);conn.close();time.sleep(.5)
|
|
check(sql('SELECT count(*) FROM attachments')==before,'aborted upload created metadata')
|
|
results.append('interrupted body creates no metadata; dependency recovers')
|
|
save('failures.json',{'complete':True,'passed':results})
|
|
print('PASS private bucket, least privilege, bootstrap, S3 outage, aborted body')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|