forked from mtusi2026/mtusi2026lab4
251 lines
12 KiB
Python
251 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade4 attachment evidence with real S3 faults, process crash and grace cleanup."""
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import urllib.error
|
|
import urllib.parse
|
|
import uuid
|
|
|
|
import cluster
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,Failure,check
|
|
|
|
|
|
def save(name,value):
|
|
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS:',name,flush=True)
|
|
|
|
|
|
def internal(data,container=None):
|
|
container=container or cluster.container_ids('api')[0]
|
|
p=subprocess.run(['docker','exec','-i',container,'python','scripts/attachment_checks.py'],cwd=ROOT,
|
|
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=30)
|
|
return json.loads(p.stdout)
|
|
|
|
|
|
def upload(c,actor,chat,raw,filename='sample.png'):
|
|
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+urllib.parse.quote(filename),
|
|
201,actor=actor,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
|
|
|
|
def private_bucket():
|
|
c=Client()
|
|
alice,bob,_,chat=c.bootstrap()
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
response=upload(c,alice,chat,raw)
|
|
att=response['json']
|
|
key=internal({'action':'lookup','id':att['id']})['object_key']
|
|
result=internal({'action':'security','key':key})
|
|
check(result['anonymous_get']==403,'private object is accessible without S3 authentication')
|
|
check(result['create_bucket_with_api_credentials']==403 and result['list_other_prefix']==403,'API S3 credentials are too broad')
|
|
check(not result['console_reachable_from_api'] and not result['root_secret_mounted_in_api'],'root/console exposed to API')
|
|
before=internal({'action':'counts'})
|
|
output=cluster.compose('run','--rm','--no-deps','minio-init')
|
|
check(internal({'action':'counts'})==before,'repeated bucket bootstrap changed data')
|
|
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
|
check(download['bytes']==raw,'bootstrap changed original bytes')
|
|
save('s3-security.json',{'result':'PASS',**result,'bootstrap_repeated_without_data_loss':True,'bootstrap_output':output})
|
|
|
|
|
|
def upload_failover():
|
|
cluster.wait_cluster()
|
|
c=Client()
|
|
alice,bob,_,chat=c.bootstrap()
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
response=upload(c,alice,chat,raw)
|
|
att=response['json']
|
|
alias=response['headers']['X-Instance-Id']
|
|
c.message(alice,chat,'',attachments=[att['id']])
|
|
target=next(node for node in cluster.replicas() if node['instance']==alias)
|
|
started=time.monotonic()
|
|
try:
|
|
stopped=cluster.stop_one(alias)
|
|
deadline=started+10
|
|
while True:
|
|
try:
|
|
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
|
break
|
|
except Failure:
|
|
check(time.monotonic()<deadline,'file download did not recover in10s')
|
|
time.sleep(.1)
|
|
check(download['headers']['X-Instance-Id']!=alias,'download came from stopped process')
|
|
check(download['bytes']==raw and hashlib.sha256(download['bytes']).hexdigest()==att['sha256'],'file changed across replicas')
|
|
check(not cluster.inspect([target['container']])[0]['State']['Running'],'target was not left stopped')
|
|
result={'result':'PASS','stopped':stopped,'download_instance':download['headers']['X-Instance-Id'],
|
|
'same_bytes':True,'same_sha256':True,'seconds_from_action_start':round(time.monotonic()-started,4)}
|
|
finally:
|
|
cluster.command(['docker','start',target['container']])
|
|
cluster.wait_healthy(target['container'])
|
|
cluster.wait_cluster()
|
|
save('attachment-failover.json',result)
|
|
|
|
|
|
def interrupted_upload():
|
|
c=Client()
|
|
alice,_,_,chat=c.bootstrap()
|
|
before=internal({'action':'counts'})
|
|
parsed=urllib.parse.urlsplit(c.base)
|
|
sock=socket.create_connection((parsed.hostname,parsed.port or 80),timeout=3)
|
|
path='/api/v1/chats/'+chat['id']+'/attachments?filename=interrupted.png'
|
|
headers=('POST '+path+' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer '+alice['token']+
|
|
'\r\nContent-Type: image/png\r\nContent-Length: 500000\r\nConnection: close\r\n\r\n')
|
|
sock.sendall(headers.encode()+b'\x89PNG\r\n\x1a\n'+b'x'*100)
|
|
sock.shutdown(socket.SHUT_RDWR)
|
|
sock.close()
|
|
time.sleep(.5)
|
|
after=internal({'action':'counts'})
|
|
check(before==after,'interrupted request left an object or metadata')
|
|
save('interrupted-upload.json',{'result':'PASS','declared_bytes':500000,'sent_bytes':108,
|
|
'metadata_unchanged':True,'s3_object_count_unchanged':True})
|
|
|
|
|
|
def crash_orphan():
|
|
cluster.wait_cluster()
|
|
c=Client()
|
|
alice,bob,_,chat=c.bootstrap()
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
nodes=cluster.replicas()
|
|
target,survivor=nodes[0],nodes[1]
|
|
filename='crash_'+uuid.uuid4().hex+'.png'
|
|
fixture={'filename':filename,'function':'crash_'+uuid.uuid4().hex,'trigger':'crash_'+uuid.uuid4().hex}
|
|
before=set(internal({'action':'list'},survivor['container'])['keys'])
|
|
internal({'action':'install',**fixture},survivor['container'])
|
|
code='''import json,base64,sys,urllib.request,urllib.error
|
|
p=json.load(sys.stdin)
|
|
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=base64.b64decode(p['bytes']),method='POST',headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'})
|
|
try:
|
|
response=urllib.request.urlopen(r,timeout=8)
|
|
print(json.dumps({'status':response.code}))
|
|
except Exception as exc:
|
|
print(json.dumps({'status':None,'error_type':type(exc).__name__}))
|
|
'''
|
|
data={'address':cluster.front_address(target['container']),'path':'/api/v1/chats/'+chat['id']+'/attachments?filename='+filename,
|
|
'bytes':base64.b64encode(raw).decode(),'token':alice['token']}
|
|
def post():
|
|
p=subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,input=json.dumps(data),capture_output=True,text=True,check=True,timeout=12)
|
|
return json.loads(p.stdout)
|
|
pool=ThreadPoolExecutor(max_workers=1)
|
|
future=pool.submit(post)
|
|
try:
|
|
deadline=time.monotonic()+5
|
|
while not internal({'action':'waiting'},survivor['container'])['waiting']:
|
|
check(time.monotonic()<deadline and not future.done(),'did not observe PUT followed by pending metadata INSERT')
|
|
time.sleep(.05)
|
|
after=set(internal({'action':'list'},survivor['container'])['keys'])
|
|
created=after-before
|
|
check(len(created)==1,'expected exactly one new S3 object during pending metadata INSERT')
|
|
orphan=created.pop()
|
|
cluster.command(['docker','kill','--signal','SIGKILL',target['container']])
|
|
crashed=cluster.inspect([target['container']])[0]['State']
|
|
check(not crashed['Running'] and crashed['ExitCode']==137,'SIGKILL did not produce the expected stopped state')
|
|
observed_exit_code=crashed['ExitCode']
|
|
response=future.result(timeout=12)
|
|
check(response['status']!=201,'crashed request returned an acknowledgement')
|
|
internal({'action':'remove',**fixture},survivor['container'])
|
|
check(internal({'action':'tracked','key':orphan},survivor['container'])['tracked']==0,'metadata unexpectedly committed after crash')
|
|
check(internal({'action':'exists','key':orphan},survivor['container'])['exists'],'crash did not leave an orphan to clean')
|
|
finally:
|
|
pool.shutdown(wait=True)
|
|
internal({'action':'remove',**fixture},survivor['container'])
|
|
cluster.command(['docker','start',target['container']])
|
|
cluster.wait_healthy(target['container'])
|
|
cluster.wait_cluster()
|
|
# A real orphan from crash, plus an old locked object and a tracked/linked attachment.
|
|
locked='originals/'+uuid.uuid4().hex
|
|
internal({'action':'put','key':locked,'bytes':base64.b64encode(raw).decode()})
|
|
tracked=upload(c,alice,chat,raw)['json']
|
|
c.message(alice,chat,'',attachments=[tracked['id']])
|
|
print('Waiting61s for the enforced60s cleanup grace (no time bypass).',flush=True)
|
|
time.sleep(61)
|
|
fresh='originals/'+uuid.uuid4().hex
|
|
internal({'action':'put','key':fresh,'bytes':base64.b64encode(raw).decode()})
|
|
lock_code='''import json,sys,psycopg
|
|
from messenger.db import setting
|
|
key=sys.stdin.readline().strip()
|
|
with psycopg.connect(setting('DATABASE_URL')) as conn:
|
|
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,))
|
|
print('locked',flush=True)
|
|
sys.stdin.readline()
|
|
'''
|
|
holder=subprocess.Popen(['docker','exec','-i',survivor['container'],'python','-u','-c',lock_code],cwd=ROOT,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
|
try:
|
|
holder.stdin.write(locked+'\n')
|
|
holder.stdin.flush()
|
|
check(holder.stdout.readline().strip()=='locked','failed to hold cleanup coordination lock')
|
|
dry=internal({'action':'sweep','grace':60,'apply':False})
|
|
check(dry['deleted']==0 and dry['orphans']>=1 and dry['locked']>=1,'dry-run/grace/lock selection failed')
|
|
check(internal({'action':'exists','key':orphan})['exists'],'dry-run deleted an object')
|
|
applied=internal({'action':'sweep','grace':60,'apply':True})
|
|
check(applied['deleted']>=1,'orphan not removed after grace')
|
|
check(not internal({'action':'exists','key':orphan})['exists'],'crash orphan remained after cleanup')
|
|
check(internal({'action':'exists','key':locked})['exists'],'active upload lock was ignored')
|
|
check(internal({'action':'exists','key':fresh})['exists'],'fresh object was deleted')
|
|
repeat=internal({'action':'sweep','grace':60,'apply':True})
|
|
check(repeat['deleted']==0,'repeated sweep was not idempotent')
|
|
check(c.call('GET','/api/v1/attachments/'+tracked['id']+'/content',actor=bob)['bytes']==raw,'tracked/linked file was deleted')
|
|
finally:
|
|
holder.stdin.write('\n')
|
|
holder.stdin.flush()
|
|
holder.stdin.close()
|
|
holder.wait(timeout=5)
|
|
released=internal({'action':'sweep','grace':60,'apply':True})
|
|
check(not internal({'action':'exists','key':locked})['exists'],'unlocked old orphan not removed')
|
|
internal({'action':'delete','key':fresh})
|
|
save('orphan-cleanup.json',{'result':'PASS','fault':'SIGKILL API after successful S3PUT, before metadata COMMIT',
|
|
'crash_exit_code':observed_exit_code,'target_container':target['container'],'upload_acknowledged':False,'orphan_confirmed':True,'grace_wait_seconds':61,
|
|
'dry_run':dry,'apply':applied,'repeat':repeat,'after_lock_release':released,
|
|
'fresh_preserved':True,'tracked_and_linked_preserved':True,'active_lock_preserved':True})
|
|
|
|
|
|
def s3_outage():
|
|
c=Client(timeout=8)
|
|
alice,bob,_,chat=c.bootstrap()
|
|
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
att=upload(c,alice,chat,raw)['json']
|
|
try:
|
|
cluster.compose('stop','-t','2','minio')
|
|
req=urllib.request.Request(c.base+'/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',data=raw,method='POST',headers={'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png'})
|
|
started=time.monotonic()
|
|
try:
|
|
response=urllib.request.urlopen(req,timeout=8)
|
|
except urllib.error.HTTPError as exc:
|
|
response=exc
|
|
with response:
|
|
status=response.code
|
|
elapsed=time.monotonic()-started
|
|
check(status in (502,503,504) and elapsed<8,'S3 outage was not bounded/safe')
|
|
finally:
|
|
cluster.compose('start','minio')
|
|
cluster.wait_healthy(cluster.container_ids('minio')[0])
|
|
cluster.wait_cluster()
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original lost after S3 restart')
|
|
save('s3-outage.json',{'result':'PASS','upload_status_during_outage':status,'response_seconds':round(elapsed,4),
|
|
'original_preserved_after_restart':True,'same_session_after_recovery':True})
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('mode',choices=['private','failover','interrupted','cleanup','outage','all'],default='all',nargs='?')
|
|
args=parser.parse_args()
|
|
modes={'private':private_bucket,'failover':upload_failover,'interrupted':interrupted_upload,'cleanup':crash_orphan,'outage':s3_outage}
|
|
if args.mode=='all':
|
|
for function in modes.values():
|
|
function()
|
|
else:
|
|
modes[args.mode]()
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|