Files
mtusi2026lab4/scripts/verify_attachments.py

251 lines
12 KiB
Python

#!/usr/bin/env python3
"""Grade4 attachment evidence with real S3 faults, process crash and grace cleanup."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import time
import urllib.request
import urllib.error
import urllib.parse
import uuid
import cluster
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,Failure,check
def save(name,value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:',name,flush=True)
def internal(data,container=None):
container=container or cluster.container_ids('api')[0]
p=subprocess.run(['docker','exec','-i',container,'python','scripts/attachment_checks.py'],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=30)
return json.loads(p.stdout)
def upload(c,actor,chat,raw,filename='sample.png'):
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+urllib.parse.quote(filename),
201,actor=actor,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
def private_bucket():
c=Client()
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=upload(c,alice,chat,raw)
att=response['json']
key=internal({'action':'lookup','id':att['id']})['object_key']
result=internal({'action':'security','key':key})
check(result['anonymous_get']==403,'private object is accessible without S3 authentication')
check(result['create_bucket_with_api_credentials']==403 and result['list_other_prefix']==403,'API S3 credentials are too broad')
check(not result['console_reachable_from_api'] and not result['root_secret_mounted_in_api'],'root/console exposed to API')
before=internal({'action':'counts'})
output=cluster.compose('run','--rm','--no-deps','minio-init')
check(internal({'action':'counts'})==before,'repeated bucket bootstrap changed data')
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(download['bytes']==raw,'bootstrap changed original bytes')
save('s3-security.json',{'result':'PASS',**result,'bootstrap_repeated_without_data_loss':True,'bootstrap_output':output})
def upload_failover():
cluster.wait_cluster()
c=Client()
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=upload(c,alice,chat,raw)
att=response['json']
alias=response['headers']['X-Instance-Id']
c.message(alice,chat,'',attachments=[att['id']])
target=next(node for node in cluster.replicas() if node['instance']==alias)
started=time.monotonic()
try:
stopped=cluster.stop_one(alias)
deadline=started+10
while True:
try:
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
break
except Failure:
check(time.monotonic()<deadline,'file download did not recover in10s')
time.sleep(.1)
check(download['headers']['X-Instance-Id']!=alias,'download came from stopped process')
check(download['bytes']==raw and hashlib.sha256(download['bytes']).hexdigest()==att['sha256'],'file changed across replicas')
check(not cluster.inspect([target['container']])[0]['State']['Running'],'target was not left stopped')
result={'result':'PASS','stopped':stopped,'download_instance':download['headers']['X-Instance-Id'],
'same_bytes':True,'same_sha256':True,'seconds_from_action_start':round(time.monotonic()-started,4)}
finally:
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('attachment-failover.json',result)
def interrupted_upload():
c=Client()
alice,_,_,chat=c.bootstrap()
before=internal({'action':'counts'})
parsed=urllib.parse.urlsplit(c.base)
sock=socket.create_connection((parsed.hostname,parsed.port or 80),timeout=3)
path='/api/v1/chats/'+chat['id']+'/attachments?filename=interrupted.png'
headers=('POST '+path+' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer '+alice['token']+
'\r\nContent-Type: image/png\r\nContent-Length: 500000\r\nConnection: close\r\n\r\n')
sock.sendall(headers.encode()+b'\x89PNG\r\n\x1a\n'+b'x'*100)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
time.sleep(.5)
after=internal({'action':'counts'})
check(before==after,'interrupted request left an object or metadata')
save('interrupted-upload.json',{'result':'PASS','declared_bytes':500000,'sent_bytes':108,
'metadata_unchanged':True,'s3_object_count_unchanged':True})
def crash_orphan():
cluster.wait_cluster()
c=Client()
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
nodes=cluster.replicas()
target,survivor=nodes[0],nodes[1]
filename='crash_'+uuid.uuid4().hex+'.png'
fixture={'filename':filename,'function':'crash_'+uuid.uuid4().hex,'trigger':'crash_'+uuid.uuid4().hex}
before=set(internal({'action':'list'},survivor['container'])['keys'])
internal({'action':'install',**fixture},survivor['container'])
code='''import json,base64,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=base64.b64decode(p['bytes']),method='POST',headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'})
try:
response=urllib.request.urlopen(r,timeout=8)
print(json.dumps({'status':response.code}))
except Exception as exc:
print(json.dumps({'status':None,'error_type':type(exc).__name__}))
'''
data={'address':cluster.front_address(target['container']),'path':'/api/v1/chats/'+chat['id']+'/attachments?filename='+filename,
'bytes':base64.b64encode(raw).decode(),'token':alice['token']}
def post():
p=subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,input=json.dumps(data),capture_output=True,text=True,check=True,timeout=12)
return json.loads(p.stdout)
pool=ThreadPoolExecutor(max_workers=1)
future=pool.submit(post)
try:
deadline=time.monotonic()+5
while not internal({'action':'waiting'},survivor['container'])['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe PUT followed by pending metadata INSERT')
time.sleep(.05)
after=set(internal({'action':'list'},survivor['container'])['keys'])
created=after-before
check(len(created)==1,'expected exactly one new S3 object during pending metadata INSERT')
orphan=created.pop()
cluster.command(['docker','kill','--signal','SIGKILL',target['container']])
crashed=cluster.inspect([target['container']])[0]['State']
check(not crashed['Running'] and crashed['ExitCode']==137,'SIGKILL did not produce the expected stopped state')
observed_exit_code=crashed['ExitCode']
response=future.result(timeout=12)
check(response['status']!=201,'crashed request returned an acknowledgement')
internal({'action':'remove',**fixture},survivor['container'])
check(internal({'action':'tracked','key':orphan},survivor['container'])['tracked']==0,'metadata unexpectedly committed after crash')
check(internal({'action':'exists','key':orphan},survivor['container'])['exists'],'crash did not leave an orphan to clean')
finally:
pool.shutdown(wait=True)
internal({'action':'remove',**fixture},survivor['container'])
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
# A real orphan from crash, plus an old locked object and a tracked/linked attachment.
locked='originals/'+uuid.uuid4().hex
internal({'action':'put','key':locked,'bytes':base64.b64encode(raw).decode()})
tracked=upload(c,alice,chat,raw)['json']
c.message(alice,chat,'',attachments=[tracked['id']])
print('Waiting61s for the enforced60s cleanup grace (no time bypass).',flush=True)
time.sleep(61)
fresh='originals/'+uuid.uuid4().hex
internal({'action':'put','key':fresh,'bytes':base64.b64encode(raw).decode()})
lock_code='''import json,sys,psycopg
from messenger.db import setting
key=sys.stdin.readline().strip()
with psycopg.connect(setting('DATABASE_URL')) as conn:
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,))
print('locked',flush=True)
sys.stdin.readline()
'''
holder=subprocess.Popen(['docker','exec','-i',survivor['container'],'python','-u','-c',lock_code],cwd=ROOT,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
try:
holder.stdin.write(locked+'\n')
holder.stdin.flush()
check(holder.stdout.readline().strip()=='locked','failed to hold cleanup coordination lock')
dry=internal({'action':'sweep','grace':60,'apply':False})
check(dry['deleted']==0 and dry['orphans']>=1 and dry['locked']>=1,'dry-run/grace/lock selection failed')
check(internal({'action':'exists','key':orphan})['exists'],'dry-run deleted an object')
applied=internal({'action':'sweep','grace':60,'apply':True})
check(applied['deleted']>=1,'orphan not removed after grace')
check(not internal({'action':'exists','key':orphan})['exists'],'crash orphan remained after cleanup')
check(internal({'action':'exists','key':locked})['exists'],'active upload lock was ignored')
check(internal({'action':'exists','key':fresh})['exists'],'fresh object was deleted')
repeat=internal({'action':'sweep','grace':60,'apply':True})
check(repeat['deleted']==0,'repeated sweep was not idempotent')
check(c.call('GET','/api/v1/attachments/'+tracked['id']+'/content',actor=bob)['bytes']==raw,'tracked/linked file was deleted')
finally:
holder.stdin.write('\n')
holder.stdin.flush()
holder.stdin.close()
holder.wait(timeout=5)
released=internal({'action':'sweep','grace':60,'apply':True})
check(not internal({'action':'exists','key':locked})['exists'],'unlocked old orphan not removed')
internal({'action':'delete','key':fresh})
save('orphan-cleanup.json',{'result':'PASS','fault':'SIGKILL API after successful S3PUT, before metadata COMMIT',
'crash_exit_code':observed_exit_code,'target_container':target['container'],'upload_acknowledged':False,'orphan_confirmed':True,'grace_wait_seconds':61,
'dry_run':dry,'apply':applied,'repeat':repeat,'after_lock_release':released,
'fresh_preserved':True,'tracked_and_linked_preserved':True,'active_lock_preserved':True})
def s3_outage():
c=Client(timeout=8)
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
att=upload(c,alice,chat,raw)['json']
try:
cluster.compose('stop','-t','2','minio')
req=urllib.request.Request(c.base+'/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',data=raw,method='POST',headers={'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png'})
started=time.monotonic()
try:
response=urllib.request.urlopen(req,timeout=8)
except urllib.error.HTTPError as exc:
response=exc
with response:
status=response.code
elapsed=time.monotonic()-started
check(status in (502,503,504) and elapsed<8,'S3 outage was not bounded/safe')
finally:
cluster.compose('start','minio')
cluster.wait_healthy(cluster.container_ids('minio')[0])
cluster.wait_cluster()
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original lost after S3 restart')
save('s3-outage.json',{'result':'PASS','upload_status_during_outage':status,'response_seconds':round(elapsed,4),
'original_preserved_after_restart':True,'same_session_after_recovery':True})
def main():
parser=argparse.ArgumentParser()
parser.add_argument('mode',choices=['private','failover','interrupted','cleanup','outage','all'],default='all',nargs='?')
args=parser.parse_args()
modes={'private':private_bucket,'failover':upload_failover,'interrupted':interrupted_upload,'cleanup':crash_orphan,'outage':s3_outage}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__=='__main__':
main()