#!/usr/bin/env python3 """Grade4 attachment evidence with real S3 faults, process crash and grace cleanup.""" import argparse import base64 from concurrent.futures import ThreadPoolExecutor import hashlib import json import os from pathlib import Path import socket import subprocess import sys import time import urllib.request import urllib.error import urllib.parse import uuid import cluster ROOT=Path(__file__).resolve().parents[1] sys.path.insert(0,str(ROOT/'tests')) from client import Client,Failure,check def save(name,value): (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') print('PASS:',name,flush=True) def internal(data,container=None): container=container or cluster.container_ids('api')[0] p=subprocess.run(['docker','exec','-i',container,'python','scripts/attachment_checks.py'],cwd=ROOT, input=json.dumps(data),capture_output=True,text=True,check=True,timeout=30) return json.loads(p.stdout) def upload(c,actor,chat,raw,filename='sample.png'): return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+urllib.parse.quote(filename), 201,actor=actor,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment') def private_bucket(): c=Client() alice,bob,_,chat=c.bootstrap() raw=(ROOT/'tests/fixtures/sample.png').read_bytes() response=upload(c,alice,chat,raw) att=response['json'] key=internal({'action':'lookup','id':att['id']})['object_key'] result=internal({'action':'security','key':key}) check(result['anonymous_get']==403,'private object is accessible without S3 authentication') check(result['create_bucket_with_api_credentials']==403 and result['list_other_prefix']==403,'API S3 credentials are too broad') check(not result['console_reachable_from_api'] and not result['root_secret_mounted_in_api'],'root/console exposed to API') before=internal({'action':'counts'}) output=cluster.compose('run','--rm','--no-deps','minio-init') check(internal({'action':'counts'})==before,'repeated bucket bootstrap changed data') download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) check(download['bytes']==raw,'bootstrap changed original bytes') save('s3-security.json',{'result':'PASS',**result,'bootstrap_repeated_without_data_loss':True,'bootstrap_output':output}) def upload_failover(): cluster.wait_cluster() c=Client() alice,bob,_,chat=c.bootstrap() raw=(ROOT/'tests/fixtures/sample.png').read_bytes() response=upload(c,alice,chat,raw) att=response['json'] alias=response['headers']['X-Instance-Id'] c.message(alice,chat,'',attachments=[att['id']]) target=next(node for node in cluster.replicas() if node['instance']==alias) started=time.monotonic() try: stopped=cluster.stop_one(alias) deadline=started+10 while True: try: download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) break except Failure: check(time.monotonic()=1 and dry['locked']>=1,'dry-run/grace/lock selection failed') check(internal({'action':'exists','key':orphan})['exists'],'dry-run deleted an object') applied=internal({'action':'sweep','grace':60,'apply':True}) check(applied['deleted']>=1,'orphan not removed after grace') check(not internal({'action':'exists','key':orphan})['exists'],'crash orphan remained after cleanup') check(internal({'action':'exists','key':locked})['exists'],'active upload lock was ignored') check(internal({'action':'exists','key':fresh})['exists'],'fresh object was deleted') repeat=internal({'action':'sweep','grace':60,'apply':True}) check(repeat['deleted']==0,'repeated sweep was not idempotent') check(c.call('GET','/api/v1/attachments/'+tracked['id']+'/content',actor=bob)['bytes']==raw,'tracked/linked file was deleted') finally: holder.stdin.write('\n') holder.stdin.flush() holder.stdin.close() holder.wait(timeout=5) released=internal({'action':'sweep','grace':60,'apply':True}) check(not internal({'action':'exists','key':locked})['exists'],'unlocked old orphan not removed') internal({'action':'delete','key':fresh}) save('orphan-cleanup.json',{'result':'PASS','fault':'SIGKILL API after successful S3PUT, before metadata COMMIT', 'crash_exit_code':observed_exit_code,'target_container':target['container'],'upload_acknowledged':False,'orphan_confirmed':True,'grace_wait_seconds':61, 'dry_run':dry,'apply':applied,'repeat':repeat,'after_lock_release':released, 'fresh_preserved':True,'tracked_and_linked_preserved':True,'active_lock_preserved':True}) def s3_outage(): c=Client(timeout=8) alice,bob,_,chat=c.bootstrap() raw=(ROOT/'tests/fixtures/sample.png').read_bytes() att=upload(c,alice,chat,raw)['json'] try: cluster.compose('stop','-t','2','minio') req=urllib.request.Request(c.base+'/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',data=raw,method='POST',headers={'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png'}) started=time.monotonic() try: response=urllib.request.urlopen(req,timeout=8) except urllib.error.HTTPError as exc: response=exc with response: status=response.code elapsed=time.monotonic()-started check(status in (502,503,504) and elapsed<8,'S3 outage was not bounded/safe') finally: cluster.compose('start','minio') cluster.wait_healthy(cluster.container_ids('minio')[0]) cluster.wait_cluster() check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original lost after S3 restart') save('s3-outage.json',{'result':'PASS','upload_status_during_outage':status,'response_seconds':round(elapsed,4), 'original_preserved_after_restart':True,'same_session_after_recovery':True}) def main(): parser=argparse.ArgumentParser() parser.add_argument('mode',choices=['private','failover','interrupted','cleanup','outage','all'],default='all',nargs='?') args=parser.parse_args() modes={'private':private_bucket,'failover':upload_failover,'interrupted':interrupted_upload,'cleanup':crash_orphan,'outage':s3_outage} if args.mode=='all': for function in modes.values(): function() else: modes[args.mode]() if __name__=='__main__': main()