#!/usr/bin/env python3 """Repeatable lab4 acceptance on an isolated synthetic Compose project. No tokens, request headers, passwords, or filled-in configuration are persisted. """ import concurrent.futures as futures import datetime import hashlib import http.client import json import os from pathlib import Path import platform import select import struct import subprocess import sys import threading import time import uuid import zlib ROOT=Path(__file__).resolve().parents[1] sys.path.insert(0,str(ROOT/'tests')) from client import Client, check OUT=ROOT/'evidence' ENV=dict(os.environ,COMPOSE_PROJECT_NAME='mtusi2026lab4-evidence',HOST_PORT='18082',UPLOAD_TIMEOUT='5s',RECONCILE_GRACE_PERIOD='15s') COMPOSE=['./scripts/compose.sh'] BASE='http://localhost:18082' RUN_ID=uuid.uuid4().hex def save(name,value): temporary=OUT/(name+'.json.tmp') temporary.write_text(json.dumps(value,indent=2)+'\n') temporary.replace(OUT/(name+'.json')) def stage(name): save('acceptance',{'passed':False,'run_id':RUN_ID,'stage':name}) print(name,flush=True) def command(args, name=None, expected=0, **kw): p=subprocess.run(args,cwd=ROOT,env=ENV,text=True,capture_output=True,timeout=kw.pop('timeout',240),**kw) if name: (OUT/(name+'.txt')).write_text('$ '+' '.join(args)+'\n'+p.stdout+p.stderr) check(p.returncode==expected,'command failed: '+' '.join(args)+' (see evidence/'+str(name)+'.txt)') return p.stdout def compose(*args,**kw): return command(COMPOSE+list(args),**kw) def operator(*args,grace=None): options=['-e','RECONCILE_GRACE_PERIOD='+grace] if grace else [] output=compose('run','--rm','-T',*options,'operator','storage',*args) return json.loads(output) def sql(query): return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query).strip() def snapshot(): ids=compose('ps','-q','api','postgres','redis','s3','lb').split() results=[] for cid in ids: info=json.loads(command(['docker','inspect','--format','{{json .State}}',cid])) name=command(['docker','inspect','--format','{{.Name}}',cid]).strip().lstrip('/') results.append({'name':name,'id':cid,'started_at':info['StartedAt'],'running':info['Running'],'health':info.get('Health',{}).get('Status')}) return results def ready(c): deadline=time.monotonic()+90 while time.monotonic()I',len(padding))+chunk+struct.pack('>I',zlib.crc32(chunk))+data[-12:] def upload(c,actor,chat,data): return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',actor=actor,raw=data,expected=(201,429),headers={'Content-Type':'image/png'},schema='Attachment') def direct_api(container,path,actor=None,body=b'',content_type='image/png'): # Feed credentials via stdin, never argv or evidence. Only controlled results # are returned to the report, not the raw HTTP request. method='POST' if body else 'GET' headers=[f'{method} {path} HTTP/1.1','Host: api','Connection: close'] if actor: headers.append('Authorization: Bearer '+actor['token']) if body: headers.extend(['Content-Type: '+content_type,'Content-Length: '+str(len(body))]) raw=('\r\n'.join(headers)+'\r\n\r\n').encode()+body p=subprocess.Popen(['docker','exec','-i',container,'nc','-w','10','127.0.0.1','8080'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) try: # BusyBox nc closes the socket on stdin EOF. Keep stdin open until the # response starts so the probe itself does not cancel the server context. p.stdin.write(raw);p.stdin.flush() readable,_,_=select.select([p.stdout],[],[],15) check(bool(readable),'internal API probe timed out') line=p.stdout.readline() return int(line.split(b' ',2)[1]) finally: p.stdin.close() try: p.wait(timeout=3) except subprocess.TimeoutExpired: p.kill();p.wait() def failures(c,actor,chat,data): findings={} att=upload(c,actor,chat,data)['json'] c.message(actor,chat,'',attachments=[att['id']]) key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'") raw=compose('exec','-T','api','/usr/local/bin/messenger','storage','private-check','--key',key) findings['private_bucket']=json.loads(raw) save('failures',findings) before=snapshot() command([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh','--base-url',BASE],name='persistence') after=snapshot() after_by_name={x['name']:x for x in after} check(all(old['name'] in after_by_name and after_by_name[old['name']]['running'] and (after_by_name[old['name']]['id']!=old['id'] or after_by_name[old['name']]['started_at']!=old['started_at']) for old in before),'restart hook did not restart or replace every store/API/LB') findings['restart']={'before':before,'after':after} save('failures',findings) findings['failover']=[] for target in compose('ps','-q','api').split(): hostname=command(['docker','inspect','--format','{{.Config.Hostname}}',target]).strip() for _ in range(20): response=upload(c,actor,chat,data) if response['headers']['X-Instance-Id'].startswith(hostname+'-'):break else: raise AssertionError('cannot route upload to intended replica') attachment=response['json'];alias=response['headers']['X-Instance-Id'] stop_env=dict(ENV,TARGET_INSTANCE_ID=alias) p=subprocess.run(['./scripts/stop-one.sh'],cwd=ROOT,env=stop_env,capture_output=True,timeout=15) check(p.returncode==0,'targeted stop failed') deadline=time.monotonic()+10 while True: try: downloaded=c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor) check(downloaded['bytes']==data,'failover changed original') check(downloaded['headers']['X-Instance-Id']!=alias,'stopped uploader served download') break except Exception: check(time.monotonic()=1,'failed DB write did not leave detectable orphan') # Missing objects retain IDs/references and can only be restored from originals. operator('remove-object','--id',att['id']) dry=operator('reconcile');check(dry['missing']>=1,'missing object not detected') apply=operator('reconcile','--apply');check(apply['missing']>=1,'missing object not marked') c.call('GET','/api/v1/attachments/'+att['id'],503,actor=actor) bad=compose('run','--rm','-T','operator','storage','restore','--id',att['id'],'--source','/fixtures/README.md',expected=1) operator('restore','--id',att['id'],'--source','/fixtures/sample.png') operator('restore','--id',att['id'],'--source','/fixtures/sample.png') downloaded=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor) check(downloaded['bytes']==data,'restored bytes changed') history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=actor,schema='MessagePage')['json'] check(any(att['id'] in m['attachment_ids'] for m in history['items']),'restoration lost message reference') findings['missing_repair']={'dry_run':dry,'apply':apply,'wrong_original_rejected':True,'restored_sha256':hashlib.sha256(data).hexdigest(),'reference_preserved':True,'repeat_safe':True} save('failures',findings) count_before=sql('SELECT count(*) FROM attachments') try: compose('stop','postgres') raw=compose('run','--rm','-T','operator','storage','reconcile','--apply',expected=1,name='reconcile-db-down') scan=json.loads(raw) check(scan['deleted']==0 and scan['marked_missing']==0,'DB outage caused reconciliation writes') findings['database_outage']={'reconciler':scan} finally: compose('up','-d','--no-deps','--wait','postgres');ready(c) check(sql('SELECT count(*) FROM attachments')==count_before,'DB restart lost attachment metadata') save('failures',findings) return findings def measurements(c,actor,chat,data): records=[] for size in [len(data),5*1024*1024]: body=data if size==len(data) else padded_png(data,size) for trial in range(1,4): compose('up','-d','--no-deps','--force-recreate','--wait','api');ready(c) upload(c,actor,chat,data) # One documented warm-up upload. ids=compose('ps','-q','api').split() cpu=subprocess.Popen(['docker','stats','--no-stream','--format','{{json .}}',*ids],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) barrier=threading.Barrier(10) def work(_): barrier.wait();return upload(Client(BASE,timeout=30),actor,chat,body) start=time.monotonic() with futures.ThreadPoolExecutor(max_workers=10) as pool: results=list(pool.map(work,range(10))) duration=time.monotonic()-start successful=[x['elapsed'] for x in results if x['status']==201] rss=[] for cid in ids: values=command(['docker','exec',cid,'sh','-c','cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak']).split() rss.append({'container':cid,'memory_current_bytes':int(values[0]),'memory_peak_bytes':int(values[1])}) stats,_=cpu.communicate(timeout=15) cpu_samples=[{'container':v.get('Name'),'cpu_percent':v.get('CPUPerc')} for v in [json.loads(line) for line in stats.splitlines()]] def quantile(p): return sorted(successful)[min(len(successful)-1,int((len(successful)-1)*p))] if successful else None records.append({'bytes':len(body),'trial':trial,'clients':10,'warmup_uploads':1,'duration_seconds':duration,'successful':len(successful),'errors':10-len(successful),'error_rate':(10-len(successful))/10,'successful_rps':len(successful)/duration,'p50':quantile(.5),'p95':quantile(.95),'p99':quantile(.99),'status_counts':{str(code):sum(x['status']==code for x in results) for code in [201,429]},'memory':rss,'cpu':cpu_samples}) save('uploads',records) return records def research(c,actor,chat,data): results=[] # Demonstrate freshness independently of the duration of mass seeding/scanning. fresh_key=operator('seed')['keys'][0] invalid_key='originals/not-a-server-uuid-'+uuid.uuid4().hex outside_key='unmanaged/'+uuid.uuid4().hex operator('seed','--key',invalid_key) operator('seed','--key',outside_key) fresh=operator('reconcile','--apply',grace='1h') check(operator('object-check','--key',fresh_key)['exists'],'fresh object deleted') # Upload and reconciliation execute concurrently; every acknowledged upload # must remain downloadable. The separate PostgreSQL test checks lock contention. with futures.ThreadPoolExecutor(max_workers=2) as workers: scan=workers.submit(operator,'reconcile','--apply') concurrent_upload=upload(c,actor,chat,data) scan_result=scan.result() check(concurrent_upload['status']==201,'concurrent upload failed') att=concurrent_upload['json'] check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)['bytes']==data,'concurrent reconciliation damaged upload') safety={'fresh_apply':fresh,'fresh_key_preserved':True,'concurrent_scan':scan_result,'concurrent_upload_sha256':hashlib.sha256(data).hexdigest()} save('reconciliation-safety',safety) for count in [100,1000]: operator('seed','--count',str(count)) # All objects exist before the grace wait starts. Duration of seeding is # intentionally excluded from the scan timing, but dataset size is recorded. time.sleep(16) scans=[operator('reconcile') for _ in range(3)] applied=operator('reconcile','--apply') repeated=operator('reconcile','--apply') check(applied['deleted']>=count,'old orphans not cleaned') check(repeated['deleted']==0 and repeated['missing']==0,'repeat was not safe or valid original deleted') check(operator('object-check','--key',invalid_key)['exists'],'invalid managed key deleted') check(operator('object-check','--key',outside_key)['exists'],'object outside prefix deleted') results.append({'seeded':count,'dry_runs':scans,'apply':applied,'repeat_apply':repeated,'invalid_key_preserved':True,'outside_prefix_preserved':True}) save('reconciliation',results) return results def main(): OUT.mkdir(exist_ok=True) stage("bootstrap") command(['./scripts/bootstrap.sh'],name='bootstrap') before_keys=hashlib.sha256((ROOT/'secrets/lab4/s3.json').read_bytes()).hexdigest() command(['./scripts/bootstrap.sh'],name='bootstrap-repeat') check(before_keys==hashlib.sha256((ROOT/'secrets/lab4/s3.json').read_bytes()).hexdigest(),'bootstrap rotated credentials') compose('config','--quiet',name='compose-config') compose('up','-d','--build','--wait','--wait-timeout','120',name='stand-up',timeout=600) meta={'run_id':RUN_ID,'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'git_head':command(['git','rev-parse','HEAD']).strip(),'working_tree_dirty':bool(command(['git','status','--porcelain']).strip()),'host':platform.platform(),'docker':json.loads(command(['docker','info','--format','{{json .}}'])),'project':ENV['COMPOSE_PROJECT_NAME'],'limits':{'api_replicas':2,'api_cpu_each':1,'api_ram_each':536870912,'upload_timeout':'5s','grace':'15s','upload_slots_each':4,'decode_slots_each':1}} # Keep a small allow-list; docker info can contain unrelated host settings. meta['docker']={k:meta['docker'][k] for k in ['ServerVersion','Architecture','NCPU','MemTotal']} (OUT/'environment.json').write_text(json.dumps(meta,indent=2)+'\n') compose('run','--rm','-T','s3-init',name='bucket-bootstrap-repeat') c=Client(BASE,timeout=20);ready(c) command(['make','check','build','vet'],name='build-checks') validator=str(ROOT/'.venv/bin/python') if (ROOT/'.venv/bin/python').exists() else sys.executable command(['make','validate','PYTHON='+validator],name='validate') command(['make','test-go','BASE_URL='+BASE],name='go-tests') command(['make','test-storage'],name='storage-tests') command(['make','test','BASE_URL='+BASE],name='smoke') actor,bob,eve,chat=c.bootstrap();data=(ROOT/'tests/fixtures/sample.png').read_bytes() stage('faults') findings=failures(c,actor,chat,data) (OUT/'failures.json').write_text(json.dumps(findings,indent=2)+'\n') stage('uploads') (OUT/'uploads.json').write_text(json.dumps(measurements(c,actor,chat,data),indent=2)+'\n') stage('reconciliation') (OUT/'reconciliation.json').write_text(json.dumps(research(c,actor,chat,data),indent=2)+'\n') result={'run_id':RUN_ID,'passed':True,'checks':['contract','go_race','transaction_rollback','smoke','private_bucket','persistence','both_uploaders_failover','public_failover','s3_down','interrupted_body','s3_put_db_timeout','missing_restore','10_client_memory','reconciler_grace','reconciler_repeat'],'remaining_metadata':int(sql('SELECT count(*) FROM attachments')),'timestamp_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()} compose('down',name='stand-down') save('acceptance',result) # Preserve all experiment volumes. print('PASS: lab4 acceptance; summaries in evidence/. Isolated stand stopped; volumes preserved.',flush=True) if __name__=='__main__': main()