Files
mtusi2026lab4/scripts/verify_stack.py
T

187 lines
10 KiB
Python

"""Black-box attachment checks against real PostgreSQL and private SeaweedFS."""
from concurrent.futures import ThreadPoolExecutor
import base64
import hashlib
import json
import os
from pathlib import Path
import socket
import struct
import subprocess
import sys
import tempfile
import threading
import time
import uuid
import zlib
import httpx
from stack import ROOT, docker, instance, ready, ready_replicas, replicas
OUT=ROOT/'evidence'
OUT.mkdir(exist_ok=True)
result={}
def sql(q): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',q)
def inside(code): return docker('compose','exec','-T','api','python','-c',code)
def user(c):
name='test_'+uuid.uuid4().hex[:20]
password='synthetic-password-123'
r=c.post('/api/v1/users',json={'username':name,'display_name':'Files','password':password}); assert r.status_code==201,r.text
uid=r.json()['id']
r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201,r.text
return uid,{'Authorization':'Bearer '+r.json()['token']}
def newchat(c,h,members):
r=c.post('/api/v1/chats',headers=h,json={'title':'Attachments','member_ids':members})
assert r.status_code==201,r.text
return r.json()['id']
def restore(c):
docker('compose','up','-d','--no-deps','--scale','api=2','api'); ready_replicas(); ready(c)
deadline=time.monotonic()+15
while time.monotonic()<deadline:
ids={c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(12)}-{None}
if len(ids)==2: return
time.sleep(.3)
raise AssertionError('LB did not recover two replicas')
def public(mode):
name='restart' if mode=='persistence' else 'stop-one'
with tempfile.TemporaryDirectory() as tmp:
if os.name=='nt':
hook=Path(tmp)/(name+'.cmd')
hook.write_text('@echo off\n"'+sys.executable+'" "'+str(ROOT/('scripts/'+name+'.py'))+'"\n',encoding='utf-8')
else: hook=ROOT/('scripts/'+name+'.sh')
run=subprocess.run([sys.executable,'-X','utf8','tests/resilience.py','--mode',mode,'--action-script',str(hook)],cwd=ROOT,text=True,encoding='utf-8',capture_output=True)
if run.returncode: raise AssertionError(run.stdout+run.stderr)
return run.stdout.strip()
def png(width,height,raw):
def chunk(kind,data): return struct.pack('!I',len(data))+kind+data+struct.pack('!I',zlib.crc32(kind+data)&0xffffffff)
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
def memory():
data={}
for cid in replicas():
status=docker('exec',cid,'python','-c',"print(open('/proc/1/status').read())")
data[cid]={k:int(next(line.split()[1] for line in status.splitlines() if line.startswith(k+':'))) for k in ('VmRSS','VmHWM')}
probe="""import json,pathlib
p=pathlib.Path('/sys/fs/cgroup')
if (p/'memory.peak').exists():
result={'version':2,'peak_bytes':int((p/'memory.peak').read_text()),'events':dict(line.split() for line in (p/'memory.events').read_text().splitlines())}
else:
p=p/'memory'
result={'version':1,'peak_bytes':int((p/'memory.max_usage_in_bytes').read_text()),'failcnt':int((p/'memory.failcnt').read_text()),'oom_control':(p/'memory.oom_control').read_text()}
print(json.dumps(result))"""
limits=json.loads(docker('exec',cid,'python','-c',probe))
data[cid]['cgroup']=limits
return data
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
restore(c)
result['public_smoke']=subprocess.check_output([sys.executable,'-X','utf8','tests/smoke.py'],cwd=ROOT,text=True).strip()
uid,h=user(c); bob,bh=user(c); eve,eh=user(c)
chat=newchat(c,h,[uid,bob]); path=f'/api/v1/chats/{chat}/attachments'
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
def upload(body=data,mime='image/png',filename='sample.png',headers=h):
return c.post(path,params={'filename':filename},headers={**headers,'Content-Type':mime},content=body)
r=upload(); assert r.status_code==201,r.text
att=r.json(); serving=r.headers['X-Instance-Id']
ap='/api/v1/attachments/'+att['id']
assert att['sha256']==hashlib.sha256(data).hexdigest()
assert c.get(ap+'/content',headers=bh).content==data
for suffix in ('','/content'): assert c.get(ap+suffix,headers=eh).status_code==404
assert upload(headers=eh).status_code==404
statuses={}
for label,body,mime,filename,expected in [
('oversize',data+b'x'*(10485761-len(data)),'image/png','big.png',413),
('corrupt',b'not an image','image/png','bad.png',415),
('truncated',data[:50],'image/png','cut.png',415),
('mismatch',data,'image/jpeg','fake.jpg',415),
('unsupported',data,'text/plain','plain.png',415),
('path',data,'image/png','../secret.png',400),
('backslash',data,'image/png','a\\b.png',400),
('control',data,'image/png','a\x00.png',400),
('pixels',png(8000,6000,b'\x00'),'image/png','bomb.png',413)]:
r=upload(body,mime,filename); statuses[label]=r.status_code
assert r.status_code==expected,(label,r.status_code,r.text[:100])
result['negative_uploads']=statuses
jpeg=base64.b64decode(inside("from PIL import Image; import io,base64; f=io.BytesIO(); Image.new('RGB',(320,200),'white').save(f,format='JPEG'); print(base64.b64encode(f.getvalue()).decode())"))
r=upload(jpeg,'image/jpeg','valid.jpg'); assert r.status_code==201,r.text
assert c.get('/api/v1/attachments/'+r.json()['id']+'/content',headers=bh).content==jpeg
r=upload(jpeg[:-32],'image/jpeg','truncated.jpg'); assert r.status_code==415,r.text
result['jpeg_validation']='valid JPEG returned byte-for-byte; truncated entropy stream rejected with 415'
other=newchat(c,h,[uid])
r=c.post(f'/api/v1/chats/{other}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id']]})
assert r.status_code==404
r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[str(uuid.uuid4())]})
assert r.status_code==404
key=uuid.uuid4().hex
msgpath=f'/api/v1/chats/{chat}/messages'
one=c.post(msgpath,headers={**h,'Idempotency-Key':key},json={'text':'Canonical'})
two=c.post(msgpath,headers={**h,'Idempotency-Key':key},json={'attachment_ids':[],'text':'Canonical'})
assert one.status_code==two.status_code==201 and one.json()==two.json()
result['attachment_acl_and_canonical_body']='foreign/unknown attachments 404; absent and [] attachment_ids replay same message'
key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'")
private=inside("import urllib.request,urllib.error\ntry:\n urllib.request.urlopen('http://s3:8333/messenger/"+key+"'); print(200)\nexcept urllib.error.HTTPError as e: print(e.code)")
assert private=='403',private
denied=inside("from src.storage import Storage\nfrom botocore.exceptions import ClientError\ns=Storage()\ntry:\n s.client.create_bucket(Bucket='denied-"+uuid.uuid4().hex+"'); print(200)\nexcept ClientError as e: print(e.response['ResponseMetadata']['HTTPStatusCode'])")
assert denied=='403',denied
denied_prefix=inside("from src.storage import Storage\nfrom botocore.exceptions import ClientError\ns=Storage()\ntry:\n s.client.put_object(Bucket=s.bucket,Key='forbidden/outside-prefix',Body=b'x'); print(200)\nexcept ClientError as e: print(e.response['ResponseMetadata']['HTTPStatusCode'])")
assert denied_prefix=='403',denied_prefix
result['private_bucket']={'anonymous_get':403,'api_create_other_bucket':403,'api_write_outside_prefix':403}
target=next(cid for cid in replicas() if instance(cid)==serving)
docker('stop','-t','5',target)
try:
deadline=time.monotonic()+10
while True:
r=c.get(ap+'/content',headers=bh)
if r.status_code==200: break
assert time.monotonic()<deadline
time.sleep(.2)
assert r.content==data and r.headers['X-Instance-Id']!=serving
result['upload_then_stop_download']={'stopped_container':target,'uploader_instance':serving,'downloader_instance':r.headers['X-Instance-Id'],'sha256_matches':True}
finally: restore(c)
# Abort a partially transmitted upload before S3 is touched.
before=sql('SELECT count(*) FROM attachments')
with socket.create_connection(('127.0.0.1',8080),timeout=3) as sock:
request=(f'POST {path}?filename=interrupted.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: {h["Authorization"]}\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n').encode()
sock.sendall(request+data[:64])
time.sleep(.5)
assert sql('SELECT count(*) FROM attachments')==before
result['interrupted_upload']='partial body closed: no attachment metadata'
docker('compose','stop','s3')
try:
r=upload(); assert r.status_code==503,r.text
result['s3_down_status']=r.status_code
finally: docker('compose','start','s3')
ready(c); restore(c)
result['bootstrap_repeat']=docker('compose','run','--rm','--no-deps','storage-init')
result['public_persistence']=public('persistence'); restore(c)
result['public_failover']=public('failover'); restore(c)
# Ten simultaneous HTTP uploads, bounded admission with retries on explicit 503.
large=png(1800,1500,b''.join(b'\x00'+os.urandom(1800*3) for _ in range(1500)))
assert 7_000_000<len(large)<10485760
barrier=threading.Barrier(10)
before_memory=memory()
def concurrent(i):
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as client:
barrier.wait(); started=time.perf_counter(); attempts=[]
for attempt in range(50):
r=client.post(path,params={'filename':f'load-{i}.png'},headers={**h,'Content-Type':'image/png'},content=large)
attempts.append(r.status_code)
if r.status_code==201:
assert r.json()['sha256']==hashlib.sha256(large).hexdigest()
return {'i':i,'seconds':round(time.perf_counter()-started,3),'statuses':attempts}
assert r.status_code==503,r.text
time.sleep(.2)
raise AssertionError('Upload did not complete within bounded retries')
with ThreadPoolExecutor(max_workers=10) as pool: uploads=list(pool.map(concurrent,range(10)))
result['concurrent_uploads']={'clients':10,'bytes_each':len(large),'before_memory_kib':before_memory,'after_memory_kib':memory(),'uploads':uploads}
result['passed']=True
(OUT/'integration.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
print(json.dumps(result,ensure_ascii=False,indent=2))