179 lines
10 KiB
Python
179 lines
10 KiB
Python
"""Black-box attachment checks against real PostgreSQL and private SeaweedFS."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
|
|
import httpx
|
|
from stack import ROOT, docker, instance, ready, ready_replicas, replicas
|
|
|
|
OUT=ROOT/'evidence'
|
|
OUT.mkdir(exist_ok=True)
|
|
result={}
|
|
|
|
def sql(q): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',q)
|
|
|
|
def inside(code): return docker('compose','exec','-T','api','python','-c',code)
|
|
|
|
def user(c):
|
|
name='test_'+uuid.uuid4().hex[:20]
|
|
password='synthetic-password-123'
|
|
r=c.post('/api/v1/users',json={'username':name,'display_name':'Files','password':password}); assert r.status_code==201,r.text
|
|
uid=r.json()['id']
|
|
r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201,r.text
|
|
return uid,{'Authorization':'Bearer '+r.json()['token']}
|
|
|
|
def newchat(c,h,members):
|
|
r=c.post('/api/v1/chats',headers=h,json={'title':'Attachments','member_ids':members})
|
|
assert r.status_code==201,r.text
|
|
return r.json()['id']
|
|
|
|
def restore(c):
|
|
docker('compose','up','-d','--no-deps','--scale','api=2','api'); ready_replicas(); ready(c)
|
|
deadline=time.monotonic()+15
|
|
while time.monotonic()<deadline:
|
|
ids={c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(12)}-{None}
|
|
if len(ids)==2: return
|
|
time.sleep(.3)
|
|
raise AssertionError('LB did not recover two replicas')
|
|
|
|
def public(mode):
|
|
name='restart' if mode=='persistence' else 'stop-one'
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
if os.name=='nt':
|
|
hook=Path(tmp)/(name+'.cmd')
|
|
hook.write_text('@echo off\n"'+sys.executable+'" "'+str(ROOT/('scripts/'+name+'.py'))+'"\n',encoding='utf-8')
|
|
else: hook=ROOT/('scripts/'+name+'.sh')
|
|
run=subprocess.run([sys.executable,'-X','utf8','tests/resilience.py','--mode',mode,'--action-script',str(hook)],cwd=ROOT,text=True,encoding='utf-8',capture_output=True)
|
|
if run.returncode: raise AssertionError(run.stdout+run.stderr)
|
|
return run.stdout.strip()
|
|
|
|
def png(width,height,raw):
|
|
def chunk(kind,data): return struct.pack('!I',len(data))+kind+data+struct.pack('!I',zlib.crc32(kind+data)&0xffffffff)
|
|
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
|
|
|
|
def memory():
|
|
data={}
|
|
for cid in replicas():
|
|
status=docker('exec',cid,'python','-c',"print(open('/proc/1/status').read())")
|
|
data[cid]={k:int(next(line.split()[1] for line in status.splitlines() if line.startswith(k+':'))) for k in ('VmRSS','VmHWM')}
|
|
limits=json.loads(docker('exec',cid,'python','-c',"import json,pathlib; p=pathlib.Path('/sys/fs/cgroup'); print(json.dumps({'peak_bytes':int((p/'memory.peak').read_text()),'events':dict(line.split() for line in (p/'memory.events').read_text().splitlines())}))"))
|
|
data[cid]['cgroup']=limits
|
|
return data
|
|
|
|
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
|
|
restore(c)
|
|
result['public_smoke']=subprocess.check_output([sys.executable,'-X','utf8','tests/smoke.py'],cwd=ROOT,text=True).strip()
|
|
uid,h=user(c); bob,bh=user(c); eve,eh=user(c)
|
|
chat=newchat(c,h,[uid,bob]); path=f'/api/v1/chats/{chat}/attachments'
|
|
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
def upload(body=data,mime='image/png',filename='sample.png',headers=h):
|
|
return c.post(path,params={'filename':filename},headers={**headers,'Content-Type':mime},content=body)
|
|
r=upload(); assert r.status_code==201,r.text
|
|
att=r.json(); serving=r.headers['X-Instance-Id']
|
|
ap='/api/v1/attachments/'+att['id']
|
|
assert att['sha256']==hashlib.sha256(data).hexdigest()
|
|
assert c.get(ap+'/content',headers=bh).content==data
|
|
for suffix in ('','/content'): assert c.get(ap+suffix,headers=eh).status_code==404
|
|
assert upload(headers=eh).status_code==404
|
|
statuses={}
|
|
for label,body,mime,filename,expected in [
|
|
('oversize',data+b'x'*(10485761-len(data)),'image/png','big.png',413),
|
|
('corrupt',b'not an image','image/png','bad.png',415),
|
|
('truncated',data[:50],'image/png','cut.png',415),
|
|
('mismatch',data,'image/jpeg','fake.jpg',415),
|
|
('unsupported',data,'text/plain','plain.png',415),
|
|
('path',data,'image/png','../secret.png',400),
|
|
('backslash',data,'image/png','a\\b.png',400),
|
|
('control',data,'image/png','a\x00.png',400),
|
|
('pixels',png(8000,6000,b'\x00'),'image/png','bomb.png',413)]:
|
|
r=upload(body,mime,filename); statuses[label]=r.status_code
|
|
assert r.status_code==expected,(label,r.status_code,r.text[:100])
|
|
result['negative_uploads']=statuses
|
|
jpeg=base64.b64decode(inside("from PIL import Image; import io,base64; f=io.BytesIO(); Image.new('RGB',(320,200),'white').save(f,format='JPEG'); print(base64.b64encode(f.getvalue()).decode())"))
|
|
r=upload(jpeg,'image/jpeg','valid.jpg'); assert r.status_code==201,r.text
|
|
assert c.get('/api/v1/attachments/'+r.json()['id']+'/content',headers=bh).content==jpeg
|
|
r=upload(jpeg[:-32],'image/jpeg','truncated.jpg'); assert r.status_code==415,r.text
|
|
result['jpeg_validation']='valid JPEG returned byte-for-byte; truncated entropy stream rejected with 415'
|
|
other=newchat(c,h,[uid])
|
|
r=c.post(f'/api/v1/chats/{other}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id']]})
|
|
assert r.status_code==404
|
|
r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[str(uuid.uuid4())]})
|
|
assert r.status_code==404
|
|
key=uuid.uuid4().hex
|
|
msgpath=f'/api/v1/chats/{chat}/messages'
|
|
one=c.post(msgpath,headers={**h,'Idempotency-Key':key},json={'text':'Canonical'})
|
|
two=c.post(msgpath,headers={**h,'Idempotency-Key':key},json={'attachment_ids':[],'text':'Canonical'})
|
|
assert one.status_code==two.status_code==201 and one.json()==two.json()
|
|
result['attachment_acl_and_canonical_body']='foreign/unknown attachments 404; absent and [] attachment_ids replay same message'
|
|
key=sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'")
|
|
private=inside("import urllib.request,urllib.error\ntry:\n urllib.request.urlopen('http://s3:8333/messenger/"+key+"'); print(200)\nexcept urllib.error.HTTPError as e: print(e.code)")
|
|
assert private=='403',private
|
|
denied=inside("from src.storage import Storage\nfrom botocore.exceptions import ClientError\ns=Storage()\ntry:\n s.client.create_bucket(Bucket='denied-"+uuid.uuid4().hex+"'); print(200)\nexcept ClientError as e: print(e.response['ResponseMetadata']['HTTPStatusCode'])")
|
|
assert denied=='403',denied
|
|
denied_prefix=inside("from src.storage import Storage\nfrom botocore.exceptions import ClientError\ns=Storage()\ntry:\n s.client.put_object(Bucket=s.bucket,Key='forbidden/outside-prefix',Body=b'x'); print(200)\nexcept ClientError as e: print(e.response['ResponseMetadata']['HTTPStatusCode'])")
|
|
assert denied_prefix=='403',denied_prefix
|
|
result['private_bucket']={'anonymous_get':403,'api_create_other_bucket':403,'api_write_outside_prefix':403}
|
|
target=next(cid for cid in replicas() if instance(cid)==serving)
|
|
docker('stop','-t','5',target)
|
|
try:
|
|
deadline=time.monotonic()+10
|
|
while True:
|
|
r=c.get(ap+'/content',headers=bh)
|
|
if r.status_code==200: break
|
|
assert time.monotonic()<deadline
|
|
time.sleep(.2)
|
|
assert r.content==data and r.headers['X-Instance-Id']!=serving
|
|
result['upload_then_stop_download']={'stopped_container':target,'uploader_instance':serving,'downloader_instance':r.headers['X-Instance-Id'],'sha256_matches':True}
|
|
finally: restore(c)
|
|
# Abort a partially transmitted upload before S3 is touched.
|
|
before=sql('SELECT count(*) FROM attachments')
|
|
with socket.create_connection(('127.0.0.1',8080),timeout=3) as sock:
|
|
request=(f'POST {path}?filename=interrupted.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: {h["Authorization"]}\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n').encode()
|
|
sock.sendall(request+data[:64])
|
|
time.sleep(.5)
|
|
assert sql('SELECT count(*) FROM attachments')==before
|
|
result['interrupted_upload']='partial body closed: no attachment metadata'
|
|
docker('compose','stop','s3')
|
|
try:
|
|
r=upload(); assert r.status_code==503,r.text
|
|
result['s3_down_status']=r.status_code
|
|
finally: docker('compose','start','s3')
|
|
ready(c); restore(c)
|
|
result['bootstrap_repeat']=docker('compose','run','--rm','--no-deps','storage-init')
|
|
result['public_persistence']=public('persistence'); restore(c)
|
|
result['public_failover']=public('failover'); restore(c)
|
|
# Ten simultaneous HTTP uploads, bounded admission with retries on explicit 503.
|
|
large=png(1800,1500,b''.join(b'\x00'+os.urandom(1800*3) for _ in range(1500)))
|
|
assert 7_000_000<len(large)<10485760
|
|
barrier=threading.Barrier(10)
|
|
before_memory=memory()
|
|
def concurrent(i):
|
|
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as client:
|
|
barrier.wait(); started=time.perf_counter(); attempts=[]
|
|
for attempt in range(50):
|
|
r=client.post(path,params={'filename':f'load-{i}.png'},headers={**h,'Content-Type':'image/png'},content=large)
|
|
attempts.append(r.status_code)
|
|
if r.status_code==201:
|
|
assert r.json()['sha256']==hashlib.sha256(large).hexdigest()
|
|
return {'i':i,'seconds':round(time.perf_counter()-started,3),'statuses':attempts}
|
|
assert r.status_code==503,r.text
|
|
time.sleep(.2)
|
|
raise AssertionError('Upload did not complete within bounded retries')
|
|
with ThreadPoolExecutor(max_workers=10) as pool: uploads=list(pool.map(concurrent,range(10)))
|
|
result['concurrent_uploads']={'clients':10,'bytes_each':len(large),'before_memory_kib':before_memory,'after_memory_kib':memory(),'uploads':uploads}
|
|
result['passed']=True
|
|
(OUT/'integration.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(result,ensure_ascii=False,indent=2))
|