forked from mtusi2026/mtusi2026lab4
172 lines
9.2 KiB
Python
172 lines
9.2 KiB
Python
"""Grade 4 integration checks against real stores, including fault injection."""
|
|
import asyncio
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import sys
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from PIL import Image
|
|
from botocore.exceptions import ClientError
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(ROOT), str(ROOT/'tests'), str(ROOT/'scripts')]
|
|
from client import Client
|
|
from ops import compose, command, containers, summary
|
|
from app.config import Settings
|
|
from app.store import Store
|
|
from app.attachments import Attachments, client, BUCKET
|
|
|
|
class AttachmentTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.c = Client(timeout=40)
|
|
self.a,self.b,self.e,self.chat = self.c.bootstrap()
|
|
self.path = '/api/v1/chats/'+self.chat['id']+'/attachments?filename=photo.png'
|
|
self.png = (ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
|
|
def upload(self, raw=None, mime='image/png', expected=201, path=None):
|
|
return self.c.call('POST', path or self.path, expected, actor=self.a,
|
|
raw=self.png if raw is None else raw, headers={'Content-Type':mime},
|
|
schema='Attachment' if expected==201 else None)
|
|
|
|
def sql(self, query, params=()):
|
|
with psycopg.connect(Settings.load().database_url, autocommit=True) as conn:
|
|
return conn.execute(query, params).fetchall() if query.lstrip().upper().startswith('SELECT') else conn.execute(query,params)
|
|
|
|
def test_validation_and_acl(self):
|
|
self.upload(b'x'*(10*1024*1024+1),expected=413)
|
|
self.upload(self.png[:30],expected=415)
|
|
self.upload(mime='image/jpeg',expected=415)
|
|
self.upload(path=self.path.replace('photo.png','..%2Fphoto.png'),expected=400)
|
|
bomb=io.BytesIO(); Image.new('1',(8000,5001)).save(bomb,format='PNG')
|
|
self.upload(bomb.getvalue(),expected=413)
|
|
jpg=io.BytesIO(); Image.new('RGB',(64,32),'red').save(jpg,format='JPEG')
|
|
self.upload(jpg.getvalue(),mime='image/jpeg')
|
|
item=self.upload()['json']; p='/api/v1/attachments/'+item['id']
|
|
for suffix in ('','/content'):
|
|
self.c.call('GET',p+suffix,404,actor=self.e)
|
|
self.c.call('GET',p,401)
|
|
self.c.call('GET',p+'/content',actor=self.b)
|
|
other=self.c.call('POST','/api/v1/chats',201,actor=self.a,body={
|
|
'title':'Other','member_ids':[self.a['id']]},schema='Chat')['json']
|
|
foreign=self.upload(path=self.path.replace(self.chat['id'],other['id']))['json']
|
|
for aid in (foreign['id'],str(uuid4())):
|
|
self.c.call(
|
|
'POST','/api/v1/chats/'+self.chat['id']+'/messages',404,actor=self.a,
|
|
body={'text':'foreign','attachment_ids':[aid]},headers={'Idempotency-Key':uuid4().hex})
|
|
message=self.c.message(self.a,self.chat,'',attachments=[item['id']])
|
|
self.assertEqual(message['json']['attachment_ids'],[item['id']])
|
|
self.c.call('POST','/api/v1/chats/'+self.chat['id']+'/messages',400,actor=self.a,
|
|
body={'text':' '},headers={'Idempotency-Key':uuid4().hex})
|
|
|
|
def test_private_bucket_and_bootstrap(self):
|
|
item=self.upload()['json']
|
|
import urllib.request, urllib.error
|
|
key=self.sql('SELECT object_key FROM attachments WHERE id=%s',(item['id'],))[0][0]
|
|
for path in ('attachments', 'attachments/'+key):
|
|
try:
|
|
urllib.request.urlopen('http://minio:9000/'+path,timeout=3)
|
|
self.fail('Anonymous access succeeded')
|
|
except urllib.error.HTTPError as error:
|
|
self.assertEqual(error.code,403)
|
|
s3=client()
|
|
for operation in (lambda:s3.create_bucket(Bucket='forbidden'),
|
|
lambda:s3.put_object(Bucket=BUCKET,Key='outside-prefix',Body=b'x')):
|
|
with self.assertRaises(ClientError) as caught:
|
|
operation()
|
|
self.assertEqual(caught.exception.response['ResponseMetadata']['HTTPStatusCode'],403)
|
|
bootstrap = containers('bucket-init', running=False)[0]['Id']
|
|
command(['docker','start','-a',bootstrap]); command(['docker','start','-a',bootstrap])
|
|
result=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)
|
|
self.assertEqual(result['bytes'],self.png)
|
|
|
|
def test_failed_commit_cleanup_and_abort(self):
|
|
# A partial client body cannot reach PUT.
|
|
before=self.sql('SELECT count(*) FROM attachments')[0][0]
|
|
sock=socket.create_connection(('lb',8080),timeout=3)
|
|
request=('POST '+self.path+' HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer '+self.a['token']+
|
|
'\r\nContent-Type: image/png\r\nContent-Length: 10000\r\n\r\n').encode()+self.png[:20]
|
|
sock.sendall(request); sock.close(); time.sleep(.3)
|
|
self.assertEqual(before,self.sql('SELECT count(*) FROM attachments')[0][0])
|
|
self.sql("""CREATE OR REPLACE FUNCTION lab4_fail_ready() RETURNS trigger LANGUAGE plpgsql AS
|
|
$$ BEGIN IF NEW.status='ready' AND NEW.filename='fault.png' THEN
|
|
RAISE EXCEPTION 'injected commit failure'; END IF; RETURN NEW; END $$""")
|
|
self.sql('CREATE TRIGGER lab4_fail_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_fail_ready()')
|
|
try:
|
|
self.upload(path=self.path.replace('photo.png','fault.png'),expected=503)
|
|
finally:
|
|
self.sql('DROP TRIGGER lab4_fail_ready ON attachments'); self.sql('DROP FUNCTION lab4_fail_ready()')
|
|
aid,key=self.sql("SELECT id,object_key FROM attachments WHERE filename='fault.png' AND chat_id=%s",(self.chat['id'],))[0]
|
|
s3=client(); s3.head_object(Bucket=BUCKET,Key=key)
|
|
async def cleanup():
|
|
store=Store(Settings.load()); await store.open()
|
|
try: return await Attachments(store).cleanup()
|
|
finally: await store.close()
|
|
asyncio.run(cleanup())
|
|
s3.head_object(Bucket=BUCKET,Key=key) # grace protects the fresh pending object
|
|
self.sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s",(aid,))
|
|
self.assertGreaterEqual(asyncio.run(cleanup()),1)
|
|
asyncio.run(cleanup()) # repeat is harmless
|
|
self.assertFalse(self.sql('SELECT id FROM attachments WHERE id=%s',(aid,)))
|
|
with self.assertRaises(ClientError): s3.head_object(Bucket=BUCKET,Key=key)
|
|
self.upload() # ready original survives cleanup
|
|
|
|
def test_s3_outage(self):
|
|
item=self.upload()['json']
|
|
direct=Client('http://api:8080',timeout=10)
|
|
compose('stop','minio')
|
|
try:
|
|
direct.call('GET','/health/live',schema='Health')
|
|
direct.call('GET','/health/ready',503)
|
|
direct.call('POST',self.path,503,actor=self.a,raw=self.png,headers={'Content-Type':'image/png'})
|
|
direct.call('GET','/api/v1/attachments/'+item['id']+'/content',503,actor=self.a)
|
|
finally:
|
|
compose('start','minio')
|
|
time.sleep(5)
|
|
self.assertEqual(self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes'],self.png)
|
|
|
|
def test_ten_parallel_uploads_memory(self):
|
|
# RGB random image is poorly compressible, roughly 8 MiB.
|
|
image=Image.frombytes('RGB',(1700,1600),os.urandom(1700*1600*3))
|
|
out=io.BytesIO(); image.save(out,format='PNG'); raw=out.getvalue()
|
|
ids=[x['Id'] for x in containers()]
|
|
barrier=threading.Barrier(11); stop=threading.Event(); samples=[]
|
|
def sample():
|
|
while not stop.is_set():
|
|
rows=[]
|
|
for cid in ids:
|
|
value=command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.current'])
|
|
rows.append(int(value))
|
|
samples.append({'elapsed':round(time.monotonic()-start,3),'bytes':rows})
|
|
stop.wait(.1)
|
|
start=time.monotonic(); sampler=threading.Thread(target=sample); sampler.start()
|
|
def upload(_):
|
|
barrier.wait()
|
|
t=time.monotonic(); item=self.upload(raw)['json']
|
|
got=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes']
|
|
self.assertEqual(hashlib.sha256(got).hexdigest(),hashlib.sha256(raw).hexdigest())
|
|
return time.monotonic()-t
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
futures=[pool.submit(upload,i) for i in range(10)]
|
|
barrier.wait(); durations=[f.result() for f in futures]
|
|
finally:
|
|
stop.set(); sampler.join()
|
|
result={'concurrency':10,'successful':len(durations),'size_bytes':len(raw),
|
|
'duration_seconds':time.monotonic()-start,'upload_and_download_seconds':durations,
|
|
'api_container_ids':ids,'memory_samples':samples,
|
|
'sampled_peak_bytes':[max(s['bytes'][i] for s in samples) for i in range(len(ids))],
|
|
'cgroup_memory_peak_bytes':[int(command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.peak'])) for cid in ids],
|
|
'memory_limit_bytes':512*1024*1024,
|
|
'note':'memory.current includes cache and runtime; memory.peak covers container lifetime, not only experiment'}
|
|
(ROOT/'evidence/uploads-memory.json').write_text(json.dumps(result,indent=2))
|
|
|
|
if __name__=='__main__': unittest.main(verbosity=2)
|