Files
mtusi2026lab4/student-tests/test_attachments.py
T

172 lines
9.2 KiB
Python

"""Grade 4 integration checks against real stores, including fault injection."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
import hashlib
import io
import json
import os
from pathlib import Path
import socket
import sys
import threading
import time
import unittest
from uuid import uuid4
import psycopg
from PIL import Image
from botocore.exceptions import ClientError
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT), str(ROOT/'tests'), str(ROOT/'scripts')]
from client import Client
from ops import compose, command, containers, summary
from app.config import Settings
from app.store import Store
from app.attachments import Attachments, client, BUCKET
class AttachmentTests(unittest.TestCase):
def setUp(self):
self.c = Client(timeout=40)
self.a,self.b,self.e,self.chat = self.c.bootstrap()
self.path = '/api/v1/chats/'+self.chat['id']+'/attachments?filename=photo.png'
self.png = (ROOT/'tests/fixtures/sample.png').read_bytes()
def upload(self, raw=None, mime='image/png', expected=201, path=None):
return self.c.call('POST', path or self.path, expected, actor=self.a,
raw=self.png if raw is None else raw, headers={'Content-Type':mime},
schema='Attachment' if expected==201 else None)
def sql(self, query, params=()):
with psycopg.connect(Settings.load().database_url, autocommit=True) as conn:
return conn.execute(query, params).fetchall() if query.lstrip().upper().startswith('SELECT') else conn.execute(query,params)
def test_validation_and_acl(self):
self.upload(b'x'*(10*1024*1024+1),expected=413)
self.upload(self.png[:30],expected=415)
self.upload(mime='image/jpeg',expected=415)
self.upload(path=self.path.replace('photo.png','..%2Fphoto.png'),expected=400)
bomb=io.BytesIO(); Image.new('1',(8000,5001)).save(bomb,format='PNG')
self.upload(bomb.getvalue(),expected=413)
jpg=io.BytesIO(); Image.new('RGB',(64,32),'red').save(jpg,format='JPEG')
self.upload(jpg.getvalue(),mime='image/jpeg')
item=self.upload()['json']; p='/api/v1/attachments/'+item['id']
for suffix in ('','/content'):
self.c.call('GET',p+suffix,404,actor=self.e)
self.c.call('GET',p,401)
self.c.call('GET',p+'/content',actor=self.b)
other=self.c.call('POST','/api/v1/chats',201,actor=self.a,body={
'title':'Other','member_ids':[self.a['id']]},schema='Chat')['json']
foreign=self.upload(path=self.path.replace(self.chat['id'],other['id']))['json']
for aid in (foreign['id'],str(uuid4())):
self.c.call(
'POST','/api/v1/chats/'+self.chat['id']+'/messages',404,actor=self.a,
body={'text':'foreign','attachment_ids':[aid]},headers={'Idempotency-Key':uuid4().hex})
message=self.c.message(self.a,self.chat,'',attachments=[item['id']])
self.assertEqual(message['json']['attachment_ids'],[item['id']])
self.c.call('POST','/api/v1/chats/'+self.chat['id']+'/messages',400,actor=self.a,
body={'text':' '},headers={'Idempotency-Key':uuid4().hex})
def test_private_bucket_and_bootstrap(self):
item=self.upload()['json']
import urllib.request, urllib.error
key=self.sql('SELECT object_key FROM attachments WHERE id=%s',(item['id'],))[0][0]
for path in ('attachments', 'attachments/'+key):
try:
urllib.request.urlopen('http://minio:9000/'+path,timeout=3)
self.fail('Anonymous access succeeded')
except urllib.error.HTTPError as error:
self.assertEqual(error.code,403)
s3=client()
for operation in (lambda:s3.create_bucket(Bucket='forbidden'),
lambda:s3.put_object(Bucket=BUCKET,Key='outside-prefix',Body=b'x')):
with self.assertRaises(ClientError) as caught:
operation()
self.assertEqual(caught.exception.response['ResponseMetadata']['HTTPStatusCode'],403)
bootstrap = containers('bucket-init', running=False)[0]['Id']
command(['docker','start','-a',bootstrap]); command(['docker','start','-a',bootstrap])
result=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)
self.assertEqual(result['bytes'],self.png)
def test_failed_commit_cleanup_and_abort(self):
# A partial client body cannot reach PUT.
before=self.sql('SELECT count(*) FROM attachments')[0][0]
sock=socket.create_connection(('lb',8080),timeout=3)
request=('POST '+self.path+' HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer '+self.a['token']+
'\r\nContent-Type: image/png\r\nContent-Length: 10000\r\n\r\n').encode()+self.png[:20]
sock.sendall(request); sock.close(); time.sleep(.3)
self.assertEqual(before,self.sql('SELECT count(*) FROM attachments')[0][0])
self.sql("""CREATE OR REPLACE FUNCTION lab4_fail_ready() RETURNS trigger LANGUAGE plpgsql AS
$$ BEGIN IF NEW.status='ready' AND NEW.filename='fault.png' THEN
RAISE EXCEPTION 'injected commit failure'; END IF; RETURN NEW; END $$""")
self.sql('CREATE TRIGGER lab4_fail_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_fail_ready()')
try:
self.upload(path=self.path.replace('photo.png','fault.png'),expected=503)
finally:
self.sql('DROP TRIGGER lab4_fail_ready ON attachments'); self.sql('DROP FUNCTION lab4_fail_ready()')
aid,key=self.sql("SELECT id,object_key FROM attachments WHERE filename='fault.png' AND chat_id=%s",(self.chat['id'],))[0]
s3=client(); s3.head_object(Bucket=BUCKET,Key=key)
async def cleanup():
store=Store(Settings.load()); await store.open()
try: return await Attachments(store).cleanup()
finally: await store.close()
asyncio.run(cleanup())
s3.head_object(Bucket=BUCKET,Key=key) # grace protects the fresh pending object
self.sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s",(aid,))
self.assertGreaterEqual(asyncio.run(cleanup()),1)
asyncio.run(cleanup()) # repeat is harmless
self.assertFalse(self.sql('SELECT id FROM attachments WHERE id=%s',(aid,)))
with self.assertRaises(ClientError): s3.head_object(Bucket=BUCKET,Key=key)
self.upload() # ready original survives cleanup
def test_s3_outage(self):
item=self.upload()['json']
direct=Client('http://api:8080',timeout=10)
compose('stop','minio')
try:
direct.call('GET','/health/live',schema='Health')
direct.call('GET','/health/ready',503)
direct.call('POST',self.path,503,actor=self.a,raw=self.png,headers={'Content-Type':'image/png'})
direct.call('GET','/api/v1/attachments/'+item['id']+'/content',503,actor=self.a)
finally:
compose('start','minio')
time.sleep(5)
self.assertEqual(self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes'],self.png)
def test_ten_parallel_uploads_memory(self):
# RGB random image is poorly compressible, roughly 8 MiB.
image=Image.frombytes('RGB',(1700,1600),os.urandom(1700*1600*3))
out=io.BytesIO(); image.save(out,format='PNG'); raw=out.getvalue()
ids=[x['Id'] for x in containers()]
barrier=threading.Barrier(11); stop=threading.Event(); samples=[]
def sample():
while not stop.is_set():
rows=[]
for cid in ids:
value=command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.current'])
rows.append(int(value))
samples.append({'elapsed':round(time.monotonic()-start,3),'bytes':rows})
stop.wait(.1)
start=time.monotonic(); sampler=threading.Thread(target=sample); sampler.start()
def upload(_):
barrier.wait()
t=time.monotonic(); item=self.upload(raw)['json']
got=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes']
self.assertEqual(hashlib.sha256(got).hexdigest(),hashlib.sha256(raw).hexdigest())
return time.monotonic()-t
try:
with ThreadPoolExecutor(max_workers=10) as pool:
futures=[pool.submit(upload,i) for i in range(10)]
barrier.wait(); durations=[f.result() for f in futures]
finally:
stop.set(); sampler.join()
result={'concurrency':10,'successful':len(durations),'size_bytes':len(raw),
'duration_seconds':time.monotonic()-start,'upload_and_download_seconds':durations,
'api_container_ids':ids,'memory_samples':samples,
'sampled_peak_bytes':[max(s['bytes'][i] for s in samples) for i in range(len(ids))],
'cgroup_memory_peak_bytes':[int(command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.peak'])) for cid in ids],
'memory_limit_bytes':512*1024*1024,
'note':'memory.current includes cache and runtime; memory.peak covers container lifetime, not only experiment'}
(ROOT/'evidence/uploads-memory.json').write_text(json.dumps(result,indent=2))
if __name__=='__main__': unittest.main(verbosity=2)