"""Grade 4 integration checks against real stores, including fault injection.""" import asyncio from concurrent.futures import ThreadPoolExecutor import hashlib import io import json import os from pathlib import Path import socket import sys import threading import time import unittest from uuid import uuid4 import psycopg from PIL import Image from botocore.exceptions import ClientError ROOT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(ROOT), str(ROOT/'tests'), str(ROOT/'scripts')] from client import Client from ops import compose, command, containers, summary from app.config import Settings from app.store import Store from app.attachments import Attachments, client, BUCKET class AttachmentTests(unittest.TestCase): def setUp(self): self.c = Client(timeout=40) self.a,self.b,self.e,self.chat = self.c.bootstrap() self.path = '/api/v1/chats/'+self.chat['id']+'/attachments?filename=photo.png' self.png = (ROOT/'tests/fixtures/sample.png').read_bytes() def upload(self, raw=None, mime='image/png', expected=201, path=None): return self.c.call('POST', path or self.path, expected, actor=self.a, raw=self.png if raw is None else raw, headers={'Content-Type':mime}, schema='Attachment' if expected==201 else None) def sql(self, query, params=()): with psycopg.connect(Settings.load().database_url, autocommit=True) as conn: return conn.execute(query, params).fetchall() if query.lstrip().upper().startswith('SELECT') else conn.execute(query,params) def test_validation_and_acl(self): self.upload(b'x'*(10*1024*1024+1),expected=413) self.upload(self.png[:30],expected=415) self.upload(mime='image/jpeg',expected=415) self.upload(path=self.path.replace('photo.png','..%2Fphoto.png'),expected=400) bomb=io.BytesIO(); Image.new('1',(8000,5001)).save(bomb,format='PNG') self.upload(bomb.getvalue(),expected=413) jpg=io.BytesIO(); Image.new('RGB',(64,32),'red').save(jpg,format='JPEG') self.upload(jpg.getvalue(),mime='image/jpeg') item=self.upload()['json']; p='/api/v1/attachments/'+item['id'] for suffix in ('','/content'): self.c.call('GET',p+suffix,404,actor=self.e) self.c.call('GET',p,401) self.c.call('GET',p+'/content',actor=self.b) other=self.c.call('POST','/api/v1/chats',201,actor=self.a,body={ 'title':'Other','member_ids':[self.a['id']]},schema='Chat')['json'] foreign=self.upload(path=self.path.replace(self.chat['id'],other['id']))['json'] for aid in (foreign['id'],str(uuid4())): self.c.call( 'POST','/api/v1/chats/'+self.chat['id']+'/messages',404,actor=self.a, body={'text':'foreign','attachment_ids':[aid]},headers={'Idempotency-Key':uuid4().hex}) message=self.c.message(self.a,self.chat,'',attachments=[item['id']]) self.assertEqual(message['json']['attachment_ids'],[item['id']]) self.c.call('POST','/api/v1/chats/'+self.chat['id']+'/messages',400,actor=self.a, body={'text':' '},headers={'Idempotency-Key':uuid4().hex}) def test_private_bucket_and_bootstrap(self): item=self.upload()['json'] import urllib.request, urllib.error key=self.sql('SELECT object_key FROM attachments WHERE id=%s',(item['id'],))[0][0] for path in ('attachments', 'attachments/'+key): try: urllib.request.urlopen('http://minio:9000/'+path,timeout=3) self.fail('Anonymous access succeeded') except urllib.error.HTTPError as error: self.assertEqual(error.code,403) s3=client() for operation in (lambda:s3.create_bucket(Bucket='forbidden'), lambda:s3.put_object(Bucket=BUCKET,Key='outside-prefix',Body=b'x')): with self.assertRaises(ClientError) as caught: operation() self.assertEqual(caught.exception.response['ResponseMetadata']['HTTPStatusCode'],403) bootstrap = containers('bucket-init', running=False)[0]['Id'] command(['docker','start','-a',bootstrap]); command(['docker','start','-a',bootstrap]) result=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b) self.assertEqual(result['bytes'],self.png) def test_failed_commit_cleanup_and_abort(self): # A partial client body cannot reach PUT. before=self.sql('SELECT count(*) FROM attachments')[0][0] sock=socket.create_connection(('lb',8080),timeout=3) request=('POST '+self.path+' HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer '+self.a['token']+ '\r\nContent-Type: image/png\r\nContent-Length: 10000\r\n\r\n').encode()+self.png[:20] sock.sendall(request); sock.close(); time.sleep(.3) self.assertEqual(before,self.sql('SELECT count(*) FROM attachments')[0][0]) self.sql("""CREATE OR REPLACE FUNCTION lab4_fail_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.status='ready' AND NEW.filename='fault.png' THEN RAISE EXCEPTION 'injected commit failure'; END IF; RETURN NEW; END $$""") self.sql('CREATE TRIGGER lab4_fail_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_fail_ready()') try: self.upload(path=self.path.replace('photo.png','fault.png'),expected=503) finally: self.sql('DROP TRIGGER lab4_fail_ready ON attachments'); self.sql('DROP FUNCTION lab4_fail_ready()') aid,key=self.sql("SELECT id,object_key FROM attachments WHERE filename='fault.png' AND chat_id=%s",(self.chat['id'],))[0] s3=client(); s3.head_object(Bucket=BUCKET,Key=key) async def cleanup(): store=Store(Settings.load()); await store.open() try: return await Attachments(store).cleanup() finally: await store.close() asyncio.run(cleanup()) s3.head_object(Bucket=BUCKET,Key=key) # grace protects the fresh pending object self.sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s",(aid,)) self.assertGreaterEqual(asyncio.run(cleanup()),1) asyncio.run(cleanup()) # repeat is harmless self.assertFalse(self.sql('SELECT id FROM attachments WHERE id=%s',(aid,))) with self.assertRaises(ClientError): s3.head_object(Bucket=BUCKET,Key=key) self.upload() # ready original survives cleanup def test_s3_outage(self): item=self.upload()['json'] direct=Client('http://api:8080',timeout=10) compose('stop','minio') try: direct.call('GET','/health/live',schema='Health') direct.call('GET','/health/ready',503) direct.call('POST',self.path,503,actor=self.a,raw=self.png,headers={'Content-Type':'image/png'}) direct.call('GET','/api/v1/attachments/'+item['id']+'/content',503,actor=self.a) finally: compose('start','minio') time.sleep(5) self.assertEqual(self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes'],self.png) def test_ten_parallel_uploads_memory(self): # RGB random image is poorly compressible, roughly 8 MiB. image=Image.frombytes('RGB',(1700,1600),os.urandom(1700*1600*3)) out=io.BytesIO(); image.save(out,format='PNG'); raw=out.getvalue() ids=[x['Id'] for x in containers()] barrier=threading.Barrier(11); stop=threading.Event(); samples=[] def sample(): while not stop.is_set(): rows=[] for cid in ids: value=command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.current']) rows.append(int(value)) samples.append({'elapsed':round(time.monotonic()-start,3),'bytes':rows}) stop.wait(.1) start=time.monotonic(); sampler=threading.Thread(target=sample); sampler.start() def upload(_): barrier.wait() t=time.monotonic(); item=self.upload(raw)['json'] got=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes'] self.assertEqual(hashlib.sha256(got).hexdigest(),hashlib.sha256(raw).hexdigest()) return time.monotonic()-t try: with ThreadPoolExecutor(max_workers=10) as pool: futures=[pool.submit(upload,i) for i in range(10)] barrier.wait(); durations=[f.result() for f in futures] finally: stop.set(); sampler.join() result={'concurrency':10,'successful':len(durations),'size_bytes':len(raw), 'duration_seconds':time.monotonic()-start,'upload_and_download_seconds':durations, 'api_container_ids':ids,'memory_samples':samples, 'sampled_peak_bytes':[max(s['bytes'][i] for s in samples) for i in range(len(ids))], 'cgroup_memory_peak_bytes':[int(command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.peak'])) for cid in ids], 'memory_limit_bytes':512*1024*1024, 'note':'memory.current includes cache and runtime; memory.peak covers container lifetime, not only experiment'} (ROOT/'evidence/uploads-memory.json').write_text(json.dumps(result,indent=2)) if __name__=='__main__': unittest.main(verbosity=2)