ЛР 4: приватное S3-хранилище и вложения сообщений

This commit is contained in:
2026-11-12 16:06:09 +03:00
parent ec540a1a72
commit 03cd380fb8
26 changed files with 1028 additions and 74 deletions
+45
View File
@@ -0,0 +1,45 @@
"""Additional grade-3 checks: concurrent replay and invalid uploads."""
from concurrent.futures import ThreadPoolExecutor
import uuid
import struct
import zlib
import subprocess
from client import Client, LAB, arguments, check, run
def main():
args = arguments('Extra acceptance checks')
c = Client(args.base_url)
alice,bob,eve,chat = c.bootstrap()
key = uuid.uuid4().hex
def send(_):
return c.message(alice,chat,'Concurrent retry',key=key)['json']
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(16)))
check(all(r==results[0] for r in results),'concurrent replay created different messages')
print('PASS: 16 concurrent retries return one message')
if LAB >= 4:
path = '/api/v1/chats/'+chat['id']+'/attachments'
content = open('tests/fixtures/sample.png','rb').read()
c.call('POST',path+'?filename=../x.png',400,actor=alice,raw=content,headers={'Content-Type':'image/png'})
c.call('POST',path+'?filename=x.jpg',415,actor=alice,raw=content,headers={'Content-Type':'image/jpeg'})
c.call('POST',path+'?filename=x.png',415,actor=alice,raw=b'broken',headers={'Content-Type':'image/png'})
c.call('POST',path+'?filename=x.png',413,actor=alice,raw=b'x'*(10485760+1),headers={'Content-Type':'image/png'})
# A real valid PNG >40 million pixels, highly compressible on the wire.
def chunk(kind, data):
return struct.pack('!I',len(data))+kind+data+struct.pack('!I',zlib.crc32(kind+data)&0xffffffff)
compressor = zlib.compressobj()
compressed = b''.join(compressor.compress(b'\0'*10001) for _ in range(4001))+compressor.flush()
huge = b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!IIBBBBB',10000,4001,8,0,0,0,0))+chunk(b'IDAT',compressed)+chunk(b'IEND',b'')
c.call('POST',path+'?filename=huge.png',413,actor=alice,raw=huge,headers={'Content-Type':'image/png'})
jpeg = subprocess.check_output(['docker','compose','exec','-T','api1','python','-c',"from PIL import Image; import sys; Image.new('RGB',(8,8),'red').save(sys.stdout.buffer,format='JPEG')"])
uploaded = c.call('POST',path+'?filename=real.jpg',201,actor=alice,raw=jpeg,headers={'Content-Type':'image/jpeg'},schema='Attachment')['json']
downloaded = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',actor=bob)
check(downloaded['bytes']==jpeg,'JPEG original changed')
other = c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Another chat','member_ids':[alice['id']]},schema='Chat')['json']
c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=alice,body={'text':'','attachment_ids':[uploaded['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
print('PASS: real JPEG roundtrip; >40 million pixels rejected; cross-chat attachment rejected')
print('PASS: unsafe filename, MIME mismatch, corruption and byte limit')
if __name__=='__main__':
run(main)
+53
View File
@@ -0,0 +1,53 @@
"""Stop exactly the API that accepted an upload, for each of the two APIs."""
import hashlib
import os
import subprocess
import time
from client import Client, arguments, check, run
def main():
args = arguments('Upload/download failover on both APIs')
c = Client(args.base_url)
alice,bob,eve,chat = c.bootstrap()
content = open('tests/fixtures/sample.png','rb').read()
path = '/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png'
for service in ('api1','api2'):
cid = subprocess.check_output(['docker','compose','ps','-q',service],text=True).strip()
target = subprocess.check_output(['docker','inspect','--format','{{.Config.Hostname}}',cid],text=True).strip()
for _ in range(20):
uploaded = c.call('POST',path,201,actor=alice,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
if uploaded['headers']['X-Instance-Id'] == target:
break
else:
raise AssertionError('Target API not observed through LB')
ident = uploaded['json']['id']
c.message(alice,chat,'',attachments=[ident])
started = time.monotonic()
try:
subprocess.run(['scripts/stop-one.sh'],env=dict(os.environ,TARGET_INSTANCE_ID=target),check=True,stdout=subprocess.DEVNULL)
while time.monotonic()-started < 10:
try:
downloaded = c.call('GET','/api/v1/attachments/'+ident+'/content',actor=bob)
except Exception:
time.sleep(.2)
continue
check(downloaded['headers']['X-Instance-Id'] != target,'stopped API still responds')
check(downloaded['bytes']==content,'original changed')
print('PASS:',service,'uploaded; stopped; other API returned SHA-256',hashlib.sha256(content).hexdigest(),'after',round(time.monotonic()-started,3),'seconds',flush=True)
break
else:
raise AssertionError('Download did not recover in 10 seconds')
finally:
subprocess.run(['docker','compose','start',service],check=True)
for _ in range(30):
health = subprocess.check_output(['docker','inspect','--format','{{.State.Health.Status}}',cid],text=True).strip()
if health == 'healthy':
time.sleep(2) # allow HAProxy rise check
break
time.sleep(1)
else:
raise AssertionError('API did not recover')
if __name__=='__main__':
run(main)
+39
View File
@@ -0,0 +1,39 @@
"""Check readiness and upload when S3 is unavailable; always restore S3."""
import subprocess
import time
from client import Client, arguments, run
def main():
args = arguments('Storage failure')
c = Client(args.base_url)
alice,bob,eve,chat = c.bootstrap()
data = open('tests/fixtures/sample.png','rb').read()
# Call an API inside its container because LB correctly removes unready APIs.
code = '''import urllib.request, urllib.error
for path, data, headers in CASES:
try:
r = urllib.request.urlopen(urllib.request.Request('http://localhost:8000'+path,data=data,headers=headers),timeout=10)
except urllib.error.HTTPError as e:
r = e
assert r.code == 503, r.code
print('PASS: S3 down returns 503 for readiness and upload')
'''
cases = [('/health/ready',None,{}),('/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',data,{'Content-Type':'image/png','Authorization':'Bearer '+alice['token']})]
try:
subprocess.run(['docker','compose','stop','-t','2','s3'],check=True)
subprocess.run(['docker','compose','exec','-T','api1','python','-'],input='CASES='+repr(cases)+'\n'+code,text=True,check=True)
finally:
subprocess.run(['docker','compose','start','s3'],check=True)
for _ in range(60):
try:
c.call('GET','/health/ready')
print('PASS: S3 restored')
break
except Exception:
time.sleep(1)
else:
raise AssertionError('S3 did not recover')
if __name__=='__main__':
run(main)