forked from mtusi2026/mtusi2026lab4
177 lines
8.8 KiB
Python
177 lines
8.8 KiB
Python
import concurrent.futures
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def docker(*args, input=None):
|
|
result = subprocess.run(['docker', *args], cwd=ROOT, input=input, text=True, capture_output=True, check=True)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def internal(code):
|
|
return docker('compose', 'exec', '-T', 'api', 'python', '-', input=code)
|
|
|
|
|
|
def png(width, height, random=False):
|
|
def chunk(kind, data):
|
|
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
|
|
raw = b''.join(b'\0' + (os.urandom(width * 3) if random else b'\x80' * (width * 3)) for _ in range(height))
|
|
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!2I5B', width, height, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
|
|
|
|
|
|
def wait_ready():
|
|
for _ in range(90):
|
|
try:
|
|
Client().call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Exception:
|
|
time.sleep(1)
|
|
raise AssertionError('Readiness timeout')
|
|
|
|
|
|
def main():
|
|
evidence = {}
|
|
c = Client(timeout=40)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=fixture.png'
|
|
sample = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
def upload(data=sample, mime='image/png', expected=201, actor=alice, route=path):
|
|
return Client(timeout=40).call('POST', route, expected, actor=actor, raw=data,
|
|
headers={'Content-Type': mime}, schema='Attachment' if expected == 201 else None)
|
|
att = upload()['json']
|
|
upload(b'x' * (10485760 + 1), expected=413)
|
|
upload(b'not an image', expected=415)
|
|
upload(sample[:-20], expected=415)
|
|
upload(sample, 'image/jpeg', 415)
|
|
upload(actor=eve, expected=404)
|
|
upload(actor=None, expected=401)
|
|
for name in ('../bad.png', 'bad\\x.png', 'bad\x00.png', 'bad\n.png', ' '):
|
|
upload(expected=400, route=path.split('?')[0] + '?filename=' + urllib.parse.quote(name))
|
|
other = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'Other', 'member_ids': [eve['id']]}, schema='Chat')['json']
|
|
foreign = upload(actor=eve, route='/api/v1/chats/' + other['id'] + '/attachments?filename=x.png')['json']
|
|
for item in (foreign['id'], str(uuid.uuid4())):
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 404, actor=alice,
|
|
body={'text': '', 'attachment_ids': [item]}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
|
huge = bytearray(sample)
|
|
huge[16:24] = struct.pack('!II', 40000001, 1)
|
|
huge[29:33] = struct.pack('!I', zlib.crc32(huge[12:29]))
|
|
upload(bytes(huge), expected=413)
|
|
jpeg = internal("from PIL import Image\nimport io,base64\nf=io.BytesIO(); Image.new('RGB',(32,32)).save(f,'JPEG'); print(base64.b64encode(f.getvalue()).decode())")
|
|
import base64
|
|
jpg = base64.b64decode(jpeg)
|
|
uploaded_jpeg = upload(jpg, 'image/jpeg')['json']
|
|
check(c.call('GET', '/api/v1/attachments/' + uploaded_jpeg['id'] + '/content', actor=bob)['bytes'] == jpg, 'JPEG changed')
|
|
evidence['negative_tests'] = 'PASS: byte/pixel limits, corrupt/truncated images, MIME mismatch, filename, ACL, foreign and unknown attachment'
|
|
evidence['private_bucket'] = json.loads(internal("""import urllib.request, urllib.error, json
|
|
from src.storage import client,BUCKET
|
|
try:
|
|
urllib.request.urlopen('http://s3:9000/'+BUCKET+'/originals/""" + att['id'] + """')
|
|
raise AssertionError('Anonymous read allowed')
|
|
except urllib.error.HTTPError as e:
|
|
assert e.code==403
|
|
print(json.dumps({'anonymous_get':e.code}))
|
|
try:
|
|
client().create_bucket(Bucket='forbidden-bucket')
|
|
raise AssertionError('API can create bucket')
|
|
except Exception as e:
|
|
assert getattr(e,'response',{}).get('Error',{}).get('Code')=='AccessDenied'
|
|
"""))
|
|
orphan_key = 'originals/' + str(uuid.uuid4())
|
|
missing = upload()['json']
|
|
internal("""from src.storage import client,BUCKET
|
|
from src.migrate import database_url
|
|
import psycopg
|
|
s=client()
|
|
s.put_object(Bucket=BUCKET,Key='""" + orphan_key + """',Body=b'orphan')
|
|
s.delete_object(Bucket=BUCKET,Key='originals/""" + missing['id'] + """')
|
|
with psycopg.connect(database_url()) as db:
|
|
db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s", ('""" + missing['id'] + """',))
|
|
""")
|
|
injected = time.monotonic()
|
|
evidence['uploads'] = []
|
|
for width in (512, 1024):
|
|
data = png(width, width, True)
|
|
baseline = docker('stats', '--no-stream', '--format', '{{json .}}').splitlines()
|
|
observations = []
|
|
stop = threading.Event()
|
|
def monitor():
|
|
while not stop.is_set():
|
|
observations.extend(json.loads(line) for line in docker('stats', '--no-stream', '--format', '{{json .}}').splitlines() if 'lab4-api' in line)
|
|
watcher = threading.Thread(target=monitor)
|
|
watcher.start()
|
|
started = time.monotonic()
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
|
|
results = list(pool.map(lambda _: upload(data), range(10)))
|
|
elapsed = time.monotonic() - started
|
|
finally:
|
|
stop.set()
|
|
watcher.join()
|
|
peak = {container: int(docker('exec', container, 'cat', '/sys/fs/cgroup/memory.peak'))
|
|
for container in docker('compose', 'ps', '-q', 'api').split()}
|
|
durations = sorted(row['elapsed'] for row in results)
|
|
evidence['uploads'].append(dict(size_bytes=len(data), concurrency=10, success=len(results), errors=0,
|
|
seconds=elapsed, successful_rps=10/elapsed, p50=statistics.median(durations), p95=durations[-1], p99=durations[-1],
|
|
baseline=[json.loads(line) for line in baseline if 'lab4-api' in line], samples=observations, lifetime_cgroup_peak_bytes=peak))
|
|
connection = http.client.HTTPConnection('localhost', 8080, timeout=5)
|
|
connection.putrequest('POST', path)
|
|
connection.putheader('Authorization', 'Bearer ' + alice['token'])
|
|
connection.putheader('Content-Type', 'image/png')
|
|
connection.putheader('Content-Length', '100000')
|
|
connection.endheaders()
|
|
connection.send(sample[:100])
|
|
connection.close()
|
|
evidence['interrupted_upload'] = 'Connection closed after 100 bytes of declared 100000; no S3 PUT before validation'
|
|
remaining = 62 - (time.monotonic() - injected)
|
|
if remaining > 0:
|
|
while remaining > 0:
|
|
time.sleep(min(remaining, 20))
|
|
remaining = 62 - (time.monotonic() - injected)
|
|
fresh_key = 'originals/' + str(uuid.uuid4())
|
|
internal("from src.storage import client,BUCKET\nclient().put_object(Bucket=BUCKET,Key='" + fresh_key + "',Body=b'fresh')")
|
|
def reconcile(apply):
|
|
return json.loads(internal('from src.reconcile import run\nimport json\nprint(json.dumps(run(' + str(apply) + ',60)))'))
|
|
dry = reconcile(False)
|
|
check(dry['orphan_candidates'] >= 1 and dry['missing_candidates'] >= 1 and dry['deleted'] == 0, 'Dry-run failed')
|
|
applied = reconcile(True)
|
|
check(applied['deleted'] >= 1 and applied['marked_failed'] >= 1, 'Repair failed')
|
|
repeated = reconcile(True)
|
|
check(repeated['deleted'] == 0 and repeated['marked_failed'] == 0, 'Reconcile is not idempotent')
|
|
check(applied['fresh_preserved'] >= 1 and applied['referenced_preserved'] >= 1, 'Safety failed')
|
|
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == sample, 'Referenced file removed')
|
|
c.call('GET', '/api/v1/attachments/' + missing['id'], 503, actor=bob)
|
|
c.call('GET', '/api/v1/attachments/' + missing['id'] + '/content', 503, actor=bob)
|
|
evidence['reconciler'] = {'dry_run': dry, 'apply': applied, 'repeat': repeated}
|
|
docker('compose', 'stop', 's3')
|
|
try:
|
|
response = c.call('GET', '/health/ready', 503)
|
|
evidence['s3_outage'] = {'ready': response['status']}
|
|
internal("from src.reconcile import run\ntry:\n run(True,60)\nexcept Exception:\n print('PASS: outage fails closed')\nelse:\n raise AssertionError('Expected failure')")
|
|
finally:
|
|
docker('compose', 'start', 's3')
|
|
wait_ready()
|
|
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == sample, 'S3 restart changed data')
|
|
internal("from src.storage import client,BUCKET\nclient().delete_object(Bucket=BUCKET,Key='" + fresh_key + "')")
|
|
(ROOT / 'evidence/acceptance.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
print('PASS: negative images, ACL, private S3, interrupted upload, concurrent uploads, reconciliation, S3 outage')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|