forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
161 lines
8.0 KiB
Python
161 lines
8.0 KiB
Python
"""Integration checks against the local Compose stack. --lifecycle restarts its services."""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check, run
|
|
|
|
|
|
def compose(*args):
|
|
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True)
|
|
check(result.returncode == 0, 'Compose action failed: ' + ' '.join(args[:2]))
|
|
return result.stdout.strip()
|
|
|
|
|
|
def wait_ready(client):
|
|
deadline = time.monotonic() + 90
|
|
observed = set()
|
|
stable_since = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response = client.call('GET', '/health/ready', schema='Health')
|
|
upstreams = compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-upstreams.conf')
|
|
# A single 200 can come from an old Nginx worker while a removal
|
|
# reload is still pending. Wait for both re-added replicas to settle.
|
|
check(sum(':8080' in line for line in upstreams.splitlines()) == 2,
|
|
'Both upstreams have not recovered')
|
|
observed.add(response['headers']['X-Instance-Id'])
|
|
if stable_since is None:
|
|
stable_since = time.monotonic()
|
|
if len(observed) == 2 and time.monotonic() - stable_since >= 4:
|
|
return
|
|
except Failure:
|
|
observed.clear()
|
|
stable_since = None
|
|
time.sleep(1)
|
|
raise Failure('API did not recover')
|
|
|
|
|
|
def direct_call(method, path, expected, actor=None, raw=None, content_type=None):
|
|
"""Probe the API process when the LB has excluded every unready replica."""
|
|
headers = {'Authorization': 'Bearer ' + actor['token']} if actor else {}
|
|
if content_type:
|
|
headers['Content-Type'] = content_type
|
|
request = {'method': method, 'path': path, 'headers': headers,
|
|
'body': base64.b64encode(raw).decode() if raw is not None else None}
|
|
script = """const q=JSON.parse(require('node:fs').readFileSync(0,'utf8'));
|
|
fetch('http://127.0.0.1:8080'+q.path,{method:q.method,headers:q.headers,
|
|
body:q.body===null?undefined:Buffer.from(q.body,'base64')}).then(async r=>{
|
|
console.log(JSON.stringify({status:r.status,body:await r.json(),
|
|
requestId:r.headers.get('x-request-id'),instanceId:r.headers.get('x-instance-id')}));
|
|
}).catch(()=>process.exit(1));"""
|
|
result = subprocess.run(['docker', 'compose', 'exec', '-T', '--index', '1',
|
|
'api', 'node', '-e', script], cwd=ROOT,
|
|
input=json.dumps(request), capture_output=True, text=True)
|
|
check(result.returncode == 0, 'direct API probe failed')
|
|
response = json.loads(result.stdout)
|
|
check(response['status'] == expected, 'direct API returned wrong status')
|
|
check(response['requestId'] and response['instanceId'], 'direct API lacks response identifiers')
|
|
if expected >= 400:
|
|
check(response['body']['error']['request_id'] == response['requestId'], 'error request ID differs')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--lifecycle', action='store_true')
|
|
args = parser.parse_args()
|
|
c = Client(args.base_url)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id']
|
|
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
|
|
def upload(data=png, name='фото.png', mime='image/png', expected=201, actor=alice):
|
|
return c.call('POST', path + '/attachments?filename=' + quote(name, safe=''), expected,
|
|
actor=actor, raw=data, headers={'Content-Type': mime},
|
|
schema='Attachment' if expected == 201 else None)
|
|
|
|
att = upload()['json']
|
|
item = '/api/v1/attachments/' + att['id']
|
|
check(c.call('GET', item, actor=bob, schema='Attachment')['json'] == att, 'metadata differs')
|
|
check(c.call('GET', item + '/content', actor=bob)['bytes'] == png, 'PNG changed')
|
|
c.call('GET', item + '/content?variant=thumbnail', 404, actor=bob)
|
|
c.call('GET', item, 401)
|
|
c.call('GET', item, 404, actor=eve)
|
|
c.call('GET', item + '/content', 404, actor=eve)
|
|
upload(actor=eve, expected=404)
|
|
upload(name='../photo.png', expected=400)
|
|
upload(name='photo\\bad.png', expected=400)
|
|
upload(name='bad\x00.png', expected=400)
|
|
upload(name=' ', expected=400)
|
|
upload(data=b'not an image', expected=415)
|
|
upload(data=png[:50], expected=415)
|
|
upload(mime='image/jpeg', expected=415)
|
|
upload(data=b'x' * (10485760 + 1), expected=413)
|
|
|
|
def chunk(kind, value):
|
|
return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
|
|
|
|
huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0))
|
|
huge += chunk(b'IDAT', zlib.compress(b'\x00\x00\x00\x00')) + chunk(b'IEND', b'')
|
|
upload(data=huge, expected=413)
|
|
jpeg = subprocess.run(['node', '-e',
|
|
"require('sharp')('tests/fixtures/sample.png').jpeg().toBuffer().then(b=>process.stdout.write(b))"],
|
|
cwd=ROOT, capture_output=True, check=True).stdout
|
|
jpg = upload(data=jpeg, name='photo.jpg', mime='image/jpeg')['json']
|
|
check(c.call('GET', '/api/v1/attachments/' + jpg['id'] + '/content', actor=bob)['bytes'] == jpeg, 'JPEG changed')
|
|
key = uuid.uuid4().hex
|
|
message = c.message(alice, chat, '', key, [att['id']])['json']
|
|
check(c.message(alice, chat, '', key, [att['id']])['json'] == message, 'replay differs')
|
|
c.call('POST', path + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [jpg['id']]}, headers={'Idempotency-Key': key})
|
|
for ids, status in (([att['id'], att['id']], 400), ([str(uuid.uuid4())], 404), (None, 400)):
|
|
c.call('POST', path + '/messages', status, actor=alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
|
other = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
|
body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
|
c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice,
|
|
body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
|
|
|
if args.lifecycle:
|
|
anonymous_url = 'http://s3:9000/messenger-attachments/originals/' + chat['id'] + '/' + att['id']
|
|
status = compose('exec', '-T', 'api', 'node', '-e',
|
|
'fetch(process.argv[1]).then(r=>console.log(r.status))', anonymous_url)
|
|
check(status == '403', 'S3 object is publicly readable')
|
|
try:
|
|
compose('stop', 's3')
|
|
direct_call('GET', '/health/ready', 503)
|
|
direct_call('GET', item, 200, actor=bob)
|
|
direct_call('GET', item + '/content', 503, actor=bob)
|
|
direct_call('POST', path + '/attachments?filename=outage.png', 503,
|
|
actor=alice, raw=png, content_type='image/png')
|
|
deadline = time.monotonic() + 15
|
|
while True:
|
|
upstreams = compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-upstreams.conf')
|
|
if ':8080' not in upstreams:
|
|
break
|
|
check(time.monotonic() < deadline, 'Nginx still routes to unready replicas')
|
|
time.sleep(0.5)
|
|
finally:
|
|
compose('start', 's3')
|
|
wait_ready(c)
|
|
check(c.call('GET', item + '/content', actor=bob)['bytes'] == png, 'original lost after storage recovery')
|
|
compose('stop')
|
|
compose('up', '-d', '--wait', '--wait-timeout', '180')
|
|
wait_ready(c)
|
|
check(c.call('GET', item + '/content', actor=bob)['bytes'] == png, 'original lost after stack restart')
|
|
check(c.message(alice, chat, '', key, [att['id']])['json'] == message, 'message lost after restart')
|
|
print('PASS: attachments, validation, ACL, idempotency' + (', private S3, outage and persistence' if args.lifecycle else ''))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|