121 lines
5.5 KiB
Python
121 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Explicit local Docker tests: private bucket, uploader failover, S3 outage."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def compose(*args, input=None):
|
|
return subprocess.check_output(['docker', 'compose', *args], cwd=ROOT, input=input,
|
|
text=True, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
def wait_ready(c):
|
|
deadline = time.monotonic() + 90
|
|
while True:
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Failure:
|
|
check(time.monotonic() < deadline, 'stack did not recover')
|
|
time.sleep(1)
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--mode', required=True, choices=['private', 'failover', 's3-outage'])
|
|
args = arguments(__doc__, configure)
|
|
c = Client(args.base_url, args.ca_file)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
|
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
att = uploaded['json']
|
|
content_path = '/api/v1/attachments/' + att['id'] + '/content'
|
|
if args.mode == 'private':
|
|
script = '''import sys, urllib.request, urllib.error
|
|
url = "http://s3:8333/messenger-attachments/originals/" + sys.stdin.read().strip()
|
|
try:
|
|
urllib.request.urlopen(url, timeout=5)
|
|
except urllib.error.HTTPError as error:
|
|
assert error.code == 403, error.code
|
|
print("PASS: anonymous S3 GET returns 403 for an existing object")
|
|
else:
|
|
raise AssertionError("private object leaked")
|
|
'''
|
|
print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, input=att['id']).strip())
|
|
elif args.mode == 'failover':
|
|
target = uploaded['headers']['X-Instance-Id']
|
|
service = None
|
|
for candidate in ('api-a', 'api-b'):
|
|
container = compose('ps', '-q', candidate).strip()
|
|
hostname = subprocess.check_output(['docker', 'inspect', '--format', '{{.Config.Hostname}}', container], text=True).strip()
|
|
if hostname == target:
|
|
service = candidate
|
|
check(service is not None, 'uploading replica not found')
|
|
try:
|
|
compose('stop', service)
|
|
deadline = time.monotonic() + 15
|
|
while True:
|
|
try:
|
|
download = c.call('GET', content_path, actor=bob)
|
|
break
|
|
except Failure:
|
|
check(time.monotonic() < deadline, 'download did not recover')
|
|
time.sleep(0.5)
|
|
check(download['headers']['X-Instance-Id'] != target, 'download used stopped uploader')
|
|
check(download['bytes'] == data, 'original changed after stopping uploader')
|
|
print('PASS: uploaded via ' + service + '; stopped that container; original downloaded through another replica')
|
|
finally:
|
|
compose('start', service)
|
|
wait_ready(c)
|
|
else:
|
|
try:
|
|
compose('stop', 's3')
|
|
# Direct API request inside the network: LB excludes unready backends.
|
|
script = '''import json, sys, urllib.request, urllib.error
|
|
payload = json.load(sys.stdin)
|
|
for method, path, body, headers in [
|
|
("GET", "/health/ready", None, {}),
|
|
("GET", payload["path"], None, {"Authorization": "Bearer " + payload["token"]}),
|
|
("POST", payload["upload_path"], bytes.fromhex(payload["bytes"]),
|
|
{"Authorization": "Bearer " + payload["token"], "Content-Type": "image/png"}),
|
|
]:
|
|
req = urllib.request.Request("http://localhost:8080" + path, data=body, headers=headers, method=method)
|
|
try:
|
|
urllib.request.urlopen(req, timeout=15)
|
|
except urllib.error.HTTPError as error:
|
|
assert error.code == 503, error.code
|
|
document = json.load(error)
|
|
assert document["error"]["code"] == "unavailable"
|
|
assert document["error"]["request_id"] == error.headers["X-Request-Id"]
|
|
else:
|
|
raise AssertionError("S3 outage was not reported")
|
|
assert urllib.request.urlopen("http://localhost:8080/health/live").status == 200
|
|
print("PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200")
|
|
'''
|
|
payload = {'path': content_path, 'token': bob['token'], 'bytes': data.hex(),
|
|
'upload_path': '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png'}
|
|
print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script,
|
|
input=json.dumps(payload)).strip())
|
|
finally:
|
|
compose('start', 's3')
|
|
# HeadBucket/read recovery alone does not prove the write path is ready.
|
|
# Use the same write-probing bootstrap as the documented restart hook.
|
|
compose('run', '--rm', '--no-deps', 's3-init')
|
|
wait_ready(c)
|
|
check(c.call('GET', content_path, actor=bob)['bytes'] == data, 'original lost after S3 outage')
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after-recovery.png',
|
|
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
print('PASS: original available and new upload accepted after S3 restart/bootstrap')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|