forked from mtusi2026/mtusi2026lab5
135 lines
5.9 KiB
Python
135 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
import hashlib
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from client import Client, Failure, arguments, check, run
|
|
from stack import wait_status
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
BUCKET = 'messenger-attachments'
|
|
|
|
|
|
def compose(*args, check_rc=True, stdin=None):
|
|
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc,
|
|
capture_output=True, text=True, input=stdin)
|
|
|
|
|
|
def secret(name):
|
|
return (ROOT / 'secrets' / name).read_text().strip()
|
|
|
|
|
|
def s3_status(method, path, user=None, data=None):
|
|
cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method]
|
|
if user:
|
|
cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]]
|
|
if data is not None:
|
|
cmd += ['--data-binary', data]
|
|
cmd.append('http://127.0.0.1:9000' + path)
|
|
return compose(*cmd).stdout.strip()
|
|
|
|
|
|
def upload(c, actor, chat, expected):
|
|
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor,
|
|
raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None)
|
|
|
|
|
|
def wait_api(c):
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
c.call('GET', '/health/ready')
|
|
return
|
|
except Failure:
|
|
time.sleep(1)
|
|
raise Failure('api did not recover')
|
|
|
|
|
|
def wait_s3(timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'):
|
|
return
|
|
time.sleep(1)
|
|
raise Failure('s3 did not recover')
|
|
|
|
|
|
def object_key(attachment_id):
|
|
out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc',
|
|
"select object_key from attachments where id='%s'" % attachment_id).stdout.strip()
|
|
check(out.startswith('attachments/'), 'object key not found')
|
|
return out
|
|
|
|
|
|
def main():
|
|
args = arguments('S3 privacy, outage and orphan cleanup checks')
|
|
c = Client(args.base_url, args.ca_file, timeout=15)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
att = upload(c, alice, chat, 202)['json']
|
|
wait_status(c, bob, att['id'], ('ready',), 60)
|
|
key = object_key(att['id'])
|
|
check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename')
|
|
|
|
api = (secret('s3_access_key_id'), secret('s3_secret_access_key'))
|
|
admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key'))
|
|
path = '/' + BUCKET + '/' + key
|
|
check(s3_status('GET', path) == '403', 'anonymous GET must be denied')
|
|
check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied')
|
|
check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied')
|
|
check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied')
|
|
check(s3_status('GET', path, api) == '200', 'api credentials cannot read')
|
|
check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets')
|
|
ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout
|
|
published = [line.split()[0] for line in ports.splitlines() if '->' in line]
|
|
check(published == ['lb'], 'only lb may publish ports, found %s' % published)
|
|
|
|
compose('stop', 's3')
|
|
try:
|
|
upload(c, alice, chat, 503)
|
|
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
|
|
c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment')
|
|
c.message(alice, chat, 'text works while s3 is down')
|
|
finally:
|
|
compose('start', 's3')
|
|
wait_s3()
|
|
wait_api(c)
|
|
deadline = time.monotonic() + 60
|
|
while True:
|
|
try:
|
|
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes']
|
|
check(body == SAMPLE, 'content differs after s3 restart')
|
|
break
|
|
except Failure:
|
|
check(time.monotonic() < deadline, 'download did not recover')
|
|
time.sleep(1)
|
|
upload(c, alice, chat, 202)
|
|
|
|
orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex
|
|
check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan')
|
|
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup')
|
|
check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period')
|
|
time.sleep(1)
|
|
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup')
|
|
check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed')
|
|
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed')
|
|
|
|
check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready')
|
|
stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4())
|
|
check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object')
|
|
compose('stop', 'worker')
|
|
try:
|
|
queued = upload(c, alice, chat, 202)['json']
|
|
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup')
|
|
check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed')
|
|
check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker')
|
|
check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging')
|
|
print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|