Files
mtusi2026lab5-fork/tests/s3_checks.py
T

135 lines
5.9 KiB
Python

#!/usr/bin/env python3
import hashlib
import subprocess
import time
import uuid
from pathlib import Path
from client import Client, Failure, arguments, check, run
from stack import wait_status
ROOT = Path(__file__).resolve().parent.parent
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
BUCKET = 'messenger-attachments'
def compose(*args, check_rc=True, stdin=None):
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc,
capture_output=True, text=True, input=stdin)
def secret(name):
return (ROOT / 'secrets' / name).read_text().strip()
def s3_status(method, path, user=None, data=None):
cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method]
if user:
cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]]
if data is not None:
cmd += ['--data-binary', data]
cmd.append('http://127.0.0.1:9000' + path)
return compose(*cmd).stdout.strip()
def upload(c, actor, chat, expected):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor,
raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None)
def wait_api(c):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready')
return
except Failure:
time.sleep(1)
raise Failure('api did not recover')
def wait_s3(timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'):
return
time.sleep(1)
raise Failure('s3 did not recover')
def object_key(attachment_id):
out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc',
"select object_key from attachments where id='%s'" % attachment_id).stdout.strip()
check(out.startswith('attachments/'), 'object key not found')
return out
def main():
args = arguments('S3 privacy, outage and orphan cleanup checks')
c = Client(args.base_url, args.ca_file, timeout=15)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat, 202)['json']
wait_status(c, bob, att['id'], ('ready',), 60)
key = object_key(att['id'])
check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename')
api = (secret('s3_access_key_id'), secret('s3_secret_access_key'))
admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key'))
path = '/' + BUCKET + '/' + key
check(s3_status('GET', path) == '403', 'anonymous GET must be denied')
check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied')
check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied')
check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied')
check(s3_status('GET', path, api) == '200', 'api credentials cannot read')
check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets')
ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout
published = [line.split()[0] for line in ports.splitlines() if '->' in line]
check(published == ['lb'], 'only lb may publish ports, found %s' % published)
compose('stop', 's3')
try:
upload(c, alice, chat, 503)
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment')
c.message(alice, chat, 'text works while s3 is down')
finally:
compose('start', 's3')
wait_s3()
wait_api(c)
deadline = time.monotonic() + 60
while True:
try:
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes']
check(body == SAMPLE, 'content differs after s3 restart')
break
except Failure:
check(time.monotonic() < deadline, 'download did not recover')
time.sleep(1)
upload(c, alice, chat, 202)
orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex
check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan')
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup')
check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period')
time.sleep(1)
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup')
check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed')
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed')
check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready')
stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4())
check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object')
compose('stop', 'worker')
try:
queued = upload(c, alice, chat, 202)['json']
compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup')
check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed')
check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed')
finally:
compose('up', '-d', '--no-deps', 'worker')
check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging')
print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup')
if __name__ == '__main__':
run(main)