Completed for a grade of four
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Abort a binary request mid-body and verify temporary files and metadata are absent."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
name = 'interrupted-' + uuid.uuid4().hex + '.png'
|
||||
base = urllib.parse.urlsplit(c.base)
|
||||
body = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
headers = ('POST /api/v1/chats/' + chat['id'] + '/attachments?filename=' + name + ' HTTP/1.1\r\n'
|
||||
'Host: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\n'
|
||||
'Content-Type: image/png\r\nContent-Length: 1048576\r\n\r\n')
|
||||
with socket.create_connection((base.hostname, base.port or 80)) as connection:
|
||||
connection.sendall(headers.encode() + body[:len(body)//2])
|
||||
time.sleep(0.5)
|
||||
# Let the server observe EOF, or the bounded receive timeout.
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', 'api'], cwd=ROOT, text=True).split()
|
||||
deadline = time.monotonic() + 20
|
||||
while True:
|
||||
files = [subprocess.check_output(['docker', 'exec', cid, 'find', '/tmp', '-name', 'messenger-upload-*'], text=True).strip() for cid in ids]
|
||||
if not any(files):
|
||||
break
|
||||
check(time.monotonic() < deadline, 'temporary upload file survived interruption/timeout')
|
||||
time.sleep(0.2)
|
||||
count = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc',
|
||||
"SELECT count(*) FROM attachments WHERE filename = '" + name + "'"], cwd=ROOT, text=True).strip()
|
||||
check(count == '0', 'interrupted upload published metadata')
|
||||
print('PASS: connection closed mid-body; temporary files removed on both replicas; metadata absent')
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ten simultaneous binary uploads and sampled API cgroup memory, without tokens in output."""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def fixture():
|
||||
width = height = 1600
|
||||
rng = random.Random(42)
|
||||
pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b'IDAT', zlib.compress(pixels, 1)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def memory(ids):
|
||||
result = {}
|
||||
for container in ids:
|
||||
value = subprocess.check_output(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/memory.current'], text=True)
|
||||
result[container[:12]] = int(value)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(timeout=60)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = fixture()
|
||||
expected = hashlib.sha256(data).hexdigest()
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', 'api'], cwd=ROOT, text=True).split()
|
||||
baseline = memory(ids)
|
||||
samples = []
|
||||
stop = threading.Event()
|
||||
start = time.monotonic()
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
samples.append({'seconds': round(time.monotonic() - start, 3), 'bytes': memory(ids)})
|
||||
stop.wait(0.2)
|
||||
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def upload(index):
|
||||
client = Client(timeout=60)
|
||||
barrier.wait(timeout=30)
|
||||
result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png',
|
||||
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
check(result['json']['sha256'] == expected, 'uploaded checksum mismatch')
|
||||
downloaded = client.call('GET', '/api/v1/attachments/' + result['json']['id'] + '/content', actor=bob)
|
||||
check(hashlib.sha256(downloaded['bytes']).hexdigest() == expected, 'download checksum mismatch')
|
||||
return {'upload': index, 'seconds': round(result['elapsed'], 3), 'instance': result['headers']['X-Instance-Id']}
|
||||
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(upload, range(10)))
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
peak = {key: max([baseline[key]] + [s['bytes'][key] for s in samples]) for key in baseline}
|
||||
print(json.dumps({'concurrency': 10, 'file_bytes': len(data), 'pixels': 1600 * 1600,
|
||||
'sha256': expected, 'baseline_bytes': baseline, 'sampled_peak_bytes': peak,
|
||||
'results': results, 'samples': samples}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Bootstrap is deliberately repeated; existing objects must survive both runs.
|
||||
docker compose run --rm s3-init
|
||||
docker compose run --rm s3-init
|
||||
docker compose run --rm --entrypoint /bin/sh s3-init -ec '
|
||||
mc alias set app http://s3:9000 "$(cat /run/secrets/s3_api_access_key_id)" "$(cat /run/secrets/s3_api_secret_access_key)" >/dev/null
|
||||
if mc admin info app >/dev/null 2>&1; then exit 1; fi
|
||||
if mc mb app/forbidden-bucket >/dev/null 2>&1; then exit 1; fi
|
||||
if mc anonymous set download "app/$S3_BUCKET" >/dev/null 2>&1; then exit 1; fi
|
||||
printf test > /tmp/test-object
|
||||
if mc cp /tmp/test-object "app/$S3_BUCKET/forbidden-prefix/object" >/dev/null 2>&1; then exit 1; fi
|
||||
mc stat "app/$S3_BUCKET" >/dev/null
|
||||
printf "PASS: app cannot administer MinIO, create buckets, change policy or write outside originals/\n"
|
||||
'
|
||||
if docker compose exec -T lb wget -q -T 2 -O /dev/null http://s3:9001; then
|
||||
echo 'FAIL: console reachable from shared network'
|
||||
exit 1
|
||||
fi
|
||||
echo 'PASS: console unavailable from shared network'
|
||||
@@ -9,7 +9,7 @@ mkdir -p secrets
|
||||
# Readable by the non-root users inside the containers (bind-mounted as is).
|
||||
umask 022
|
||||
|
||||
for name in db_password redis_password s3_access_key_id s3_secret_access_key; do
|
||||
for name in db_password redis_password s3_access_key_id s3_secret_access_key s3_api_access_key_id s3_api_secret_access_key; do
|
||||
if [ ! -s "secrets/$name" ]; then
|
||||
openssl rand -hex 24 > "secrets/$name"
|
||||
echo "created secrets/$name"
|
||||
|
||||
Reference in New Issue
Block a user