lab4 done
This commit is contained in:
@@ -4,12 +4,17 @@ cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
for name in postgres_password cursor_key; do
|
||||
for name in postgres_password cursor_key s3_root_secret_key s3_secret_key; do
|
||||
if [ ! -s "secrets/$name" ]; then
|
||||
openssl rand -hex 32 > "secrets/$name"
|
||||
fi
|
||||
done
|
||||
for name in s3_root_access_key s3_access_key; do
|
||||
if [ ! -s "secrets/$name" ]; then
|
||||
openssl rand -hex 8 > "secrets/$name"
|
||||
fi
|
||||
done
|
||||
# Compose mounts files directly on Linux; the unprivileged API must read them.
|
||||
# The host directory remains private (0700).
|
||||
chmod 444 secrets/postgres_password secrets/cursor_key
|
||||
chmod 444 secrets/postgres_password secrets/cursor_key secrets/s3_root_access_key secrets/s3_root_secret_key secrets/s3_access_key secrets/s3_secret_key
|
||||
printf '%s\n' 'Local secrets are ready.'
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ten simultaneous proxy uploads, checksums, latency and cgroup memory samples."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT/'tests/own'))
|
||||
from helpers import Client, check, containers, docker, png, save, upload
|
||||
|
||||
|
||||
def memory(ids):
|
||||
result = {}
|
||||
for container in ids:
|
||||
raw = docker('exec',container,'cat','/sys/fs/cgroup/memory.current').stdout
|
||||
result[container] = int(raw)
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
return sorted(values)[max(0, math.ceil(len(values)*p)-1)]
|
||||
|
||||
|
||||
def kernel_peak(ids):
|
||||
return {container:int(docker('exec',container,'cat','/sys/fs/cgroup/memory.peak').stdout)
|
||||
for container in ids}
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(timeout=55)
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
upload(c,alice,chat,png())
|
||||
ids = containers('api')
|
||||
results = {'concurrency':10, 'warmup_uploads':1, 'memory_method':'cgroup v2 memory.current sampled every >=100ms; values include file cache', 'runs':[]}
|
||||
for width,height in ((512,512),(1024,768)):
|
||||
content = png(width,height,noisy=True)
|
||||
sha = hashlib.sha256(content).hexdigest()
|
||||
baseline = memory(ids)
|
||||
baseline_kernel_peak = kernel_peak(ids)
|
||||
samples = []
|
||||
barrier = threading.Barrier(11)
|
||||
def worker(number):
|
||||
local = Client(timeout=55)
|
||||
barrier.wait()
|
||||
return upload(local,alice,chat,content,filename='concurrent-'+str(number)+'.png')
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
futures = [pool.submit(worker,n) for n in range(10)]
|
||||
barrier.wait()
|
||||
while not all(f.done() for f in futures):
|
||||
samples.append({'seconds':round(time.monotonic()-started,3), 'bytes':memory(ids)})
|
||||
time.sleep(.1)
|
||||
responses = [f.result() for f in futures]
|
||||
elapsed = time.monotonic()-started
|
||||
samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)})
|
||||
for response in responses:
|
||||
att = response['json']
|
||||
check(att['sha256']==sha,'concurrent upload checksum mismatch')
|
||||
downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
||||
check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed')
|
||||
latency = [r['elapsed']*1000 for r in responses]
|
||||
peak = {container:max(sample['bytes'][container] for sample in samples) for container in ids}
|
||||
run = {'width':width, 'height':height, 'size_bytes':len(content), 'sha256':sha,
|
||||
'requests':10, 'successful':len(responses), 'errors':0, 'duration_seconds':round(elapsed,3),
|
||||
'successful_rps':round(len(responses)/elapsed,3),
|
||||
'latency_ms':{str(p):round(percentile(latency,p),3) for p in (.5,.95,.99)},
|
||||
'baseline_bytes':baseline, 'sampled_peak_bytes':peak,
|
||||
'kernel_lifetime_peak_before_bytes':baseline_kernel_peak,
|
||||
'kernel_lifetime_peak_after_bytes':kernel_peak(ids),
|
||||
'sampled_peak_sum_bytes':max(sum(sample['bytes'].values()) for sample in samples),
|
||||
'samples':samples}
|
||||
check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit')
|
||||
results['runs'].append(run)
|
||||
print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True)
|
||||
save('lab4-upload-memory.json',results)
|
||||
print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
+48
-7
@@ -14,7 +14,11 @@ STACK = os.environ.get('STACK_NAME', 'mtusi-lab4')
|
||||
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4')
|
||||
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4')
|
||||
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4')
|
||||
S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z')
|
||||
MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z')
|
||||
PORT = os.environ.get('PORT', '8080')
|
||||
SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key',
|
||||
's3_root_secret_key', 's3_access_key', 's3_secret_key')
|
||||
|
||||
|
||||
def docker(*args, capture=False, **kwargs):
|
||||
@@ -65,7 +69,7 @@ def wait_api():
|
||||
def job(kind, image, command, integration=False):
|
||||
name = STACK+'_'+kind
|
||||
# Fixed service name is a deployment lock: concurrent runners fail to create it.
|
||||
args = ['service', 'create', '--detach', '--name', name, '--mode', 'replicated-job',
|
||||
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
|
||||
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
||||
'--constraint', 'node.labels.mtusi_lab4_storage==true',
|
||||
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
|
||||
@@ -73,15 +77,23 @@ def job(kind, image, command, integration=False):
|
||||
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
|
||||
if integration:
|
||||
for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger',
|
||||
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key']:
|
||||
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key',
|
||||
'S3_ENDPOINT=minio:9000', 'S3_BUCKET=attachments',
|
||||
'S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key', 'S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key']:
|
||||
args += ['--env', value]
|
||||
args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key']
|
||||
for secret in ('s3_access_key', 's3_secret_key'):
|
||||
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
|
||||
else:
|
||||
args += ['--env', 'GOOSE_DRIVER=postgres', '--env',
|
||||
'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5']
|
||||
docker(*args, image, *command)
|
||||
wait_job(name, kind, 300 if integration else 90)
|
||||
|
||||
|
||||
def wait_job(name, kind, timeout=90):
|
||||
try:
|
||||
deadline = time.monotonic()+(300 if integration else 90)
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
ids = docker('service', 'ps', '-q', name, capture=True).split()
|
||||
states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids]
|
||||
@@ -97,7 +109,31 @@ def job(kind, image, command, integration=False):
|
||||
docker('service', 'rm', name)
|
||||
|
||||
|
||||
def bootstrap_s3():
|
||||
name = STACK+'_s3-bootstrap'
|
||||
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
|
||||
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
||||
'--constraint', 'node.labels.mtusi_lab4_storage==true',
|
||||
'--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh',
|
||||
'--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh',
|
||||
'--config', 'source='+STACK+'_s3_policy,target=/policy.json']
|
||||
for secret in SECRETS[2:]:
|
||||
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
|
||||
docker(*args, MC, '/bootstrap.sh')
|
||||
wait_job(name, 'S3 bootstrap')
|
||||
|
||||
|
||||
def reconcile():
|
||||
active = tasks('api')
|
||||
if not active:
|
||||
raise RuntimeError('no API task available for reconciliation')
|
||||
container = active[0]['Status']['ContainerStatus']['ContainerID']
|
||||
docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:])
|
||||
|
||||
|
||||
def build():
|
||||
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
|
||||
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
|
||||
docker('build', '-t', API, '.')
|
||||
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
|
||||
|
||||
@@ -113,7 +149,7 @@ def up():
|
||||
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
|
||||
docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID'])
|
||||
subprocess.run(['./scripts/bootstrap.sh'], check=True)
|
||||
for secret in ('postgres_password', 'cursor_key'):
|
||||
for secret in SECRETS:
|
||||
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
|
||||
if STACK+'_'+secret not in known:
|
||||
docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret)
|
||||
@@ -131,6 +167,8 @@ def up():
|
||||
docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env)
|
||||
wait_service('postgres')
|
||||
wait_service('redis')
|
||||
wait_service('minio')
|
||||
bootstrap_s3()
|
||||
if fresh:
|
||||
job('migrations', MIGRATIONS, ['up'])
|
||||
docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2'))
|
||||
@@ -143,7 +181,7 @@ def up():
|
||||
|
||||
def restart():
|
||||
# Force replaces every task while retaining the same Swarm secrets and volumes.
|
||||
services = ('postgres', 'redis', 'api', 'lb')
|
||||
services = ('postgres', 'redis', 'minio', 'api', 'lb')
|
||||
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
|
||||
for service in services:
|
||||
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
|
||||
@@ -155,6 +193,7 @@ def restart():
|
||||
wait_service('lb')
|
||||
wait_service('postgres')
|
||||
wait_service('redis')
|
||||
wait_service('minio')
|
||||
wait_api()
|
||||
|
||||
|
||||
@@ -171,7 +210,7 @@ def stop_one():
|
||||
|
||||
def update():
|
||||
previous = {t['ID'] for t in tasks('api')}
|
||||
docker('service', 'update', '--detach', '--force', '--image', API, STACK+'_api')
|
||||
docker('service', 'update', '--detach', '--force', '--no-resolve-image', '--image', API, STACK+'_api')
|
||||
deadline = time.monotonic()+180
|
||||
while time.monotonic() < deadline:
|
||||
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
|
||||
@@ -187,7 +226,7 @@ def update():
|
||||
|
||||
def status():
|
||||
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
|
||||
for name in ('lb','api','postgres','redis'):
|
||||
for name in ('lb','api','postgres','redis','minio'):
|
||||
service = inspect('service', STACK+'_'+name)
|
||||
spec = service['Spec']
|
||||
template = spec['TaskTemplate']
|
||||
@@ -207,6 +246,8 @@ def main():
|
||||
if action == 'status': status()
|
||||
elif action == 'up': up()
|
||||
elif action == 'build': build()
|
||||
elif action == 'bootstrap-s3': bootstrap_s3()
|
||||
elif action == 'reconcile': reconcile()
|
||||
elif action == 'wait': wait_api()
|
||||
elif action == 'restart': restart()
|
||||
elif action == 'stop-one': stop_one()
|
||||
|
||||
Reference in New Issue
Block a user