lab 3 done
This commit is contained in:
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
for name in postgres_password cursor_key; do
|
||||
if [ ! -s "secrets/$name" ]; then
|
||||
openssl rand -hex 32 > "secrets/$name"
|
||||
fi
|
||||
done
|
||||
# Compose mounts files directly on Linux; the unprivileged API must read them.
|
||||
# The host directory remains private (0700).
|
||||
chmod 444 secrets/postgres_password secrets/cursor_key
|
||||
printf '%s\n' 'Local secrets are ready.'
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
exec docker compose --env-file /dev/null -f compose.yaml "$@"
|
||||
fi
|
||||
exec docker-compose --env-file /dev/null -f compose.yaml "$@"
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible local benchmark with resource samples; no credentials in output."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STACK = os.environ.get('STACK_NAME', 'mtusi-lab3')
|
||||
|
||||
def containers(service=None):
|
||||
label = 'com.docker.swarm.service.name='+STACK+'_'+service if service else 'com.docker.stack.namespace='+STACK
|
||||
return subprocess.check_output(['docker','ps','--filter','label='+label,'-q'],text=True).split()
|
||||
|
||||
|
||||
def counts():
|
||||
sql = 'SELECT json_build_object(\'users\',(SELECT count(*) FROM users),\'chats\',(SELECT count(*) FROM chats),\'messages\',(SELECT count(*) FROM messages))'
|
||||
return json.loads(subprocess.check_output(['docker', 'exec', containers('postgres')[0], 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql], text=True))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--duration', type=int, default=30)
|
||||
parser.add_argument('--concurrency', type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
command = [sys.executable, str(ROOT / 'tests/load.py'), '--base-url', args.base_url, '--concurrency', str(args.concurrency)]
|
||||
subprocess.run(command + ['--duration', '3'], check=True, stdout=subprocess.DEVNULL)
|
||||
before = counts()
|
||||
ids = containers()
|
||||
samples = []
|
||||
process = subprocess.Popen(command + ['--duration', str(args.duration), '--output', str(ROOT / 'evidence/swarm-load.json')], stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
while process.poll() is None:
|
||||
stats = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True)
|
||||
samples.append({'elapsed_sample': len(samples), 'containers': [json.loads(line) for line in stats.splitlines()]})
|
||||
time.sleep(1)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
code = process.wait()
|
||||
if code:
|
||||
raise SystemExit(code)
|
||||
report = {'warmup_seconds': 3, 'dataset_before': before, 'dataset_after': counts(), 'samples': samples}
|
||||
(ROOT / 'evidence/swarm-resources.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
print((ROOT / 'evidence/swarm-load.json').read_text())
|
||||
print('Resource samples and dataset counts: evidence/swarm-resources.json')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/swarm.py restart
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/swarm.py stop-one
|
||||
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Single-node Swarm lab lifecycle; never deletes volumes or prints credentials."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
os.chdir(ROOT)
|
||||
STACK = os.environ.get('STACK_NAME', 'mtusi-lab3')
|
||||
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab3')
|
||||
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab3')
|
||||
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab3')
|
||||
PORT = os.environ.get('PORT', '8080')
|
||||
|
||||
|
||||
def docker(*args, capture=False, **kwargs):
|
||||
return subprocess.run(['docker', *args], check=True, text=True,
|
||||
stdout=subprocess.PIPE if capture else None, **kwargs).stdout
|
||||
|
||||
|
||||
def inspect(kind, name):
|
||||
return json.loads(docker(kind, 'inspect', name, capture=True))[0]
|
||||
|
||||
|
||||
def tasks(service):
|
||||
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', STACK+'_'+service, capture=True).split()
|
||||
return [json.loads(docker('inspect', task, capture=True))[0] for task in ids]
|
||||
|
||||
|
||||
def wait_service(service, count=1, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
active = tasks(service)
|
||||
if len(active) == count and all(t['Status']['State'] == 'running' for t in active):
|
||||
# This lab pins storage to this manager and runs on one Docker engine.
|
||||
healthy = True
|
||||
for task in active:
|
||||
container = task['Status'].get('ContainerStatus', {}).get('ContainerID')
|
||||
state = json.loads(docker('inspect', container, capture=True))[0]['State']
|
||||
healthy &= state.get('Health', {}).get('Status', 'healthy') == 'healthy'
|
||||
if healthy:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise RuntimeError('service did not become healthy: '+service)
|
||||
|
||||
|
||||
def wait_api():
|
||||
wait_service('api', int(os.environ.get('API_REPLICAS', '2')))
|
||||
deadline = time.monotonic()+45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen('http://localhost:'+PORT+'/health/ready', timeout=3) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('LB readiness timeout')
|
||||
|
||||
|
||||
def job(kind, image, command, integration=False):
|
||||
name = STACK+'_'+kind
|
||||
# Fixed service name is a deployment lock: concurrent runners fail to create it.
|
||||
args = ['service', 'create', '--detach', '--name', name, '--mode', 'replicated-job',
|
||||
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
||||
'--constraint', 'node.labels.mtusi_lab3_storage==true',
|
||||
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
|
||||
'--secret', 'source='+STACK+'_postgres_password,target=postgres_password',
|
||||
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
|
||||
if integration:
|
||||
for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger',
|
||||
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key']:
|
||||
args += ['--env', value]
|
||||
args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key']
|
||||
else:
|
||||
args += ['--env', 'GOOSE_DRIVER=postgres', '--env',
|
||||
'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5']
|
||||
docker(*args, image, *command)
|
||||
try:
|
||||
deadline = time.monotonic()+(300 if integration else 90)
|
||||
while time.monotonic() < deadline:
|
||||
ids = docker('service', 'ps', '-q', name, capture=True).split()
|
||||
states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids]
|
||||
if 'complete' in states:
|
||||
docker('service', 'logs', name)
|
||||
return
|
||||
if any(s in ('failed', 'rejected') for s in states):
|
||||
docker('service', 'logs', name)
|
||||
raise RuntimeError(kind+' failed')
|
||||
time.sleep(1)
|
||||
raise RuntimeError(kind+' timed out')
|
||||
finally:
|
||||
docker('service', 'rm', name)
|
||||
|
||||
|
||||
def build():
|
||||
docker('build', '-t', API, '.')
|
||||
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
|
||||
|
||||
|
||||
def up():
|
||||
swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True))
|
||||
if swarm['LocalNodeState'] == 'inactive':
|
||||
docker('swarm', 'init', capture=True) # join token stays out of logs
|
||||
swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True))
|
||||
if not swarm['ControlAvailable']:
|
||||
raise RuntimeError('run on the Swarm manager')
|
||||
if len(docker('node', 'ls', '-q', capture=True).split()) != 1:
|
||||
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
|
||||
docker('node', 'update', '--label-add', 'mtusi_lab3_storage=true', swarm['NodeID'])
|
||||
subprocess.run(['./scripts/bootstrap.sh'], check=True)
|
||||
for secret in ('postgres_password', 'cursor_key'):
|
||||
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
|
||||
if STACK+'_'+secret not in known:
|
||||
docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret)
|
||||
build()
|
||||
existing = docker('service', 'ls', '--format', '{{.Name}}', capture=True).split()
|
||||
# Upgrade the shared schema before replacing any running API image.
|
||||
fresh = STACK+'_api' not in existing
|
||||
if not fresh:
|
||||
job('migrations', MIGRATIONS, ['up'])
|
||||
# Initial deployment keeps API at zero until the one-shot migration succeeds.
|
||||
env = dict(os.environ)
|
||||
env['STACK_NAME'] = STACK
|
||||
if fresh:
|
||||
env['API_REPLICAS'] = '0'
|
||||
docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env)
|
||||
wait_service('postgres')
|
||||
wait_service('redis')
|
||||
if fresh:
|
||||
job('migrations', MIGRATIONS, ['up'])
|
||||
docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2'))
|
||||
if fresh:
|
||||
wait_api()
|
||||
else:
|
||||
update()
|
||||
print('PASS: Swarm stack ready at http://localhost:'+PORT)
|
||||
|
||||
|
||||
def restart():
|
||||
# Force replaces every task while retaining the same Swarm secrets and volumes.
|
||||
services = ('postgres', 'redis', 'api', 'lb')
|
||||
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
|
||||
for service in services:
|
||||
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
|
||||
deadline = time.monotonic()+75
|
||||
while any(previous[name].intersection(t['ID'] for t in tasks(name)) for name in services):
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError('restart did not replace every original task')
|
||||
time.sleep(1)
|
||||
wait_service('lb')
|
||||
wait_service('postgres')
|
||||
wait_service('redis')
|
||||
wait_api()
|
||||
|
||||
|
||||
def stop_one():
|
||||
target = os.environ['TARGET_INSTANCE_ID']
|
||||
matches = [t for t in tasks('api') if t['ID'] == target]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError('TARGET_INSTANCE_ID is not a running API task in this stack')
|
||||
task = matches[0]
|
||||
container = task['Status']['ContainerStatus']['ContainerID']
|
||||
print('Stopping task '+target+' container '+container, flush=True)
|
||||
docker('kill', container) # Swarm replaces it with a different task/instance ID.
|
||||
|
||||
|
||||
def update():
|
||||
previous = {t['ID'] for t in tasks('api')}
|
||||
docker('service', 'update', '--detach', '--force', '--image', API, STACK+'_api')
|
||||
deadline = time.monotonic()+180
|
||||
while time.monotonic() < deadline:
|
||||
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
|
||||
if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')):
|
||||
wait_api()
|
||||
print('PASS: rolling update completed')
|
||||
return
|
||||
if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'):
|
||||
raise RuntimeError('rolling update: '+state)
|
||||
time.sleep(1)
|
||||
raise RuntimeError('rolling update timed out')
|
||||
|
||||
|
||||
def status():
|
||||
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
|
||||
for name in ('lb','api','postgres','redis'):
|
||||
service = inspect('service', STACK+'_'+name)
|
||||
spec = service['Spec']
|
||||
template = spec['TaskTemplate']
|
||||
result['services'][name] = {
|
||||
'image': template['ContainerSpec']['Image'],
|
||||
'limits': template.get('Resources',{}).get('Limits',{}),
|
||||
'published_ports': spec.get('EndpointSpec',{}).get('Ports',[]),
|
||||
'secret_names': [x['SecretName'] for x in template['ContainerSpec'].get('Secrets',[])],
|
||||
'mounts': template['ContainerSpec'].get('Mounts',[]),
|
||||
'tasks': [{'id':t['ID'],'state':t['Status']['State'],
|
||||
'container':t['Status'].get('ContainerStatus',{}).get('ContainerID')}
|
||||
for t in tasks(name)]}
|
||||
print(json.dumps(result,indent=2))
|
||||
|
||||
def main():
|
||||
action = sys.argv[1]
|
||||
if action == 'status': status()
|
||||
elif action == 'up': up()
|
||||
elif action == 'build': build()
|
||||
elif action == 'wait': wait_api()
|
||||
elif action == 'restart': restart()
|
||||
elif action == 'stop-one': stop_one()
|
||||
elif action == 'update': update()
|
||||
elif action == 'migrate': job('migrations', MIGRATIONS, ['up'])
|
||||
elif action == 'migrate-status': job('migrations', MIGRATIONS, ['status'])
|
||||
elif action == 'test-integration':
|
||||
docker('build', '--target', 'tests', '-t', TESTS, '.')
|
||||
job('tests', TESTS, [], integration=True)
|
||||
elif action == 'down': docker('stack', 'rm', STACK)
|
||||
else: raise RuntimeError('unknown action: '+action)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (RuntimeError, subprocess.CalledProcessError) as error:
|
||||
print('FAIL: '+str(error), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user