300 lines
14 KiB
Python
300 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Single-node Swarm lab lifecycle; never deletes volumes or prints credentials."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
os.chdir(ROOT)
|
|
STACK = os.environ.get('STACK_NAME', 'mtusi-lab5')
|
|
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab5')
|
|
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab5')
|
|
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab5')
|
|
S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z')
|
|
MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z')
|
|
PORT = os.environ.get('PORT', '18085')
|
|
SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key',
|
|
's3_root_secret_key', 's3_access_key', 's3_secret_key')
|
|
|
|
|
|
def docker(*args, capture=False, **kwargs):
|
|
return subprocess.run(['docker', *args], check=True, text=True,
|
|
stdout=subprocess.PIPE if capture else None, **kwargs).stdout
|
|
|
|
|
|
def inspect(kind, name):
|
|
return json.loads(docker(kind, 'inspect', name, capture=True))[0]
|
|
|
|
|
|
def tasks(service):
|
|
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', STACK+'_'+service, capture=True).split()
|
|
return [json.loads(docker('inspect', task, capture=True))[0] for task in ids]
|
|
|
|
|
|
def wait_service(service, count=1, timeout=120):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
active = tasks(service)
|
|
if len(active) == count and all(t['Status']['State'] == 'running' for t in active):
|
|
# This lab pins storage to this manager and runs on one Docker engine.
|
|
healthy = True
|
|
for task in active:
|
|
container = task['Status'].get('ContainerStatus', {}).get('ContainerID')
|
|
state = json.loads(docker('inspect', container, capture=True))[0]['State']
|
|
healthy &= state.get('Health', {}).get('Status', 'healthy') == 'healthy'
|
|
if healthy:
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError('service did not become healthy: '+service)
|
|
|
|
|
|
def wait_api():
|
|
wait_service('api', int(os.environ.get('API_REPLICAS', '2')))
|
|
deadline = time.monotonic()+45
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen('http://localhost:'+PORT+'/health/ready', timeout=3) as response:
|
|
if response.status == 200:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(.5)
|
|
raise RuntimeError('LB readiness timeout')
|
|
|
|
|
|
def job(kind, image, command, integration=False):
|
|
name = STACK+'_'+kind
|
|
# Fixed service name is a deployment lock: concurrent runners fail to create it.
|
|
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
|
|
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
|
'--constraint', 'node.labels.mtusi_lab5_storage==true',
|
|
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
|
|
'--secret', 'source='+STACK+'_postgres_password,target=postgres_password',
|
|
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
|
|
if integration:
|
|
for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger',
|
|
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key',
|
|
'S3_ENDPOINT=minio:9000', 'S3_BUCKET=attachments',
|
|
'S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key', 'S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key']:
|
|
args += ['--env', value]
|
|
args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key']
|
|
for secret in ('s3_access_key', 's3_secret_key'):
|
|
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
|
|
else:
|
|
args += ['--env', 'GOOSE_DRIVER=postgres', '--env',
|
|
'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5']
|
|
docker(*args, image, *command)
|
|
wait_job(name, kind, 300 if integration else 90)
|
|
|
|
|
|
def wait_job(name, kind, timeout=90):
|
|
try:
|
|
deadline = time.monotonic()+timeout
|
|
while time.monotonic() < deadline:
|
|
ids = docker('service', 'ps', '-q', name, capture=True).split()
|
|
states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids]
|
|
if 'complete' in states:
|
|
docker('service', 'logs', name)
|
|
return
|
|
if any(s in ('failed', 'rejected') for s in states):
|
|
docker('service', 'logs', name)
|
|
raise RuntimeError(kind+' failed')
|
|
time.sleep(1)
|
|
raise RuntimeError(kind+' timed out')
|
|
finally:
|
|
docker('service', 'rm', name)
|
|
|
|
|
|
def bootstrap_s3():
|
|
name = STACK+'_s3-bootstrap'
|
|
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
|
|
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
|
'--constraint', 'node.labels.mtusi_lab5_storage==true',
|
|
'--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh',
|
|
'--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh',
|
|
'--config', 'source='+STACK+'_s3_policy_v5,target=/policy.json']
|
|
for secret in SECRETS[2:]:
|
|
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
|
|
docker(*args, MC, '/bootstrap.sh')
|
|
wait_job(name, 'S3 bootstrap')
|
|
|
|
|
|
def reconcile():
|
|
active = tasks('api')
|
|
if not active:
|
|
raise RuntimeError('no API task available for reconciliation')
|
|
container = active[0]['Status']['ContainerStatus']['ContainerID']
|
|
docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:])
|
|
|
|
|
|
def worker_cli(args):
|
|
active=[t for t in tasks('worker') if t['Status']['State']=='running']
|
|
if not active: raise RuntimeError('start worker before using its operator CLI')
|
|
docker('exec',active[0]['Status']['ContainerStatus']['ContainerID'],'/usr/local/bin/worker',*args)
|
|
|
|
|
|
def build():
|
|
if os.environ.get('SKIP_STORAGE_BUILD') != '1':
|
|
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
|
|
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
|
|
else:
|
|
docker('image','inspect',S3,MC,capture=True)
|
|
docker('build', '-t', API, '.')
|
|
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
|
|
|
|
|
|
def up():
|
|
swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True))
|
|
if swarm['LocalNodeState'] == 'inactive':
|
|
docker('swarm', 'init', capture=True) # join token stays out of logs
|
|
swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True))
|
|
if not swarm['ControlAvailable']:
|
|
raise RuntimeError('run on the Swarm manager')
|
|
if len(docker('node', 'ls', '-q', capture=True).split()) != 1:
|
|
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
|
|
docker('node', 'update', '--label-add', 'mtusi_lab5_storage=true', swarm['NodeID'])
|
|
subprocess.run(['./scripts/bootstrap.sh'], check=True)
|
|
for secret in SECRETS:
|
|
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
|
|
if STACK+'_'+secret not in known:
|
|
docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret)
|
|
build()
|
|
existing = docker('service', 'ls', '--format', '{{.Name}}', capture=True).split()
|
|
# Upgrade the shared schema before replacing any running API image.
|
|
fresh = STACK+'_api' not in existing
|
|
if not fresh:
|
|
# A maintenance barrier prevents old lab4 writers inserting ready-only rows
|
|
# after the one-shot async backfill but before the new API image starts.
|
|
docker('service','scale','--detach',STACK+'_api=0')
|
|
deadline=time.monotonic()+45
|
|
while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_api','-q',capture=True).strip():
|
|
if time.monotonic()>deadline: raise RuntimeError('old API did not drain before migrations')
|
|
time.sleep(.5)
|
|
job('migrations', MIGRATIONS, ['up'])
|
|
# Initial deployment keeps API at zero until the one-shot migration succeeds.
|
|
env = dict(os.environ)
|
|
env['STACK_NAME'] = STACK
|
|
if fresh:
|
|
env['API_REPLICAS'] = '0'
|
|
docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env)
|
|
wait_service('postgres')
|
|
wait_service('redis')
|
|
wait_service('minio')
|
|
bootstrap_s3()
|
|
if fresh:
|
|
job('migrations', MIGRATIONS, ['up'])
|
|
docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2'))
|
|
if fresh:
|
|
wait_api()
|
|
else:
|
|
update()
|
|
wait_service('worker')
|
|
print('PASS: Swarm stack ready at http://localhost:'+PORT)
|
|
|
|
|
|
def restart():
|
|
# Force replaces every task while retaining the same Swarm secrets and volumes.
|
|
services = ('postgres', 'redis', 'minio', 'api', 'lb', 'worker')
|
|
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
|
|
for service in services:
|
|
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
|
|
deadline = time.monotonic()+75
|
|
while any(previous[name].intersection(t['ID'] for t in tasks(name)) for name in services):
|
|
if time.monotonic() > deadline:
|
|
raise RuntimeError('restart did not replace every original task')
|
|
time.sleep(1)
|
|
wait_service('lb')
|
|
wait_service('postgres')
|
|
wait_service('redis')
|
|
wait_service('minio')
|
|
wait_api()
|
|
|
|
|
|
def stop_one():
|
|
target = os.environ['TARGET_INSTANCE_ID']
|
|
matches = [t for t in tasks('api') if t['ID'] == target]
|
|
if len(matches) != 1:
|
|
raise RuntimeError('TARGET_INSTANCE_ID is not a running API task in this stack')
|
|
task = matches[0]
|
|
container = task['Status']['ContainerStatus']['ContainerID']
|
|
print('Stopping task '+target+' container '+container, flush=True)
|
|
docker('kill', container) # Swarm replaces it with a different task/instance ID.
|
|
|
|
|
|
def update():
|
|
previous = {t['ID'] for t in tasks('api')}
|
|
docker('service', 'update', '--detach', '--force', '--no-resolve-image', '--image', API, STACK+'_api')
|
|
deadline = time.monotonic()+180
|
|
while time.monotonic() < deadline:
|
|
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
|
|
if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')):
|
|
wait_api()
|
|
docker('service','update','--detach','--force','--no-resolve-image','--image',API,STACK+'_worker')
|
|
wait_service('worker')
|
|
print('PASS: rolling update completed (API and worker image refreshed)')
|
|
return
|
|
if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'):
|
|
raise RuntimeError('rolling update: '+state)
|
|
time.sleep(1)
|
|
raise RuntimeError('rolling update timed out')
|
|
|
|
|
|
def status():
|
|
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
|
|
for name in ('lb','api','worker','postgres','redis','minio'):
|
|
service = inspect('service', STACK+'_'+name)
|
|
spec = service['Spec']
|
|
template = spec['TaskTemplate']
|
|
result['services'][name] = {
|
|
'image': template['ContainerSpec']['Image'],
|
|
'limits': template.get('Resources',{}).get('Limits',{}),
|
|
'published_ports': spec.get('EndpointSpec',{}).get('Ports',[]),
|
|
'secret_names': [x['SecretName'] for x in template['ContainerSpec'].get('Secrets',[])],
|
|
'mounts': template['ContainerSpec'].get('Mounts',[]),
|
|
'tasks': [{'id':t['ID'],'state':t['Status']['State'],
|
|
'container':t['Status'].get('ContainerStatus',{}).get('ContainerID')}
|
|
for t in tasks(name)]}
|
|
print(json.dumps(result,indent=2))
|
|
|
|
def main():
|
|
action = sys.argv[1]
|
|
if action == 'status': status()
|
|
elif action == 'up': up()
|
|
elif action == 'build': build()
|
|
elif action == 'bootstrap-s3': bootstrap_s3()
|
|
elif action == 'reconcile': reconcile()
|
|
elif action == 'worker-stop':
|
|
docker('service','scale','--detach',STACK+'_worker=0')
|
|
deadline=time.monotonic()+30
|
|
while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_worker','-q',capture=True).strip():
|
|
if time.monotonic()>deadline: raise RuntimeError('worker did not stop')
|
|
time.sleep(.2)
|
|
elif action == 'worker-start':
|
|
docker('service','scale','--detach',STACK+'_worker=1')
|
|
wait_service('worker')
|
|
elif action == 'failed-jobs': worker_cli(['--failed'])
|
|
elif action == 'replay': worker_cli(sys.argv[2:])
|
|
elif action == 'wait': wait_api()
|
|
elif action == 'restart': restart()
|
|
elif action == 'stop-one': stop_one()
|
|
elif action == 'update': update()
|
|
elif action == 'migrate': job('migrations', MIGRATIONS, ['up'])
|
|
elif action == 'migrate-status': job('migrations', MIGRATIONS, ['status'])
|
|
elif action == 'test-integration':
|
|
docker('build', '--target', 'tests', '-t', TESTS, '.')
|
|
job('tests', TESTS, [], integration=True)
|
|
elif action == 'down': docker('stack', 'rm', STACK)
|
|
else: raise RuntimeError('unknown action: '+action)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (RuntimeError, subprocess.CalledProcessError) as error:
|
|
print('FAIL: '+str(error), file=sys.stderr)
|
|
sys.exit(1)
|