#!/usr/bin/env python3 """Single-node Swarm lab lifecycle; never deletes volumes or prints credentials.""" import json import os from pathlib import Path import subprocess import sys import time import urllib.request ROOT = Path(__file__).resolve().parents[1] os.chdir(ROOT) STACK = os.environ.get('STACK_NAME', 'mtusi-lab5') API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab5') MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab5') TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab5') S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z') MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') PORT = os.environ.get('PORT', '18085') SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key', 's3_root_secret_key', 's3_access_key', 's3_secret_key') def docker(*args, capture=False, **kwargs): return subprocess.run(['docker', *args], check=True, text=True, stdout=subprocess.PIPE if capture else None, **kwargs).stdout def inspect(kind, name): return json.loads(docker(kind, 'inspect', name, capture=True))[0] def tasks(service): ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', STACK+'_'+service, capture=True).split() return [json.loads(docker('inspect', task, capture=True))[0] for task in ids] def wait_service(service, count=1, timeout=120): deadline = time.monotonic() + timeout while time.monotonic() < deadline: active = tasks(service) if len(active) == count and all(t['Status']['State'] == 'running' for t in active): # This lab pins storage to this manager and runs on one Docker engine. healthy = True for task in active: container = task['Status'].get('ContainerStatus', {}).get('ContainerID') state = json.loads(docker('inspect', container, capture=True))[0]['State'] healthy &= state.get('Health', {}).get('Status', 'healthy') == 'healthy' if healthy: return time.sleep(1) raise RuntimeError('service did not become healthy: '+service) def wait_api(): wait_service('api', int(os.environ.get('API_REPLICAS', '2'))) deadline = time.monotonic()+45 while time.monotonic() < deadline: try: with urllib.request.urlopen('http://localhost:'+PORT+'/health/ready', timeout=3) as response: if response.status == 200: return except OSError: pass time.sleep(.5) raise RuntimeError('LB readiness timeout') def job(kind, image, command, integration=False): name = STACK+'_'+kind # Fixed service name is a deployment lock: concurrent runners fail to create it. args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', '--constraint', 'node.labels.mtusi_lab5_storage==true', '--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m', '--secret', 'source='+STACK+'_postgres_password,target=postgres_password', '--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password'] if integration: for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger', 'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key', 'S3_ENDPOINT=minio:9000', 'S3_BUCKET=attachments', 'S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key', 'S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key']: args += ['--env', value] args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key'] for secret in ('s3_access_key', 's3_secret_key'): args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] else: args += ['--env', 'GOOSE_DRIVER=postgres', '--env', 'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5'] docker(*args, image, *command) wait_job(name, kind, 300 if integration else 90) def wait_job(name, kind, timeout=90): try: deadline = time.monotonic()+timeout while time.monotonic() < deadline: ids = docker('service', 'ps', '-q', name, capture=True).split() states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids] if 'complete' in states: docker('service', 'logs', name) return if any(s in ('failed', 'rejected') for s in states): docker('service', 'logs', name) raise RuntimeError(kind+' failed') time.sleep(1) raise RuntimeError(kind+' timed out') finally: docker('service', 'rm', name) def bootstrap_s3(): name = STACK+'_s3-bootstrap' args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', '--constraint', 'node.labels.mtusi_lab5_storage==true', '--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh', '--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh', '--config', 'source='+STACK+'_s3_policy_v5,target=/policy.json'] for secret in SECRETS[2:]: args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] docker(*args, MC, '/bootstrap.sh') wait_job(name, 'S3 bootstrap') def reconcile(): active = tasks('api') if not active: raise RuntimeError('no API task available for reconciliation') container = active[0]['Status']['ContainerStatus']['ContainerID'] docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:]) def worker_cli(args): active=[t for t in tasks('worker') if t['Status']['State']=='running'] if not active: raise RuntimeError('start worker before using its operator CLI') docker('exec',active[0]['Status']['ContainerStatus']['ContainerID'],'/usr/local/bin/worker',*args) def build(): if os.environ.get('SKIP_STORAGE_BUILD') != '1': docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') else: docker('image','inspect',S3,MC,capture=True) docker('build', '-t', API, '.') docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.') def up(): swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True)) if swarm['LocalNodeState'] == 'inactive': docker('swarm', 'init', capture=True) # join token stays out of logs swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True)) if not swarm['ControlAvailable']: raise RuntimeError('run on the Swarm manager') if len(docker('node', 'ls', '-q', capture=True).split()) != 1: raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution') docker('node', 'update', '--label-add', 'mtusi_lab5_storage=true', swarm['NodeID']) subprocess.run(['./scripts/bootstrap.sh'], check=True) for secret in SECRETS: known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split() if STACK+'_'+secret not in known: docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret) build() existing = docker('service', 'ls', '--format', '{{.Name}}', capture=True).split() # Upgrade the shared schema before replacing any running API image. fresh = STACK+'_api' not in existing if not fresh: # A maintenance barrier prevents old lab4 writers inserting ready-only rows # after the one-shot async backfill but before the new API image starts. docker('service','scale','--detach',STACK+'_api=0') deadline=time.monotonic()+45 while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_api','-q',capture=True).strip(): if time.monotonic()>deadline: raise RuntimeError('old API did not drain before migrations') time.sleep(.5) job('migrations', MIGRATIONS, ['up']) # Initial deployment keeps API at zero until the one-shot migration succeeds. env = dict(os.environ) env['STACK_NAME'] = STACK if fresh: env['API_REPLICAS'] = '0' docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env) wait_service('postgres') wait_service('redis') wait_service('minio') bootstrap_s3() if fresh: job('migrations', MIGRATIONS, ['up']) docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2')) if fresh: wait_api() else: update() wait_service('worker') print('PASS: Swarm stack ready at http://localhost:'+PORT) def restart(): # Force replaces every task while retaining the same Swarm secrets and volumes. services = ('postgres', 'redis', 'minio', 'api', 'lb', 'worker') previous = {name: {t['ID'] for t in tasks(name)} for name in services} for service in services: docker('service', 'update', '--detach', '--force', STACK+'_'+service) deadline = time.monotonic()+75 while any(previous[name].intersection(t['ID'] for t in tasks(name)) for name in services): if time.monotonic() > deadline: raise RuntimeError('restart did not replace every original task') time.sleep(1) wait_service('lb') wait_service('postgres') wait_service('redis') wait_service('minio') wait_api() def stop_one(): target = os.environ['TARGET_INSTANCE_ID'] matches = [t for t in tasks('api') if t['ID'] == target] if len(matches) != 1: raise RuntimeError('TARGET_INSTANCE_ID is not a running API task in this stack') task = matches[0] container = task['Status']['ContainerStatus']['ContainerID'] print('Stopping task '+target+' container '+container, flush=True) docker('kill', container) # Swarm replaces it with a different task/instance ID. def update(): previous = {t['ID'] for t in tasks('api')} docker('service', 'update', '--detach', '--force', '--no-resolve-image', '--image', API, STACK+'_api') deadline = time.monotonic()+180 while time.monotonic() < deadline: state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State') if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')): wait_api() docker('service','update','--detach','--force','--no-resolve-image','--image',API,STACK+'_worker') wait_service('worker') print('PASS: rolling update completed (API and worker image refreshed)') return if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'): raise RuntimeError('rolling update: '+state) time.sleep(1) raise RuntimeError('rolling update timed out') def status(): result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}} for name in ('lb','api','worker','postgres','redis','minio'): service = inspect('service', STACK+'_'+name) spec = service['Spec'] template = spec['TaskTemplate'] result['services'][name] = { 'image': template['ContainerSpec']['Image'], 'limits': template.get('Resources',{}).get('Limits',{}), 'published_ports': spec.get('EndpointSpec',{}).get('Ports',[]), 'secret_names': [x['SecretName'] for x in template['ContainerSpec'].get('Secrets',[])], 'mounts': template['ContainerSpec'].get('Mounts',[]), 'tasks': [{'id':t['ID'],'state':t['Status']['State'], 'container':t['Status'].get('ContainerStatus',{}).get('ContainerID')} for t in tasks(name)]} print(json.dumps(result,indent=2)) def main(): action = sys.argv[1] if action == 'status': status() elif action == 'up': up() elif action == 'build': build() elif action == 'bootstrap-s3': bootstrap_s3() elif action == 'reconcile': reconcile() elif action == 'worker-stop': docker('service','scale','--detach',STACK+'_worker=0') deadline=time.monotonic()+30 while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_worker','-q',capture=True).strip(): if time.monotonic()>deadline: raise RuntimeError('worker did not stop') time.sleep(.2) elif action == 'worker-start': docker('service','scale','--detach',STACK+'_worker=1') wait_service('worker') elif action == 'failed-jobs': worker_cli(['--failed']) elif action == 'replay': worker_cli(sys.argv[2:]) elif action == 'wait': wait_api() elif action == 'restart': restart() elif action == 'stop-one': stop_one() elif action == 'update': update() elif action == 'migrate': job('migrations', MIGRATIONS, ['up']) elif action == 'migrate-status': job('migrations', MIGRATIONS, ['status']) elif action == 'test-integration': docker('build', '--target', 'tests', '-t', TESTS, '.') job('tests', TESTS, [], integration=True) elif action == 'down': docker('stack', 'rm', STACK) else: raise RuntimeError('unknown action: '+action) if __name__ == '__main__': try: main() except (RuntimeError, subprocess.CalledProcessError) as error: print('FAIL: '+str(error), file=sys.stderr) sys.exit(1)