Files
mtusi2026lab3/scripts/swarm.py
T
2026-10-20 20:46:51 +03:00

229 lines
9.8 KiB
Python

#!/usr/bin/env python3
"""Single-node Swarm lab lifecycle; never deletes volumes or prints credentials."""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
os.chdir(ROOT)
STACK = os.environ.get('STACK_NAME', 'mtusi-lab3')
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab3')
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab3')
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab3')
PORT = os.environ.get('PORT', '8080')
def docker(*args, capture=False, **kwargs):
return subprocess.run(['docker', *args], check=True, text=True,
stdout=subprocess.PIPE if capture else None, **kwargs).stdout
def inspect(kind, name):
return json.loads(docker(kind, 'inspect', name, capture=True))[0]
def tasks(service):
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', STACK+'_'+service, capture=True).split()
return [json.loads(docker('inspect', task, capture=True))[0] for task in ids]
def wait_service(service, count=1, timeout=120):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
active = tasks(service)
if len(active) == count and all(t['Status']['State'] == 'running' for t in active):
# This lab pins storage to this manager and runs on one Docker engine.
healthy = True
for task in active:
container = task['Status'].get('ContainerStatus', {}).get('ContainerID')
state = json.loads(docker('inspect', container, capture=True))[0]['State']
healthy &= state.get('Health', {}).get('Status', 'healthy') == 'healthy'
if healthy:
return
time.sleep(1)
raise RuntimeError('service did not become healthy: '+service)
def wait_api():
wait_service('api', int(os.environ.get('API_REPLICAS', '2')))
deadline = time.monotonic()+45
while time.monotonic() < deadline:
try:
with urllib.request.urlopen('http://localhost:'+PORT+'/health/ready', timeout=3) as response:
if response.status == 200:
return
except OSError:
pass
time.sleep(.5)
raise RuntimeError('LB readiness timeout')
def job(kind, image, command, integration=False):
name = STACK+'_'+kind
# Fixed service name is a deployment lock: concurrent runners fail to create it.
args = ['service', 'create', '--detach', '--name', name, '--mode', 'replicated-job',
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
'--constraint', 'node.labels.mtusi_lab3_storage==true',
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
'--secret', 'source='+STACK+'_postgres_password,target=postgres_password',
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
if integration:
for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger',
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key']:
args += ['--env', value]
args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key']
else:
args += ['--env', 'GOOSE_DRIVER=postgres', '--env',
'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5']
docker(*args, image, *command)
try:
deadline = time.monotonic()+(300 if integration else 90)
while time.monotonic() < deadline:
ids = docker('service', 'ps', '-q', name, capture=True).split()
states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids]
if 'complete' in states:
docker('service', 'logs', name)
return
if any(s in ('failed', 'rejected') for s in states):
docker('service', 'logs', name)
raise RuntimeError(kind+' failed')
time.sleep(1)
raise RuntimeError(kind+' timed out')
finally:
docker('service', 'rm', name)
def build():
docker('build', '-t', API, '.')
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
def up():
swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True))
if swarm['LocalNodeState'] == 'inactive':
docker('swarm', 'init', capture=True) # join token stays out of logs
swarm = json.loads(docker('info', '--format', '{{json .Swarm}}', capture=True))
if not swarm['ControlAvailable']:
raise RuntimeError('run on the Swarm manager')
if len(docker('node', 'ls', '-q', capture=True).split()) != 1:
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
docker('node', 'update', '--label-add', 'mtusi_lab3_storage=true', swarm['NodeID'])
subprocess.run(['./scripts/bootstrap.sh'], check=True)
for secret in ('postgres_password', 'cursor_key'):
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
if STACK+'_'+secret not in known:
docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret)
build()
existing = docker('service', 'ls', '--format', '{{.Name}}', capture=True).split()
# Upgrade the shared schema before replacing any running API image.
fresh = STACK+'_api' not in existing
if not fresh:
job('migrations', MIGRATIONS, ['up'])
# Initial deployment keeps API at zero until the one-shot migration succeeds.
env = dict(os.environ)
env['STACK_NAME'] = STACK
if fresh:
env['API_REPLICAS'] = '0'
docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env)
wait_service('postgres')
wait_service('redis')
if fresh:
job('migrations', MIGRATIONS, ['up'])
docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2'))
if fresh:
wait_api()
else:
update()
print('PASS: Swarm stack ready at http://localhost:'+PORT)
def restart():
# Force replaces every task while retaining the same Swarm secrets and volumes.
services = ('postgres', 'redis', 'api', 'lb')
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
for service in services:
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
deadline = time.monotonic()+75
while any(previous[name].intersection(t['ID'] for t in tasks(name)) for name in services):
if time.monotonic() > deadline:
raise RuntimeError('restart did not replace every original task')
time.sleep(1)
wait_service('lb')
wait_service('postgres')
wait_service('redis')
wait_api()
def stop_one():
target = os.environ['TARGET_INSTANCE_ID']
matches = [t for t in tasks('api') if t['ID'] == target]
if len(matches) != 1:
raise RuntimeError('TARGET_INSTANCE_ID is not a running API task in this stack')
task = matches[0]
container = task['Status']['ContainerStatus']['ContainerID']
print('Stopping task '+target+' container '+container, flush=True)
docker('kill', container) # Swarm replaces it with a different task/instance ID.
def update():
previous = {t['ID'] for t in tasks('api')}
docker('service', 'update', '--detach', '--force', '--image', API, STACK+'_api')
deadline = time.monotonic()+180
while time.monotonic() < deadline:
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')):
wait_api()
print('PASS: rolling update completed')
return
if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'):
raise RuntimeError('rolling update: '+state)
time.sleep(1)
raise RuntimeError('rolling update timed out')
def status():
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
for name in ('lb','api','postgres','redis'):
service = inspect('service', STACK+'_'+name)
spec = service['Spec']
template = spec['TaskTemplate']
result['services'][name] = {
'image': template['ContainerSpec']['Image'],
'limits': template.get('Resources',{}).get('Limits',{}),
'published_ports': spec.get('EndpointSpec',{}).get('Ports',[]),
'secret_names': [x['SecretName'] for x in template['ContainerSpec'].get('Secrets',[])],
'mounts': template['ContainerSpec'].get('Mounts',[]),
'tasks': [{'id':t['ID'],'state':t['Status']['State'],
'container':t['Status'].get('ContainerStatus',{}).get('ContainerID')}
for t in tasks(name)]}
print(json.dumps(result,indent=2))
def main():
action = sys.argv[1]
if action == 'status': status()
elif action == 'up': up()
elif action == 'build': build()
elif action == 'wait': wait_api()
elif action == 'restart': restart()
elif action == 'stop-one': stop_one()
elif action == 'update': update()
elif action == 'migrate': job('migrations', MIGRATIONS, ['up'])
elif action == 'migrate-status': job('migrations', MIGRATIONS, ['status'])
elif action == 'test-integration':
docker('build', '--target', 'tests', '-t', TESTS, '.')
job('tests', TESTS, [], integration=True)
elif action == 'down': docker('stack', 'rm', STACK)
else: raise RuntimeError('unknown action: '+action)
if __name__ == '__main__':
try:
main()
except (RuntimeError, subprocess.CalledProcessError) as error:
print('FAIL: '+str(error), file=sys.stderr)
sys.exit(1)