lab5 done

This commit is contained in:
Khakimov Dmitrij
2026-11-10 22:13:40 +03:00
parent b2bd7501d3
commit cbe9d8748c
63 changed files with 8907 additions and 307 deletions
+332
View File
@@ -0,0 +1,332 @@
#!/usr/bin/env python3
"""Repeatable lab5 track 1 experiment against the isolated Swarm stack.
Uses synthetic users and PNGs. Changes only worker knobs and admission limits,
then restores their original values. Never deletes volumes or accepted jobs.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import csv
import datetime as dt
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import random
import struct
import subprocess
import sys
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def docker(*args):
return subprocess.check_output(['docker', *args], text=True).strip()
def service_spec(name):
return json.loads(docker('service', 'inspect', name))[0]['Spec']
def containers(service):
return docker('ps', '--filter', 'label=com.docker.swarm.service.name=' + service,
'--format', '{{.ID}}').split()
def wait_service(name, timeout=120):
deadline, stable, previous = time.monotonic() + timeout, None, None
while time.monotonic() < deadline:
inspected = json.loads(docker('service', 'inspect', name))[0]
spec = inspected['Spec']
update_state = inspected.get('UpdateStatus', {}).get('State')
check(update_state not in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'),
'service configuration update failed: ' + name)
expected = spec['Mode']['Replicated']['Replicas']
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', name).split()
healthy = expected > 0 and len(ids) == expected and update_state in (None, 'completed')
if healthy:
tasks = json.loads(docker('inspect', *ids))
healthy = all(t['Status']['State'] == 'running' for t in tasks)
if healthy:
states = json.loads(docker('inspect', *[t['Status']['ContainerStatus']['ContainerID'] for t in tasks]))
healthy = all(c['State'].get('Health', {}).get('Status', 'healthy') == 'healthy' for c in states)
now, identity = time.monotonic(), frozenset(ids)
if healthy:
if stable is None or identity != previous:
stable = now
if stable is not None and now - stable >= 5:
return
else:
stable = None
previous = identity
time.sleep(.5)
raise AssertionError('service did not stabilize: ' + name)
def update_env(service, values):
args = ['service', 'update', '--detach']
for name, value in values.items():
args += ['--env-rm', name] if value is None else ['--env-add', name + '=' + str(value)]
docker(*args, service)
wait_service(service)
def old_env(spec, names):
values = dict(entry.split('=', 1) for entry in spec['TaskTemplate']['ContainerSpec'].get('Env', []))
return {name: values.get(name) for name in names}
def png(width, height):
rng = random.Random(20261110 + width)
def chunk(kind, value):
return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(pixels)) + chunk(b'IEND', b''))
def percentile(values, fraction):
check(bool(values), 'no observations for percentile')
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
class Experiment:
def __init__(self, args):
self.args = args
self.api = args.stack + '_api'
self.worker = args.stack + '_worker'
self.pg = args.stack + '_postgres'
self.client = Client(args.base_url, timeout=60)
def sql(self, statement):
ids = containers(self.pg)
check(len(ids) == 1, 'expected one PostgreSQL container')
return docker('exec', ids[0], 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-v', 'ON_ERROR_STOP=1', '-c', statement)
def outstanding(self):
return int(self.sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')"))
def sample(self, chat):
# chat is a UUID generated by the API, never an arbitrary SQL fragment.
from uuid import UUID
chat_id = str(UUID(chat['id']))
query = """SELECT json_build_object(
'outstanding', count(*) FILTER (WHERE j.state IN ('queued','processing')),
'queued', count(*) FILTER (WHERE j.state='queued'),
'processing', count(*) FILTER (WHERE j.state='processing'),
'queue_lag_seconds', COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)
FILTER (WHERE j.state='queued')),0))
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE a.chat_id='""" + chat_id + "'"
result = json.loads(self.sql(query))
total_rss = 0
for container in containers(self.worker):
# Only process status counters; no command lines, environment or credentials.
raw = docker('exec', container, 'sh', '-c',
"awk '/^VmRSS:/{s+=$2} END{printf \"%.0f\\n\",s*1024}' /proc/[0-9]*/status 2>/dev/null || true")
total_rss += int(raw or 0)
result['rss_bytes'] = total_rss
return result
def upload(self, actor, chat, content):
started = time.monotonic()
response = Client(self.args.base_url, timeout=60).call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=research.png',
[202, 429], actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
return {'started': started, 'accepted': time.monotonic(), 'status': response['status'],
'instance': response['headers'].get('X-Instance-Id'),
'attachment': response['json'] if response['status'] == 202 else None}
def validate_result(self, actor, attachment, content):
check({v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'},
'ready must publish all three variants')
original = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)
check(original['bytes'] == content, 'original changed during pipeline')
for variant in attachment['variants']:
response = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + variant['name'], actor=actor)
check(len(response['bytes']) == variant['size_bytes'], 'variant size differs')
check(hashlib.sha256(response['bytes']).hexdigest() == variant['sha256'], 'variant checksum differs')
def run_case(self, concurrency, prefetch, width, height):
check(self.outstanding() == 0, 'drain other workloads before research')
update_env(self.worker, {'WORKER_CONCURRENCY': concurrency, 'WORKER_PREFETCH': prefetch})
alice, bob, _, chat = self.client.bootstrap()
content = png(width, height)
warm = self.upload(alice, chat, content)
check(warm['status'] == 202, 'warmup unexpectedly rejected')
self.client.wait_ready(alice, warm['attachment'], wait_seconds=self.args.timeout)
records, samples, rejected = [], [], 0
started, deadline = time.monotonic(), time.monotonic() + self.args.timeout
baseline = self.sample(chat)
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
pending = {pool.submit(self.upload, alice, chat, content) for _ in range(self.args.requests)}
polling = {}
while pending or polling:
check(time.monotonic() < deadline, 'pipeline did not drain before research timeout')
done = {future for future in pending if future.done()}
pending -= done
for future in done:
row = future.result()
if row['status'] == 429:
rejected += 1
else:
check(row['attachment']['status'] == 'queued', 'upload must return a queued snapshot')
polling[row['attachment']['id']] = row
for attachment_id, row in list(polling.items()):
attachment = self.client.call('GET', '/api/v1/attachments/' + attachment_id,
actor=bob, schema='Attachment')['json']
check(attachment['status'] != 'failed', 'research attachment failed')
if attachment['status'] == 'ready':
now = time.monotonic()
row.update(attachment=attachment, upload_ms=(row['accepted']-row['started'])*1000,
ready_ms=(now-row['started'])*1000,
after_accept_ms=(now-row['accepted'])*1000)
records.append(row)
del polling[attachment_id]
sample = self.sample(chat)
samples.append({'seconds': round(time.monotonic() - started, 3), **sample})
if pending or polling:
time.sleep(self.args.interval)
elapsed = time.monotonic() - started
check(len(records) == self.args.requests and rejected == 0, 'unexpected rejection in comparison matrix')
check(self.outstanding() == 0, 'comparison workload not drained')
self.validate_result(bob, records[-1]['attachment'], content)
result = {'concurrency': concurrency, 'prefetch': prefetch, 'width': width, 'height': height,
'size_bytes': len(content), 'input_sha256': hashlib.sha256(content).hexdigest(),
'offered': self.args.requests, 'accepted': len(records),
'rejected': rejected, 'completed': len(records), 'makespan_seconds': round(elapsed, 3),
'throughput_rps': round(len(records) / elapsed, 3),
'ready_p50_ms': round(percentile([r['ready_ms'] for r in records], .5), 3),
'ready_p95_ms': round(percentile([r['ready_ms'] for r in records], .95), 3),
'after_accept_p95_ms': round(percentile([r['after_accept_ms'] for r in records], .95), 3),
'upload_p95_ms': round(percentile([r['upload_ms'] for r in records], .95), 3),
'queue_lag_peak_seconds': max(s['queue_lag_seconds'] for s in samples),
'outstanding_peak': max(s['outstanding'] for s in samples),
'rss_baseline_bytes': baseline['rss_bytes'],
'rss_peak_bytes': max([baseline['rss_bytes']] + [s['rss_bytes'] for s in samples]),
'samples': samples,
'observations': [{'attachment_id': r['attachment']['id'], 'ready_ms': round(r['ready_ms'], 3),
'upload_ms': round(r['upload_ms'], 3)} for r in records]}
print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'observations')}), flush=True)
return result
def overload(self, limit):
check(self.outstanding() == 0, 'drain other workloads before overload test')
docker('service', 'scale', '--detach', self.worker + '=0')
deadline = time.monotonic() + 60
while containers(self.worker):
check(time.monotonic() < deadline, 'worker did not stop')
time.sleep(.5)
update_env(self.api, {'MAX_OUTSTANDING_JOBS': limit})
alice, bob, _, chat = self.client.bootstrap()
content = png(320, 200)
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
rows = list(pool.map(lambda _: self.upload(alice, chat, content), range(limit * 2)))
accepted = [r for r in rows if r['status'] == 202]
rejected = len(rows) - len(accepted)
outstanding = self.outstanding()
check(len({row['instance'] for row in rows}) >= 2, 'overload must exercise both API replicas')
check(len(accepted) == limit and rejected == limit and outstanding == limit,
'admission limit did not bound accepted work while worker was stopped')
self.client.message(alice, chat, 'text messages remain available during overload')
docker('service', 'scale', '--detach', self.worker + '=1')
wait_service(self.worker)
started = time.monotonic()
for row in accepted:
attachment = self.client.wait_ready(bob, row['attachment'], wait_seconds=self.args.timeout)
check(attachment['error_code'] is None, 'accepted work failed after overload')
recovery = time.monotonic() - started
check(self.outstanding() == 0, 'accepted work remained outstanding after recovery')
followup = self.upload(alice, chat, content)
check(followup['status'] == 202, 'API did not resume admission after drain')
self.client.wait_ready(bob, followup['attachment'], wait_seconds=self.args.timeout)
result = {'limit': limit, 'offered': len(rows), 'accepted': len(accepted), 'rejected_429': rejected,
'observed_api_instances': len({row['instance'] for row in rows}),
'outstanding_worker_stopped': outstanding, 'ready_after_resume': len(accepted),
'observed_drain_after_worker_healthy_seconds': round(recovery, 3),
'outstanding_after_recovery': self.outstanding(), 'followup_upload': 202,
'text_messages_during_overload': 'PASS'}
print('OVERLOAD ' + json.dumps(result), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:18085'))
parser.add_argument('--stack', default=os.environ.get('STACK_NAME', 'mtusi-lab5'))
parser.add_argument('--requests', type=int, default=20)
parser.add_argument('--uploaders', type=int, default=8)
parser.add_argument('--interval', type=float, default=.25)
parser.add_argument('--timeout', type=int, default=240)
parser.add_argument('--limit', type=int, default=8)
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/lab5-research.json')
args = parser.parse_args()
check(args.requests >= 5 and args.uploaders > 0 and 0 < args.limit <= 100,
'invalid experiment limits')
check(args.interval > 0 and args.timeout > 0, 'sampling interval and timeout must be positive')
experiment = Experiment(args)
initial_worker, initial_api = service_spec(experiment.worker), service_spec(experiment.api)
original_worker_env = old_env(initial_worker, ('WORKER_CONCURRENCY', 'WORKER_PREFETCH'))
original_api_env = old_env(initial_api, ('MAX_OUTSTANDING_JOBS',))
original_replicas = initial_worker['Mode']['Replicated']['Replicas']
check(original_replicas == 1, 'experiment expects one worker service replica; concurrency is internal')
report = {'track': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'stack': args.stack, 'worker_image': initial_worker['TaskTemplate']['ContainerSpec']['Image'],
'worker_image_id': docker('inspect', containers(experiment.worker)[0], '--format', '{{.Image}}'),
'environment': {'docker_server': docker('version', '--format', '{{.Server.Version}}'),
'docker_cpus': int(docker('info', '--format', '{{.NCPU}}')),
'docker_memory_bytes': int(docker('info', '--format', '{{.MemTotal}}')),
'architecture': platform.machine(), 'python': platform.python_version()},
'worker_limits': initial_worker['TaskTemplate'].get('Resources', {}).get('Limits', {}),
'initial_settings': {'worker': original_worker_env, 'api': original_api_env},
'requests_per_case': args.requests, 'upload_concurrency': args.uploaders,
'sampling_interval_min_seconds': args.interval,
'method': {'throughput': 'completed / burst makespan including client upload and polling',
'ready_latency': 'client POST start to first observed ready; nearest-rank p95',
'queue_lag': 'age of oldest queued PostgreSQL job in this chat; excludes processing',
'rss': 'sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly',
'limitations': 'single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA'},
'cases': []}
args.output.parent.mkdir(parents=True, exist_ok=True)
def save():
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + '\n')
try:
for width, height in ((512, 512), (1536, 1536)):
for concurrency, prefetch in ((1, 1), (2, 4), (4, 8)):
report['cases'].append(experiment.run_case(concurrency, prefetch, width, height))
save()
report['overload'] = experiment.overload(args.limit)
save()
finally:
# Keep every accepted job recoverable, including when an assertion fails.
try:
if not containers(experiment.worker):
docker('service', 'scale', '--detach', experiment.worker + '=' + str(original_replicas))
wait_service(experiment.worker)
update_env(experiment.worker, original_worker_env)
finally:
update_env(experiment.api, original_api_env)
report['result'] = 'PASS'
report['restored_initial_settings'] = True
report['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat()
save()
summary = args.output.with_suffix('.csv')
keys = ['width', 'height', 'concurrency', 'prefetch', 'offered', 'accepted', 'completed',
'throughput_rps', 'ready_p95_ms', 'after_accept_p95_ms', 'upload_p95_ms',
'queue_lag_peak_seconds', 'rss_peak_bytes']
with summary.open('w', newline='') as file:
writer = csv.DictWriter(file, fieldnames=keys, extrasaction='ignore')
writer.writeheader()
writer.writerows(report['cases'])
print('PASS: six measured configurations, bounded overload and recovery; ' + str(args.output))
if __name__ == '__main__':
main()
+1 -1
View File
@@ -9,7 +9,7 @@ import sys
import time
ROOT = Path(__file__).resolve().parents[1]
STACK = os.environ.get('STACK_NAME', 'mtusi-lab4')
STACK = os.environ.get('STACK_NAME', 'mtusi-lab5')
def containers(service=None):
label = 'com.docker.swarm.service.name='+STACK+'_'+service if service else 'com.docker.stack.namespace='+STACK
+2 -2
View File
@@ -59,7 +59,7 @@ def main():
elapsed = time.monotonic()-started
samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)})
for response in responses:
att = response['json']
att = c.wait_ready(bob,response['json'])
check(att['sha256']==sha,'concurrent upload checksum mismatch')
downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed')
@@ -77,7 +77,7 @@ def main():
check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit')
results['runs'].append(run)
print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True)
save('lab4-upload-memory.json',results)
save('lab5-upload-memory.json',results)
print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.')
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env python3
"""Render measured pipeline results; install requirements-research.txt first."""
import argparse
import json
import os
from pathlib import Path
import tempfile
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('input', nargs='?', type=Path, default=Path('evidence/lab5-research.json'))
args = parser.parse_args()
os.environ['MPLCONFIGDIR'] = str(Path(tempfile.gettempdir()) / 'mtusi-lab5-matplotlib')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
report = json.loads(args.input.read_text())
assert report.get('result') == 'PASS', 'plot only completed experiments'
cases = report['cases']
metrics = [
('throughput_rps', 'Throughput, images/s', 1),
('ready_p95_ms', 'p95 POST start → ready, s', .001),
('queue_lag_peak_seconds', 'Peak queued job age, s', 1),
('rss_peak_bytes', 'Sampled worker + child RSS, MiB', 1 / 1024**2),
]
plt.rcParams.update({'font.size': 10, 'axes.spines.top': False, 'axes.spines.right': False})
figure, axes = plt.subplots(2, 2, figsize=(10, 7), layout='constrained')
for axis, (metric, label, factor) in zip(axes.flat, metrics):
for width in sorted({case['width'] for case in cases}):
series = sorted([case for case in cases if case['width'] == width], key=lambda c: c['concurrency'])
axis.plot(range(len(series)), [case[metric] * factor for case in series],
marker='o', linewidth=2, label=str(width) + ' × ' + str(series[0]['height']))
axis.set_xticks(range(len(series)), [str(c['concurrency']) + '/' + str(c['prefetch']) for c in series])
axis.set_xlabel('Concurrency / prefetch')
axis.set_ylabel(label)
axis.set_ylim(bottom=0)
axis.grid(axis='y', alpha=.25)
axis.legend(title='PNG pixels')
figure.suptitle('Redis Streams image pipeline — finite burst on one host\n'
+ str(report['requests_per_case']) + ' images per case; sampled RSS and queue lag', fontsize=13)
for suffix in ('.svg', '.png'):
target = args.input.with_suffix(suffix)
figure.savefig(target, dpi=180, metadata={'Creator': 'Matplotlib'})
print(target)
plt.close(figure)
if __name__ == '__main__':
main()
+44 -14
View File
@@ -10,13 +10,13 @@ import urllib.request
ROOT = Path(__file__).resolve().parents[1]
os.chdir(ROOT)
STACK = os.environ.get('STACK_NAME', 'mtusi-lab4')
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4')
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4')
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4')
STACK = os.environ.get('STACK_NAME', 'mtusi-lab5')
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab5')
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab5')
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab5')
S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z')
MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z')
PORT = os.environ.get('PORT', '8080')
PORT = os.environ.get('PORT', '18085')
SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key',
's3_root_secret_key', 's3_access_key', 's3_secret_key')
@@ -71,7 +71,7 @@ def job(kind, image, command, integration=False):
# Fixed service name is a deployment lock: concurrent runners fail to create it.
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
'--constraint', 'node.labels.mtusi_lab4_storage==true',
'--constraint', 'node.labels.mtusi_lab5_storage==true',
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
'--secret', 'source='+STACK+'_postgres_password,target=postgres_password',
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
@@ -113,10 +113,10 @@ def bootstrap_s3():
name = STACK+'_s3-bootstrap'
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
'--constraint', 'node.labels.mtusi_lab4_storage==true',
'--constraint', 'node.labels.mtusi_lab5_storage==true',
'--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh',
'--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh',
'--config', 'source='+STACK+'_s3_policy,target=/policy.json']
'--config', 'source='+STACK+'_s3_policy_v5,target=/policy.json']
for secret in SECRETS[2:]:
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
docker(*args, MC, '/bootstrap.sh')
@@ -131,9 +131,18 @@ def reconcile():
docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:])
def worker_cli(args):
active=[t for t in tasks('worker') if t['Status']['State']=='running']
if not active: raise RuntimeError('start worker before using its operator CLI')
docker('exec',active[0]['Status']['ContainerStatus']['ContainerID'],'/usr/local/bin/worker',*args)
def build():
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
if os.environ.get('SKIP_STORAGE_BUILD') != '1':
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
else:
docker('image','inspect',S3,MC,capture=True)
docker('build', '-t', API, '.')
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
@@ -147,7 +156,7 @@ def up():
raise RuntimeError('run on the Swarm manager')
if len(docker('node', 'ls', '-q', capture=True).split()) != 1:
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID'])
docker('node', 'update', '--label-add', 'mtusi_lab5_storage=true', swarm['NodeID'])
subprocess.run(['./scripts/bootstrap.sh'], check=True)
for secret in SECRETS:
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
@@ -158,6 +167,13 @@ def up():
# Upgrade the shared schema before replacing any running API image.
fresh = STACK+'_api' not in existing
if not fresh:
# A maintenance barrier prevents old lab4 writers inserting ready-only rows
# after the one-shot async backfill but before the new API image starts.
docker('service','scale','--detach',STACK+'_api=0')
deadline=time.monotonic()+45
while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_api','-q',capture=True).strip():
if time.monotonic()>deadline: raise RuntimeError('old API did not drain before migrations')
time.sleep(.5)
job('migrations', MIGRATIONS, ['up'])
# Initial deployment keeps API at zero until the one-shot migration succeeds.
env = dict(os.environ)
@@ -176,12 +192,13 @@ def up():
wait_api()
else:
update()
wait_service('worker')
print('PASS: Swarm stack ready at http://localhost:'+PORT)
def restart():
# Force replaces every task while retaining the same Swarm secrets and volumes.
services = ('postgres', 'redis', 'minio', 'api', 'lb')
services = ('postgres', 'redis', 'minio', 'api', 'lb', 'worker')
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
for service in services:
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
@@ -216,7 +233,9 @@ def update():
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')):
wait_api()
print('PASS: rolling update completed')
docker('service','update','--detach','--force','--no-resolve-image','--image',API,STACK+'_worker')
wait_service('worker')
print('PASS: rolling update completed (API and worker image refreshed)')
return
if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'):
raise RuntimeError('rolling update: '+state)
@@ -226,7 +245,7 @@ def update():
def status():
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
for name in ('lb','api','postgres','redis','minio'):
for name in ('lb','api','worker','postgres','redis','minio'):
service = inspect('service', STACK+'_'+name)
spec = service['Spec']
template = spec['TaskTemplate']
@@ -248,6 +267,17 @@ def main():
elif action == 'build': build()
elif action == 'bootstrap-s3': bootstrap_s3()
elif action == 'reconcile': reconcile()
elif action == 'worker-stop':
docker('service','scale','--detach',STACK+'_worker=0')
deadline=time.monotonic()+30
while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_worker','-q',capture=True).strip():
if time.monotonic()>deadline: raise RuntimeError('worker did not stop')
time.sleep(.2)
elif action == 'worker-start':
docker('service','scale','--detach',STACK+'_worker=1')
wait_service('worker')
elif action == 'failed-jobs': worker_cli(['--failed'])
elif action == 'replay': worker_cli(sys.argv[2:])
elif action == 'wait': wait_api()
elif action == 'restart': restart()
elif action == 'stop-one': stop_one()