lab5 done
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repeatable lab5 track 1 experiment against the isolated Swarm stack.
|
||||
|
||||
Uses synthetic users and PNGs. Changes only worker knobs and admission limits,
|
||||
then restores their original values. Never deletes volumes or accepted jobs.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import csv
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], text=True).strip()
|
||||
|
||||
|
||||
def service_spec(name):
|
||||
return json.loads(docker('service', 'inspect', name))[0]['Spec']
|
||||
|
||||
|
||||
def containers(service):
|
||||
return docker('ps', '--filter', 'label=com.docker.swarm.service.name=' + service,
|
||||
'--format', '{{.ID}}').split()
|
||||
|
||||
|
||||
def wait_service(name, timeout=120):
|
||||
deadline, stable, previous = time.monotonic() + timeout, None, None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = json.loads(docker('service', 'inspect', name))[0]
|
||||
spec = inspected['Spec']
|
||||
update_state = inspected.get('UpdateStatus', {}).get('State')
|
||||
check(update_state not in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'),
|
||||
'service configuration update failed: ' + name)
|
||||
expected = spec['Mode']['Replicated']['Replicas']
|
||||
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', name).split()
|
||||
healthy = expected > 0 and len(ids) == expected and update_state in (None, 'completed')
|
||||
if healthy:
|
||||
tasks = json.loads(docker('inspect', *ids))
|
||||
healthy = all(t['Status']['State'] == 'running' for t in tasks)
|
||||
if healthy:
|
||||
states = json.loads(docker('inspect', *[t['Status']['ContainerStatus']['ContainerID'] for t in tasks]))
|
||||
healthy = all(c['State'].get('Health', {}).get('Status', 'healthy') == 'healthy' for c in states)
|
||||
now, identity = time.monotonic(), frozenset(ids)
|
||||
if healthy:
|
||||
if stable is None or identity != previous:
|
||||
stable = now
|
||||
if stable is not None and now - stable >= 5:
|
||||
return
|
||||
else:
|
||||
stable = None
|
||||
previous = identity
|
||||
time.sleep(.5)
|
||||
raise AssertionError('service did not stabilize: ' + name)
|
||||
|
||||
|
||||
def update_env(service, values):
|
||||
args = ['service', 'update', '--detach']
|
||||
for name, value in values.items():
|
||||
args += ['--env-rm', name] if value is None else ['--env-add', name + '=' + str(value)]
|
||||
docker(*args, service)
|
||||
wait_service(service)
|
||||
|
||||
|
||||
def old_env(spec, names):
|
||||
values = dict(entry.split('=', 1) for entry in spec['TaskTemplate']['ContainerSpec'].get('Env', []))
|
||||
return {name: values.get(name) for name in names}
|
||||
|
||||
|
||||
def png(width, height):
|
||||
rng = random.Random(20261110 + width)
|
||||
def chunk(kind, value):
|
||||
return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
|
||||
pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b'IDAT', zlib.compress(pixels)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
check(bool(values), 'no observations for percentile')
|
||||
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
|
||||
|
||||
|
||||
class Experiment:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.api = args.stack + '_api'
|
||||
self.worker = args.stack + '_worker'
|
||||
self.pg = args.stack + '_postgres'
|
||||
self.client = Client(args.base_url, timeout=60)
|
||||
|
||||
def sql(self, statement):
|
||||
ids = containers(self.pg)
|
||||
check(len(ids) == 1, 'expected one PostgreSQL container')
|
||||
return docker('exec', ids[0], 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-At', '-v', 'ON_ERROR_STOP=1', '-c', statement)
|
||||
|
||||
def outstanding(self):
|
||||
return int(self.sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')"))
|
||||
|
||||
def sample(self, chat):
|
||||
# chat is a UUID generated by the API, never an arbitrary SQL fragment.
|
||||
from uuid import UUID
|
||||
chat_id = str(UUID(chat['id']))
|
||||
query = """SELECT json_build_object(
|
||||
'outstanding', count(*) FILTER (WHERE j.state IN ('queued','processing')),
|
||||
'queued', count(*) FILTER (WHERE j.state='queued'),
|
||||
'processing', count(*) FILTER (WHERE j.state='processing'),
|
||||
'queue_lag_seconds', COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)
|
||||
FILTER (WHERE j.state='queued')),0))
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
||||
WHERE a.chat_id='""" + chat_id + "'"
|
||||
result = json.loads(self.sql(query))
|
||||
total_rss = 0
|
||||
for container in containers(self.worker):
|
||||
# Only process status counters; no command lines, environment or credentials.
|
||||
raw = docker('exec', container, 'sh', '-c',
|
||||
"awk '/^VmRSS:/{s+=$2} END{printf \"%.0f\\n\",s*1024}' /proc/[0-9]*/status 2>/dev/null || true")
|
||||
total_rss += int(raw or 0)
|
||||
result['rss_bytes'] = total_rss
|
||||
return result
|
||||
|
||||
def upload(self, actor, chat, content):
|
||||
started = time.monotonic()
|
||||
response = Client(self.args.base_url, timeout=60).call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=research.png',
|
||||
[202, 429], actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
return {'started': started, 'accepted': time.monotonic(), 'status': response['status'],
|
||||
'instance': response['headers'].get('X-Instance-Id'),
|
||||
'attachment': response['json'] if response['status'] == 202 else None}
|
||||
|
||||
def validate_result(self, actor, attachment, content):
|
||||
check({v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'},
|
||||
'ready must publish all three variants')
|
||||
original = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)
|
||||
check(original['bytes'] == content, 'original changed during pipeline')
|
||||
for variant in attachment['variants']:
|
||||
response = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + variant['name'], actor=actor)
|
||||
check(len(response['bytes']) == variant['size_bytes'], 'variant size differs')
|
||||
check(hashlib.sha256(response['bytes']).hexdigest() == variant['sha256'], 'variant checksum differs')
|
||||
|
||||
def run_case(self, concurrency, prefetch, width, height):
|
||||
check(self.outstanding() == 0, 'drain other workloads before research')
|
||||
update_env(self.worker, {'WORKER_CONCURRENCY': concurrency, 'WORKER_PREFETCH': prefetch})
|
||||
alice, bob, _, chat = self.client.bootstrap()
|
||||
content = png(width, height)
|
||||
warm = self.upload(alice, chat, content)
|
||||
check(warm['status'] == 202, 'warmup unexpectedly rejected')
|
||||
self.client.wait_ready(alice, warm['attachment'], wait_seconds=self.args.timeout)
|
||||
records, samples, rejected = [], [], 0
|
||||
started, deadline = time.monotonic(), time.monotonic() + self.args.timeout
|
||||
baseline = self.sample(chat)
|
||||
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
|
||||
pending = {pool.submit(self.upload, alice, chat, content) for _ in range(self.args.requests)}
|
||||
polling = {}
|
||||
while pending or polling:
|
||||
check(time.monotonic() < deadline, 'pipeline did not drain before research timeout')
|
||||
done = {future for future in pending if future.done()}
|
||||
pending -= done
|
||||
for future in done:
|
||||
row = future.result()
|
||||
if row['status'] == 429:
|
||||
rejected += 1
|
||||
else:
|
||||
check(row['attachment']['status'] == 'queued', 'upload must return a queued snapshot')
|
||||
polling[row['attachment']['id']] = row
|
||||
for attachment_id, row in list(polling.items()):
|
||||
attachment = self.client.call('GET', '/api/v1/attachments/' + attachment_id,
|
||||
actor=bob, schema='Attachment')['json']
|
||||
check(attachment['status'] != 'failed', 'research attachment failed')
|
||||
if attachment['status'] == 'ready':
|
||||
now = time.monotonic()
|
||||
row.update(attachment=attachment, upload_ms=(row['accepted']-row['started'])*1000,
|
||||
ready_ms=(now-row['started'])*1000,
|
||||
after_accept_ms=(now-row['accepted'])*1000)
|
||||
records.append(row)
|
||||
del polling[attachment_id]
|
||||
sample = self.sample(chat)
|
||||
samples.append({'seconds': round(time.monotonic() - started, 3), **sample})
|
||||
if pending or polling:
|
||||
time.sleep(self.args.interval)
|
||||
elapsed = time.monotonic() - started
|
||||
check(len(records) == self.args.requests and rejected == 0, 'unexpected rejection in comparison matrix')
|
||||
check(self.outstanding() == 0, 'comparison workload not drained')
|
||||
self.validate_result(bob, records[-1]['attachment'], content)
|
||||
result = {'concurrency': concurrency, 'prefetch': prefetch, 'width': width, 'height': height,
|
||||
'size_bytes': len(content), 'input_sha256': hashlib.sha256(content).hexdigest(),
|
||||
'offered': self.args.requests, 'accepted': len(records),
|
||||
'rejected': rejected, 'completed': len(records), 'makespan_seconds': round(elapsed, 3),
|
||||
'throughput_rps': round(len(records) / elapsed, 3),
|
||||
'ready_p50_ms': round(percentile([r['ready_ms'] for r in records], .5), 3),
|
||||
'ready_p95_ms': round(percentile([r['ready_ms'] for r in records], .95), 3),
|
||||
'after_accept_p95_ms': round(percentile([r['after_accept_ms'] for r in records], .95), 3),
|
||||
'upload_p95_ms': round(percentile([r['upload_ms'] for r in records], .95), 3),
|
||||
'queue_lag_peak_seconds': max(s['queue_lag_seconds'] for s in samples),
|
||||
'outstanding_peak': max(s['outstanding'] for s in samples),
|
||||
'rss_baseline_bytes': baseline['rss_bytes'],
|
||||
'rss_peak_bytes': max([baseline['rss_bytes']] + [s['rss_bytes'] for s in samples]),
|
||||
'samples': samples,
|
||||
'observations': [{'attachment_id': r['attachment']['id'], 'ready_ms': round(r['ready_ms'], 3),
|
||||
'upload_ms': round(r['upload_ms'], 3)} for r in records]}
|
||||
print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'observations')}), flush=True)
|
||||
return result
|
||||
|
||||
def overload(self, limit):
|
||||
check(self.outstanding() == 0, 'drain other workloads before overload test')
|
||||
docker('service', 'scale', '--detach', self.worker + '=0')
|
||||
deadline = time.monotonic() + 60
|
||||
while containers(self.worker):
|
||||
check(time.monotonic() < deadline, 'worker did not stop')
|
||||
time.sleep(.5)
|
||||
update_env(self.api, {'MAX_OUTSTANDING_JOBS': limit})
|
||||
alice, bob, _, chat = self.client.bootstrap()
|
||||
content = png(320, 200)
|
||||
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
|
||||
rows = list(pool.map(lambda _: self.upload(alice, chat, content), range(limit * 2)))
|
||||
accepted = [r for r in rows if r['status'] == 202]
|
||||
rejected = len(rows) - len(accepted)
|
||||
outstanding = self.outstanding()
|
||||
check(len({row['instance'] for row in rows}) >= 2, 'overload must exercise both API replicas')
|
||||
check(len(accepted) == limit and rejected == limit and outstanding == limit,
|
||||
'admission limit did not bound accepted work while worker was stopped')
|
||||
self.client.message(alice, chat, 'text messages remain available during overload')
|
||||
docker('service', 'scale', '--detach', self.worker + '=1')
|
||||
wait_service(self.worker)
|
||||
started = time.monotonic()
|
||||
for row in accepted:
|
||||
attachment = self.client.wait_ready(bob, row['attachment'], wait_seconds=self.args.timeout)
|
||||
check(attachment['error_code'] is None, 'accepted work failed after overload')
|
||||
recovery = time.monotonic() - started
|
||||
check(self.outstanding() == 0, 'accepted work remained outstanding after recovery')
|
||||
followup = self.upload(alice, chat, content)
|
||||
check(followup['status'] == 202, 'API did not resume admission after drain')
|
||||
self.client.wait_ready(bob, followup['attachment'], wait_seconds=self.args.timeout)
|
||||
result = {'limit': limit, 'offered': len(rows), 'accepted': len(accepted), 'rejected_429': rejected,
|
||||
'observed_api_instances': len({row['instance'] for row in rows}),
|
||||
'outstanding_worker_stopped': outstanding, 'ready_after_resume': len(accepted),
|
||||
'observed_drain_after_worker_healthy_seconds': round(recovery, 3),
|
||||
'outstanding_after_recovery': self.outstanding(), 'followup_upload': 202,
|
||||
'text_messages_during_overload': 'PASS'}
|
||||
print('OVERLOAD ' + json.dumps(result), flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:18085'))
|
||||
parser.add_argument('--stack', default=os.environ.get('STACK_NAME', 'mtusi-lab5'))
|
||||
parser.add_argument('--requests', type=int, default=20)
|
||||
parser.add_argument('--uploaders', type=int, default=8)
|
||||
parser.add_argument('--interval', type=float, default=.25)
|
||||
parser.add_argument('--timeout', type=int, default=240)
|
||||
parser.add_argument('--limit', type=int, default=8)
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/lab5-research.json')
|
||||
args = parser.parse_args()
|
||||
check(args.requests >= 5 and args.uploaders > 0 and 0 < args.limit <= 100,
|
||||
'invalid experiment limits')
|
||||
check(args.interval > 0 and args.timeout > 0, 'sampling interval and timeout must be positive')
|
||||
experiment = Experiment(args)
|
||||
initial_worker, initial_api = service_spec(experiment.worker), service_spec(experiment.api)
|
||||
original_worker_env = old_env(initial_worker, ('WORKER_CONCURRENCY', 'WORKER_PREFETCH'))
|
||||
original_api_env = old_env(initial_api, ('MAX_OUTSTANDING_JOBS',))
|
||||
original_replicas = initial_worker['Mode']['Replicated']['Replicas']
|
||||
check(original_replicas == 1, 'experiment expects one worker service replica; concurrency is internal')
|
||||
report = {'track': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'stack': args.stack, 'worker_image': initial_worker['TaskTemplate']['ContainerSpec']['Image'],
|
||||
'worker_image_id': docker('inspect', containers(experiment.worker)[0], '--format', '{{.Image}}'),
|
||||
'environment': {'docker_server': docker('version', '--format', '{{.Server.Version}}'),
|
||||
'docker_cpus': int(docker('info', '--format', '{{.NCPU}}')),
|
||||
'docker_memory_bytes': int(docker('info', '--format', '{{.MemTotal}}')),
|
||||
'architecture': platform.machine(), 'python': platform.python_version()},
|
||||
'worker_limits': initial_worker['TaskTemplate'].get('Resources', {}).get('Limits', {}),
|
||||
'initial_settings': {'worker': original_worker_env, 'api': original_api_env},
|
||||
'requests_per_case': args.requests, 'upload_concurrency': args.uploaders,
|
||||
'sampling_interval_min_seconds': args.interval,
|
||||
'method': {'throughput': 'completed / burst makespan including client upload and polling',
|
||||
'ready_latency': 'client POST start to first observed ready; nearest-rank p95',
|
||||
'queue_lag': 'age of oldest queued PostgreSQL job in this chat; excludes processing',
|
||||
'rss': 'sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly',
|
||||
'limitations': 'single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA'},
|
||||
'cases': []}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
def save():
|
||||
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + '\n')
|
||||
try:
|
||||
for width, height in ((512, 512), (1536, 1536)):
|
||||
for concurrency, prefetch in ((1, 1), (2, 4), (4, 8)):
|
||||
report['cases'].append(experiment.run_case(concurrency, prefetch, width, height))
|
||||
save()
|
||||
report['overload'] = experiment.overload(args.limit)
|
||||
save()
|
||||
finally:
|
||||
# Keep every accepted job recoverable, including when an assertion fails.
|
||||
try:
|
||||
if not containers(experiment.worker):
|
||||
docker('service', 'scale', '--detach', experiment.worker + '=' + str(original_replicas))
|
||||
wait_service(experiment.worker)
|
||||
update_env(experiment.worker, original_worker_env)
|
||||
finally:
|
||||
update_env(experiment.api, original_api_env)
|
||||
report['result'] = 'PASS'
|
||||
report['restored_initial_settings'] = True
|
||||
report['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat()
|
||||
save()
|
||||
summary = args.output.with_suffix('.csv')
|
||||
keys = ['width', 'height', 'concurrency', 'prefetch', 'offered', 'accepted', 'completed',
|
||||
'throughput_rps', 'ready_p95_ms', 'after_accept_p95_ms', 'upload_p95_ms',
|
||||
'queue_lag_peak_seconds', 'rss_peak_bytes']
|
||||
with summary.open('w', newline='') as file:
|
||||
writer = csv.DictWriter(file, fieldnames=keys, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
writer.writerows(report['cases'])
|
||||
print('PASS: six measured configurations, bounded overload and recovery; ' + str(args.output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -9,7 +9,7 @@ import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STACK = os.environ.get('STACK_NAME', 'mtusi-lab4')
|
||||
STACK = os.environ.get('STACK_NAME', 'mtusi-lab5')
|
||||
|
||||
def containers(service=None):
|
||||
label = 'com.docker.swarm.service.name='+STACK+'_'+service if service else 'com.docker.stack.namespace='+STACK
|
||||
|
||||
@@ -59,7 +59,7 @@ def main():
|
||||
elapsed = time.monotonic()-started
|
||||
samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)})
|
||||
for response in responses:
|
||||
att = response['json']
|
||||
att = c.wait_ready(bob,response['json'])
|
||||
check(att['sha256']==sha,'concurrent upload checksum mismatch')
|
||||
downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
||||
check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed')
|
||||
@@ -77,7 +77,7 @@ def main():
|
||||
check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit')
|
||||
results['runs'].append(run)
|
||||
print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True)
|
||||
save('lab4-upload-memory.json',results)
|
||||
save('lab5-upload-memory.json',results)
|
||||
print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.')
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render measured pipeline results; install requirements-research.txt first."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('input', nargs='?', type=Path, default=Path('evidence/lab5-research.json'))
|
||||
args = parser.parse_args()
|
||||
os.environ['MPLCONFIGDIR'] = str(Path(tempfile.gettempdir()) / 'mtusi-lab5-matplotlib')
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
report = json.loads(args.input.read_text())
|
||||
assert report.get('result') == 'PASS', 'plot only completed experiments'
|
||||
cases = report['cases']
|
||||
metrics = [
|
||||
('throughput_rps', 'Throughput, images/s', 1),
|
||||
('ready_p95_ms', 'p95 POST start → ready, s', .001),
|
||||
('queue_lag_peak_seconds', 'Peak queued job age, s', 1),
|
||||
('rss_peak_bytes', 'Sampled worker + child RSS, MiB', 1 / 1024**2),
|
||||
]
|
||||
plt.rcParams.update({'font.size': 10, 'axes.spines.top': False, 'axes.spines.right': False})
|
||||
figure, axes = plt.subplots(2, 2, figsize=(10, 7), layout='constrained')
|
||||
for axis, (metric, label, factor) in zip(axes.flat, metrics):
|
||||
for width in sorted({case['width'] for case in cases}):
|
||||
series = sorted([case for case in cases if case['width'] == width], key=lambda c: c['concurrency'])
|
||||
axis.plot(range(len(series)), [case[metric] * factor for case in series],
|
||||
marker='o', linewidth=2, label=str(width) + ' × ' + str(series[0]['height']))
|
||||
axis.set_xticks(range(len(series)), [str(c['concurrency']) + '/' + str(c['prefetch']) for c in series])
|
||||
axis.set_xlabel('Concurrency / prefetch')
|
||||
axis.set_ylabel(label)
|
||||
axis.set_ylim(bottom=0)
|
||||
axis.grid(axis='y', alpha=.25)
|
||||
axis.legend(title='PNG pixels')
|
||||
figure.suptitle('Redis Streams image pipeline — finite burst on one host\n'
|
||||
+ str(report['requests_per_case']) + ' images per case; sampled RSS and queue lag', fontsize=13)
|
||||
for suffix in ('.svg', '.png'):
|
||||
target = args.input.with_suffix(suffix)
|
||||
figure.savefig(target, dpi=180, metadata={'Creator': 'Matplotlib'})
|
||||
print(target)
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+44
-14
@@ -10,13 +10,13 @@ import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
os.chdir(ROOT)
|
||||
STACK = os.environ.get('STACK_NAME', 'mtusi-lab4')
|
||||
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4')
|
||||
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4')
|
||||
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4')
|
||||
STACK = os.environ.get('STACK_NAME', 'mtusi-lab5')
|
||||
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab5')
|
||||
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab5')
|
||||
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab5')
|
||||
S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z')
|
||||
MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z')
|
||||
PORT = os.environ.get('PORT', '8080')
|
||||
PORT = os.environ.get('PORT', '18085')
|
||||
SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key',
|
||||
's3_root_secret_key', 's3_access_key', 's3_secret_key')
|
||||
|
||||
@@ -71,7 +71,7 @@ def job(kind, image, command, integration=False):
|
||||
# Fixed service name is a deployment lock: concurrent runners fail to create it.
|
||||
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
|
||||
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
||||
'--constraint', 'node.labels.mtusi_lab4_storage==true',
|
||||
'--constraint', 'node.labels.mtusi_lab5_storage==true',
|
||||
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
|
||||
'--secret', 'source='+STACK+'_postgres_password,target=postgres_password',
|
||||
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
|
||||
@@ -113,10 +113,10 @@ def bootstrap_s3():
|
||||
name = STACK+'_s3-bootstrap'
|
||||
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
|
||||
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
|
||||
'--constraint', 'node.labels.mtusi_lab4_storage==true',
|
||||
'--constraint', 'node.labels.mtusi_lab5_storage==true',
|
||||
'--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh',
|
||||
'--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh',
|
||||
'--config', 'source='+STACK+'_s3_policy,target=/policy.json']
|
||||
'--config', 'source='+STACK+'_s3_policy_v5,target=/policy.json']
|
||||
for secret in SECRETS[2:]:
|
||||
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
|
||||
docker(*args, MC, '/bootstrap.sh')
|
||||
@@ -131,9 +131,18 @@ def reconcile():
|
||||
docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:])
|
||||
|
||||
|
||||
def worker_cli(args):
|
||||
active=[t for t in tasks('worker') if t['Status']['State']=='running']
|
||||
if not active: raise RuntimeError('start worker before using its operator CLI')
|
||||
docker('exec',active[0]['Status']['ContainerStatus']['ContainerID'],'/usr/local/bin/worker',*args)
|
||||
|
||||
|
||||
def build():
|
||||
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
|
||||
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
|
||||
if os.environ.get('SKIP_STORAGE_BUILD') != '1':
|
||||
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
|
||||
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
|
||||
else:
|
||||
docker('image','inspect',S3,MC,capture=True)
|
||||
docker('build', '-t', API, '.')
|
||||
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
|
||||
|
||||
@@ -147,7 +156,7 @@ def up():
|
||||
raise RuntimeError('run on the Swarm manager')
|
||||
if len(docker('node', 'ls', '-q', capture=True).split()) != 1:
|
||||
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
|
||||
docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID'])
|
||||
docker('node', 'update', '--label-add', 'mtusi_lab5_storage=true', swarm['NodeID'])
|
||||
subprocess.run(['./scripts/bootstrap.sh'], check=True)
|
||||
for secret in SECRETS:
|
||||
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
|
||||
@@ -158,6 +167,13 @@ def up():
|
||||
# Upgrade the shared schema before replacing any running API image.
|
||||
fresh = STACK+'_api' not in existing
|
||||
if not fresh:
|
||||
# A maintenance barrier prevents old lab4 writers inserting ready-only rows
|
||||
# after the one-shot async backfill but before the new API image starts.
|
||||
docker('service','scale','--detach',STACK+'_api=0')
|
||||
deadline=time.monotonic()+45
|
||||
while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_api','-q',capture=True).strip():
|
||||
if time.monotonic()>deadline: raise RuntimeError('old API did not drain before migrations')
|
||||
time.sleep(.5)
|
||||
job('migrations', MIGRATIONS, ['up'])
|
||||
# Initial deployment keeps API at zero until the one-shot migration succeeds.
|
||||
env = dict(os.environ)
|
||||
@@ -176,12 +192,13 @@ def up():
|
||||
wait_api()
|
||||
else:
|
||||
update()
|
||||
wait_service('worker')
|
||||
print('PASS: Swarm stack ready at http://localhost:'+PORT)
|
||||
|
||||
|
||||
def restart():
|
||||
# Force replaces every task while retaining the same Swarm secrets and volumes.
|
||||
services = ('postgres', 'redis', 'minio', 'api', 'lb')
|
||||
services = ('postgres', 'redis', 'minio', 'api', 'lb', 'worker')
|
||||
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
|
||||
for service in services:
|
||||
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
|
||||
@@ -216,7 +233,9 @@ def update():
|
||||
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
|
||||
if state == 'completed' and previous.isdisjoint(t['ID'] for t in tasks('api')):
|
||||
wait_api()
|
||||
print('PASS: rolling update completed')
|
||||
docker('service','update','--detach','--force','--no-resolve-image','--image',API,STACK+'_worker')
|
||||
wait_service('worker')
|
||||
print('PASS: rolling update completed (API and worker image refreshed)')
|
||||
return
|
||||
if state in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'):
|
||||
raise RuntimeError('rolling update: '+state)
|
||||
@@ -226,7 +245,7 @@ def update():
|
||||
|
||||
def status():
|
||||
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
|
||||
for name in ('lb','api','postgres','redis','minio'):
|
||||
for name in ('lb','api','worker','postgres','redis','minio'):
|
||||
service = inspect('service', STACK+'_'+name)
|
||||
spec = service['Spec']
|
||||
template = spec['TaskTemplate']
|
||||
@@ -248,6 +267,17 @@ def main():
|
||||
elif action == 'build': build()
|
||||
elif action == 'bootstrap-s3': bootstrap_s3()
|
||||
elif action == 'reconcile': reconcile()
|
||||
elif action == 'worker-stop':
|
||||
docker('service','scale','--detach',STACK+'_worker=0')
|
||||
deadline=time.monotonic()+30
|
||||
while docker('ps','--filter','label=com.docker.swarm.service.name='+STACK+'_worker','-q',capture=True).strip():
|
||||
if time.monotonic()>deadline: raise RuntimeError('worker did not stop')
|
||||
time.sleep(.2)
|
||||
elif action == 'worker-start':
|
||||
docker('service','scale','--detach',STACK+'_worker=1')
|
||||
wait_service('worker')
|
||||
elif action == 'failed-jobs': worker_cli(['--failed'])
|
||||
elif action == 'replay': worker_cli(sys.argv[2:])
|
||||
elif action == 'wait': wait_api()
|
||||
elif action == 'restart': restart()
|
||||
elif action == 'stop-one': stop_one()
|
||||
|
||||
Reference in New Issue
Block a user