forked from mtusi2026/mtusi2026lab5
333 lines
18 KiB
Python
333 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable lab5 track 1 experiment against the isolated Swarm stack.
|
|
|
|
Uses synthetic users and PNGs. Changes only worker knobs and admission limits,
|
|
then restores their original values. Never deletes volumes or accepted jobs.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import csv
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import random
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(['docker', *args], text=True).strip()
|
|
|
|
|
|
def service_spec(name):
|
|
return json.loads(docker('service', 'inspect', name))[0]['Spec']
|
|
|
|
|
|
def containers(service):
|
|
return docker('ps', '--filter', 'label=com.docker.swarm.service.name=' + service,
|
|
'--format', '{{.ID}}').split()
|
|
|
|
|
|
def wait_service(name, timeout=120):
|
|
deadline, stable, previous = time.monotonic() + timeout, None, None
|
|
while time.monotonic() < deadline:
|
|
inspected = json.loads(docker('service', 'inspect', name))[0]
|
|
spec = inspected['Spec']
|
|
update_state = inspected.get('UpdateStatus', {}).get('State')
|
|
check(update_state not in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'),
|
|
'service configuration update failed: ' + name)
|
|
expected = spec['Mode']['Replicated']['Replicas']
|
|
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', name).split()
|
|
healthy = expected > 0 and len(ids) == expected and update_state in (None, 'completed')
|
|
if healthy:
|
|
tasks = json.loads(docker('inspect', *ids))
|
|
healthy = all(t['Status']['State'] == 'running' for t in tasks)
|
|
if healthy:
|
|
states = json.loads(docker('inspect', *[t['Status']['ContainerStatus']['ContainerID'] for t in tasks]))
|
|
healthy = all(c['State'].get('Health', {}).get('Status', 'healthy') == 'healthy' for c in states)
|
|
now, identity = time.monotonic(), frozenset(ids)
|
|
if healthy:
|
|
if stable is None or identity != previous:
|
|
stable = now
|
|
if stable is not None and now - stable >= 5:
|
|
return
|
|
else:
|
|
stable = None
|
|
previous = identity
|
|
time.sleep(.5)
|
|
raise AssertionError('service did not stabilize: ' + name)
|
|
|
|
|
|
def update_env(service, values):
|
|
args = ['service', 'update', '--detach']
|
|
for name, value in values.items():
|
|
args += ['--env-rm', name] if value is None else ['--env-add', name + '=' + str(value)]
|
|
docker(*args, service)
|
|
wait_service(service)
|
|
|
|
|
|
def old_env(spec, names):
|
|
values = dict(entry.split('=', 1) for entry in spec['TaskTemplate']['ContainerSpec'].get('Env', []))
|
|
return {name: values.get(name) for name in names}
|
|
|
|
|
|
def png(width, height):
|
|
rng = random.Random(20261110 + width)
|
|
def chunk(kind, value):
|
|
return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
|
|
pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
|
|
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
|
+ chunk(b'IDAT', zlib.compress(pixels)) + chunk(b'IEND', b''))
|
|
|
|
|
|
def percentile(values, fraction):
|
|
check(bool(values), 'no observations for percentile')
|
|
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
|
|
|
|
|
|
class Experiment:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.api = args.stack + '_api'
|
|
self.worker = args.stack + '_worker'
|
|
self.pg = args.stack + '_postgres'
|
|
self.client = Client(args.base_url, timeout=60)
|
|
|
|
def sql(self, statement):
|
|
ids = containers(self.pg)
|
|
check(len(ids) == 1, 'expected one PostgreSQL container')
|
|
return docker('exec', ids[0], 'psql', '-U', 'messenger', '-d', 'messenger',
|
|
'-At', '-v', 'ON_ERROR_STOP=1', '-c', statement)
|
|
|
|
def outstanding(self):
|
|
return int(self.sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')"))
|
|
|
|
def sample(self, chat):
|
|
# chat is a UUID generated by the API, never an arbitrary SQL fragment.
|
|
from uuid import UUID
|
|
chat_id = str(UUID(chat['id']))
|
|
query = """SELECT json_build_object(
|
|
'outstanding', count(*) FILTER (WHERE j.state IN ('queued','processing')),
|
|
'queued', count(*) FILTER (WHERE j.state='queued'),
|
|
'processing', count(*) FILTER (WHERE j.state='processing'),
|
|
'queue_lag_seconds', COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)
|
|
FILTER (WHERE j.state='queued')),0))
|
|
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
|
WHERE a.chat_id='""" + chat_id + "'"
|
|
result = json.loads(self.sql(query))
|
|
total_rss = 0
|
|
for container in containers(self.worker):
|
|
# Only process status counters; no command lines, environment or credentials.
|
|
raw = docker('exec', container, 'sh', '-c',
|
|
"awk '/^VmRSS:/{s+=$2} END{printf \"%.0f\\n\",s*1024}' /proc/[0-9]*/status 2>/dev/null || true")
|
|
total_rss += int(raw or 0)
|
|
result['rss_bytes'] = total_rss
|
|
return result
|
|
|
|
def upload(self, actor, chat, content):
|
|
started = time.monotonic()
|
|
response = Client(self.args.base_url, timeout=60).call(
|
|
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=research.png',
|
|
[202, 429], actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
return {'started': started, 'accepted': time.monotonic(), 'status': response['status'],
|
|
'instance': response['headers'].get('X-Instance-Id'),
|
|
'attachment': response['json'] if response['status'] == 202 else None}
|
|
|
|
def validate_result(self, actor, attachment, content):
|
|
check({v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'},
|
|
'ready must publish all three variants')
|
|
original = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)
|
|
check(original['bytes'] == content, 'original changed during pipeline')
|
|
for variant in attachment['variants']:
|
|
response = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + variant['name'], actor=actor)
|
|
check(len(response['bytes']) == variant['size_bytes'], 'variant size differs')
|
|
check(hashlib.sha256(response['bytes']).hexdigest() == variant['sha256'], 'variant checksum differs')
|
|
|
|
def run_case(self, concurrency, prefetch, width, height):
|
|
check(self.outstanding() == 0, 'drain other workloads before research')
|
|
update_env(self.worker, {'WORKER_CONCURRENCY': concurrency, 'WORKER_PREFETCH': prefetch})
|
|
alice, bob, _, chat = self.client.bootstrap()
|
|
content = png(width, height)
|
|
warm = self.upload(alice, chat, content)
|
|
check(warm['status'] == 202, 'warmup unexpectedly rejected')
|
|
self.client.wait_ready(alice, warm['attachment'], wait_seconds=self.args.timeout)
|
|
records, samples, rejected = [], [], 0
|
|
started, deadline = time.monotonic(), time.monotonic() + self.args.timeout
|
|
baseline = self.sample(chat)
|
|
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
|
|
pending = {pool.submit(self.upload, alice, chat, content) for _ in range(self.args.requests)}
|
|
polling = {}
|
|
while pending or polling:
|
|
check(time.monotonic() < deadline, 'pipeline did not drain before research timeout')
|
|
done = {future for future in pending if future.done()}
|
|
pending -= done
|
|
for future in done:
|
|
row = future.result()
|
|
if row['status'] == 429:
|
|
rejected += 1
|
|
else:
|
|
check(row['attachment']['status'] == 'queued', 'upload must return a queued snapshot')
|
|
polling[row['attachment']['id']] = row
|
|
for attachment_id, row in list(polling.items()):
|
|
attachment = self.client.call('GET', '/api/v1/attachments/' + attachment_id,
|
|
actor=bob, schema='Attachment')['json']
|
|
check(attachment['status'] != 'failed', 'research attachment failed')
|
|
if attachment['status'] == 'ready':
|
|
now = time.monotonic()
|
|
row.update(attachment=attachment, upload_ms=(row['accepted']-row['started'])*1000,
|
|
ready_ms=(now-row['started'])*1000,
|
|
after_accept_ms=(now-row['accepted'])*1000)
|
|
records.append(row)
|
|
del polling[attachment_id]
|
|
sample = self.sample(chat)
|
|
samples.append({'seconds': round(time.monotonic() - started, 3), **sample})
|
|
if pending or polling:
|
|
time.sleep(self.args.interval)
|
|
elapsed = time.monotonic() - started
|
|
check(len(records) == self.args.requests and rejected == 0, 'unexpected rejection in comparison matrix')
|
|
check(self.outstanding() == 0, 'comparison workload not drained')
|
|
self.validate_result(bob, records[-1]['attachment'], content)
|
|
result = {'concurrency': concurrency, 'prefetch': prefetch, 'width': width, 'height': height,
|
|
'size_bytes': len(content), 'input_sha256': hashlib.sha256(content).hexdigest(),
|
|
'offered': self.args.requests, 'accepted': len(records),
|
|
'rejected': rejected, 'completed': len(records), 'makespan_seconds': round(elapsed, 3),
|
|
'throughput_rps': round(len(records) / elapsed, 3),
|
|
'ready_p50_ms': round(percentile([r['ready_ms'] for r in records], .5), 3),
|
|
'ready_p95_ms': round(percentile([r['ready_ms'] for r in records], .95), 3),
|
|
'after_accept_p95_ms': round(percentile([r['after_accept_ms'] for r in records], .95), 3),
|
|
'upload_p95_ms': round(percentile([r['upload_ms'] for r in records], .95), 3),
|
|
'queue_lag_peak_seconds': max(s['queue_lag_seconds'] for s in samples),
|
|
'outstanding_peak': max(s['outstanding'] for s in samples),
|
|
'rss_baseline_bytes': baseline['rss_bytes'],
|
|
'rss_peak_bytes': max([baseline['rss_bytes']] + [s['rss_bytes'] for s in samples]),
|
|
'samples': samples,
|
|
'observations': [{'attachment_id': r['attachment']['id'], 'ready_ms': round(r['ready_ms'], 3),
|
|
'upload_ms': round(r['upload_ms'], 3)} for r in records]}
|
|
print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'observations')}), flush=True)
|
|
return result
|
|
|
|
def overload(self, limit):
|
|
check(self.outstanding() == 0, 'drain other workloads before overload test')
|
|
docker('service', 'scale', '--detach', self.worker + '=0')
|
|
deadline = time.monotonic() + 60
|
|
while containers(self.worker):
|
|
check(time.monotonic() < deadline, 'worker did not stop')
|
|
time.sleep(.5)
|
|
update_env(self.api, {'MAX_OUTSTANDING_JOBS': limit})
|
|
alice, bob, _, chat = self.client.bootstrap()
|
|
content = png(320, 200)
|
|
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
|
|
rows = list(pool.map(lambda _: self.upload(alice, chat, content), range(limit * 2)))
|
|
accepted = [r for r in rows if r['status'] == 202]
|
|
rejected = len(rows) - len(accepted)
|
|
outstanding = self.outstanding()
|
|
check(len({row['instance'] for row in rows}) >= 2, 'overload must exercise both API replicas')
|
|
check(len(accepted) == limit and rejected == limit and outstanding == limit,
|
|
'admission limit did not bound accepted work while worker was stopped')
|
|
self.client.message(alice, chat, 'text messages remain available during overload')
|
|
docker('service', 'scale', '--detach', self.worker + '=1')
|
|
wait_service(self.worker)
|
|
started = time.monotonic()
|
|
for row in accepted:
|
|
attachment = self.client.wait_ready(bob, row['attachment'], wait_seconds=self.args.timeout)
|
|
check(attachment['error_code'] is None, 'accepted work failed after overload')
|
|
recovery = time.monotonic() - started
|
|
check(self.outstanding() == 0, 'accepted work remained outstanding after recovery')
|
|
followup = self.upload(alice, chat, content)
|
|
check(followup['status'] == 202, 'API did not resume admission after drain')
|
|
self.client.wait_ready(bob, followup['attachment'], wait_seconds=self.args.timeout)
|
|
result = {'limit': limit, 'offered': len(rows), 'accepted': len(accepted), 'rejected_429': rejected,
|
|
'observed_api_instances': len({row['instance'] for row in rows}),
|
|
'outstanding_worker_stopped': outstanding, 'ready_after_resume': len(accepted),
|
|
'observed_drain_after_worker_healthy_seconds': round(recovery, 3),
|
|
'outstanding_after_recovery': self.outstanding(), 'followup_upload': 202,
|
|
'text_messages_during_overload': 'PASS'}
|
|
print('OVERLOAD ' + json.dumps(result), flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:18085'))
|
|
parser.add_argument('--stack', default=os.environ.get('STACK_NAME', 'mtusi-lab5'))
|
|
parser.add_argument('--requests', type=int, default=20)
|
|
parser.add_argument('--uploaders', type=int, default=8)
|
|
parser.add_argument('--interval', type=float, default=.25)
|
|
parser.add_argument('--timeout', type=int, default=240)
|
|
parser.add_argument('--limit', type=int, default=8)
|
|
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/lab5-research.json')
|
|
args = parser.parse_args()
|
|
check(args.requests >= 5 and args.uploaders > 0 and 0 < args.limit <= 100,
|
|
'invalid experiment limits')
|
|
check(args.interval > 0 and args.timeout > 0, 'sampling interval and timeout must be positive')
|
|
experiment = Experiment(args)
|
|
initial_worker, initial_api = service_spec(experiment.worker), service_spec(experiment.api)
|
|
original_worker_env = old_env(initial_worker, ('WORKER_CONCURRENCY', 'WORKER_PREFETCH'))
|
|
original_api_env = old_env(initial_api, ('MAX_OUTSTANDING_JOBS',))
|
|
original_replicas = initial_worker['Mode']['Replicated']['Replicas']
|
|
check(original_replicas == 1, 'experiment expects one worker service replica; concurrency is internal')
|
|
report = {'track': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
'stack': args.stack, 'worker_image': initial_worker['TaskTemplate']['ContainerSpec']['Image'],
|
|
'worker_image_id': docker('inspect', containers(experiment.worker)[0], '--format', '{{.Image}}'),
|
|
'environment': {'docker_server': docker('version', '--format', '{{.Server.Version}}'),
|
|
'docker_cpus': int(docker('info', '--format', '{{.NCPU}}')),
|
|
'docker_memory_bytes': int(docker('info', '--format', '{{.MemTotal}}')),
|
|
'architecture': platform.machine(), 'python': platform.python_version()},
|
|
'worker_limits': initial_worker['TaskTemplate'].get('Resources', {}).get('Limits', {}),
|
|
'initial_settings': {'worker': original_worker_env, 'api': original_api_env},
|
|
'requests_per_case': args.requests, 'upload_concurrency': args.uploaders,
|
|
'sampling_interval_min_seconds': args.interval,
|
|
'method': {'throughput': 'completed / burst makespan including client upload and polling',
|
|
'ready_latency': 'client POST start to first observed ready; nearest-rank p95',
|
|
'queue_lag': 'age of oldest queued PostgreSQL job in this chat; excludes processing',
|
|
'rss': 'sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly',
|
|
'limitations': 'single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA'},
|
|
'cases': []}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
def save():
|
|
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + '\n')
|
|
try:
|
|
for width, height in ((512, 512), (1536, 1536)):
|
|
for concurrency, prefetch in ((1, 1), (2, 4), (4, 8)):
|
|
report['cases'].append(experiment.run_case(concurrency, prefetch, width, height))
|
|
save()
|
|
report['overload'] = experiment.overload(args.limit)
|
|
save()
|
|
finally:
|
|
# Keep every accepted job recoverable, including when an assertion fails.
|
|
try:
|
|
if not containers(experiment.worker):
|
|
docker('service', 'scale', '--detach', experiment.worker + '=' + str(original_replicas))
|
|
wait_service(experiment.worker)
|
|
update_env(experiment.worker, original_worker_env)
|
|
finally:
|
|
update_env(experiment.api, original_api_env)
|
|
report['result'] = 'PASS'
|
|
report['restored_initial_settings'] = True
|
|
report['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
save()
|
|
summary = args.output.with_suffix('.csv')
|
|
keys = ['width', 'height', 'concurrency', 'prefetch', 'offered', 'accepted', 'completed',
|
|
'throughput_rps', 'ready_p95_ms', 'after_accept_p95_ms', 'upload_p95_ms',
|
|
'queue_lag_peak_seconds', 'rss_peak_bytes']
|
|
with summary.open('w', newline='') as file:
|
|
writer = csv.DictWriter(file, fieldnames=keys, extrasaction='ignore')
|
|
writer.writeheader()
|
|
writer.writerows(report['cases'])
|
|
print('PASS: six measured configurations, bounded overload and recovery; ' + str(args.output))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|