Files
mtusi2026lab5/scripts/benchmark_pipeline.py
T
2026-11-10 22:13:40 +03:00

333 lines
18 KiB
Python

#!/usr/bin/env python3
"""Repeatable lab5 track 1 experiment against the isolated Swarm stack.
Uses synthetic users and PNGs. Changes only worker knobs and admission limits,
then restores their original values. Never deletes volumes or accepted jobs.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import csv
import datetime as dt
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import random
import struct
import subprocess
import sys
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def docker(*args):
return subprocess.check_output(['docker', *args], text=True).strip()
def service_spec(name):
return json.loads(docker('service', 'inspect', name))[0]['Spec']
def containers(service):
return docker('ps', '--filter', 'label=com.docker.swarm.service.name=' + service,
'--format', '{{.ID}}').split()
def wait_service(name, timeout=120):
deadline, stable, previous = time.monotonic() + timeout, None, None
while time.monotonic() < deadline:
inspected = json.loads(docker('service', 'inspect', name))[0]
spec = inspected['Spec']
update_state = inspected.get('UpdateStatus', {}).get('State')
check(update_state not in ('paused', 'rollback_started', 'rollback_paused', 'rollback_completed'),
'service configuration update failed: ' + name)
expected = spec['Mode']['Replicated']['Replicas']
ids = docker('service', 'ps', '--filter', 'desired-state=running', '-q', name).split()
healthy = expected > 0 and len(ids) == expected and update_state in (None, 'completed')
if healthy:
tasks = json.loads(docker('inspect', *ids))
healthy = all(t['Status']['State'] == 'running' for t in tasks)
if healthy:
states = json.loads(docker('inspect', *[t['Status']['ContainerStatus']['ContainerID'] for t in tasks]))
healthy = all(c['State'].get('Health', {}).get('Status', 'healthy') == 'healthy' for c in states)
now, identity = time.monotonic(), frozenset(ids)
if healthy:
if stable is None or identity != previous:
stable = now
if stable is not None and now - stable >= 5:
return
else:
stable = None
previous = identity
time.sleep(.5)
raise AssertionError('service did not stabilize: ' + name)
def update_env(service, values):
args = ['service', 'update', '--detach']
for name, value in values.items():
args += ['--env-rm', name] if value is None else ['--env-add', name + '=' + str(value)]
docker(*args, service)
wait_service(service)
def old_env(spec, names):
values = dict(entry.split('=', 1) for entry in spec['TaskTemplate']['ContainerSpec'].get('Env', []))
return {name: values.get(name) for name in names}
def png(width, height):
rng = random.Random(20261110 + width)
def chunk(kind, value):
return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
pixels = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(pixels)) + chunk(b'IEND', b''))
def percentile(values, fraction):
check(bool(values), 'no observations for percentile')
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
class Experiment:
def __init__(self, args):
self.args = args
self.api = args.stack + '_api'
self.worker = args.stack + '_worker'
self.pg = args.stack + '_postgres'
self.client = Client(args.base_url, timeout=60)
def sql(self, statement):
ids = containers(self.pg)
check(len(ids) == 1, 'expected one PostgreSQL container')
return docker('exec', ids[0], 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-v', 'ON_ERROR_STOP=1', '-c', statement)
def outstanding(self):
return int(self.sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')"))
def sample(self, chat):
# chat is a UUID generated by the API, never an arbitrary SQL fragment.
from uuid import UUID
chat_id = str(UUID(chat['id']))
query = """SELECT json_build_object(
'outstanding', count(*) FILTER (WHERE j.state IN ('queued','processing')),
'queued', count(*) FILTER (WHERE j.state='queued'),
'processing', count(*) FILTER (WHERE j.state='processing'),
'queue_lag_seconds', COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)
FILTER (WHERE j.state='queued')),0))
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE a.chat_id='""" + chat_id + "'"
result = json.loads(self.sql(query))
total_rss = 0
for container in containers(self.worker):
# Only process status counters; no command lines, environment or credentials.
raw = docker('exec', container, 'sh', '-c',
"awk '/^VmRSS:/{s+=$2} END{printf \"%.0f\\n\",s*1024}' /proc/[0-9]*/status 2>/dev/null || true")
total_rss += int(raw or 0)
result['rss_bytes'] = total_rss
return result
def upload(self, actor, chat, content):
started = time.monotonic()
response = Client(self.args.base_url, timeout=60).call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=research.png',
[202, 429], actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
return {'started': started, 'accepted': time.monotonic(), 'status': response['status'],
'instance': response['headers'].get('X-Instance-Id'),
'attachment': response['json'] if response['status'] == 202 else None}
def validate_result(self, actor, attachment, content):
check({v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'},
'ready must publish all three variants')
original = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)
check(original['bytes'] == content, 'original changed during pipeline')
for variant in attachment['variants']:
response = self.client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + variant['name'], actor=actor)
check(len(response['bytes']) == variant['size_bytes'], 'variant size differs')
check(hashlib.sha256(response['bytes']).hexdigest() == variant['sha256'], 'variant checksum differs')
def run_case(self, concurrency, prefetch, width, height):
check(self.outstanding() == 0, 'drain other workloads before research')
update_env(self.worker, {'WORKER_CONCURRENCY': concurrency, 'WORKER_PREFETCH': prefetch})
alice, bob, _, chat = self.client.bootstrap()
content = png(width, height)
warm = self.upload(alice, chat, content)
check(warm['status'] == 202, 'warmup unexpectedly rejected')
self.client.wait_ready(alice, warm['attachment'], wait_seconds=self.args.timeout)
records, samples, rejected = [], [], 0
started, deadline = time.monotonic(), time.monotonic() + self.args.timeout
baseline = self.sample(chat)
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
pending = {pool.submit(self.upload, alice, chat, content) for _ in range(self.args.requests)}
polling = {}
while pending or polling:
check(time.monotonic() < deadline, 'pipeline did not drain before research timeout')
done = {future for future in pending if future.done()}
pending -= done
for future in done:
row = future.result()
if row['status'] == 429:
rejected += 1
else:
check(row['attachment']['status'] == 'queued', 'upload must return a queued snapshot')
polling[row['attachment']['id']] = row
for attachment_id, row in list(polling.items()):
attachment = self.client.call('GET', '/api/v1/attachments/' + attachment_id,
actor=bob, schema='Attachment')['json']
check(attachment['status'] != 'failed', 'research attachment failed')
if attachment['status'] == 'ready':
now = time.monotonic()
row.update(attachment=attachment, upload_ms=(row['accepted']-row['started'])*1000,
ready_ms=(now-row['started'])*1000,
after_accept_ms=(now-row['accepted'])*1000)
records.append(row)
del polling[attachment_id]
sample = self.sample(chat)
samples.append({'seconds': round(time.monotonic() - started, 3), **sample})
if pending or polling:
time.sleep(self.args.interval)
elapsed = time.monotonic() - started
check(len(records) == self.args.requests and rejected == 0, 'unexpected rejection in comparison matrix')
check(self.outstanding() == 0, 'comparison workload not drained')
self.validate_result(bob, records[-1]['attachment'], content)
result = {'concurrency': concurrency, 'prefetch': prefetch, 'width': width, 'height': height,
'size_bytes': len(content), 'input_sha256': hashlib.sha256(content).hexdigest(),
'offered': self.args.requests, 'accepted': len(records),
'rejected': rejected, 'completed': len(records), 'makespan_seconds': round(elapsed, 3),
'throughput_rps': round(len(records) / elapsed, 3),
'ready_p50_ms': round(percentile([r['ready_ms'] for r in records], .5), 3),
'ready_p95_ms': round(percentile([r['ready_ms'] for r in records], .95), 3),
'after_accept_p95_ms': round(percentile([r['after_accept_ms'] for r in records], .95), 3),
'upload_p95_ms': round(percentile([r['upload_ms'] for r in records], .95), 3),
'queue_lag_peak_seconds': max(s['queue_lag_seconds'] for s in samples),
'outstanding_peak': max(s['outstanding'] for s in samples),
'rss_baseline_bytes': baseline['rss_bytes'],
'rss_peak_bytes': max([baseline['rss_bytes']] + [s['rss_bytes'] for s in samples]),
'samples': samples,
'observations': [{'attachment_id': r['attachment']['id'], 'ready_ms': round(r['ready_ms'], 3),
'upload_ms': round(r['upload_ms'], 3)} for r in records]}
print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'observations')}), flush=True)
return result
def overload(self, limit):
check(self.outstanding() == 0, 'drain other workloads before overload test')
docker('service', 'scale', '--detach', self.worker + '=0')
deadline = time.monotonic() + 60
while containers(self.worker):
check(time.monotonic() < deadline, 'worker did not stop')
time.sleep(.5)
update_env(self.api, {'MAX_OUTSTANDING_JOBS': limit})
alice, bob, _, chat = self.client.bootstrap()
content = png(320, 200)
with ThreadPoolExecutor(max_workers=self.args.uploaders) as pool:
rows = list(pool.map(lambda _: self.upload(alice, chat, content), range(limit * 2)))
accepted = [r for r in rows if r['status'] == 202]
rejected = len(rows) - len(accepted)
outstanding = self.outstanding()
check(len({row['instance'] for row in rows}) >= 2, 'overload must exercise both API replicas')
check(len(accepted) == limit and rejected == limit and outstanding == limit,
'admission limit did not bound accepted work while worker was stopped')
self.client.message(alice, chat, 'text messages remain available during overload')
docker('service', 'scale', '--detach', self.worker + '=1')
wait_service(self.worker)
started = time.monotonic()
for row in accepted:
attachment = self.client.wait_ready(bob, row['attachment'], wait_seconds=self.args.timeout)
check(attachment['error_code'] is None, 'accepted work failed after overload')
recovery = time.monotonic() - started
check(self.outstanding() == 0, 'accepted work remained outstanding after recovery')
followup = self.upload(alice, chat, content)
check(followup['status'] == 202, 'API did not resume admission after drain')
self.client.wait_ready(bob, followup['attachment'], wait_seconds=self.args.timeout)
result = {'limit': limit, 'offered': len(rows), 'accepted': len(accepted), 'rejected_429': rejected,
'observed_api_instances': len({row['instance'] for row in rows}),
'outstanding_worker_stopped': outstanding, 'ready_after_resume': len(accepted),
'observed_drain_after_worker_healthy_seconds': round(recovery, 3),
'outstanding_after_recovery': self.outstanding(), 'followup_upload': 202,
'text_messages_during_overload': 'PASS'}
print('OVERLOAD ' + json.dumps(result), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:18085'))
parser.add_argument('--stack', default=os.environ.get('STACK_NAME', 'mtusi-lab5'))
parser.add_argument('--requests', type=int, default=20)
parser.add_argument('--uploaders', type=int, default=8)
parser.add_argument('--interval', type=float, default=.25)
parser.add_argument('--timeout', type=int, default=240)
parser.add_argument('--limit', type=int, default=8)
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/lab5-research.json')
args = parser.parse_args()
check(args.requests >= 5 and args.uploaders > 0 and 0 < args.limit <= 100,
'invalid experiment limits')
check(args.interval > 0 and args.timeout > 0, 'sampling interval and timeout must be positive')
experiment = Experiment(args)
initial_worker, initial_api = service_spec(experiment.worker), service_spec(experiment.api)
original_worker_env = old_env(initial_worker, ('WORKER_CONCURRENCY', 'WORKER_PREFETCH'))
original_api_env = old_env(initial_api, ('MAX_OUTSTANDING_JOBS',))
original_replicas = initial_worker['Mode']['Replicated']['Replicas']
check(original_replicas == 1, 'experiment expects one worker service replica; concurrency is internal')
report = {'track': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'stack': args.stack, 'worker_image': initial_worker['TaskTemplate']['ContainerSpec']['Image'],
'worker_image_id': docker('inspect', containers(experiment.worker)[0], '--format', '{{.Image}}'),
'environment': {'docker_server': docker('version', '--format', '{{.Server.Version}}'),
'docker_cpus': int(docker('info', '--format', '{{.NCPU}}')),
'docker_memory_bytes': int(docker('info', '--format', '{{.MemTotal}}')),
'architecture': platform.machine(), 'python': platform.python_version()},
'worker_limits': initial_worker['TaskTemplate'].get('Resources', {}).get('Limits', {}),
'initial_settings': {'worker': original_worker_env, 'api': original_api_env},
'requests_per_case': args.requests, 'upload_concurrency': args.uploaders,
'sampling_interval_min_seconds': args.interval,
'method': {'throughput': 'completed / burst makespan including client upload and polling',
'ready_latency': 'client POST start to first observed ready; nearest-rank p95',
'queue_lag': 'age of oldest queued PostgreSQL job in this chat; excludes processing',
'rss': 'sum VmRSS for worker, renderers and cwebp; shared pages may count repeatedly',
'limitations': 'single host; warmup per case; sampled peaks and polling latency; finite burst, no production SLA'},
'cases': []}
args.output.parent.mkdir(parents=True, exist_ok=True)
def save():
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + '\n')
try:
for width, height in ((512, 512), (1536, 1536)):
for concurrency, prefetch in ((1, 1), (2, 4), (4, 8)):
report['cases'].append(experiment.run_case(concurrency, prefetch, width, height))
save()
report['overload'] = experiment.overload(args.limit)
save()
finally:
# Keep every accepted job recoverable, including when an assertion fails.
try:
if not containers(experiment.worker):
docker('service', 'scale', '--detach', experiment.worker + '=' + str(original_replicas))
wait_service(experiment.worker)
update_env(experiment.worker, original_worker_env)
finally:
update_env(experiment.api, original_api_env)
report['result'] = 'PASS'
report['restored_initial_settings'] = True
report['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat()
save()
summary = args.output.with_suffix('.csv')
keys = ['width', 'height', 'concurrency', 'prefetch', 'offered', 'accepted', 'completed',
'throughput_rps', 'ready_p95_ms', 'after_accept_p95_ms', 'upload_p95_ms',
'queue_lag_peak_seconds', 'rss_peak_bytes']
with summary.open('w', newline='') as file:
writer = csv.DictWriter(file, fieldnames=keys, extrasaction='ignore')
writer.writeheader()
writer.writerows(report['cases'])
print('PASS: six measured configurations, bounded overload and recovery; ' + str(args.output))
if __name__ == '__main__':
main()