85 lines
3.6 KiB
Python
Executable File
85 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from client import Client, arguments, check, run
|
|
from image_gen import png
|
|
from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status
|
|
|
|
LIMIT_MIB = 768
|
|
|
|
|
|
def sample(stop, state):
|
|
while not stop.is_set():
|
|
out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split()
|
|
if len(out) > 1:
|
|
state['rss'] = max(state['rss'], int(out[1]) / 1024)
|
|
n = psql("select count(*) from jobs where status = 'processing'")
|
|
state['processing'] = max(state['processing'], int(n or 0))
|
|
time.sleep(0.2)
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--huge', type=int, default=6)
|
|
p.add_argument('--small', type=int, default=20)
|
|
p.add_argument('--output')
|
|
args = arguments('Worker concurrency, memory and time limits', configure)
|
|
c = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip())
|
|
huge = png(6320, 6320)
|
|
|
|
state, stop = {'rss': 0, 'processing': 0}, threading.Event()
|
|
t = threading.Thread(target=sample, args=(stop, state))
|
|
t.start()
|
|
started = time.monotonic()
|
|
ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)]
|
|
ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)]
|
|
accepted = time.monotonic() - started
|
|
results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids]
|
|
elapsed = time.monotonic() - started
|
|
stop.set()
|
|
t.join()
|
|
container = compose('ps', '-q', 'worker').stdout.strip()
|
|
oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container],
|
|
capture_output=True, text=True).stdout.split()
|
|
|
|
compose('stop', 'worker')
|
|
slow = upload(c, alice, chat, huge)['json']
|
|
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'})
|
|
timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120)
|
|
_, attempts, last_error = job_row(slow['id'])
|
|
restore_worker()
|
|
|
|
report = {
|
|
'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)},
|
|
'small_images': args.small,
|
|
'worker_concurrency': concurrency,
|
|
'max_processing_jobs_observed': state['processing'],
|
|
'all_ready': results.count('ready') == len(results),
|
|
'accept_seconds': round(accepted, 2),
|
|
'all_ready_seconds': round(elapsed, 1),
|
|
'worker_peak_rss_mib': round(state['rss'], 1),
|
|
'worker_memory_limit_mib': LIMIT_MIB,
|
|
'worker_restarts_oom': oom,
|
|
'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'],
|
|
'attempts': attempts, 'last_error': last_error},
|
|
}
|
|
text = json.dumps(report, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + '\n')
|
|
print(text)
|
|
check(report['all_ready'], 'some jobs failed under load')
|
|
check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY')
|
|
check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit')
|
|
check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error,
|
|
'timeout must be a bounded transient failure')
|
|
print('PASS: bounded worker concurrency, memory below limit, job timeout enforced')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|