#!/usr/bin/env python3 import json import subprocess import threading import time from pathlib import Path from client import Client, arguments, check, run from image_gen import png from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status LIMIT_MIB = 768 def sample(stop, state): while not stop.is_set(): out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split() if len(out) > 1: state['rss'] = max(state['rss'], int(out[1]) / 1024) n = psql("select count(*) from jobs where status = 'processing'") state['processing'] = max(state['processing'], int(n or 0)) time.sleep(0.2) def main(): def configure(p): p.add_argument('--huge', type=int, default=6) p.add_argument('--small', type=int, default=20) p.add_argument('--output') args = arguments('Worker concurrency, memory and time limits', configure) c = Client(args.base_url, args.ca_file, timeout=30) alice, bob, eve, chat = c.bootstrap() concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip()) huge = png(6320, 6320) state, stop = {'rss': 0, 'processing': 0}, threading.Event() t = threading.Thread(target=sample, args=(stop, state)) t.start() started = time.monotonic() ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)] ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)] accepted = time.monotonic() - started results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids] elapsed = time.monotonic() - started stop.set() t.join() container = compose('ps', '-q', 'worker').stdout.strip() oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], capture_output=True, text=True).stdout.split() compose('stop', 'worker') slow = upload(c, alice, chat, huge)['json'] compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'}) timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120) _, attempts, last_error = job_row(slow['id']) restore_worker() report = { 'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)}, 'small_images': args.small, 'worker_concurrency': concurrency, 'max_processing_jobs_observed': state['processing'], 'all_ready': results.count('ready') == len(results), 'accept_seconds': round(accepted, 2), 'all_ready_seconds': round(elapsed, 1), 'worker_peak_rss_mib': round(state['rss'], 1), 'worker_memory_limit_mib': LIMIT_MIB, 'worker_restarts_oom': oom, 'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'], 'attempts': attempts, 'last_error': last_error}, } text = json.dumps(report, indent=2) if args.output: Path(args.output).write_text(text + '\n') print(text) check(report['all_ready'], 'some jobs failed under load') check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY') check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit') check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error, 'timeout must be a bounded transient failure') print('PASS: bounded worker concurrency, memory below limit, job timeout enforced') if __name__ == '__main__': run(main)