Files
mtusi2026lab6-fork/tests/worker_load.py

85 lines
3.6 KiB
Python
Executable File

#!/usr/bin/env python3
import json
import subprocess
import threading
import time
from pathlib import Path
from client import Client, arguments, check, run
from image_gen import png
from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status
LIMIT_MIB = 768
def sample(stop, state):
while not stop.is_set():
out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split()
if len(out) > 1:
state['rss'] = max(state['rss'], int(out[1]) / 1024)
n = psql("select count(*) from jobs where status = 'processing'")
state['processing'] = max(state['processing'], int(n or 0))
time.sleep(0.2)
def main():
def configure(p):
p.add_argument('--huge', type=int, default=6)
p.add_argument('--small', type=int, default=20)
p.add_argument('--output')
args = arguments('Worker concurrency, memory and time limits', configure)
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = c.bootstrap()
concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip())
huge = png(6320, 6320)
state, stop = {'rss': 0, 'processing': 0}, threading.Event()
t = threading.Thread(target=sample, args=(stop, state))
t.start()
started = time.monotonic()
ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)]
ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)]
accepted = time.monotonic() - started
results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids]
elapsed = time.monotonic() - started
stop.set()
t.join()
container = compose('ps', '-q', 'worker').stdout.strip()
oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container],
capture_output=True, text=True).stdout.split()
compose('stop', 'worker')
slow = upload(c, alice, chat, huge)['json']
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'})
timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120)
_, attempts, last_error = job_row(slow['id'])
restore_worker()
report = {
'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)},
'small_images': args.small,
'worker_concurrency': concurrency,
'max_processing_jobs_observed': state['processing'],
'all_ready': results.count('ready') == len(results),
'accept_seconds': round(accepted, 2),
'all_ready_seconds': round(elapsed, 1),
'worker_peak_rss_mib': round(state['rss'], 1),
'worker_memory_limit_mib': LIMIT_MIB,
'worker_restarts_oom': oom,
'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'],
'attempts': attempts, 'last_error': last_error},
}
text = json.dumps(report, indent=2)
if args.output:
Path(args.output).write_text(text + '\n')
print(text)
check(report['all_ready'], 'some jobs failed under load')
check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY')
check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit')
check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error,
'timeout must be a bounded transient failure')
print('PASS: bounded worker concurrency, memory below limit, job timeout enforced')
if __name__ == '__main__':
run(main)