Files
mtusi2026lab5/tests/cleanup_and_limits.py
T

76 lines
4.1 KiB
Python

"""Conservative cleanup and actual subprocess timeout boundary; worker stopped."""
from datetime import datetime,timedelta,timezone
import json
from pathlib import Path
import subprocess
import sys
import time
import unittest.mock as mock
import uuid
sys.path.insert(0,str(Path(__file__).resolve().parent))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client,check
from app import store,object_store
from pipeline import envelope,fetch_job,replay
from worker import handle
from janitor import sweep
def main():
store.pool.open(wait=True,timeout=20)
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap()
def upload():
att,data=c.attachment(alice,chat)
with store.connect() as conn:
jid=conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0]
msg=envelope(fetch_job(conn,jid))
return att,msg
rows=[]
try:
att,msg=upload()
real_run=subprocess.run
def slow_decoder(*args,**kwargs):
return real_run([sys.executable,'-c','import time; time.sleep(2)'],timeout=.05,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
start=time.monotonic()
with mock.patch('worker.subprocess.run',side_effect=slow_decoder):
check(handle(msg)=='retry','decoder timeout not classified as transient')
elapsed=time.monotonic()-start
with store.connect() as conn:
job=fetch_job(conn,msg['job_id']);check(job['state']=='queued' and job['error_code']=='processing_timeout','timeout not persisted')
conn.execute('UPDATE jobs SET next_at=now() WHERE id=%s',(job['id'],))
check(handle(msg)=='ready','timeout recovery failed')
rows.append(dict(case='real_child_timeout_with_test_budget_50ms',status='PASS',elapsed_ms=round(elapsed*1000,3)))
# Validate memory-exit classification without risking an OOM on the host.
att2,msg2=upload()
def exhausted(*args,**kwargs):
return real_run([sys.executable,'-c',"import resource; resource.setrlimit(resource.RLIMIT_AS,(32*1024**2,32*1024**2));\ntry: x=bytearray(128*1024**2)\nexcept MemoryError: raise SystemExit(3)"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
with mock.patch('worker.subprocess.run',side_effect=exhausted):
check(handle(msg2)=='failed','memory failure not terminal')
with store.connect() as conn:check(fetch_job(conn,msg2['job_id'])['error_code']=='resource_limit','memory failure leaked details')
replay(store,msg2['job_id']);check(handle(msg2)=='ready','memory-failed replay did not complete')
rows.append(dict(case='real_child_address_space_limit_32MiB',status='PASS',error_code='resource_limit'))
active,msg3=upload()
orphan='staging/'+str(uuid.uuid4())
object_store.client.put_object(Bucket=object_store.bucket,Key=orphan,Body=b'orphan')
# Advance janitor's clock, leaving actual DB state/worker leases intact.
class Future(datetime):
@classmethod
def now(cls,tz=None):return datetime.now(tz)+timedelta(days=2)
with mock.patch('janitor.datetime',Future):
dry=sweep(False,86400)
check(orphan in dry['staging_candidates'],'orphan not detected')
check('staging/'+active['id'] not in dry['staging_candidates'],'active source scheduled for deletion')
object_store.client.head_object(Bucket=object_store.bucket,Key=orphan)
applied=sweep(True,86400)
check(orphan in applied['staging_candidates'],'orphan not deleted')
repeat=sweep(True,86400)
check(not repeat['staging_candidates'],'cleanup is not idempotent')
check(handle(msg3)=='ready','cleanup lost current job')
rows.append(dict(case='staging_dry_run_apply_repeat',status='PASS',candidates=len(applied['staging_candidates']),
repeat_candidates=0,active_retained=True,clock='advanced_2_days'))
print(json.dumps({'status':'PASS','rows':rows},indent=2))
finally:store.pool.close();object_store.close()
if __name__=='__main__':main()