76 lines
4.1 KiB
Python
76 lines
4.1 KiB
Python
"""Conservative cleanup and actual subprocess timeout boundary; worker stopped."""
|
|
from datetime import datetime,timedelta,timezone
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import unittest.mock as mock
|
|
import uuid
|
|
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
|
from client import Client,check
|
|
from app import store,object_store
|
|
from pipeline import envelope,fetch_job,replay
|
|
from worker import handle
|
|
from janitor import sweep
|
|
|
|
|
|
def main():
|
|
store.pool.open(wait=True,timeout=20)
|
|
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap()
|
|
def upload():
|
|
att,data=c.attachment(alice,chat)
|
|
with store.connect() as conn:
|
|
jid=conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0]
|
|
msg=envelope(fetch_job(conn,jid))
|
|
return att,msg
|
|
rows=[]
|
|
try:
|
|
att,msg=upload()
|
|
real_run=subprocess.run
|
|
def slow_decoder(*args,**kwargs):
|
|
return real_run([sys.executable,'-c','import time; time.sleep(2)'],timeout=.05,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
start=time.monotonic()
|
|
with mock.patch('worker.subprocess.run',side_effect=slow_decoder):
|
|
check(handle(msg)=='retry','decoder timeout not classified as transient')
|
|
elapsed=time.monotonic()-start
|
|
with store.connect() as conn:
|
|
job=fetch_job(conn,msg['job_id']);check(job['state']=='queued' and job['error_code']=='processing_timeout','timeout not persisted')
|
|
conn.execute('UPDATE jobs SET next_at=now() WHERE id=%s',(job['id'],))
|
|
check(handle(msg)=='ready','timeout recovery failed')
|
|
rows.append(dict(case='real_child_timeout_with_test_budget_50ms',status='PASS',elapsed_ms=round(elapsed*1000,3)))
|
|
# Validate memory-exit classification without risking an OOM on the host.
|
|
att2,msg2=upload()
|
|
def exhausted(*args,**kwargs):
|
|
return real_run([sys.executable,'-c',"import resource; resource.setrlimit(resource.RLIMIT_AS,(32*1024**2,32*1024**2));\ntry: x=bytearray(128*1024**2)\nexcept MemoryError: raise SystemExit(3)"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
with mock.patch('worker.subprocess.run',side_effect=exhausted):
|
|
check(handle(msg2)=='failed','memory failure not terminal')
|
|
with store.connect() as conn:check(fetch_job(conn,msg2['job_id'])['error_code']=='resource_limit','memory failure leaked details')
|
|
replay(store,msg2['job_id']);check(handle(msg2)=='ready','memory-failed replay did not complete')
|
|
rows.append(dict(case='real_child_address_space_limit_32MiB',status='PASS',error_code='resource_limit'))
|
|
active,msg3=upload()
|
|
orphan='staging/'+str(uuid.uuid4())
|
|
object_store.client.put_object(Bucket=object_store.bucket,Key=orphan,Body=b'orphan')
|
|
# Advance janitor's clock, leaving actual DB state/worker leases intact.
|
|
class Future(datetime):
|
|
@classmethod
|
|
def now(cls,tz=None):return datetime.now(tz)+timedelta(days=2)
|
|
with mock.patch('janitor.datetime',Future):
|
|
dry=sweep(False,86400)
|
|
check(orphan in dry['staging_candidates'],'orphan not detected')
|
|
check('staging/'+active['id'] not in dry['staging_candidates'],'active source scheduled for deletion')
|
|
object_store.client.head_object(Bucket=object_store.bucket,Key=orphan)
|
|
applied=sweep(True,86400)
|
|
check(orphan in applied['staging_candidates'],'orphan not deleted')
|
|
repeat=sweep(True,86400)
|
|
check(not repeat['staging_candidates'],'cleanup is not idempotent')
|
|
check(handle(msg3)=='ready','cleanup lost current job')
|
|
rows.append(dict(case='staging_dry_run_apply_repeat',status='PASS',candidates=len(applied['staging_candidates']),
|
|
repeat_candidates=0,active_retained=True,clock='advanced_2_days'))
|
|
print(json.dumps({'status':'PASS','rows':rows},indent=2))
|
|
finally:store.pool.close();object_store.close()
|
|
|
|
|
|
if __name__=='__main__':main()
|