"""Real durable queue, crash, retry, duplicate and version-reordering experiments.""" import hashlib import atexit import json import os from pathlib import Path import subprocess import sys import tempfile import time import uuid import httpx from stack import ROOT, docker, ready, ready_replicas, replicas, instance OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True) result={} atexit.register(lambda: (OUT/('integration.json' if result.get('passed') else 'interrupted-integration.json')).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')) def sql(query): return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',query) def inside(code): return subprocess.check_output(['docker','compose','exec','-T','dispatcher','python','-'], input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip() def job(aid): return json.loads(sql("SELECT row_to_json(j) FROM jobs j WHERE attachment_id='"+aid+"' ORDER BY pipeline_version DESC LIMIT 1")) def objects(aid): return json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps(sorted(o['Key'] for prefix in ('originals/','variants/') for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix=prefix+"+repr(aid)+") for o in p.get('Contents',[]))))")) def emit_job(identifier): inside("from src.jobs import database,payload\nfrom src.broker import connect,publish\nwith database() as db, connect() as c:\n j=db.execute('SELECT * FROM jobs WHERE id=%s',("+repr(identifier)+",)).fetchone(); print(publish(c,payload(j)))") def operator(*args): return json.loads(docker('compose','exec','-T','dispatcher','python','-m','src.operator',*args)) def restore(c): docker('compose','up','-d','--no-deps','--scale','api=2','api') ready_replicas(); ready(c) def public(mode): name='restart' if mode=='persistence' else 'stop-one' with tempfile.TemporaryDirectory() as tmp: if os.name=='nt': hook=Path(tmp)/(name+'.cmd') hook.write_text('@echo off\n"'+sys.executable+'" "'+str(ROOT/('scripts/'+name+'.py'))+'"\n',encoding='utf-8') else: hook=ROOT/('scripts/'+name+'.sh') proc=subprocess.run([sys.executable,'-X','utf8','tests/resilience.py','--mode',mode,'--action-script',str(hook)],cwd=ROOT,capture_output=True,text=True,encoding='utf-8') assert proc.returncode==0,proc.stdout+proc.stderr return proc.stdout.strip() with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c: ready(c) result['public_smoke']=subprocess.check_output([sys.executable,'-X','utf8','tests/smoke.py'],cwd=ROOT,text=True).strip() name='jobs_'+uuid.uuid4().hex[:20]; password='synthetic-jobs-password' r=c.post('/api/v1/users',json={'username':name,'display_name':'Queue research','password':password}); assert r.status_code==201,r.text uid=r.json()['id'] r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201 h={'Authorization':'Bearer '+r.json()['token']} r=c.post('/api/v1/chats',headers=h,json={'title':'Jobs','member_ids':[uid]}); assert r.status_code==201 chat=r.json()['id']; data=(ROOT/'tests/fixtures/sample.png').read_bytes() def upload(): r=c.post(f'/api/v1/chats/{chat}/attachments',params={'filename':'fixture.png'},headers={**h,'Content-Type':'image/png'},content=data) assert r.status_code==202,r.text assert r.json()['status']=='queued' and r.json()['variants']==[] return r def wait(aid,status='ready',timeout=140): deadline=time.monotonic()+timeout while time.monotonic()