Implement observable messenger, SLO alerts and measured incident scenarios
This commit is contained in:
@@ -0,0 +1,189 @@
|
||||
"""Real durable queue, crash, retry, duplicate and version-reordering experiments."""
|
||||
import hashlib
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
from stack import ROOT, docker, ready, ready_replicas, replicas, instance
|
||||
|
||||
OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True)
|
||||
result={}
|
||||
atexit.register(lambda: (OUT/('integration.json' if result.get('passed') else 'interrupted-integration.json')).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8'))
|
||||
|
||||
|
||||
def sql(query):
|
||||
return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',query)
|
||||
|
||||
|
||||
def inside(code):
|
||||
return subprocess.check_output(['docker','compose','exec','-T','dispatcher','python','-'],
|
||||
input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip()
|
||||
|
||||
|
||||
def job(aid):
|
||||
return json.loads(sql("SELECT row_to_json(j) FROM jobs j WHERE attachment_id='"+aid+"' ORDER BY pipeline_version DESC LIMIT 1"))
|
||||
|
||||
|
||||
def objects(aid):
|
||||
return json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps(sorted(o['Key'] for prefix in ('originals/','variants/') for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix=prefix+"+repr(aid)+") for o in p.get('Contents',[]))))"))
|
||||
|
||||
|
||||
def emit_job(identifier):
|
||||
inside("from src.jobs import database,payload\nfrom src.broker import connect,publish\nwith database() as db, connect() as c:\n j=db.execute('SELECT * FROM jobs WHERE id=%s',("+repr(identifier)+",)).fetchone(); print(publish(c,payload(j)))")
|
||||
|
||||
|
||||
def operator(*args):
|
||||
return json.loads(docker('compose','exec','-T','dispatcher','python','-m','src.operator',*args))
|
||||
|
||||
|
||||
def restore(c):
|
||||
docker('compose','up','-d','--no-deps','--scale','api=2','api')
|
||||
ready_replicas(); ready(c)
|
||||
|
||||
|
||||
def public(mode):
|
||||
name='restart' if mode=='persistence' else 'stop-one'
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
if os.name=='nt':
|
||||
hook=Path(tmp)/(name+'.cmd')
|
||||
hook.write_text('@echo off\n"'+sys.executable+'" "'+str(ROOT/('scripts/'+name+'.py'))+'"\n',encoding='utf-8')
|
||||
else: hook=ROOT/('scripts/'+name+'.sh')
|
||||
proc=subprocess.run([sys.executable,'-X','utf8','tests/resilience.py','--mode',mode,'--action-script',str(hook)],cwd=ROOT,capture_output=True,text=True,encoding='utf-8')
|
||||
assert proc.returncode==0,proc.stdout+proc.stderr
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
|
||||
ready(c)
|
||||
result['public_smoke']=subprocess.check_output([sys.executable,'-X','utf8','tests/smoke.py'],cwd=ROOT,text=True).strip()
|
||||
name='jobs_'+uuid.uuid4().hex[:20]; password='synthetic-jobs-password'
|
||||
r=c.post('/api/v1/users',json={'username':name,'display_name':'Queue research','password':password}); assert r.status_code==201,r.text
|
||||
uid=r.json()['id']
|
||||
r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201
|
||||
h={'Authorization':'Bearer '+r.json()['token']}
|
||||
r=c.post('/api/v1/chats',headers=h,json={'title':'Jobs','member_ids':[uid]}); assert r.status_code==201
|
||||
chat=r.json()['id']; data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
def upload():
|
||||
r=c.post(f'/api/v1/chats/{chat}/attachments',params={'filename':'fixture.png'},headers={**h,'Content-Type':'image/png'},content=data)
|
||||
assert r.status_code==202,r.text
|
||||
assert r.json()['status']=='queued' and r.json()['variants']==[]
|
||||
return r
|
||||
|
||||
def wait(aid,status='ready',timeout=140):
|
||||
deadline=time.monotonic()+timeout
|
||||
while time.monotonic()<deadline:
|
||||
r=c.get('/api/v1/attachments/'+aid,headers=h)
|
||||
if r.status_code==200:
|
||||
row=r.json()
|
||||
if row['status']==status: return row
|
||||
if row['status']=='failed' and status!='failed': raise AssertionError(row)
|
||||
time.sleep(.3)
|
||||
raise AssertionError('Job did not reach '+status+': '+str(job(aid)))
|
||||
|
||||
def verify_bytes(aid):
|
||||
row=wait(aid)
|
||||
assert {v['name'] for v in row['variants']}=={'thumbnail','crop','optimized'}
|
||||
assert c.get('/api/v1/attachments/'+aid+'/content',headers=h).content==data
|
||||
for variant in row['variants']:
|
||||
r=c.get('/api/v1/attachments/'+aid+'/content',headers=h,params={'variant':variant['name']})
|
||||
assert r.status_code==200 and hashlib.sha256(r.content).hexdigest()==variant['sha256']
|
||||
assert len(r.content)==variant['size_bytes']
|
||||
dims={v['name']:(v['width'],v['height'],v['content_type']) for v in row['variants']}
|
||||
assert dims['thumbnail']==(256,160,'image/jpeg')
|
||||
assert dims['crop']==(128,128,'image/jpeg')
|
||||
assert dims['optimized']==(320,200,'image/webp')
|
||||
return row
|
||||
|
||||
# Hold publication, durably accept, then kill the accepting API before publish.
|
||||
docker('compose','stop','worker','dispatcher')
|
||||
try:
|
||||
r=upload(); aid=r.json()['id']; serving=r.headers['X-Instance-Id']; j=job(aid)
|
||||
assert sql("SELECT published_at IS NULL FROM outbox WHERE job_id='"+j['id']+"'")=='t'
|
||||
assert c.get('/api/v1/attachments/'+aid+'/content',headers=h).status_code==409
|
||||
message=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[aid]})
|
||||
assert message.status_code==201,message.text
|
||||
target=next(cid for cid in replicas() if instance(cid)==serving)
|
||||
docker('kill',target)
|
||||
docker('compose','restart','broker')
|
||||
restore(c)
|
||||
assert c.get('/api/v1/attachments/'+aid,headers=h).json()['status']=='queued'
|
||||
finally:
|
||||
docker('compose','start','dispatcher','worker')
|
||||
accepted=verify_bytes(aid)
|
||||
result['commit_before_publish']={'attachment_id':aid,'job_id':j['id'],'killed_api':target,
|
||||
'broker_restarted':True,'queued_message_accepted':True,'original_and_variants_verified':True}
|
||||
result['public_persistence']=public('persistence'); restore(c)
|
||||
result['public_failover']=public('failover'); restore(c)
|
||||
|
||||
# Terminal duplicate is a no-op. Do not create a new logical job/effect set.
|
||||
before=verify_bytes(aid); keys=objects(aid)
|
||||
for _ in range(3): emit_job(j['id'])
|
||||
time.sleep(2)
|
||||
assert verify_bytes(aid)==before and objects(aid)==keys and len(keys)==4
|
||||
result['terminal_duplicates']={'deliveries':3,'effect_objects':4,'metadata_unchanged':True}
|
||||
|
||||
result['crashes']=[]
|
||||
for point in ('before_effect','after_effect','after_commit'):
|
||||
docker('compose','stop','worker')
|
||||
try:
|
||||
aid=upload().json()['id']; j=job(aid)
|
||||
sql("INSERT INTO job_faults(job_id,point,remaining) VALUES ('"+j['id']+"','"+point+"',1)")
|
||||
before_restart=int(docker('inspect','--format','{{.RestartCount}}',docker('compose','ps','-aq','worker')))
|
||||
finally: docker('compose','start','worker')
|
||||
started=time.monotonic(); row=verify_bytes(aid)
|
||||
# Force one extra delivery after ready to exercise the done guard.
|
||||
emit_job(j['id']); time.sleep(1)
|
||||
assert verify_bytes(aid)==row and len(objects(aid))==4
|
||||
current=job(aid)
|
||||
crash_lines=[line[line.index('{'):] for line in docker('compose','logs','--no-color','worker').splitlines()
|
||||
if '"event": "injected_crash"' in line and j['id'] in line]
|
||||
assert len(crash_lines)==1,crash_lines
|
||||
audit_rows=json.loads(sql("SELECT coalesce(json_agg(x),'[]') FROM (SELECT event,details FROM job_audit WHERE job_id='"+j['id']+"' ORDER BY id) x"))
|
||||
result['crashes'].append({'point':point,'seconds_to_ready':round(time.monotonic()-started,3),
|
||||
'attempts':current['attempts'],'effect_objects':4,'audit':audit_rows,'crash_event':json.loads(crash_lines[0]),
|
||||
'restart_count_before':before_restart,'restart_count_after':int(docker('inspect','--format','{{.RestartCount}}',docker('compose','ps','-q','worker')))})
|
||||
|
||||
result['retries']=[]
|
||||
for point,count,expected in [('transient',2,'ready'),('permanent',1,'failed'),('transient',5,'failed')]:
|
||||
docker('compose','stop','worker')
|
||||
try:
|
||||
aid=upload().json()['id']; j=job(aid)
|
||||
sql(f"INSERT INTO job_faults(job_id,point,remaining) VALUES ('{j['id']}','{point}',{count})")
|
||||
finally: docker('compose','start','worker')
|
||||
row=wait(aid,expected)
|
||||
failed=job(aid)
|
||||
assert failed['attempts']==(3 if expected=='ready' else count),failed
|
||||
if expected=='failed':
|
||||
assert c.get('/api/v1/attachments/'+aid+'/content',headers=h).status_code==409
|
||||
r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[aid]})
|
||||
assert r.status_code==409
|
||||
operator('replay',j['id']); verify_bytes(aid)
|
||||
result['retries'].append({'fault':point,'injected':count,'terminal_status':row['status'],
|
||||
'error_code':row['error_code'],'attempts':failed['attempts'],'replay_ready':expected=='failed'})
|
||||
|
||||
# Explicit reorder: enqueue version 2 before version 1, then deliver stale v1 again.
|
||||
docker('compose','stop','worker','dispatcher')
|
||||
try:
|
||||
aid=upload().json()['id']; old=job(aid)
|
||||
# Operator runs in a one-shot container while dispatcher is stopped.
|
||||
newer=json.loads(docker('compose','run','--rm','--no-deps','dispatcher','python','-m','src.operator','upgrade',aid,'2'))
|
||||
order_code="from src.jobs import database,payload\nfrom src.broker import connect,publish\nwith database() as db, connect() as client:\n for identifier in "+repr([newer['job_id'],old['id']])+":\n row=db.execute('SELECT * FROM jobs WHERE id=%s',(identifier,)).fetchone(); publish(client,payload(row))"
|
||||
docker('compose','run','--rm','--no-deps','dispatcher','python','-c',order_code)
|
||||
finally: docker('compose','start','dispatcher','worker')
|
||||
row=verify_bytes(aid); keys=objects(aid)
|
||||
assert all('/v2' in key for key in keys) and len(keys)==4
|
||||
emit_job(old['id']); time.sleep(2)
|
||||
assert verify_bytes(aid)==row and objects(aid)==keys
|
||||
result['version_reorder']={'old_job_id':old['id'],'new_job_id':newer['job_id'],
|
||||
'final_pipeline_version':job(aid)['pipeline_version'],'keys':keys,'stale_delivery_did_not_change_result':True}
|
||||
result['passed']=True
|
||||
(OUT/'integration.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
print(json.dumps(result,ensure_ascii=False,indent=2))
|
||||
Reference in New Issue
Block a user