Implement observable messenger, SLO alerts and measured incident scenarios

This commit is contained in:
2026-10-23 13:09:22 +03:00
parent 470d9d6f1e
commit b7540190c1
66 changed files with 96519 additions and 39 deletions
+189
View File
@@ -0,0 +1,189 @@
"""Real durable queue, crash, retry, duplicate and version-reordering experiments."""
import hashlib
import atexit
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import uuid
import httpx
from stack import ROOT, docker, ready, ready_replicas, replicas, instance
OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True)
result={}
atexit.register(lambda: (OUT/('integration.json' if result.get('passed') else 'interrupted-integration.json')).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8'))
def sql(query):
return docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',query)
def inside(code):
return subprocess.check_output(['docker','compose','exec','-T','dispatcher','python','-'],
input=code,cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip()
def job(aid):
return json.loads(sql("SELECT row_to_json(j) FROM jobs j WHERE attachment_id='"+aid+"' ORDER BY pipeline_version DESC LIMIT 1"))
def objects(aid):
return json.loads(inside("from src.storage import Storage\nimport json\ns=Storage(); print(json.dumps(sorted(o['Key'] for prefix in ('originals/','variants/') for p in s.client.get_paginator('list_objects_v2').paginate(Bucket=s.bucket,Prefix=prefix+"+repr(aid)+") for o in p.get('Contents',[]))))"))
def emit_job(identifier):
inside("from src.jobs import database,payload\nfrom src.broker import connect,publish\nwith database() as db, connect() as c:\n j=db.execute('SELECT * FROM jobs WHERE id=%s',("+repr(identifier)+",)).fetchone(); print(publish(c,payload(j)))")
def operator(*args):
return json.loads(docker('compose','exec','-T','dispatcher','python','-m','src.operator',*args))
def restore(c):
docker('compose','up','-d','--no-deps','--scale','api=2','api')
ready_replicas(); ready(c)
def public(mode):
name='restart' if mode=='persistence' else 'stop-one'
with tempfile.TemporaryDirectory() as tmp:
if os.name=='nt':
hook=Path(tmp)/(name+'.cmd')
hook.write_text('@echo off\n"'+sys.executable+'" "'+str(ROOT/('scripts/'+name+'.py'))+'"\n',encoding='utf-8')
else: hook=ROOT/('scripts/'+name+'.sh')
proc=subprocess.run([sys.executable,'-X','utf8','tests/resilience.py','--mode',mode,'--action-script',str(hook)],cwd=ROOT,capture_output=True,text=True,encoding='utf-8')
assert proc.returncode==0,proc.stdout+proc.stderr
return proc.stdout.strip()
with httpx.Client(base_url='http://127.0.0.1:8080',timeout=35) as c:
ready(c)
result['public_smoke']=subprocess.check_output([sys.executable,'-X','utf8','tests/smoke.py'],cwd=ROOT,text=True).strip()
name='jobs_'+uuid.uuid4().hex[:20]; password='synthetic-jobs-password'
r=c.post('/api/v1/users',json={'username':name,'display_name':'Queue research','password':password}); assert r.status_code==201,r.text
uid=r.json()['id']
r=c.post('/api/v1/auth/sessions',auth=(name,password)); assert r.status_code==201
h={'Authorization':'Bearer '+r.json()['token']}
r=c.post('/api/v1/chats',headers=h,json={'title':'Jobs','member_ids':[uid]}); assert r.status_code==201
chat=r.json()['id']; data=(ROOT/'tests/fixtures/sample.png').read_bytes()
def upload():
r=c.post(f'/api/v1/chats/{chat}/attachments',params={'filename':'fixture.png'},headers={**h,'Content-Type':'image/png'},content=data)
assert r.status_code==202,r.text
assert r.json()['status']=='queued' and r.json()['variants']==[]
return r
def wait(aid,status='ready',timeout=140):
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
r=c.get('/api/v1/attachments/'+aid,headers=h)
if r.status_code==200:
row=r.json()
if row['status']==status: return row
if row['status']=='failed' and status!='failed': raise AssertionError(row)
time.sleep(.3)
raise AssertionError('Job did not reach '+status+': '+str(job(aid)))
def verify_bytes(aid):
row=wait(aid)
assert {v['name'] for v in row['variants']}=={'thumbnail','crop','optimized'}
assert c.get('/api/v1/attachments/'+aid+'/content',headers=h).content==data
for variant in row['variants']:
r=c.get('/api/v1/attachments/'+aid+'/content',headers=h,params={'variant':variant['name']})
assert r.status_code==200 and hashlib.sha256(r.content).hexdigest()==variant['sha256']
assert len(r.content)==variant['size_bytes']
dims={v['name']:(v['width'],v['height'],v['content_type']) for v in row['variants']}
assert dims['thumbnail']==(256,160,'image/jpeg')
assert dims['crop']==(128,128,'image/jpeg')
assert dims['optimized']==(320,200,'image/webp')
return row
# Hold publication, durably accept, then kill the accepting API before publish.
docker('compose','stop','worker','dispatcher')
try:
r=upload(); aid=r.json()['id']; serving=r.headers['X-Instance-Id']; j=job(aid)
assert sql("SELECT published_at IS NULL FROM outbox WHERE job_id='"+j['id']+"'")=='t'
assert c.get('/api/v1/attachments/'+aid+'/content',headers=h).status_code==409
message=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[aid]})
assert message.status_code==201,message.text
target=next(cid for cid in replicas() if instance(cid)==serving)
docker('kill',target)
docker('compose','restart','broker')
restore(c)
assert c.get('/api/v1/attachments/'+aid,headers=h).json()['status']=='queued'
finally:
docker('compose','start','dispatcher','worker')
accepted=verify_bytes(aid)
result['commit_before_publish']={'attachment_id':aid,'job_id':j['id'],'killed_api':target,
'broker_restarted':True,'queued_message_accepted':True,'original_and_variants_verified':True}
result['public_persistence']=public('persistence'); restore(c)
result['public_failover']=public('failover'); restore(c)
# Terminal duplicate is a no-op. Do not create a new logical job/effect set.
before=verify_bytes(aid); keys=objects(aid)
for _ in range(3): emit_job(j['id'])
time.sleep(2)
assert verify_bytes(aid)==before and objects(aid)==keys and len(keys)==4
result['terminal_duplicates']={'deliveries':3,'effect_objects':4,'metadata_unchanged':True}
result['crashes']=[]
for point in ('before_effect','after_effect','after_commit'):
docker('compose','stop','worker')
try:
aid=upload().json()['id']; j=job(aid)
sql("INSERT INTO job_faults(job_id,point,remaining) VALUES ('"+j['id']+"','"+point+"',1)")
before_restart=int(docker('inspect','--format','{{.RestartCount}}',docker('compose','ps','-aq','worker')))
finally: docker('compose','start','worker')
started=time.monotonic(); row=verify_bytes(aid)
# Force one extra delivery after ready to exercise the done guard.
emit_job(j['id']); time.sleep(1)
assert verify_bytes(aid)==row and len(objects(aid))==4
current=job(aid)
crash_lines=[line[line.index('{'):] for line in docker('compose','logs','--no-color','worker').splitlines()
if '"event": "injected_crash"' in line and j['id'] in line]
assert len(crash_lines)==1,crash_lines
audit_rows=json.loads(sql("SELECT coalesce(json_agg(x),'[]') FROM (SELECT event,details FROM job_audit WHERE job_id='"+j['id']+"' ORDER BY id) x"))
result['crashes'].append({'point':point,'seconds_to_ready':round(time.monotonic()-started,3),
'attempts':current['attempts'],'effect_objects':4,'audit':audit_rows,'crash_event':json.loads(crash_lines[0]),
'restart_count_before':before_restart,'restart_count_after':int(docker('inspect','--format','{{.RestartCount}}',docker('compose','ps','-q','worker')))})
result['retries']=[]
for point,count,expected in [('transient',2,'ready'),('permanent',1,'failed'),('transient',5,'failed')]:
docker('compose','stop','worker')
try:
aid=upload().json()['id']; j=job(aid)
sql(f"INSERT INTO job_faults(job_id,point,remaining) VALUES ('{j['id']}','{point}',{count})")
finally: docker('compose','start','worker')
row=wait(aid,expected)
failed=job(aid)
assert failed['attempts']==(3 if expected=='ready' else count),failed
if expected=='failed':
assert c.get('/api/v1/attachments/'+aid+'/content',headers=h).status_code==409
r=c.post(f'/api/v1/chats/{chat}/messages',headers={**h,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[aid]})
assert r.status_code==409
operator('replay',j['id']); verify_bytes(aid)
result['retries'].append({'fault':point,'injected':count,'terminal_status':row['status'],
'error_code':row['error_code'],'attempts':failed['attempts'],'replay_ready':expected=='failed'})
# Explicit reorder: enqueue version 2 before version 1, then deliver stale v1 again.
docker('compose','stop','worker','dispatcher')
try:
aid=upload().json()['id']; old=job(aid)
# Operator runs in a one-shot container while dispatcher is stopped.
newer=json.loads(docker('compose','run','--rm','--no-deps','dispatcher','python','-m','src.operator','upgrade',aid,'2'))
order_code="from src.jobs import database,payload\nfrom src.broker import connect,publish\nwith database() as db, connect() as client:\n for identifier in "+repr([newer['job_id'],old['id']])+":\n row=db.execute('SELECT * FROM jobs WHERE id=%s',(identifier,)).fetchone(); publish(client,payload(row))"
docker('compose','run','--rm','--no-deps','dispatcher','python','-c',order_code)
finally: docker('compose','start','dispatcher','worker')
row=verify_bytes(aid); keys=objects(aid)
assert all('/v2' in key for key in keys) and len(keys)==4
emit_job(old['id']); time.sleep(2)
assert verify_bytes(aid)==row and objects(aid)==keys
result['version_reorder']={'old_job_id':old['id'],'new_job_id':newer['job_id'],
'final_pipeline_version':job(aid)['pipeline_version'],'keys':keys,'stale_delivery_did_not_change_result':True}
result['passed']=True
(OUT/'integration.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
print(json.dumps(result,ensure_ascii=False,indent=2))