82 lines
6.6 KiB
Python
Executable File
82 lines
6.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Black-box fault experiment. Mutates only this synthetic Compose project."""
|
|
import hashlib,json,os,subprocess,sys,time,uuid
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,check,Failure
|
|
BASE=os.environ.get('BASE_URL','http://localhost:8086');C=Client(BASE)
|
|
RESULT={'method':'real process/container faults; no forged timestamps','cases':[]}
|
|
def dc(*args):return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip()
|
|
def sql(query):return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query)
|
|
def record(name,**kwargs):RESULT['cases'].append(dict(name=name,**kwargs));print('PASS:',name,flush=True)
|
|
def meta(a,id):return C.call('GET','/api/v1/attachments/'+id,actor=a,schema='Attachment')['json']
|
|
def wait_status(a,id,status,timeout=130):
|
|
start=time.monotonic()
|
|
while time.monotonic()-start<timeout:
|
|
try:item=meta(a,id)
|
|
except Failure:time.sleep(.25);continue
|
|
if item['status']==status:return item,round(time.monotonic()-start,3)
|
|
time.sleep(.25)
|
|
raise AssertionError('timeout waiting '+status+' last='+item['status'])
|
|
def job(id,version=1):return json.loads(sql("SELECT row_to_json(x) FROM (SELECT 1 schema_version,job_id,'attachment.process.v1' type,attachment_id,pipeline_version,1 attempt,to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"') created_at FROM pipeline_jobs WHERE attachment_id='"+id+"' AND pipeline_version="+str(version)+") x"))
|
|
def publish(payload):
|
|
# The URL is read inside the container; credentials never enter output/argv.
|
|
script='valkey-cli -u "$(cat /run/secrets/broker_url)" --no-auth-warning XADD attachment.jobs.v1 "*" payload "$1" >/dev/null'
|
|
return dc('exec','-T','broker','sh','-c',script,'sh',json.dumps(payload))
|
|
def content(a,id,variant='original'):return C.call('GET','/api/v1/attachments/'+id+'/content?variant='+variant,actor=a)['bytes']
|
|
def upload(a,chat):return C.attachment(a,chat)
|
|
def start_worker():dc('up','-d','--no-deps','worker')
|
|
def fault(a,chat,point):
|
|
dc('stop','worker');att,data=upload(a,chat);name='lab-fault-'+uuid.uuid4().hex[:10]
|
|
dc('run','-d','--no-deps','--name',name,'-e','LAB_FAULT_POINT='+point,'worker')
|
|
try:
|
|
deadline=time.monotonic()+45
|
|
while time.monotonic()<deadline:
|
|
if subprocess.run(['docker','exec',name,'test','-f','/tmp/fault-reached'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode==0:break
|
|
time.sleep(.2)
|
|
else:raise AssertionError('fault boundary never reached '+point)
|
|
status=meta(a,att['id'])['status'];check(status==('ready' if point=='after_commit' else 'processing'),'wrong crash boundary')
|
|
subprocess.run(['docker','kill',name],check=True,stdout=subprocess.DEVNULL)
|
|
start_worker();ready,recovery=wait_status(a,att['id'],'ready');check(content(a,att['id'])==data,'original changed after crash')
|
|
time.sleep(2)
|
|
check(int(sql("SELECT count(*) FROM pipeline_jobs WHERE attachment_id='"+att['id']+"'"))==1,'extra logical job')
|
|
record('crash_'+point,attachment_id=att['id'],status_at_crash=status,recovery_seconds=recovery,versions=len(ready['variants']))
|
|
return att,data
|
|
finally:subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL);start_worker()
|
|
def main():
|
|
a,b,e,chat=C.bootstrap()
|
|
dc('stop','worker','dispatcher');att,data=upload(a,chat)
|
|
C.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=a)
|
|
C.message(a,chat,'',attachments=[att['id']]);C.message(a,chat,'text while worker is down')
|
|
check(sql("SELECT published_at IS NULL FROM pipeline_jobs WHERE attachment_id='"+att['id']+"'")=='t','expected committed unpublished outbox')
|
|
dc('restart','api','broker');dc('up','-d','--no-deps','dispatcher','worker');ready,recovery=wait_status(a,att['id'],'ready')
|
|
check(content(a,att['id'])==data,'durable stage lost');check({v['name'] for v in ready['variants']}=={'thumbnail','crop','optimized'},'missing variants')
|
|
for v in ready['variants']:check(hashlib.sha256(content(a,att['id'],v['name'])).hexdigest()==v['sha256'],'variant checksum')
|
|
C.call('GET','/api/v1/attachments/'+att['id'],404,actor=e)
|
|
record('commit_before_publish_api_broker_restart',attachment_id=att['id'],recovery_seconds=recovery,original_identical=True)
|
|
# No ACK occurred for these real crashes. Lease + stream reclaim recover them.
|
|
for point in ['before_objects','after_objects','after_commit']:fault(a,chat,point)
|
|
before=meta(a,att['id']);p=job(att['id'])
|
|
for _ in range(5):publish(p)
|
|
time.sleep(2);check(meta(a,att['id'])==before,'ready duplicate changed metadata');record('five_duplicate_deliveries',unchanged=True)
|
|
dc('run','--rm','--no-deps','worker','reprocess',att['id']);v2,_=wait_status(a,att['id'],'ready');check(sql("SELECT pipeline_version FROM attachments WHERE id='"+att['id']+"'")=='2','version not incremented')
|
|
v2bytes={v['name']:content(a,att['id'],v['name']) for v in v2['variants']}
|
|
# Old version redelivery is reordered behind a fully committed new version.
|
|
for _ in range(3):publish(p)
|
|
time.sleep(2);check(meta(a,att['id'])==v2,'old version changed metadata');check(all(content(a,att['id'],k)==v for k,v in v2bytes.items()),'old version overwrote bytes')
|
|
record('reordered_v1_after_v2',metadata_unchanged=True,checksums_unchanged=True)
|
|
# A permanent checksum mismatch is observable and cannot be attached. Restore
|
|
# the expected digest and explicitly replay; the source itself stays durable.
|
|
dc('stop','worker');bad,data=upload(a,chat);original=bad['sha256'];sql("UPDATE attachments SET sha256=repeat('0',64) WHERE id='"+bad['id']+"'");start_worker();failed,_=wait_status(a,bad['id'],'failed')
|
|
check(failed['error_code']=='invalid_image','permanent error code')
|
|
C.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=a,body={'text':'','attachment_ids':[bad['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
|
sql("UPDATE attachments SET sha256='"+original+"' WHERE id='"+bad['id']+"'");dc('run','--rm','--no-deps','worker','replay',bad['id']);replayed,_=wait_status(a,bad['id'],'ready')
|
|
check(content(a,bad['id'])==data,'replay lost original');check(int(sql("SELECT count(*) FROM pipeline_audit WHERE job_id=(SELECT job_id FROM pipeline_jobs WHERE attachment_id='"+bad['id']+"') AND event='operator_replay'"))==1,'missing replay audit')
|
|
record('failed_registry_explicit_audited_replay',error_code=failed['error_code'],ready=True)
|
|
print('PASS: versioned at-least-once pipeline invariants; NOT exactly-once delivery')
|
|
if __name__=='__main__':
|
|
try:main()
|
|
finally:
|
|
dc('up','-d','--no-deps','worker','dispatcher')
|
|
(ROOT/'evidence/pipeline.json').write_text(json.dumps(RESULT,indent=2)+'\n')
|