Files
mtusi2026lab5/scripts/check-pipeline.py
T

146 lines
7.5 KiB
Python

#!/usr/bin/env python3
"""Short local fault checks; requires a running Compose stand. No load test."""
import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
def dc(*args, ok=True):
r = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True, timeout=90)
if ok and r.returncode:
raise RuntimeError('compose action failed: ' + ' '.join(args[:2]))
return r
def sql(query):
return dc('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', '-c', query).stdout.strip()
def wait_for(predicate, message, timeout=100):
end = time.monotonic()+timeout
while time.monotonic()<end:
if predicate():
return
time.sleep(.3)
raise AssertionError(message)
def wait_http_ready(client):
# Probe the same address used by clients: a healthy API process may still
# be marked down by the load balancer after a restart.
def available():
try:
return client.call('GET', '/health/ready', expected=[200, 503])['status'] == 200
except Failure:
# Connection failures are expected while the listener restarts.
return False
wait_for(available, 'HTTP endpoint did not become ready: ' + client.base)
def ready(c, actor, att):
end = time.monotonic()+45
while time.monotonic()<end:
value=c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
if value['status']=='ready':return value
check(value['status']!='failed','unexpected failed')
time.sleep(.3)
raise AssertionError('ready timeout')
def verify(c, actor, att, original):
value=ready(c,actor,att)
path='/api/v1/attachments/'+att['id']+'/content'
check(c.call('GET',path,actor=actor)['bytes']==original,'original changed')
variants={v['name']:v for v in value['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'missing variants')
for name,v in variants.items():
data=c.call('GET',path+'?variant='+name,actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest()==v['sha256'],'checksum mismatch')
if name=='crop':check(jpeg_size(data)==(128,128),'crop dimensions')
if name=='optimized':check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','not WebP')
return value
c=Client()
wait_http_ready(c)
a,b,e,chat=c.bootstrap()
try:
dc('stop','worker','broker')
# A separate API exits immediately after the upload transaction COMMIT.
dc('run','--rm','-d','--no-deps','--name','lab5-fault-api','-p','127.0.0.1:18081:8080','-e','FAULT_AFTER_UPLOAD_COMMIT=1','api')
fault=Client('http://127.0.0.1:18081')
wait_http_ready(fault)
try:
fault.attachment(a,chat)
raise AssertionError('fault did not terminate API')
except Failure:pass
aid=sql("SELECT id FROM attachments WHERE chat_id='%s' ORDER BY created_at DESC LIMIT 1"%chat['id'])
check(bool(aid),'upload COMMIT missing')
check(sql("SELECT count(*) FROM attachment_outbox WHERE attachment_id='%s'"%aid)=='1','outbox missing')
dc('start','broker','worker')
ready(c,a,{'id':aid})
print('PASS: crash after COMMIT before publish recovered from outbox')
dc('stop','worker')
att,content=c.attachment(a,chat)
def pending():
return att['id'] in dc('exec','-T','broker','valkey-cli','LRANGE','attachments:pending','0','-1').stdout
wait_for(pending,'job not published')
dc('restart','broker')
wait_for(lambda: dc('exec','-T','broker','valkey-cli','PING',ok=False).stdout.strip()=='PONG','broker unavailable')
check(pending(),'broker restart lost durable job')
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
dc('restart','api')
wait_http_ready(c)
dc('start','worker')
verify(c,a,att,content)
print('PASS: queued job survives broker/API restart; content blocked until ready')
for flag in ('FAULT_AFTER_OBJECTS','FAULT_BEFORE_ACK'):
dc('stop','worker')
att,content=c.attachment(a,chat)
r=dc('run','--rm','--no-deps','-e',flag+'=1','worker',ok=False)
check(r.returncode==86,'worker fault not triggered')
# Advance expired lease in this controlled test instead of waiting 60 seconds.
sql("UPDATE attachments SET lease_until=now()-interval '1 second' WHERE id='%s' AND status='processing'"%att['id'])
dc('start','worker')
value=verify(c,a,att,content)
before=sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])
sql("INSERT INTO attachment_outbox(attachment_id) VALUES('%s') ON CONFLICT DO NOTHING"%att['id'])
# Explicit duplicate of a previously successful logical job.
job={'schema_version':1,'job_id':att['id'],'type':'attachment.process.v1','attachment_id':att['id'],'pipeline_version':1,'attempt':1,'created_at':att['created_at']}
dc('exec','-T','broker','valkey-cli','LPUSH','attachments:pending',json.dumps(job))
time.sleep(1)
check(ready(c,a,att)==value,'duplicate changed result')
check(sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])==before,'duplicate executed')
check(dc('exec','-T','worker','/usr/local/bin/worker','replay',att['id'],ok=False).returncode!=0,'ready replay allowed')
print('PASS: '+flag+', recovery, duplicate no-op, ready replay rejected')
# A real temporary storage failure must exhaust bounded retries, then replay safely.
dc('stop','worker')
att,content=c.attachment(a,chat)
dc('run','-d','--no-deps','--name','lab5-retry-worker','-e','JOB_MAX_ATTEMPTS=2',
'-e','S3_ENDPOINT=http://127.0.0.1:1','worker')
wait_for(lambda: sql("SELECT status FROM attachments WHERE id='%s'"%att['id'])=='failed','retry exhaustion timeout')
check(sql("SELECT attempts::text || ':' || error_code FROM attachments WHERE id='%s'"%att['id'])=='2:retry_exhausted','incorrect retry limit/error')
subprocess.run(['docker','stop','lab5-retry-worker'],check=True,capture_output=True)
subprocess.run(['docker','rm','lab5-retry-worker'],check=True,capture_output=True)
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
dc('run','--rm','--no-deps','worker','replay',att['id'])
check(sql("SELECT count(*) FROM attachment_replays WHERE attachment_id='%s'"%att['id'])=='1','replay audit missing')
dc('start','worker')
verify(c,a,att,content)
print('PASS: temporary I/O failure, bounded retries, failed content 409, audited replay to ready')
worker_id=dc('ps','-q','worker').stdout.strip()
before=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
check(sql("SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype='advisory' AND objid=505001 AND granted")=='t','worker lock not found')
def restarted():
value=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
return int(value)>int(before)
wait_for(restarted,'worker did not restart after losing DB session')
att,content=c.attachment(a,chat)
verify(c,a,att,content)
print('PASS: lost DB session automatically restarts worker and processing resumes')
finally:
subprocess.run(['docker','rm','-f','lab5-retry-worker'],capture_output=True)
dc('start','broker','worker',ok=False)