146 lines
7.5 KiB
Python
146 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Short local fault checks; requires a running Compose stand. No load test."""
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
|
|
def dc(*args, ok=True):
|
|
r = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True, timeout=90)
|
|
if ok and r.returncode:
|
|
raise RuntimeError('compose action failed: ' + ' '.join(args[:2]))
|
|
return r
|
|
|
|
def sql(query):
|
|
return dc('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', '-c', query).stdout.strip()
|
|
|
|
def wait_for(predicate, message, timeout=100):
|
|
end = time.monotonic()+timeout
|
|
while time.monotonic()<end:
|
|
if predicate():
|
|
return
|
|
time.sleep(.3)
|
|
raise AssertionError(message)
|
|
|
|
def wait_http_ready(client):
|
|
# Probe the same address used by clients: a healthy API process may still
|
|
# be marked down by the load balancer after a restart.
|
|
def available():
|
|
try:
|
|
return client.call('GET', '/health/ready', expected=[200, 503])['status'] == 200
|
|
except Failure:
|
|
# Connection failures are expected while the listener restarts.
|
|
return False
|
|
wait_for(available, 'HTTP endpoint did not become ready: ' + client.base)
|
|
|
|
def ready(c, actor, att):
|
|
end = time.monotonic()+45
|
|
while time.monotonic()<end:
|
|
value=c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
|
|
if value['status']=='ready':return value
|
|
check(value['status']!='failed','unexpected failed')
|
|
time.sleep(.3)
|
|
raise AssertionError('ready timeout')
|
|
|
|
def verify(c, actor, att, original):
|
|
value=ready(c,actor,att)
|
|
path='/api/v1/attachments/'+att['id']+'/content'
|
|
check(c.call('GET',path,actor=actor)['bytes']==original,'original changed')
|
|
variants={v['name']:v for v in value['variants']}
|
|
check(set(variants)=={'thumbnail','crop','optimized'},'missing variants')
|
|
for name,v in variants.items():
|
|
data=c.call('GET',path+'?variant='+name,actor=actor)['bytes']
|
|
check(hashlib.sha256(data).hexdigest()==v['sha256'],'checksum mismatch')
|
|
if name=='crop':check(jpeg_size(data)==(128,128),'crop dimensions')
|
|
if name=='optimized':check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','not WebP')
|
|
return value
|
|
|
|
c=Client()
|
|
wait_http_ready(c)
|
|
a,b,e,chat=c.bootstrap()
|
|
try:
|
|
dc('stop','worker','broker')
|
|
# A separate API exits immediately after the upload transaction COMMIT.
|
|
dc('run','--rm','-d','--no-deps','--name','lab5-fault-api','-p','127.0.0.1:18081:8080','-e','FAULT_AFTER_UPLOAD_COMMIT=1','api')
|
|
fault=Client('http://127.0.0.1:18081')
|
|
wait_http_ready(fault)
|
|
try:
|
|
fault.attachment(a,chat)
|
|
raise AssertionError('fault did not terminate API')
|
|
except Failure:pass
|
|
aid=sql("SELECT id FROM attachments WHERE chat_id='%s' ORDER BY created_at DESC LIMIT 1"%chat['id'])
|
|
check(bool(aid),'upload COMMIT missing')
|
|
check(sql("SELECT count(*) FROM attachment_outbox WHERE attachment_id='%s'"%aid)=='1','outbox missing')
|
|
dc('start','broker','worker')
|
|
ready(c,a,{'id':aid})
|
|
print('PASS: crash after COMMIT before publish recovered from outbox')
|
|
|
|
dc('stop','worker')
|
|
att,content=c.attachment(a,chat)
|
|
def pending():
|
|
return att['id'] in dc('exec','-T','broker','valkey-cli','LRANGE','attachments:pending','0','-1').stdout
|
|
wait_for(pending,'job not published')
|
|
dc('restart','broker')
|
|
wait_for(lambda: dc('exec','-T','broker','valkey-cli','PING',ok=False).stdout.strip()=='PONG','broker unavailable')
|
|
check(pending(),'broker restart lost durable job')
|
|
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
|
|
dc('restart','api')
|
|
wait_http_ready(c)
|
|
dc('start','worker')
|
|
verify(c,a,att,content)
|
|
print('PASS: queued job survives broker/API restart; content blocked until ready')
|
|
for flag in ('FAULT_AFTER_OBJECTS','FAULT_BEFORE_ACK'):
|
|
dc('stop','worker')
|
|
att,content=c.attachment(a,chat)
|
|
r=dc('run','--rm','--no-deps','-e',flag+'=1','worker',ok=False)
|
|
check(r.returncode==86,'worker fault not triggered')
|
|
# Advance expired lease in this controlled test instead of waiting 60 seconds.
|
|
sql("UPDATE attachments SET lease_until=now()-interval '1 second' WHERE id='%s' AND status='processing'"%att['id'])
|
|
dc('start','worker')
|
|
value=verify(c,a,att,content)
|
|
before=sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])
|
|
sql("INSERT INTO attachment_outbox(attachment_id) VALUES('%s') ON CONFLICT DO NOTHING"%att['id'])
|
|
# Explicit duplicate of a previously successful logical job.
|
|
job={'schema_version':1,'job_id':att['id'],'type':'attachment.process.v1','attachment_id':att['id'],'pipeline_version':1,'attempt':1,'created_at':att['created_at']}
|
|
dc('exec','-T','broker','valkey-cli','LPUSH','attachments:pending',json.dumps(job))
|
|
time.sleep(1)
|
|
check(ready(c,a,att)==value,'duplicate changed result')
|
|
check(sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])==before,'duplicate executed')
|
|
check(dc('exec','-T','worker','/usr/local/bin/worker','replay',att['id'],ok=False).returncode!=0,'ready replay allowed')
|
|
print('PASS: '+flag+', recovery, duplicate no-op, ready replay rejected')
|
|
|
|
# A real temporary storage failure must exhaust bounded retries, then replay safely.
|
|
dc('stop','worker')
|
|
att,content=c.attachment(a,chat)
|
|
dc('run','-d','--no-deps','--name','lab5-retry-worker','-e','JOB_MAX_ATTEMPTS=2',
|
|
'-e','S3_ENDPOINT=http://127.0.0.1:1','worker')
|
|
wait_for(lambda: sql("SELECT status FROM attachments WHERE id='%s'"%att['id'])=='failed','retry exhaustion timeout')
|
|
check(sql("SELECT attempts::text || ':' || error_code FROM attachments WHERE id='%s'"%att['id'])=='2:retry_exhausted','incorrect retry limit/error')
|
|
subprocess.run(['docker','stop','lab5-retry-worker'],check=True,capture_output=True)
|
|
subprocess.run(['docker','rm','lab5-retry-worker'],check=True,capture_output=True)
|
|
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
|
|
dc('run','--rm','--no-deps','worker','replay',att['id'])
|
|
check(sql("SELECT count(*) FROM attachment_replays WHERE attachment_id='%s'"%att['id'])=='1','replay audit missing')
|
|
dc('start','worker')
|
|
verify(c,a,att,content)
|
|
print('PASS: temporary I/O failure, bounded retries, failed content 409, audited replay to ready')
|
|
|
|
worker_id=dc('ps','-q','worker').stdout.strip()
|
|
before=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
|
|
check(sql("SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype='advisory' AND objid=505001 AND granted")=='t','worker lock not found')
|
|
def restarted():
|
|
value=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
|
|
return int(value)>int(before)
|
|
wait_for(restarted,'worker did not restart after losing DB session')
|
|
att,content=c.attachment(a,chat)
|
|
verify(c,a,att,content)
|
|
print('PASS: lost DB session automatically restarts worker and processing resumes')
|
|
finally:
|
|
subprocess.run(['docker','rm','-f','lab5-retry-worker'],capture_output=True)
|
|
dc('start','broker','worker',ok=False)
|