162 lines
7.7 KiB
Python
162 lines
7.7 KiB
Python
"""Explicit, recoverable failure tests for THIS Compose project only."""
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client
|
|
|
|
|
|
def compose(*args,input=None):
|
|
return subprocess.run(['docker','compose',*args],cwd=ROOT,input=input,text=True,capture_output=True,check=True).stdout.strip()
|
|
|
|
|
|
def internal(code):
|
|
return json.loads(compose('exec','-T','api','python','-',input=code))
|
|
|
|
|
|
def ready(client):
|
|
deadline=time.monotonic()+30
|
|
while True:
|
|
try:
|
|
client.call('GET','/health/ready',schema='Health')
|
|
return
|
|
except AssertionError:
|
|
if time.monotonic()>deadline:
|
|
raise
|
|
time.sleep(.2)
|
|
|
|
|
|
def main():
|
|
base=os.environ.get('BASE_URL','http://localhost:8082')
|
|
client=Client(base,timeout=8)
|
|
alice,bob,eve,chat=client.bootstrap()
|
|
report={'date_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'checks':{}}
|
|
checks=report['checks']
|
|
def state_counts():
|
|
return internal("""import json,os,psycopg
|
|
from pathlib import Path
|
|
with psycopg.connect(Path(os.environ['DATABASE_URL_FILE']).read_text().strip()) as c:
|
|
print(json.dumps({name:c.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('chats','chat_members')}))
|
|
""")
|
|
prior=state_counts()
|
|
client.call('POST','/api/v1/chats',400,actor=alice,body={'title':'Rollback probe','member_ids':[alice['id'],bob['id'],str(uuid.uuid4())]})
|
|
assert prior==state_counts()
|
|
checks['failed_chat_transaction_leaves_no_chat_or_members']=True
|
|
client.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
|
cache=internal('''import json,os,redis
|
|
from pathlib import Path
|
|
r=redis.Redis.from_url(Path(os.environ['CACHE_URL_FILE']).read_text().strip(),decode_responses=True)
|
|
keys=r.keys('profile:v1:*')
|
|
values=[json.loads(r.get(k)) for k in keys]
|
|
assert keys and all(set(v)=={'id','username','display_name','created_at'} for v in values)
|
|
print(json.dumps({'cached_profiles':len(keys),'all_profiles_public_fields_only':True,'positive_ttl':all(r.ttl(k)>0 for k in keys)}))
|
|
''')
|
|
assert cache['positive_ttl']
|
|
checks['valkey_really_caches_profiles']=cache
|
|
checks['password_kdf_and_hashed_tokens']=internal('''import json,os,psycopg
|
|
from pathlib import Path
|
|
with psycopg.connect(Path(os.environ['DATABASE_URL_FILE']).read_text().strip()) as c:
|
|
p=c.execute("SELECT bool_and(password_hash LIKE 'scrypt$16384$8$1$%') FROM users").fetchone()[0]
|
|
t=c.execute('SELECT bool_and(octet_length(token_hash)=32) FROM sessions').fetchone()[0]
|
|
print(json.dumps({'all_passwords_scrypt':p,'all_token_digests_32_bytes':t}))
|
|
''')
|
|
assert all(checks['password_kdf_and_hashed_tokens'].values())
|
|
try:
|
|
compose('stop','valkey')
|
|
started=time.monotonic()
|
|
me=client.call('GET','/api/v1/users/me',actor=alice,schema='User')['json']
|
|
assert me==alice['user']
|
|
client.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']})
|
|
message=client.message(alice,chat,'cache outage synthetic')['json']
|
|
page=client.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage')['json']
|
|
assert message in page['items']
|
|
# Revocation remains authoritative while the cache is down.
|
|
client.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice)
|
|
client.call('GET','/api/v1/users/me',401,actor=alice)
|
|
checks['cache_outage']={'fallback_profile_and_history':True,'anonymous_rejected':True,'logout_effective':True,'scenario_seconds':round(time.monotonic()-started,3)}
|
|
finally:
|
|
compose('up','-d','--wait','valkey')
|
|
client.call('GET','/api/v1/users/me',401,actor=alice)
|
|
client.call('GET','/api/v1/users/me',actor=bob,schema='User')
|
|
checks['cache_restart_no_revoked_session_resurrection']=True
|
|
internal('''import json,os,redis
|
|
from pathlib import Path
|
|
r=redis.Redis.from_url(Path(os.environ['CACHE_URL_FILE']).read_text().strip())
|
|
r.flushdb()
|
|
print(json.dumps({'flushed':True}))
|
|
''')
|
|
assert client.call('GET','/api/v1/users/me',actor=bob,schema='User')['json']==bob['user']
|
|
checks['cache_flush_repopulates_from_postgres']=True
|
|
try:
|
|
compose('stop','postgres')
|
|
started=time.monotonic()
|
|
unavailable=client.call('GET','/api/v1/users/me',503,actor=bob)
|
|
elapsed=time.monotonic()-started
|
|
assert elapsed<7
|
|
assert unavailable['json']['error']['code']=='unavailable'
|
|
client.call('GET','/health/live',schema='Health')
|
|
client.call('GET','/health/ready',503)
|
|
client.call('GET','/api/v1/users/me',401)
|
|
checks['postgres_outage']={'authenticated_request_503_seconds':round(elapsed,3),'live_200':True,'ready_503':True,'no_anonymous_trust':True}
|
|
finally:
|
|
compose('up','-d','--wait','postgres')
|
|
ready(client)
|
|
assert client.call('GET','/api/v1/users/me',actor=bob,schema='User')['json']==bob['user']
|
|
checks['postgres_recovery_same_session']=True
|
|
# A second API container uses the same PostgreSQL and Valkey, short TTL only.
|
|
name='mtusi-lab2-short-ttl-'+uuid.uuid4().hex[:8]
|
|
try:
|
|
compose('run','-d','--no-deps','--name',name,'-p','127.0.0.1::8080','-e','SESSION_TTL_SECONDS=2','api')
|
|
mapping=subprocess.check_output(['docker','port',name,'8080/tcp'],text=True).strip()
|
|
short=Client('http://127.0.0.1:'+mapping.rsplit(':',1)[1])
|
|
ready(short)
|
|
actor=short.user('expiry')
|
|
short.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
|
time.sleep(2.2)
|
|
short.call('GET','/api/v1/users/me',401,actor=actor)
|
|
checks['expiry_ttl_2_seconds']=True
|
|
finally:
|
|
subprocess.run(['docker','rm','-f',name],capture_output=True)
|
|
migrations_before=internal('''import json,os,psycopg
|
|
from pathlib import Path
|
|
with psycopg.connect(Path(os.environ['DATABASE_URL_FILE']).read_text().strip()) as c:
|
|
print(json.dumps({'migrations':c.execute('SELECT count(*) FROM schema_migrations').fetchone()[0],'users':c.execute('SELECT count(*) FROM users').fetchone()[0]}))
|
|
''')
|
|
compose('exec','-T','api','python','src/app.py','--migrate')
|
|
migrations_after=internal('''import json,os,psycopg
|
|
from pathlib import Path
|
|
with psycopg.connect(Path(os.environ['DATABASE_URL_FILE']).read_text().strip()) as c:
|
|
print(json.dumps({'migrations':c.execute('SELECT count(*) FROM schema_migrations').fetchone()[0],'users':c.execute('SELECT count(*) FROM users').fetchone()[0]}))
|
|
''')
|
|
assert migrations_before==migrations_after
|
|
checks['repeat_migration_does_not_duplicate_data']=migrations_after
|
|
# Actual process start times around the public all-service restart scenario.
|
|
def states():
|
|
values={}
|
|
for service in ('api','postgres','valkey'):
|
|
cid=compose('ps','-q',service)
|
|
values[service]=json.loads(subprocess.check_output(['docker','inspect',cid],text=True))[0]['State']['StartedAt']
|
|
return values
|
|
before=states()
|
|
started=time.monotonic()
|
|
completed=subprocess.run(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL='+base],cwd=ROOT,text=True,capture_output=True,check=True)
|
|
(ROOT/'evidence/persistence.txt').write_text(completed.stdout+completed.stderr)
|
|
after=states()
|
|
assert all(before[s]!=after[s] for s in before)
|
|
checks['full_stack_persistence']={'api_and_all_stores_restarted':True,'before':before,'after':after,'elapsed_seconds':round(time.monotonic()-started,3)}
|
|
report['status']='PASS'
|
|
(ROOT/'evidence/infrastructure.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
|
print(json.dumps(report,ensure_ascii=False,indent=2))
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|