558 lines
32 KiB
Python
558 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify lab2 levels 3/4 on an isolated Compose project; retain its volumes.
|
|
|
|
Uses Python stdlib, Docker Compose and OpenSSL. Existing public tests are run
|
|
unchanged. Credentials stay in ignored secrets/; evidence contains no tokens.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import concurrent.futures as futures
|
|
import contextlib
|
|
import csv
|
|
import datetime as dt
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import secrets
|
|
import select
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def utc():
|
|
return dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z')
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise RuntimeError(message)
|
|
|
|
|
|
class Verification:
|
|
def __init__(self, port, output):
|
|
self.port = port
|
|
with socket.socket() as probe:
|
|
probe.bind(('127.0.0.1', port))
|
|
self.run_id = dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:6]
|
|
self.project = 'lab2-verify-' + self.run_id.lower()
|
|
self.private = ROOT / 'secrets' / 'levels-3-4' / self.run_id
|
|
self.private.mkdir(parents=True, mode=0o700)
|
|
self.out = ROOT / output
|
|
self.out.mkdir(parents=True, exist_ok=True)
|
|
self.envfile, self.override = self.private / '.env', self.private / 'compose.json'
|
|
self.env = {k: v for k, v in os.environ.items() if not k.startswith(
|
|
('COMPOSE_', 'POSTGRES_', 'REDIS_', 'JWT_', 'SESSION_', 'APP_', 'APPLICATION_')) and k != 'HOST_PORT'}
|
|
self.env.update(COMPOSE_PROJECT_NAME=self.project, COMPOSE_ENV_FILE=str(self.envfile), COMPOSE_OVERRIDE_FILE=str(self.override))
|
|
self.cmd = ['docker', 'compose', '--project-name', self.project, '--env-file', str(self.envfile),
|
|
'-f', str(ROOT / 'docker-compose.yml'), '-f', str(self.override)]
|
|
self.image = 'mtusi-messenger:' + self.project
|
|
self.passwords = [secrets.token_urlsafe(32), secrets.token_urlsafe(32)]
|
|
self.tokens = []
|
|
self.http_rows, self.checks, self.transcript = [], [], []
|
|
self.summary = dict(run_id=self.run_id, started_utc=utc(), project=self.project,
|
|
port=port, commit=self.run(['git','rev-parse','HEAD']), dirty=True,
|
|
checks=self.checks, scope='isolated Compose; original stack is not restarted')
|
|
self.api_env = {'SESSION_TTL_SECONDS': '3600'}
|
|
self.started = False
|
|
|
|
def safe(self, text):
|
|
for value in self.passwords + self.tokens:
|
|
text = text.replace(value, '<redacted>')
|
|
return re.sub(r'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+', '<redacted-jwt>', text)
|
|
|
|
def run(self, args, data=None, timeout=180, record=None):
|
|
result = subprocess.run(args, input=data, capture_output=True, text=True, cwd=ROOT, env=self.env, timeout=timeout)
|
|
if record:
|
|
self.transcript.append(f'\n$ {record}\n' + self.safe(result.stdout + result.stderr) + f'\nexit_code={result.returncode}\n')
|
|
self.flush()
|
|
require(result.returncode == 0, 'Command failed: ' + (record or ' '.join(args[:3])))
|
|
return result.stdout.strip()
|
|
|
|
def compose(self, *args, record=None):
|
|
return self.run(self.cmd + list(args), record=record)
|
|
|
|
def sql(self, statement):
|
|
return self.compose_sql(statement)
|
|
|
|
def compose_sql(self, statement):
|
|
return self.run(self.cmd + ['exec','-T','postgres','sh','-c',
|
|
'exec psql -X -q -At -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"'], data=statement)
|
|
|
|
def redis(self, *args):
|
|
return self.run(self.cmd + ['exec','-T','redis','sh','-c',
|
|
'REDISCLI_AUTH="$REDIS_PASSWORD" exec redis-cli --no-auth-warning -n 0 "$@"', 'redis-cli'] + list(args))
|
|
|
|
def flush(self):
|
|
(self.out/'summary.json').write_text(json.dumps(self.summary, ensure_ascii=False, indent=2)+'\n')
|
|
(self.out/'checks.txt').write_text(''.join(self.transcript))
|
|
if self.http_rows:
|
|
with (self.out/'http.csv').open('w', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=list(self.http_rows[0]), lineterminator="\n")
|
|
writer.writeheader()
|
|
writer.writerows(self.http_rows)
|
|
|
|
def check(self, name, function):
|
|
started = time.monotonic()
|
|
try:
|
|
details = function() or {}
|
|
except Exception as error:
|
|
self.checks.append(dict(name=name, status='FAIL', duration_seconds=round(time.monotonic()-started,3), reason=self.safe(str(error))))
|
|
self.flush()
|
|
raise
|
|
self.checks.append(dict(name=name, status='PASS', duration_seconds=round(time.monotonic()-started,3), details=details))
|
|
self.flush()
|
|
print('PASS: ' + name, flush=True)
|
|
|
|
def call(self, method, path, expected=200, actor=None, body=None, basic=None, headers=None, record=True, raw=None):
|
|
conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=5)
|
|
h = dict(headers or {})
|
|
if actor:
|
|
h['Authorization'] = 'Bearer ' + actor['token']
|
|
if basic:
|
|
h['Authorization'] = 'Basic ' + base64.b64encode(basic.encode()).decode()
|
|
if body is not None:
|
|
h['Content-Type'] = 'application/json'
|
|
started, stamp = time.monotonic(), utc()
|
|
try:
|
|
conn.request(method,path,body=raw if raw is not None else (None if body is None else json.dumps(body)),headers=h)
|
|
response = conn.getresponse()
|
|
raw = response.read()
|
|
data = json.loads(raw) if raw else None
|
|
elapsed = round((time.monotonic()-started)*1000,3)
|
|
rid = response.getheader('X-Request-Id','')
|
|
if record:
|
|
self.http_rows.append(dict(timestamp_utc=stamp, method=method, path=path, status=response.status,
|
|
duration_ms=elapsed, request_id=rid))
|
|
if expected is not None:
|
|
allowed = [expected] if isinstance(expected,int) else expected
|
|
require(response.status in allowed, f'{method} {path}: expected {allowed}, received {response.status}')
|
|
require(bool(rid),'missing request ID')
|
|
if response.status >= 400:
|
|
require(data['error']['request_id']==rid,'error request ID mismatch')
|
|
require(not any(v in json.dumps(data) for v in self.passwords+self.tokens),'credential leak in error')
|
|
if response.status==401:
|
|
scheme='Basic' if path=='/api/v1/auth/sessions' else 'Bearer'
|
|
require(response.getheader('WWW-Authenticate','').startswith(scheme),'missing auth challenge')
|
|
return dict(status=response.status, data=data, duration_ms=elapsed, request_id=rid)
|
|
finally:
|
|
conn.close()
|
|
|
|
def wait_ready(self):
|
|
deadline=time.monotonic()+45
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
if self.call('GET','/health/ready',expected=None,record=False)['status']==200:
|
|
return
|
|
except (OSError,http.client.HTTPException):
|
|
pass
|
|
time.sleep(.25)
|
|
raise RuntimeError('API readiness timeout')
|
|
|
|
def wait_db(self):
|
|
deadline=time.monotonic()+45
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
if self.sql('SELECT 1;')=='1':
|
|
return
|
|
except RuntimeError:
|
|
pass
|
|
time.sleep(.25)
|
|
raise RuntimeError('PostgreSQL readiness timeout')
|
|
|
|
def write_override(self):
|
|
self.override.write_text(json.dumps({'services':{'api':{
|
|
'container_name':self.project+'-api', 'image':self.image,
|
|
'environment':self.api_env,
|
|
'volumes':[{'type':'bind','source':str(self.private/'certs'),'target':'/run/certs','read_only':True}]}}}))
|
|
|
|
def recreate_api(self):
|
|
self.write_override()
|
|
self.compose('up','-d','--no-build','--no-deps','--force-recreate','api')
|
|
self.wait_ready()
|
|
|
|
def prepare(self):
|
|
certs=self.private/'certs'
|
|
certs.mkdir(mode=0o700)
|
|
self.run(['openssl','genrsa','-out',str(certs/'private_key.pem'),'2048'])
|
|
self.run(['openssl','rsa','-in',str(certs/'private_key.pem'),'-pubout','-out',str(certs/'public_key.pem')])
|
|
(certs/'private_key.pem').chmod(0o600)
|
|
(certs/'public_key.pem').chmod(0o644)
|
|
if platform.system()=='Linux':
|
|
self.run(['setfacl','-m','u:65532:rx',str(certs)])
|
|
self.run(['setfacl','-m','u:65532:r',str(certs/'private_key.pem'),str(certs/'public_key.pem')])
|
|
content=(ROOT/'.env.example').read_text()
|
|
values={'HOST_PORT':str(self.port),'POSTGRES_USER':'acceptance','POSTGRES_DATABASE':'acceptance',
|
|
'POSTGRES_PASSWORD':self.passwords[0],'REDIS_PASSWORD':self.passwords[1],
|
|
'POSTGRES_RUN_MIGRATIONS':'true','POSTGRES_MAX_CONNECTIONS':'10','SESSION_TTL_SECONDS':'3600',
|
|
'POSTGRES_CONNECT_TIMEOUT':'500ms','POSTGRES_QUERY_TIMEOUT':'2s'}
|
|
for key,value in values.items():
|
|
content=re.sub(r'^'+key+r'=.*$',key+'='+value,content,flags=re.MULTILINE)
|
|
fd=os.open(self.envfile,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600)
|
|
with os.fdopen(fd,'w') as stream:
|
|
stream.write(content)
|
|
self.write_override()
|
|
self.summary['docker']=self.run(['docker','version','--format','{{.Server.Version}}'])
|
|
self.summary['resources']=self.run(['docker','info','--format','{{.OSType}}/{{.Architecture}} CPUs={{.NCPU}} memory_bytes={{.MemTotal}}'])
|
|
self.compose('config','--quiet',record='docker compose [isolated project] config --quiet')
|
|
self.compose('build','api',record='docker compose [isolated project] build api')
|
|
self.summary['image']=self.run(['docker','image','inspect','--format','{{.Id}}',self.image])
|
|
source_files=sorted((ROOT/'service').rglob('*.go'))+sorted((ROOT/'service/migrations').glob('*.sql'))
|
|
digest=hashlib.sha256()
|
|
for p in source_files:
|
|
digest.update(str(p.relative_to(ROOT)).encode()+b'\0'+p.read_bytes())
|
|
self.summary['service_source_sha256']=digest.hexdigest()
|
|
self.started=True
|
|
self.compose('up','-d','--no-build','--wait','--wait-timeout','120',record='docker compose [isolated project] up -d --no-build --wait --wait-timeout 120')
|
|
self.wait_ready()
|
|
return {'new_project':self.project,'initial_volumes':self.volumes()}
|
|
|
|
def ids(self):
|
|
return {name:self.compose('ps','-q',name) for name in ('api','postgres','redis')}
|
|
|
|
def volumes(self):
|
|
return self.run(['docker','volume','ls','--filter','label=com.docker.compose.project='+self.project,'--format','{{.Name}}']).splitlines()
|
|
|
|
def public_checks(self):
|
|
self.run(['make','check'],record='make check')
|
|
self.run(['make','test','BASE_URL=http://127.0.0.1:'+str(self.port)],record='make test BASE_URL=http://127.0.0.1:'+str(self.port))
|
|
return {'check_exit':0,'smoke_exit':0,'covers':['wrong password','Basic rejected on private API','X-User-Id rejected/ignored','logout and revoked token','chat access and pagination']}
|
|
|
|
def actor(self,label):
|
|
username='verify_'+uuid.uuid4().hex[:16]
|
|
password=secrets.token_urlsafe(24)
|
|
user=self.call('POST','/api/v1/users',201,body=dict(username=username,display_name='Verify '+label,password=password))['data']
|
|
session=self.call('POST','/api/v1/auth/sessions',201,basic=username+':'+password)['data']
|
|
self.tokens.append(session['token'])
|
|
return dict(label=label,id=user['id'],token=session['token'],username=username,password=password,expires_at=session['expires_at'])
|
|
|
|
def schema(self):
|
|
names=self.sql("SELECT string_agg(tablename, ',' ORDER BY tablename) FROM pg_tables WHERE schemaname='public';")
|
|
require(set(names.split(','))=={'users','chats','chat_members','messages','refresh_tokens','versions'},'unexpected schema tables')
|
|
version=self.sql('SELECT version FROM versions;')
|
|
require(version=='5','migration version not 5')
|
|
constraints=self.sql("SELECT conrelid::regclass::text || '|' || conname || '|' || contype::text || '|' || pg_get_constraintdef(oid) FROM pg_constraint WHERE connamespace='public'::regnamespace ORDER BY 1;")
|
|
(self.out/'schema.txt').write_text('tables='+names+'\nversion='+version+'\n'+constraints+'\n')
|
|
require('users_username_unique|u|' in constraints,'username uniqueness missing')
|
|
require(len([line for line in constraints.splitlines() if '|f|' in line])==4,'expected four FK constraints')
|
|
self.alice,self.bob,self.eve=[self.actor(label) for label in ('alice','bob','eve')]
|
|
self.chat=self.call('POST','/api/v1/chats',201,actor=self.alice,
|
|
body={'title':'Verification chat','member_ids':[self.alice['id'],self.bob['id']]})['data']
|
|
self.path='/api/v1/chats/'+self.chat['id']
|
|
self.key='chat:'+self.chat['id']
|
|
self.call('POST',self.path+'/messages',201,actor=self.alice,body={'text':'Persistent verification message'})
|
|
algorithm=self.sql(f"SELECT split_part(password_hash,'$',2) FROM users WHERE id='{self.alice['id']}';")
|
|
profile=self.sql(f"SELECT split_part(password_hash,'$',4) FROM users WHERE id='{self.alice['id']}';")
|
|
require(algorithm=='argon2id' and profile=='m=65536,t=3,p=4','unexpected password KDF')
|
|
# Do not export claims or nonce values; verify the entropy source's output size.
|
|
payload=json.loads(base64.urlsafe_b64decode(self.alice['token'].split('.')[1]+'=='))
|
|
nonce=base64.urlsafe_b64decode(payload['nonce']+'==')
|
|
require(len(nonce)==32,'expected 32 random nonce bytes')
|
|
return dict(version=int(version),fk_count=4,password_kdf=algorithm,password_profile=profile,nonce_bytes=len(nonce))
|
|
|
|
def counts(self):
|
|
return json.loads(self.sql("SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM refresh_tokens),'version',(SELECT version FROM versions));"))
|
|
|
|
def migration_repeat(self):
|
|
before=self.counts()
|
|
self.recreate_api()
|
|
after=self.counts()
|
|
require(before==after,'re-running migrations changed counts or version')
|
|
return dict(before=before,after=after)
|
|
|
|
def persistence(self):
|
|
before,volumes=self.ids(),self.volumes()
|
|
self.run(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh',
|
|
'BASE_URL=http://127.0.0.1:'+str(self.port)],record='make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:'+str(self.port))
|
|
after=self.ids()
|
|
require(all(before[n]!=after[n] for n in before),'a dependency was not recreated')
|
|
require(volumes==self.volumes(),'volumes changed during restart')
|
|
me=self.call('GET','/api/v1/users/me',actor=self.alice)['data']
|
|
require(me['id']==self.alice['id'],'own previous session lost')
|
|
history=self.call('GET',self.path+'/messages?limit=100',actor=self.bob)['data']
|
|
require(any(x['text']=='Persistent verification message' for x in history['items']),'own message lost')
|
|
return dict(before=before,after=after,volumes_preserved=volumes,previous_session_valid=True)
|
|
|
|
def registration_race(self):
|
|
username='race_'+uuid.uuid4().hex[:16]
|
|
barrier=threading.Barrier(8)
|
|
body={'username':username,'display_name':'Concurrent registration','password':secrets.token_urlsafe(24)}
|
|
def worker(_):
|
|
barrier.wait(timeout=10)
|
|
return self.call('POST','/api/v1/users',expected=[201,409],body=body)['status']
|
|
with futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
statuses=list(pool.map(worker,range(8)))
|
|
count=int(self.sql(f"SELECT count(*) FROM users WHERE username='{username}';"))
|
|
require(statuses.count(201)==1 and statuses.count(409)==7 and count==1,'registration race mismatch')
|
|
return dict(concurrency=8,created=1,conflicts=7,rows=count)
|
|
|
|
def rollback(self):
|
|
title='rollback_'+uuid.uuid4().hex
|
|
before=self.counts()
|
|
self.call('POST','/api/v1/chats',400,actor=self.alice,
|
|
body={'title':title,'member_ids':[self.alice['id'],str(uuid.uuid4())]})
|
|
after=self.counts()
|
|
require(before==after,'partial chat or member data survived rollback')
|
|
require(self.sql(f"SELECT count(*) FROM chats WHERE title='{title}';")=='0','partial chat persisted')
|
|
return dict(before=before,after=after,failed_step='member INSERT FK after chat INSERT',http_status=400)
|
|
|
|
def history(self):
|
|
chat=self.chat['id']; user=self.alice['id']
|
|
self.sql(f"INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),'{chat}','{user}','History fixture '||i, '2020-01-01'::timestamptz+i*interval '1 microsecond' FROM generate_series(1,20000) i; ANALYZE messages;")
|
|
require(self.sql(f"SELECT count(*) FROM messages WHERE chat_id='{chat}';")=='20001','history fixture size mismatch')
|
|
cursor=self.sql(f"SELECT created_at::text || '|' || id::text FROM messages WHERE chat_id='{chat}' ORDER BY created_at,id OFFSET 10000 LIMIT 1;").split('|')
|
|
plans={}
|
|
for label,condition in [('first',''),('deep',f" AND (created_at,id)>('{cursor[0]}'::timestamptz,'{cursor[1]}'::uuid)")]:
|
|
query=f"SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id='{chat}'{condition} ORDER BY created_at,id LIMIT 101"
|
|
plan=json.loads(self.sql('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) '+query+';'))
|
|
(self.out/('explain-'+label+'.json')).write_text(json.dumps(plan,indent=2)+'\n')
|
|
(self.out/('explain-'+label+'.txt')).write_text(self.sql('EXPLAIN (ANALYZE, BUFFERS) '+query+';')+'\n')
|
|
nodes=[]
|
|
def walk(node):
|
|
nodes.append(node)
|
|
for child in node.get('Plans',[]): walk(child)
|
|
walk(plan[0]['Plan'])
|
|
require(any(n.get('Index Name')=='idx_messages_chat_id_created_at_id' for n in nodes),'history index not selected')
|
|
require(not any(n.get('Node Type')=='Seq Scan' and n.get('Relation Name')=='messages' for n in nodes),'history full scan')
|
|
plans[label]=dict(execution_ms=plan[0]['Execution Time'],returned=plan[0]['Plan']['Actual Rows'],nodes=[n['Node Type'] for n in nodes])
|
|
expected=self.sql(f"SELECT string_agg(id::text,',' ORDER BY created_at,id) FROM messages WHERE chat_id='{chat}';").split(',')
|
|
ids=[]; path=self.path+'/messages?limit=100'
|
|
for _ in range(205):
|
|
page=self.call('GET',path,actor=self.bob)['data']
|
|
ids.extend(x['id'] for x in page['items'])
|
|
if page['next_cursor'] is None: break
|
|
path=self.path+'/messages?limit=100&cursor='+quote(page['next_cursor'],safe='')
|
|
require(ids==expected and len(set(ids))==20001,'history traversal lost, duplicated or reordered rows')
|
|
return dict(messages=len(ids),page_size=100,plans=plans)
|
|
|
|
def expiry(self):
|
|
self.api_env['SESSION_TTL_SECONDS']='2'
|
|
self.recreate_api()
|
|
try:
|
|
actor=self.actor('expiry')
|
|
self.call('GET','/api/v1/users/me',actor=actor)
|
|
expires=dt.datetime.fromisoformat(actor['expires_at'].replace('Z','+00:00'))
|
|
wait=max(0,(expires-dt.datetime.now(dt.timezone.utc)).total_seconds())+.25
|
|
time.sleep(wait)
|
|
self.call('GET','/api/v1/users/me',401,actor=actor)
|
|
return dict(session_ttl_seconds=2,legacy_jwt_refresh='24h',before=200,after=401,wait_seconds=round(wait,3))
|
|
finally:
|
|
self.api_env['SESSION_TTL_SECONDS']='3600'
|
|
self.recreate_api()
|
|
|
|
def redis_failure(self):
|
|
since=utc()
|
|
self.redis('DEL',self.key)
|
|
self.call('GET',self.path,actor=self.alice)
|
|
self.call('GET',self.path,actor=self.alice)
|
|
self.call('GET',self.path,404,actor=self.eve)
|
|
raw=self.compose('logs','--no-color','--since',since,'api')
|
|
require(raw.count('chat_db_lookup')==1 and 'chat_cache_hit' in raw,'cache miss/hit behavior mismatch')
|
|
try:
|
|
self.compose('stop','redis')
|
|
response=self.call('GET',self.path,actor=self.alice)
|
|
require(response['duration_ms']<1000,'Redis failure delayed request >=1s')
|
|
self.call('GET',self.path,401)
|
|
self.recreate_api()
|
|
self.call('GET',self.path,actor=self.alice)
|
|
finally:
|
|
self.compose('start','redis')
|
|
deadline=time.monotonic()+30
|
|
while True:
|
|
try:
|
|
if self.redis('PING')=='PONG': break
|
|
except RuntimeError: pass
|
|
require(time.monotonic()<deadline,'Redis recovery timed out')
|
|
time.sleep(.2)
|
|
self.redis('DEL',self.key)
|
|
self.call('GET',self.path,actor=self.bob)
|
|
self.call('GET',self.path,actor=self.bob)
|
|
return dict(cold_sql=1,warm_cache=True,outsider_status=404,outage_ms=response['duration_ms'],startup_without_redis=True,recovered=True)
|
|
|
|
def postgres_failure(self):
|
|
self.call('GET',self.path,actor=self.alice)
|
|
try:
|
|
self.compose('stop','postgres')
|
|
response=self.call('GET',self.path,503,actor=self.alice)
|
|
require(response['data']['error']['code']=='unavailable','wrong database outage error code')
|
|
require(response['duration_ms']<3000,'PostgreSQL failure exceeded bounded response')
|
|
self.call('GET',self.path,401)
|
|
self.call('GET',self.path,401,headers={'Authorization':'Bearer invalid-token'})
|
|
readiness=self.call('GET','/health/ready',503)
|
|
self.call('GET','/health/live',200)
|
|
finally:
|
|
self.compose('start','postgres')
|
|
self.wait_db()
|
|
self.wait_ready()
|
|
self.call('GET',self.path,actor=self.alice)
|
|
return dict(valid_session_status=503,anonymous_status=401,invalid_token_status=401,ready=503,live=200,
|
|
outage_ms=response['duration_ms'],readiness_ms=readiness['duration_ms'],previous_session_recovered=True)
|
|
|
|
@contextlib.contextmanager
|
|
def table_lock(self):
|
|
proc=subprocess.Popen(self.cmd+['exec','-T','postgres','sh','-c',
|
|
'exec psql -X -q -At -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"'],
|
|
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,cwd=ROOT,env=self.env)
|
|
try:
|
|
proc.stdin.write("SET statement_timeout='5s'; SET idle_in_transaction_session_timeout='5s'; BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE;\n\\echo LOCKED\n")
|
|
proc.stdin.flush()
|
|
require(bool(select.select([proc.stdout],[],[],8)[0]) and proc.stdout.readline().strip()=='LOCKED','could not acquire table lock')
|
|
yield
|
|
finally:
|
|
if proc.poll() is None:
|
|
try: proc.communicate('ROLLBACK;\n\\q\n',timeout=8)
|
|
except (subprocess.TimeoutExpired,BrokenPipeError):
|
|
proc.kill(); proc.communicate()
|
|
|
|
def pool_and_deadline(self):
|
|
self.api_env['POSTGRES_MAX_CONNECTIONS']='2'
|
|
self.recreate_api()
|
|
self.redis('DEL',self.key)
|
|
peak=0
|
|
with self.table_lock():
|
|
with futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
barrier=threading.Barrier(8)
|
|
def worker(_):
|
|
barrier.wait(timeout=10)
|
|
return self.call('GET',self.path,503,actor=self.alice)
|
|
jobs=[pool.submit(worker,i) for i in range(8)]
|
|
while not all(job.done() for job in jobs):
|
|
count=int(self.sql("SELECT count(*) FROM pg_stat_activity WHERE application_name='mtusi-api';"))
|
|
peak=max(peak,count)
|
|
time.sleep(.05)
|
|
results=[job.result() for job in jobs]
|
|
require(peak==2,'pool maximum not applied/observed')
|
|
require(all(1500<r['duration_ms']<3000 for r in results),'request budget not enforced')
|
|
self.api_env['POSTGRES_MAX_CONNECTIONS']='10'
|
|
self.recreate_api()
|
|
self.call('GET',self.path,actor=self.alice)
|
|
raw=self.compose('logs','--no-color','api')
|
|
require('max_conns=10' in raw and 'statement_timeout_ms=2000' in raw and 'connect_timeout=500ms' in raw,'pool startup configuration missing')
|
|
return dict(test_limit=2,observed_peak=peak,requests=8,status=503,durations_ms=[r['duration_ms'] for r in results],
|
|
restored_limit=10,query_timeout_ms=2000,connect_timeout_ms=500)
|
|
|
|
def configuration(self):
|
|
observed={}
|
|
try:
|
|
for label,session,legacy,expected in [('default','','',3600),('legacy','','24h',86400),('primary','2','invalid-legacy',2)]:
|
|
self.api_env.update(SESSION_TTL_SECONDS=session,JWT_REFRESH_TOKEN_LIFETIME=legacy)
|
|
self.recreate_api()
|
|
result=self.call('POST','/api/v1/auth/sessions',201,basic=self.alice['username']+':'+self.alice['password'])['data']
|
|
self.tokens.append(result['token'])
|
|
claims=json.loads(base64.urlsafe_b64decode(result['token'].split('.')[1]+'=='))
|
|
require(claims['exp']-claims['iat']==expected,'session lifetime precedence mismatch')
|
|
observed[label]=expected
|
|
invalid=[('SESSION_TTL_SECONDS','0'),('SESSION_TTL_SECONDS','abc'),
|
|
('POSTGRES_MAX_CONNECTIONS','0'),('POSTGRES_CONNECT_TIMEOUT','0s'),('POSTGRES_QUERY_TIMEOUT','0s')]
|
|
self.api_env={'SESSION_TTL_SECONDS':'3600'}
|
|
self.write_override()
|
|
for name,value in invalid:
|
|
proc=subprocess.run(self.cmd+['run','--rm','--no-deps','-e',name+'='+value,'api'],capture_output=True,text=True,cwd=ROOT,env=self.env,timeout=15)
|
|
require(proc.returncode!=0 and 'load server config' in proc.stderr,'invalid config was not rejected')
|
|
return dict(lifetime_seconds=observed,invalid_configurations_rejected=len(invalid))
|
|
finally:
|
|
self.api_env={'SESSION_TTL_SECONDS':'3600'}
|
|
self.recreate_api()
|
|
|
|
def contract_edges(self):
|
|
before=len(self.http_rows)
|
|
self.call('GET','/api/v1/chats/not-a-uuid',400,actor=self.alice)
|
|
self.call('GET','/api/v1/chats?limit=101',400,actor=self.alice)
|
|
self.call('GET',self.path+'/messages?cursor=invalid',400,actor=self.alice)
|
|
self.call('GET',self.path+'/messages?limit=1&limit=2',400,actor=self.alice)
|
|
for members,title in [([self.alice['id']],' '),([self.bob['id']],'No creator'),([self.alice['id']]*2,'Duplicates')]:
|
|
self.call('POST','/api/v1/chats',400,actor=self.alice,body={'title':title,'member_ids':members})
|
|
self.call('POST',self.path+'/messages',400,actor=self.alice,body={'text':'Spoofed','sender_id':self.bob['id']})
|
|
for fields in [dict(username='UPPER',display_name='Valid',password='Valid-password-123'),
|
|
dict(username='valid_name',display_name=' ',password='Valid-password-123'),
|
|
dict(username='valid_name',display_name='Valid',password='short'),
|
|
dict(username='valid_name',display_name='Valid',password='Valid-password-123',extra=True)]:
|
|
self.call('POST','/api/v1/users',400,body=fields)
|
|
self.call('POST','/api/v1/users',400,headers={'Content-Type':'application/json'},raw='{')
|
|
self.call('POST','/api/v1/users',415,headers={'Content-Type':'text/plain'},raw='{}')
|
|
return dict(cases=len(self.http_rows)-before,invalid_uuid_status=400,unknown_fields_status=400,malformed_json_status=400,unsupported_media_status=415)
|
|
|
|
def auth_additional(self):
|
|
# Explicit checks beyond the public smoke: forged signatures and unknown user.
|
|
bad=self.alice['token'][:-8]+('A' if self.alice['token'][-8]!='A' else 'B')+self.alice['token'][-7:]
|
|
self.call('GET','/api/v1/users/me',401,headers={'Authorization':'Bearer '+bad})
|
|
self.call('POST','/api/v1/auth/sessions',401,basic='unknown_'+uuid.uuid4().hex+':wrong-password')
|
|
return dict(forged_signature=401,unknown_username=401)
|
|
|
|
def audit(self):
|
|
tracked=self.run(['git','ls-files','-z']).split('\0')
|
|
dangerous=[name for name in tracked if name and (name=='.env' or name.startswith(('certs/','secrets/','volumes/')) or name.endswith(('.pem','.key','.p12','.pfx')))]
|
|
require(not dangerous,'private state tracked in Git')
|
|
self.run(['git','diff','--exit-code','upstream/master','--','README.md','contracts','tests'])
|
|
findings=[]
|
|
for name in tracked:
|
|
path=ROOT/name
|
|
if not name or not path.is_file(): continue
|
|
try: content=path.read_text()
|
|
except (UnicodeError,OSError): continue
|
|
if re.search(r'-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+',content):
|
|
findings.append(name)
|
|
require(not findings,'credential-like material tracked: '+', '.join(findings))
|
|
return dict(private_paths_tracked=0,jwt_or_private_key_findings=0,readme_contract_tests_unchanged=True,
|
|
scope='current tracked tree; complete Git history is not certified')
|
|
|
|
def finish(self):
|
|
if not self.started: return
|
|
# Restore temporary API overrides, ensure dependencies recover, then stop only our project.
|
|
self.api_env={'SESSION_TTL_SECONDS':'3600'}
|
|
self.write_override()
|
|
self.compose('start','postgres','redis')
|
|
self.wait_db()
|
|
self.recreate_api()
|
|
self.compose('up','-d','--no-build','--wait','--wait-timeout','60')
|
|
self.transcript.append('\nFinal isolated state before shutdown:\n'+self.compose('ps','-a')+'\n')
|
|
raw=self.compose('logs','--no-color','--timestamps','api')
|
|
selected=[line for line in raw.splitlines() if any(marker in line for marker in
|
|
('postgres_pool_config','HTTP server started','redis unavailable at startup'))]
|
|
(self.out/'startup.txt').write_text(self.safe('\n'.join(selected))+'\n')
|
|
volumes=self.volumes()
|
|
self.compose('down')
|
|
require(volumes==self.volumes(),'cleanup deleted volumes')
|
|
self.summary['retained_volumes']=volumes
|
|
self.summary['isolated_stack_stopped']=True
|
|
self.summary['ended_utc']=utc()
|
|
self.flush()
|
|
|
|
def execute(self):
|
|
try:
|
|
for name,action in [('fresh_build_and_start',self.prepare),('public_check_and_smoke',self.public_checks),
|
|
('schema_kdf_and_session_randomness',self.schema),('configuration_compatibility',self.configuration),
|
|
('contract_validation_edges',self.contract_edges),('repeat_migrations',self.migration_repeat),
|
|
('full_persistence',self.persistence),('username_registration_race',self.registration_race),
|
|
('chat_creation_rollback',self.rollback),('history_20000_and_explain',self.history),
|
|
('session_expiry_seconds',self.expiry),('additional_authentication',self.auth_additional),
|
|
('redis_cache_and_failure',self.redis_failure),('postgres_failure_and_recovery',self.postgres_failure),
|
|
('pool_limit_and_sql_deadline',self.pool_and_deadline),('tracked_secrets_and_contract_audit',self.audit)]:
|
|
self.check(name,action)
|
|
self.summary['success']=True
|
|
finally:
|
|
self.finish()
|
|
self.flush()
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--port',type=int,default=18084)
|
|
parser.add_argument('--output',default='evidence/levels-3-4')
|
|
args=parser.parse_args()
|
|
Verification(args.port,args.output).execute()
|
|
print('PASS: levels 3/4 verification finished; isolated volumes retained; original stack unchanged.',flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|