295 lines
18 KiB
Python
295 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Black-box and real database/fault tests. Synthetic data; never print credentials.
|
|
Run inside Compose tools (Docker socket needed only for DB inspection/faults).
|
|
"""
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from urllib.parse import quote
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, check, timestamp
|
|
|
|
|
|
def compose(*args, input=None):
|
|
result = subprocess.run(['docker', 'compose', '--env-file', '.env', *args],
|
|
input=input, text=True, capture_output=True, timeout=75)
|
|
check(result.returncode == 0, 'compose ' + args[0] + ' failed (inspect service health locally)')
|
|
return result.stdout.strip()
|
|
|
|
|
|
def db(code):
|
|
source = "import json, psycopg\nfrom src.config import secret\nfrom src.store import public\nwith psycopg.connect(secret('DATABASE_URL')) as conn:\n" + ''.join(' ' + line + '\n' for line in code.splitlines())
|
|
return compose('exec', '-T', 'api', 'python', '-', input=source)
|
|
|
|
|
|
def counts():
|
|
return json.loads(db("print(json.dumps([conn.execute('SELECT count(*) FROM chats').fetchone()[0], conn.execute('SELECT count(*) FROM chat_members').fetchone()[0]]))"))
|
|
|
|
|
|
def walk(c, actor, chat, limit=17):
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=' + str(limit)
|
|
items = []
|
|
for _ in range(1000):
|
|
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
|
items.extend(page['items'])
|
|
if page['next_cursor'] is None:
|
|
return items
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=' + str(limit) + '&cursor=' + quote(page['next_cursor'], safe='')
|
|
raise AssertionError('pagination failed to terminate')
|
|
|
|
|
|
def core(base):
|
|
c = Client(base)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id']
|
|
body = dict(alice['registration'], username='race_' + uuid.uuid4().hex[:16])
|
|
def register(_):
|
|
return Client(base).call('POST', '/api/v1/users', [201, 409], body=body)['status']
|
|
with ThreadPoolExecutor(max_workers=12) as executor:
|
|
statuses = list(executor.map(register, range(12)))
|
|
check(statuses.count(201) == 1 and statuses.count(409) == 11, 'duplicate registration race not serialized')
|
|
print('PASS: concurrent same username: 12 requests, 1 x 201, 11 x 409')
|
|
|
|
before = counts()
|
|
c.call('POST', '/api/v1/chats', 400, actor=alice,
|
|
body={'title': 'rollback', 'member_ids': [alice['id'], str(uuid.uuid4())]})
|
|
check(counts() == before, 'failed composite operation left a parent or membership behind')
|
|
for members in ([alice['id'], alice['id']], [bob['id']]):
|
|
c.call('POST', '/api/v1/chats', 400, actor=alice, body={'title': 'invalid', 'member_ids': members})
|
|
print('PASS: unknown member FK after parent INSERT rolls back chat and memberships; duplicate/missing owner -> 400')
|
|
|
|
for body_bad in ({'text': 'a\x00b'}, {'text': 'x', 'sender_id': eve['id']}, {'text': ' '}, {'text': 123}, {'text': 'x' * 4001}):
|
|
c.call('POST', path + '/messages', 400, actor=alice, body=body_bad)
|
|
for bad_user in (dict(alice['registration'], password='short'), dict(alice['registration'], password=' ' * 12), dict(alice['registration'], admin=True),
|
|
dict(alice['registration'], display_name='a\x00b'), dict(alice['registration'], display_name=' '), dict(alice['registration'], username='BAD')):
|
|
c.call('POST', '/api/v1/users', 400, body=bad_user)
|
|
c.call('POST', '/api/v1/chats', 400, actor=alice, body={'title': 'a\x00b', 'member_ids': [alice['id']]})
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
print('PASS: NUL in display_name/title/text -> 400; PostgreSQL readiness remains 200')
|
|
c.call('POST', '/api/v1/users', 400, raw=b'{', headers={'Content-Type': 'application/json'})
|
|
for limit in ('0', '101', 'abc', '1.5'):
|
|
c.call('GET', path + '/messages?limit=' + limit, 400, actor=alice)
|
|
for cursor in ('bad', '!', base64.urlsafe_b64encode(b'{}').decode()):
|
|
c.call('GET', path + '/messages?cursor=' + quote(cursor), 400, actor=alice)
|
|
for suffix in ('', '/messages'):
|
|
c.call('GET', path + suffix, 404, actor=eve)
|
|
c.call('POST', path + '/messages', 404, actor=eve, body={'text': 'denied'})
|
|
c.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=alice)
|
|
print('PASS: strict types/extra fields, malformed JSON, UUID, limits/cursors, blank/oversized text, outsider isolation')
|
|
|
|
def send(i):
|
|
return Client(base).message(alice if i % 2 == 0 else bob, chat, 'parallel-' + str(i))['json']
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
sent = list(executor.map(send, range(40)))
|
|
# Deliberately force equal timestamps to exercise the UUID tie-breaker.
|
|
db("conn.execute('UPDATE messages SET created_at=%s WHERE chat_id=%s', ('2026-01-01T00:00:00Z', '" + chat['id'] + "'))")
|
|
items = walk(c, bob, chat)
|
|
check({m['id'] for m in items} == {m['id'] for m in sent}, 'lost concurrent message')
|
|
check(len(items) == 40 and len({m['id'] for m in items}) == 40, 'duplicate page/message')
|
|
check(items == sorted(items, key=lambda x: (timestamp(x['created_at']), uuid.UUID(x['id']).int)), 'incorrect tuple ordering')
|
|
first = c.call('GET', path + '/messages?limit=1', actor=alice, schema='MessagePage')['json']
|
|
c.call('GET', path + '/messages?cursor=' + quote(first['next_cursor'] + '!'), 400, actor=alice)
|
|
second_chat = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'cursor scope', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
|
c.call('GET', '/api/v1/chats/' + second_chat['id'] + '/messages?cursor=' + quote(first['next_cursor']), 400, actor=alice)
|
|
continuation = c.call('GET', path + '/messages?limit=100&cursor=' + quote(first['next_cursor']), actor=bob, schema='MessagePage')['json']
|
|
check(len(continuation['items']) == 39 and continuation['next_cursor'] is None, 'changing limit failed')
|
|
chats_page = c.call('GET', '/api/v1/chats?limit=1', actor=alice, schema='ChatPage')['json']
|
|
check(chats_page['next_cursor'], 'missing chat cursor')
|
|
c.call('GET', '/api/v1/chats?cursor=' + quote(chats_page['next_cursor']), 400, actor=bob)
|
|
confirmed = c.message(alice, chat, 'read-after-write')['json']
|
|
check(confirmed in walk(c, bob, chat), 'confirmed write invisible on next read')
|
|
print('PASS: 40 concurrent writes; tied timestamp UUID ordering; complete keyset traversal, scoped cursors, changing limit, read-after-write')
|
|
|
|
for header in ('Basic invalid!', 'Bearer unknown', 'Digest xyz'):
|
|
c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': header})
|
|
wrong = base64.b64encode((alice['registration']['username'] + ':wrong-password').encode()).decode()
|
|
unknown = base64.b64encode(('unknown_' + uuid.uuid4().hex[:10] + ':wrong-password').encode()).decode()
|
|
a = c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + wrong})
|
|
b = c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + unknown})
|
|
check(a['json']['error']['message'] == b['json']['error']['message'], 'credential error leaks user existence')
|
|
c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': alice['id']})
|
|
c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + wrong})
|
|
c.call('GET', '/api/v1/users/me', actor=alice, headers={'X-User-Id': eve['id']}, schema='User')
|
|
|
|
# Entire UTF-8 range and colon survive the KDF and Basic split(':', 1).
|
|
registration = {'username': 'utf_' + uuid.uuid4().hex[:16], 'display_name': 'Юникод', 'password': '🙂:' * 64}
|
|
user = c.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
|
|
credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
|
|
session = c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
|
|
check(session['user'] == user, 'UTF8/128-code-point password failed')
|
|
c.call('POST', '/api/v1/auth/sessions', 400, body={}, headers={'Authorization': 'Basic ' + credential})
|
|
hash_status = db("row = conn.execute('SELECT password_hash FROM users WHERE id=%s', ('" + user['id'] + "',)).fetchone()\nprint(row[0].startswith('$argon2id$'))")
|
|
check(hash_status == 'True', 'password is not an Argon2id hash')
|
|
print('PASS: malformed Basic, wrong/unknown credentials uniform 401, X-User-Id ignored, 128-code-point UTF8 colon password, stored Argon2id')
|
|
|
|
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
key = 'profile:' + alice['id']
|
|
ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key))
|
|
check(0 < ttl <= 60, 'Valkey profile missing TTL')
|
|
cached = json.loads(compose('exec', '-T', 'valkey', 'valkey-cli', 'GET', key))
|
|
check(cached == alice['user'], 'Valkey public profile cache mismatch')
|
|
def hits():
|
|
info = compose('exec', '-T', 'valkey', 'valkey-cli', 'INFO', 'stats')
|
|
return int(next(line.split(':')[1] for line in info.splitlines() if line.startswith('keyspace_hits:')))
|
|
hits_before = hits()
|
|
for _ in range(3):
|
|
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
check(hits() >= hits_before + 3, 'Valkey is not serving profile hits')
|
|
check(int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key)) <= ttl, 'cache hits extend TTL')
|
|
compose('exec', '-T', 'valkey', 'valkey-cli', 'EXPIRE', key, '1')
|
|
time.sleep(1.1)
|
|
check(compose('exec', '-T', 'valkey', 'valkey-cli', 'EXISTS', key) == '0', 'cache TTL did not expire')
|
|
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
check(int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key)) > 0, 'expired profile not refilled')
|
|
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
|
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
|
c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=alice)
|
|
print('PASS: real Valkey GET hits, TTL <=60 without sliding; expiry/refill; logout rejects cached profile and repeated logout')
|
|
|
|
|
|
def expiry(base):
|
|
c = Client(base)
|
|
actor = c.user('ttl')
|
|
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
started = time.monotonic()
|
|
time.sleep(2.2)
|
|
c.call('GET', '/api/v1/users/me', 401, actor=actor)
|
|
c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=actor)
|
|
print('PASS: SESSION_TTL_SECONDS=2: immediate authentication succeeds; after %.2fs cached profile cannot bypass expiry/logout' % (time.monotonic() - started))
|
|
|
|
|
|
def wait_ready(c):
|
|
for _ in range(90):
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except AssertionError:
|
|
time.sleep(0.5)
|
|
raise AssertionError('dependency did not recover')
|
|
|
|
|
|
def failures(base):
|
|
c = Client(base, timeout=8)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
real_call = c.call
|
|
observed_latencies = []
|
|
def bounded_call(*args, **kwargs):
|
|
response = real_call(*args, **kwargs)
|
|
observed_latencies.append(response['elapsed'])
|
|
check(response['elapsed'] < 4.5, 'dependency request exceeded bounded 4.5 second budget')
|
|
return response
|
|
c.call = bounded_call
|
|
before = c.message(alice, chat, 'before failures')['json']
|
|
try:
|
|
compose('stop', 'valkey')
|
|
start = time.monotonic()
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Bearer ' + 'a' * 43})
|
|
c.call('GET', '/api/v1/users/me', 401)
|
|
c.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve)
|
|
after = c.message(bob, chat, 'write without cache')['json']
|
|
check(before in walk(c, bob, chat) and after in walk(c, alice, chat), 'cache outage lost data')
|
|
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
|
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
|
print('PASS: Valkey stopped: ready=200, authenticated read/write via PG, unknown/missing/revoked=401, outsider=404; scenario %.3fs' % (time.monotonic() - start))
|
|
finally:
|
|
compose('start', 'valkey')
|
|
# Cache recovery must not resurrect the revoked token.
|
|
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
|
try:
|
|
compose('stop', 'postgres')
|
|
start = time.monotonic()
|
|
response = c.call('GET', '/health/ready', 503)
|
|
c.call('GET', '/health/live', schema='Health')
|
|
c.call('GET', '/api/v1/users/me', 503, actor=bob)
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 503, actor=bob, body={'text': 'must not commit'})
|
|
c.call('GET', '/api/v1/users/me', 503, headers={'Authorization': 'Bearer ' + 'a' * 43})
|
|
c.call('GET', '/api/v1/users/me', 401)
|
|
check(response['json']['error']['code'] == 'unavailable', 'wrong bounded dependency error')
|
|
print('PASS: PostgreSQL stopped: ready/auth/write=503, live=200, no credentials=401, no cached authentication; scenario %.3fs' % (time.monotonic() - start))
|
|
finally:
|
|
compose('start', 'postgres')
|
|
recovery = time.monotonic()
|
|
wait_ready(c)
|
|
c.call('GET', '/api/v1/users/me', actor=bob, schema='User')
|
|
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
|
messages = walk(c, bob, chat)
|
|
check(before in messages and after in messages, 'confirmed data lost after database restart')
|
|
check(all(m['text'] != 'must not commit' for m in messages), 'rejected write committed')
|
|
print('PASS: PostgreSQL recovery %.3fs, same valid token/data survive, revoked token remains revoked, rejected write absent' % (time.monotonic() - recovery))
|
|
print('PASS: all %d HTTP requests bounded below 4.5s; maximum %.3fs' % (len(observed_latencies), max(observed_latencies)))
|
|
|
|
|
|
def explain(base):
|
|
c = Client(base)
|
|
alice = c.user('explain')
|
|
chat = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': '10000 history', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
|
source = r"""
|
|
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(), %s, %s, 'synthetic-' || i, '2026-01-01'::timestamptz + i * interval '1 microsecond' FROM generate_series(1,10000) i", (CHAT, USER))
|
|
conn.execute('ANALYZE messages')
|
|
print('Rows in test chat:', conn.execute('SELECT count(*) FROM messages WHERE chat_id=%s', (CHAT,)).fetchone()[0])
|
|
for position in (0,5000,9900):
|
|
if position:
|
|
key = conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET %s LIMIT 1', (CHAT,position)).fetchone()
|
|
sql = 'SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s'
|
|
args = (CHAT,*key,51)
|
|
else:
|
|
sql = 'SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s'
|
|
args = (CHAT,51)
|
|
rows = conn.execute('EXPLAIN (ANALYZE, BUFFERS) ' + sql, args).fetchall()
|
|
plan = '\n'.join(row[0] for row in rows)
|
|
assert 'Index Scan using messages_history' in plan and 'Seq Scan' not in plan, 'history index not used'
|
|
print('\nPosition:',position, '\n'+plan)
|
|
print('PostgreSQL:',conn.execute('SHOW server_version').fetchone()[0])
|
|
print('Durability:',conn.execute('SHOW fsync').fetchone()[0],conn.execute('SHOW synchronous_commit').fetchone()[0])
|
|
""".replace('CHAT', repr(chat['id'])).replace('USER', repr(alice['id']))
|
|
print(db(source.strip()))
|
|
start = time.monotonic()
|
|
items = walk(c, alice, chat, limit=100)
|
|
check(len(items) == 10000 and len({m['id'] for m in items}) == 10000, '10000-row API traversal lost/duplicated items')
|
|
print('PASS: actual EXPLAIN ANALYZE BUFFERS at first/middle/tail; API traversal 10000 rows, 100 pages, %.3fs' % (time.monotonic() - start))
|
|
|
|
|
|
def migrations(base):
|
|
def snapshot():
|
|
return db("tables = ['users','chats','chat_members','messages','sessions','schema_migrations']\ncounts = {name: conn.execute('SELECT count(*) FROM ' + name).fetchone()[0] for name in tables}\nexpiry = conn.execute(\"SELECT md5(coalesce(string_agg(token_hash::text || expires_at::text, ',' ORDER BY token_hash), '')) FROM sessions\").fetchone()[0]\nprint(json.dumps([counts,expiry]))")
|
|
before = snapshot()
|
|
for _ in range(2):
|
|
print(compose('exec', '-T', 'api', 'python', '-m', 'src.db'))
|
|
check(snapshot() == before, 'repeated migrations changed counts/session expiries')
|
|
print('PASS: two repeated migrations; all table counts and absolute session expiries unchanged:', json.loads(before)[0])
|
|
|
|
|
|
def persistence_meta(base):
|
|
c = Client(base)
|
|
actor = c.user('persistmeta')
|
|
digest = hashlib.sha256(actor['token'].encode()).hexdigest()
|
|
query = "row = conn.execute('SELECT expires_at,octet_length(token_hash) FROM sessions WHERE token_hash=%s', (bytes.fromhex('" + digest + "'),)).fetchone()\nprint(json.dumps(public(list(row))))"
|
|
initial = db(query)
|
|
check(json.loads(initial)[1] == 32, 'token hash not 256 bits')
|
|
# Resolve hooks explicitly, just like the public resilience test.
|
|
result = subprocess.run(['sh', 'scripts/restart.sh'], capture_output=True, timeout=75)
|
|
check(result.returncode == 0, 'restart failed')
|
|
wait_ready(c)
|
|
check(db(query) == initial, 'restart changed original expiry/hash length')
|
|
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
print('PASS: same opaque token authenticates after all services restart; absolute PG expires_at unchanged; only 32-byte token hash stored')
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('mode', choices=['core', 'expiry', 'failures', 'explain', 'migrations', 'persistence_meta'])
|
|
parser.add_argument('--base-url', default='http://api:8080')
|
|
args = parser.parse_args()
|
|
globals()[args.mode](args.base_url)
|