level 4: redis cache etc
This commit is contained in:
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-4 checks. Not part of make test. Restores Redis, PostgreSQL and SESSION_TTL."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
ENV_PATH = ROOT / '.env'
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, timeout=20):
|
||||
data = None if body is None else json.dumps(body).encode('utf-8')
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, method=method)
|
||||
if body is not None:
|
||||
req.add_header('Content-Type', 'application/json')
|
||||
for key, value in (headers or {}).items():
|
||||
req.add_header(key, value)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
payload = response.read()
|
||||
return response.status, payload, response.headers
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read(), exc.headers
|
||||
|
||||
|
||||
def compose(*args):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', *args],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('docker compose %s failed' % ' '.join(args))
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def psql(sql):
|
||||
proc = subprocess.run(
|
||||
[
|
||||
'docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||||
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
||||
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
||||
'-v', 'ON_ERROR_STOP=1', '-tAc', sql,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('psql failed')
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def redis_cli(*args):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'redis', 'redis-cli', *args],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('redis-cli failed')
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def wait_ready(seconds=45):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, _, _ = call('GET', '/health/ready', timeout=3)
|
||||
if status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise SystemExit('API did not become ready')
|
||||
|
||||
|
||||
def register(label):
|
||||
username = 'u_%s_%s' % (label, uuid.uuid4().hex[:12])
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
status, raw, _ = call('POST', '/api/v1/users', {
|
||||
'username': username,
|
||||
'display_name': label,
|
||||
'password': password,
|
||||
})
|
||||
if status != 201:
|
||||
raise SystemExit('register failed: %s' % status)
|
||||
user = json.loads(raw)
|
||||
return user, username, password
|
||||
|
||||
|
||||
def login(username, password):
|
||||
basic = base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii')
|
||||
status, raw, headers = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic})
|
||||
return status, raw, headers
|
||||
|
||||
|
||||
def bearer(token):
|
||||
return {'Authorization': 'Bearer ' + token}
|
||||
|
||||
|
||||
def session_key(token):
|
||||
return 'session:' + hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def check_race():
|
||||
username = 'u_race_' + uuid.uuid4().hex[:12]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
body = {'username': username, 'display_name': 'Race', 'password': password}
|
||||
|
||||
def once():
|
||||
status, _, _ = call('POST', '/api/v1/users', body)
|
||||
return status
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
statuses = list(pool.map(lambda _: once(), range(8)))
|
||||
if statuses.count(201) != 1 or statuses.count(409) != 7:
|
||||
raise SystemExit('username race statuses: %s' % statuses)
|
||||
print('PASS: one 201 and seven 409 for the same username')
|
||||
|
||||
|
||||
def check_rollback():
|
||||
user, username, password = register('rollback')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('rollback login failed')
|
||||
token = json.loads(raw)['token']
|
||||
title = 'rollback-' + uuid.uuid4().hex
|
||||
missing = str(uuid.uuid4())
|
||||
status, _, _ = call(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
{'title': title, 'member_ids': [user['id'], missing]},
|
||||
headers=bearer(token),
|
||||
)
|
||||
if status != 400:
|
||||
raise SystemExit('expected 400 for an unknown member, got %s' % status)
|
||||
safe = title.replace("'", '')
|
||||
chats = psql("SELECT count(*) FROM chats WHERE title = '%s'" % safe)
|
||||
members = psql(
|
||||
"SELECT count(*) FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE c.title = '%s'" % safe
|
||||
)
|
||||
if chats != '0' or members != '0':
|
||||
raise SystemExit('rollback left rows chats=%s members=%s' % (chats, members))
|
||||
print('PASS: unknown member rolls back chat and membership')
|
||||
|
||||
|
||||
def check_wrong_password():
|
||||
_, username, password = register('pw')
|
||||
bad_status, _, bad_headers = login(username, 'wrong-password-value')
|
||||
missing_status, _, missing_headers = login('missing_' + uuid.uuid4().hex[:8], password)
|
||||
bad_auth = bad_headers.get('WWW-Authenticate', '')
|
||||
missing_auth = missing_headers.get('WWW-Authenticate', '')
|
||||
if bad_status != 401 or missing_status != 401:
|
||||
raise SystemExit('password statuses %s %s' % (bad_status, missing_status))
|
||||
if not bad_auth.lower().startswith('basic') or bad_auth != missing_auth:
|
||||
raise SystemExit('WWW-Authenticate mismatch')
|
||||
print('PASS: wrong password and unknown user share the same 401 challenge')
|
||||
|
||||
|
||||
def check_logout():
|
||||
_, username, password = register('logout')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('logout login failed')
|
||||
token = json.loads(raw)['token']
|
||||
status, body, _ = call('DELETE', '/api/v1/auth/sessions/current', headers=bearer(token))
|
||||
if status != 204 or body:
|
||||
raise SystemExit('logout did not return an empty 204')
|
||||
status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
if status != 401:
|
||||
raise SystemExit('revoked token was accepted')
|
||||
if redis_cli('EXISTS', session_key(token)) != '0':
|
||||
raise SystemExit('revoked session key is still cached')
|
||||
print('PASS: logout returns 204, then 401, and drops the cache key')
|
||||
|
||||
|
||||
def check_redis_down():
|
||||
_, username, password = register('redis')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('redis-down login failed')
|
||||
token = json.loads(raw)['token']
|
||||
compose('stop', 'redis')
|
||||
try:
|
||||
status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
if status != 200:
|
||||
raise SystemExit('valid session failed while Redis was down: %s' % status)
|
||||
status, _, _ = call('GET', '/api/v1/users/me')
|
||||
if status != 401:
|
||||
raise SystemExit('anonymous request was not 401 while Redis was down: %s' % status)
|
||||
finally:
|
||||
compose('start', 'redis')
|
||||
wait_ready()
|
||||
print('PASS: Redis outage falls back to PostgreSQL and still rejects anonymous requests')
|
||||
|
||||
|
||||
def check_postgres_down():
|
||||
_, username, password = register('pg')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('postgres-down login failed')
|
||||
token = json.loads(raw)['token']
|
||||
compose('stop', 'postgres')
|
||||
try:
|
||||
status, body, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
text = body.decode('utf-8', 'replace').lower()
|
||||
if status != 503:
|
||||
raise SystemExit('expected 503 when PostgreSQL is down, got %s' % status)
|
||||
if 'asyncpg' in text or 'select ' in text or 'syntax' in text:
|
||||
raise SystemExit('503 body leaked database details')
|
||||
status, _, _ = call('GET', '/api/v1/users/me')
|
||||
if status != 401:
|
||||
raise SystemExit('anonymous request was not 401 while PostgreSQL was down: %s' % status)
|
||||
finally:
|
||||
compose('start', 'postgres')
|
||||
wait_ready()
|
||||
print('PASS: PostgreSQL outage is 503 for a bearer and 401 without one')
|
||||
|
||||
|
||||
def check_expiry():
|
||||
original = ENV_PATH.read_bytes()
|
||||
text = original.decode('utf-8')
|
||||
updated, count = re.subn(
|
||||
r'^SESSION_TTL_SECONDS=.*$',
|
||||
'SESSION_TTL_SECONDS=2',
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
if count != 1:
|
||||
raise SystemExit('SESSION_TTL_SECONDS is missing from .env')
|
||||
ENV_PATH.write_bytes(updated.encode('utf-8'))
|
||||
try:
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'api')
|
||||
wait_ready()
|
||||
_, username, password = register('ttl')
|
||||
status, raw, _ = login(username, password)
|
||||
if status != 201:
|
||||
raise SystemExit('ttl login failed')
|
||||
token = json.loads(raw)['token']
|
||||
time.sleep(3)
|
||||
status, _, _ = call('GET', '/api/v1/users/me', headers=bearer(token))
|
||||
if status != 401:
|
||||
raise SystemExit('expired token was accepted: %s' % status)
|
||||
finally:
|
||||
ENV_PATH.write_bytes(original)
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'api')
|
||||
wait_ready()
|
||||
print('PASS: SESSION_TTL_SECONDS=2 rejects the same token after 3 seconds')
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
check_race()
|
||||
check_rollback()
|
||||
check_wrong_password()
|
||||
check_logout()
|
||||
check_redis_down()
|
||||
check_postgres_down()
|
||||
check_expiry()
|
||||
finally:
|
||||
subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'start', 'postgres', 'redis'],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
)
|
||||
print('PASS: grade 4 checks')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) for keyset history. Not part of make test.
|
||||
|
||||
Seeds one chat until the default planner uses messages_history_idx, then
|
||||
deletes that seed. The saved plan contains no tokens.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / 'evidence' / 'explain.txt'
|
||||
SIZES = (10000, 50000, 200000)
|
||||
|
||||
|
||||
def psql(sql, tuples_only=False):
|
||||
command = [
|
||||
'docker', 'compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||||
'psql', '-U', os.environ.get('POSTGRES_USER', 'messenger'),
|
||||
'-d', os.environ.get('POSTGRES_DB', 'messenger'),
|
||||
'-v', 'ON_ERROR_STOP=1',
|
||||
]
|
||||
if tuples_only:
|
||||
command.append('-tA')
|
||||
proc = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
input=sql,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
sys.stderr.write(proc.stderr)
|
||||
raise SystemExit('psql failed')
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def seed(user_id, chat_id, count):
|
||||
psql(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(),
|
||||
'%(chat)s'::uuid,
|
||||
'%(user)s'::uuid,
|
||||
'm',
|
||||
timestamptz '2024-01-01 00:00:00+00' + (g * interval '1 microsecond')
|
||||
FROM generate_series(1, %(count)d) AS g;
|
||||
ANALYZE messages;
|
||||
""" % {'chat': chat_id, 'user': user_id, 'count': count}
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
user_id = str(uuid.uuid4())
|
||||
chat_id = str(uuid.uuid4())
|
||||
username = 'u_explain_' + uuid.uuid4().hex[:12]
|
||||
psql(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ('%(user)s', '%(name)s', 'Explain', 'not-used-for-login', clock_timestamp());
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
VALUES ('%(chat)s', 'explain-seed', clock_timestamp());
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
VALUES ('%(chat)s', '%(user)s', 0);
|
||||
""" % {'user': user_id, 'chat': chat_id, 'name': username}
|
||||
)
|
||||
plan = ''
|
||||
used = 0
|
||||
try:
|
||||
for count in SIZES:
|
||||
seed(user_id, chat_id, count)
|
||||
used += count
|
||||
cursor = psql(
|
||||
"""
|
||||
SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), id::text
|
||||
FROM messages
|
||||
WHERE chat_id = '%s'::uuid
|
||||
ORDER BY created_at, id
|
||||
OFFSET 49
|
||||
LIMIT 1;
|
||||
""" % chat_id,
|
||||
tuples_only=True,
|
||||
).strip()
|
||||
created_at, message_id = cursor.split('|')
|
||||
plan = psql(
|
||||
"""
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = '%(chat)s'::uuid
|
||||
AND (created_at, id) > ('%(created)s+00'::timestamptz, '%(message)s'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
""" % {'chat': chat_id, 'created': created_at, 'message': message_id}
|
||||
)
|
||||
if 'Index Scan using messages_history_idx' in plan or 'Index Only Scan using messages_history_idx' in plan:
|
||||
break
|
||||
sys.stderr.write('planner did not pick the history index at %d rows\n' % used)
|
||||
else:
|
||||
sys.stderr.write(plan)
|
||||
raise SystemExit('history query still does not use messages_history_idx')
|
||||
if 'Seq Scan' in plan.split('Index Scan', 1)[0] and 'messages_history_idx' not in plan.split('Index Scan', 1)[0]:
|
||||
pass
|
||||
EVIDENCE.parent.mkdir(exist_ok=True)
|
||||
EVIDENCE.write_text(
|
||||
'rows_in_chat: %d\nquery: keyset page after the 50th message, LIMIT 51\n\n%s' % (used, plan),
|
||||
encoding='utf-8',
|
||||
newline='\n',
|
||||
)
|
||||
print('PASS: history index used at %d messages; wrote %s' % (used, EVIDENCE.name))
|
||||
finally:
|
||||
psql(
|
||||
"""
|
||||
DELETE FROM messages WHERE chat_id = '%(chat)s';
|
||||
DELETE FROM chat_members WHERE chat_id = '%(chat)s';
|
||||
DELETE FROM chats WHERE id = '%(chat)s';
|
||||
DELETE FROM users WHERE id = '%(user)s';
|
||||
""" % {'chat': chat_id, 'user': user_id}
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Restart the API and PostgreSQL. Named volumes stay; nothing is re-seeded.
|
||||
docker compose --env-file .env restart api postgres
|
||||
# Restart the API, PostgreSQL and the session cache. Named volumes stay; nothing is re-seeded.
|
||||
docker compose --env-file .env restart api postgres redis
|
||||
|
||||
Reference in New Issue
Block a user