lab2
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
mkdir -p evidence/cache
|
||||
|
||||
{
|
||||
echo "# API cache performance experiment"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \
|
||||
NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 \
|
||||
python3 tests/cache_api_performance.py
|
||||
} > evidence/cache/api-performance.txt 2>&1
|
||||
cat evidence/cache/api-performance.txt
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUT="$ROOT/evidence/cache"
|
||||
GO="${GO:-go}"
|
||||
|
||||
cd "$ROOT"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
{
|
||||
echo "# Cache consistency evidence (track 1)"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "# redis: ${TEST_REDIS_HOST:-localhost}:${TEST_REDIS_PORT:-6379} db=${TEST_REDIS_DB:-1}"
|
||||
echo
|
||||
"$GO" test -tags=integration -count=1 -v -timeout=5m \
|
||||
-run 'TestChatRepositorySuite|TestSessionRepositorySuite' \
|
||||
./internal/storage/redis/...
|
||||
} | tee "$OUT/consistency.txt"
|
||||
|
||||
echo "wrote $OUT/consistency.txt"
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
mkdir -p evidence/cache
|
||||
|
||||
{
|
||||
echo "# Cache latency experiment"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
go version
|
||||
uname -sm
|
||||
go test -tags=integration -count=3 -timeout=5m -v \
|
||||
-run '^TestCacheLatencyExperiment$' ./internal/storage/postgres/...
|
||||
} > evidence/cache/latency.txt 2>&1
|
||||
cat evidence/cache/latency.txt
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
mkdir -p evidence/cache
|
||||
|
||||
{
|
||||
echo "# Cache performance experiment"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
go version
|
||||
uname -sm
|
||||
go test -tags=integration -count=3 -timeout=5m -v \
|
||||
-run '^TestCachePerformanceExperiment$' ./internal/storage/postgres/...
|
||||
} > evidence/cache/performance.txt 2>&1
|
||||
cat evidence/cache/performance.txt
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible Redis-outage check: stop and pause, latency budget, create-during-outage.
|
||||
|
||||
Exit code is non-zero when any assertion fails. Always unpauses/starts Redis.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
import base64
|
||||
|
||||
for proxy in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
||||
os.environ.pop(proxy, None)
|
||||
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
|
||||
os.environ['no_proxy'] = os.environ['NO_PROXY']
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
COMPOSE = os.environ.get('COMPOSE_BIN', 'docker-compose')
|
||||
COMPOSE_FILE = 'docker-compose.local.yaml'
|
||||
BUDGET_MS = int(os.environ.get('OUTAGE_BUDGET_MS', '2000'))
|
||||
PARALLEL = int(os.environ.get('OUTAGE_PARALLEL', '30'))
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run([COMPOSE, '-f', COMPOSE_FILE, *args], check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
h = {'Accept': 'application/json'}
|
||||
if body is not None:
|
||||
h['Content-Type'] = 'application/json'
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
return resp.status, resp.read(), (time.monotonic() - started) * 1000
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read(), (time.monotonic() - started) * 1000
|
||||
|
||||
|
||||
def user(label):
|
||||
username = label + '_' + uuid.uuid4().hex[:8]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
call('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
_, body, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
return json.loads(body)
|
||||
|
||||
|
||||
def list_chats(actor):
|
||||
status, body, ms = call('GET', '/api/v1/chats?limit=50', headers={'Authorization': 'Bearer ' + actor['token']})
|
||||
assert status == 200, 'GET /chats expected 200, got %d' % status
|
||||
return json.loads(body)['items'], ms
|
||||
|
||||
|
||||
def assert_budget(label, samples):
|
||||
worst = max(samples)
|
||||
p95 = sorted(samples)[max(0, (len(samples) * 95 + 99) // 100 - 1)]
|
||||
print(' %s: n=%d max=%.0fms p95=%.0fms budget=%dms' % (label, len(samples), worst, p95, BUDGET_MS))
|
||||
assert worst <= BUDGET_MS, '%s exceeded budget: %.0fms > %dms' % (label, worst, BUDGET_MS)
|
||||
|
||||
|
||||
def main():
|
||||
alice = user('alice')
|
||||
bob = user('bob')
|
||||
ah = {'Authorization': 'Bearer ' + alice['token']}
|
||||
|
||||
try:
|
||||
_, ms = list_chats(alice)
|
||||
print('baseline: %.0fms' % ms)
|
||||
|
||||
compose('stop', 'redis')
|
||||
_, ms = list_chats(alice)
|
||||
print('after stop: %.0fms' % ms)
|
||||
assert_budget('stop', [ms])
|
||||
|
||||
compose('start', 'redis')
|
||||
time.sleep(2)
|
||||
compose('pause', 'redis')
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=PARALLEL) as pool:
|
||||
results = [f for f in pool.map(lambda _: list_chats(alice), range(PARALLEL))]
|
||||
assert_budget('pause parallel', [ms for _, ms in results])
|
||||
|
||||
title = 'during-outage-' + uuid.uuid4().hex[:8]
|
||||
status, _, _ = call('POST', '/api/v1/chats', {
|
||||
'title': title,
|
||||
'member_ids': [alice['user']['id'], bob['user']['id']],
|
||||
}, headers=ah)
|
||||
assert status == 201, 'chat create during pause expected 201, got %d' % status
|
||||
print('create during pause: 201')
|
||||
|
||||
compose('unpause', 'redis')
|
||||
time.sleep(2)
|
||||
|
||||
chats, ms = list_chats(alice)
|
||||
assert any(c['title'] == title for c in chats), 'new chat missing after recovery (stale cache)'
|
||||
print('after recovery: new chat visible, %.0fms' % ms)
|
||||
|
||||
stuck = subprocess.run(
|
||||
[COMPOSE, '-f', COMPOSE_FILE, 'exec', '-T', 'redis', 'redis-cli', '--scan', '--pattern', 'lock:*'],
|
||||
check=True, capture_output=True, text=True,
|
||||
).stdout.split()
|
||||
for key in stuck:
|
||||
ttl = subprocess.run(
|
||||
[COMPOSE, '-f', COMPOSE_FILE, 'exec', '-T', 'redis', 'redis-cli', 'ttl', key],
|
||||
check=True, capture_output=True, text=True,
|
||||
).stdout.strip()
|
||||
assert ttl != '-1', 'stuck lock without TTL: %s' % key
|
||||
print('lock keys without TTL: 0')
|
||||
print('PASS: redis outage handled within budget')
|
||||
finally:
|
||||
for action in ('unpause', 'start'):
|
||||
subprocess.run([COMPOSE, '-f', COMPOSE_FILE, action, 'redis'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except AssertionError as exc:
|
||||
print('FAIL: %s' % exc)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker-compose}
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
"$COMPOSE_BIN" -f docker-compose.local.yaml restart postgres redis
|
||||
"$COMPOSE_BIN" -f docker-compose.local.yaml up -d --wait postgres redis
|
||||
"$COMPOSE_BIN" -f docker-compose.local.yaml up -d --force-recreate api
|
||||
Reference in New Issue
Block a user