lab2 fix
This commit is contained in:
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible Redis-outage check: stop and pause, latency budget, create-during-outage.
|
||||
|
||||
Exit code is non-zero when any assertion fails. Always unpauses/starts Redis.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
import base64
|
||||
|
||||
for proxy in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
||||
os.environ.pop(proxy, None)
|
||||
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
|
||||
os.environ['no_proxy'] = os.environ['NO_PROXY']
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
COMPOSE = os.environ.get('COMPOSE_BIN', 'docker-compose')
|
||||
COMPOSE_FILE = 'docker-compose.local.yaml'
|
||||
BUDGET_MS = int(os.environ.get('OUTAGE_BUDGET_MS', '2000'))
|
||||
PARALLEL = int(os.environ.get('OUTAGE_PARALLEL', '30'))
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run([COMPOSE, '-f', COMPOSE_FILE, *args], check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
h = {'Accept': 'application/json'}
|
||||
if body is not None:
|
||||
h['Content-Type'] = 'application/json'
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
return resp.status, resp.read(), (time.monotonic() - started) * 1000
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read(), (time.monotonic() - started) * 1000
|
||||
|
||||
|
||||
def user(label):
|
||||
username = label + '_' + uuid.uuid4().hex[:8]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
call('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
_, body, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
return json.loads(body)
|
||||
|
||||
|
||||
def list_chats(actor):
|
||||
status, body, ms = call('GET', '/api/v1/chats?limit=50', headers={'Authorization': 'Bearer ' + actor['token']})
|
||||
assert status == 200, 'GET /chats expected 200, got %d' % status
|
||||
return json.loads(body)['items'], ms
|
||||
|
||||
|
||||
def assert_budget(label, samples):
|
||||
worst = max(samples)
|
||||
p95 = sorted(samples)[max(0, (len(samples) * 95 + 99) // 100 - 1)]
|
||||
print(' %s: n=%d max=%.0fms p95=%.0fms budget=%dms' % (label, len(samples), worst, p95, BUDGET_MS))
|
||||
assert worst <= BUDGET_MS, '%s exceeded budget: %.0fms > %dms' % (label, worst, BUDGET_MS)
|
||||
|
||||
|
||||
def main():
|
||||
alice = user('alice')
|
||||
bob = user('bob')
|
||||
ah = {'Authorization': 'Bearer ' + alice['token']}
|
||||
|
||||
try:
|
||||
_, ms = list_chats(alice)
|
||||
print('baseline: %.0fms' % ms)
|
||||
|
||||
compose('stop', 'redis')
|
||||
_, ms = list_chats(alice)
|
||||
print('after stop: %.0fms' % ms)
|
||||
assert_budget('stop', [ms])
|
||||
|
||||
compose('start', 'redis')
|
||||
time.sleep(2)
|
||||
compose('pause', 'redis')
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=PARALLEL) as pool:
|
||||
results = [f for f in pool.map(lambda _: list_chats(alice), range(PARALLEL))]
|
||||
assert_budget('pause parallel', [ms for _, ms in results])
|
||||
|
||||
title = 'during-outage-' + uuid.uuid4().hex[:8]
|
||||
status, _, _ = call('POST', '/api/v1/chats', {
|
||||
'title': title,
|
||||
'member_ids': [alice['user']['id'], bob['user']['id']],
|
||||
}, headers=ah)
|
||||
assert status == 201, 'chat create during pause expected 201, got %d' % status
|
||||
print('create during pause: 201')
|
||||
|
||||
compose('unpause', 'redis')
|
||||
time.sleep(2)
|
||||
|
||||
chats, ms = list_chats(alice)
|
||||
assert any(c['title'] == title for c in chats), 'new chat missing after recovery (stale cache)'
|
||||
print('after recovery: new chat visible, %.0fms' % ms)
|
||||
|
||||
stuck = subprocess.run(
|
||||
[COMPOSE, '-f', COMPOSE_FILE, 'exec', '-T', 'redis', 'redis-cli', '--scan', '--pattern', 'lock:*'],
|
||||
check=True, capture_output=True, text=True,
|
||||
).stdout.split()
|
||||
for key in stuck:
|
||||
ttl = subprocess.run(
|
||||
[COMPOSE, '-f', COMPOSE_FILE, 'exec', '-T', 'redis', 'redis-cli', 'ttl', key],
|
||||
check=True, capture_output=True, text=True,
|
||||
).stdout.strip()
|
||||
assert ttl != '-1', 'stuck lock without TTL: %s' % key
|
||||
print('lock keys without TTL: 0')
|
||||
print('PASS: redis outage handled within budget')
|
||||
finally:
|
||||
for action in ('unpause', 'start'):
|
||||
subprocess.run([COMPOSE, '-f', COMPOSE_FILE, action, 'redis'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except AssertionError as exc:
|
||||
print('FAIL: %s' % exc)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user