Files
mtusi2026lab6/student-tests/replication.py
T

102 lines
4.1 KiB
Python

"""Concurrent idempotency, both replica failures, and rolling recreation under writes."""
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check, Failure
def compose(*args):
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, timeout=120)
def history(c, actor, chat):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
result = []
while path:
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
result.extend(page['items'])
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') if page['next_cursor'] else None
return result
def main():
c = Client(timeout=5)
alice, bob, eve, chat = c.bootstrap()
key = uuid.uuid4().hex
def send(_):
return Client().message(alice, chat, 'One logical message', key=key)
with ThreadPoolExecutor(max_workers=12) as executor:
responses = list(executor.map(send, range(24)))
ids = {item['json']['id'] for item in responses}
instances = {item['headers']['X-Instance-Id'] for item in responses}
check(len(ids) == 1 and len(instances) >= 2, 'Concurrency did not cover both replicas or duplicated message')
check(len(history(c, alice, chat)) == 1, 'Duplicate row committed')
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
headers={'Idempotency-Key': key}, body={'text': 'Different'})
results = [{'scenario': 'concurrent_idempotency', 'requests': 24,
'replicas': sorted(instances), 'unique_messages': 1}]
def experiment(label, action):
stop = threading.Event()
writes, errors, times = [], [], []
def traffic():
client = Client(timeout=3)
retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex
while not stop.is_set():
try:
response = client.message(alice, chat, text, key=retry_key)
writes.append(response['json'])
times.append(time.monotonic())
retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex
except Failure:
errors.append(time.monotonic())
time.sleep(0.02)
thread = threading.Thread(target=traffic)
thread.start()
started = time.monotonic()
try:
action()
time.sleep(5)
finally:
stop.set()
thread.join(timeout=10)
duration = time.monotonic() - started
stored = {item['id']: item for item in history(c, bob, chat)}
check(writes and all(stored.get(item['id']) == item for item in writes), 'Confirmed writes lost')
bodies = [item['text'] for item in stored.values()]
check(len(bodies) == len(set(bodies)), 'Retry duplicated a logical message')
gaps = [b-a for a,b in zip([started]+times, times+[time.monotonic()])]
check(max(gaps) < 10, 'Recovery exceeded 10 seconds')
results.append({'scenario': label, 'duration_seconds': round(duration,3),
'writes': len(writes), 'errors': len(errors),
'rps': round(len(writes)/duration,3), 'max_success_gap_seconds': round(max(gaps),3)})
for service in ('api1', 'api2'):
try:
experiment('stop_' + service, lambda: compose('stop', '-t', '2', service))
finally:
compose('up', '-d', '--no-deps', '--wait', service)
time.sleep(2)
def rolling():
for service in ('api1', 'api2'):
compose('up', '-d', '--no-deps', '--force-recreate', '--wait', service)
time.sleep(2)
experiment('rolling_recreation', rolling)
Path(ROOT / 'evidence/replication.json').write_text(json.dumps(results, indent=2), encoding='utf-8')
print(json.dumps(results, indent=2))
if __name__ == '__main__':
main()