Files
mtusi2026lab3/scripts/verify_lab3.py
T
2026-10-23 15:36:38 +03:00

200 lines
9.8 KiB
Python

"""Repeatable grade-4 experiments. Run on host with Docker access and UTF-8 mode."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from ops import ROOT, compose, restore, rolling, save, snapshot, stop_instance, wait_api
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
from resilience import history
def command(name, args):
result = subprocess.run(args, cwd=ROOT, text=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180)
(ROOT / 'evidence' / name).write_text(result.stdout, encoding='utf-8')
print(result.stdout, flush=True)
check(result.returncode == 0, 'Command failed: ' + name)
def concurrency(base):
c = Client(base)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
key = uuid4().hex
barrier = threading.Barrier(8)
def send(_):
client = Client(base)
barrier.wait(timeout=10)
return client.message(alice, chat, 'one concurrent message', key=key)
responses = []
with ThreadPoolExecutor(max_workers=8) as executor:
for _ in range(3):
responses.extend(executor.map(send, range(8)))
first = responses[0]['json']
check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replies differ')
instances = sorted({r['headers']['X-Instance-Id'] for r in responses})
check(len(instances) >= 2, 'Concurrent test did not reach both replicas')
check(history(c, alice, chat) == [first], 'Concurrent retry duplicated message')
c.call('POST', path, 409, actor=alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
c.call('POST', path, 401, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
c.call('POST', path, 404, actor=eve, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
for invalid in (None, '', 'bad key', 'x' * 129):
c.call('POST', path, 400, actor=alice, body={'text': 'invalid key'},
headers={} if invalid is None else {'Idempotency-Key': invalid})
other = c.message(bob, chat, 'independent user', key=key)['json']
check(other['id'] != first['id'], 'Key scope must include actor')
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Scope', 'member_ids': [alice['id']]}, schema='Chat')['json']
other = c.message(alice, chat2, 'independent chat', key=key)['json']
check(other['id'] != first['id'], 'Key scope must include chat')
save('idempotency.json', {'requests': 24, 'concurrency': 8, 'instances': instances,
'unique_messages': 1, 'same_response': True,
'conflict_auth_access_key_scope': 'PASS'})
return c, alice, chat, key, first
def experiment(base, name, action, target=None):
c = Client(base, timeout=9)
alice, bob, _, chat = c.bootstrap()
confirmed_before = c.message(alice, chat, 'before ' + name)['json']
events = []
stop = threading.Event()
started = time.monotonic()
def worker():
client = Client(base, timeout=9)
key = uuid4().hex
while not stop.is_set():
begin = time.monotonic()
try:
response = client.message(alice, chat, key, key=key)
msg = response['json']
events.append({'kind': 'write', 'time': time.monotonic() - started,
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id'],
'message': msg})
key = uuid4().hex
except Failure as exc:
events.append({'kind': 'error', 'operation': 'write', 'time': time.monotonic() - started,
'latency': time.monotonic() - begin, 'error': str(exc)})
continue # Keep original key/body after an uncertain write.
begin = time.monotonic()
try:
response = client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')
events.append({'kind': 'read', 'time': time.monotonic() - started,
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id']})
except Failure as exc:
events.append({'kind': 'error', 'operation': 'read', 'time': time.monotonic() - started,
'latency': time.monotonic() - begin, 'error': str(exc)})
time.sleep(0.02)
with ThreadPoolExecutor(max_workers=2) as executor:
futures = [executor.submit(worker) for _ in range(2)]
try:
time.sleep(3)
action_started = time.monotonic() - started
effect = action()
action_finished = time.monotonic() - started
time.sleep(max(5, 20 - (time.monotonic() - started)))
finally:
stop.set()
for future in futures:
future.result()
duration = time.monotonic() - started
successes = [e for e in events if e['kind'] != 'error']
errors = [e for e in events if e['kind'] == 'error']
writes = [e for e in successes if e['kind'] == 'write']
recovery = {}
for kind in ('read', 'write'):
after = [e for e in successes if e['kind'] == kind and e['time'] >= action_finished]
check(bool(after), 'No ' + kind + ' after action')
recovery[kind] = round(min(e['time'] for e in after) - action_started, 3)
if target:
check(all(e['instance'] != target for e in after), 'Stopped target still serves')
check(recovery[kind] <= 10, 'Recovery exceeded 10 seconds')
messages = history(c, bob, chat)
by_id = {m['id']: m for m in messages}
check(by_id.get(confirmed_before['id']) == confirmed_before, 'Pre-failure message lost')
check(all(by_id.get(e['message']['id']) == e['message'] for e in writes), 'Confirmed write lost')
check(len({m['text'] for m in messages}) == len(messages), 'Retry duplicated a message')
# Retry a pre-failure key through the surviving/new processes.
if writes:
item = writes[0]['message']
check(c.message(alice, chat, item['text'], key=item['text'])['json'] == item, 'Replay changed after action')
latencies = sorted(e['latency'] for e in successes)
gaps = {}
for kind in ('read', 'write'):
times = sorted(e['time'] for e in successes if e['kind'] == kind)
gaps[kind] = round(max((b - a for a, b in zip(times, times[1:])), default=0), 3)
summary = {'duration_seconds': round(duration, 3), 'concurrency': 2,
'successful_http_rps': round(len(successes) / duration, 2),
'successful_write_rps': round(len(writes) / duration, 2),
'successful_requests': len(successes), 'errors': len(errors),
'action_started_seconds': round(action_started, 3),
'action_finished_seconds': round(action_finished, 3),
'first_read_write_after_action_seconds_from_start': recovery,
'max_success_gap_seconds': gaps,
'latency_ms': {str(p): round(latencies[min(int(len(latencies)*p/100), len(latencies)-1)]*1000, 2)
for p in (50, 95, 99)},
'confirmed_messages_preserved': True, 'duplicate_messages': 0,
'instances': sorted({e['instance'] for e in successes}), 'target': target}
save(name + '.json', {'summary': summary, 'containers': effect, 'errors': errors})
print(name + ': ' + json.dumps(summary), flush=True)
return summary
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
args = parser.parse_args()
os.environ['PYTHONUTF8'] = '1'
base = args.base_url
wait_api()
command('check.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
c, actor, chat, key, original = concurrency(base)
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
command('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence',
'--base-url', base, '--action-script', hook])
wait_api()
check(c.message(actor, chat, original['text'], key=key)['json'] == original,
'Idempotency/session not preserved across full restart')
save('idempotency-restart.json', {'same_token': True, 'same_201_message': True})
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
for index in range(2):
try:
command('public-failover-' + str(index+1) + '.txt',
[sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover',
'--base-url', base, '--action-script', hook])
finally:
restore()
results = {}
for index, row in enumerate(snapshot('api'), start=1):
target = row['instance']
try:
results['failover-' + str(index)] = experiment(base, 'failover-' + str(index),
lambda: stop_instance(target), target)
finally:
restore()
# Build one image first; apply compatible migrations once before replacing replicas.
compose('build', 'api')
compose('run', '--rm', '--no-deps', 'migrate')
results['rolling'] = experiment(base, 'rolling-load', rolling)
save('grade4-summary.json', results)
save('final-containers.json', snapshot())
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
print('PASS: all grade-4 scenarios', flush=True)
if __name__ == '__main__':
main()