forked from mtusi2026/mtusi2026lab3
Grade 4
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
"""Project-scoped Docker operations; never read/export container environments."""
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, timeout=120).strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return docker('compose', *args)
|
||||
|
||||
|
||||
def snapshot(service=None):
|
||||
ids = compose('ps', '-a', '-q', *([service] if service else [])).split()
|
||||
if not ids:
|
||||
return []
|
||||
# Select fields before returning; no secrets in evidence.
|
||||
template = ('{"id":{{json .Id}},"name":{{json .Name}},"instance":{{json .Config.Hostname}},'
|
||||
'"image":{{json .Image}},"state":{{json .State.Status}},'
|
||||
'"started":{{json .State.StartedAt}},'
|
||||
'"health":{{with index .State "Health"}}{{json .Status}}{{else}}null{{end}},'
|
||||
'"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},'
|
||||
'"ports":{{json .HostConfig.PortBindings}}}')
|
||||
return [json.loads(line) for line in docker('inspect', '--format', template, *ids).splitlines()]
|
||||
|
||||
|
||||
def wait_api(count=2, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
rows = snapshot('api')
|
||||
if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows):
|
||||
time.sleep(3) # Docker DNS validity 2s, plus resolver budget 1s.
|
||||
return rows
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError('API replicas did not become healthy')
|
||||
|
||||
|
||||
def save(name, value):
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
(ROOT / 'evidence' / name).write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def stop_instance(instance):
|
||||
before = snapshot('api')
|
||||
matches = [r for r in before if r['instance'] == instance and r['state'] == 'running']
|
||||
if len(matches) != 1 or sum(r['state'] == 'running' for r in before) < 2:
|
||||
raise RuntimeError('Target must identify one running API with a running peer')
|
||||
docker('stop', '--time', '8', matches[0]['id'])
|
||||
result = {'target': instance, 'before': before, 'after': snapshot('api')}
|
||||
if next(r for r in result['after'] if r['instance'] == instance)['state'] != 'exited':
|
||||
raise RuntimeError('Target was not stopped')
|
||||
return result
|
||||
|
||||
|
||||
def restore():
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
||||
return wait_api()
|
||||
|
||||
|
||||
def rolling():
|
||||
before = wait_api()
|
||||
if len(before) != 2:
|
||||
raise RuntimeError('This rolling scenario requires exactly two replicas')
|
||||
steps = []
|
||||
for old in before:
|
||||
stop_instance(old['instance'])
|
||||
# Only remove the selected, already stopped API; never remove a volume.
|
||||
docker('rm', old['id'])
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api')
|
||||
steps.append({'replaced': old['id'], 'ready': wait_api()})
|
||||
result = {'before': before, 'steps': steps, 'after': snapshot('api')}
|
||||
if {r['id'] for r in before} & {r['id'] for r in result['after']}:
|
||||
raise RuntimeError('Not all original API containers were replaced')
|
||||
return result
|
||||
+2
-2
@@ -15,12 +15,12 @@ def snapshot():
|
||||
|
||||
def main():
|
||||
before = snapshot()
|
||||
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
|
||||
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 'nginx'],
|
||||
cwd=ROOT, check=True, timeout=60)
|
||||
after = snapshot()
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
(ROOT / 'evidence' / 'restart.txt').write_text(
|
||||
'Command: docker compose restart api postgres redis\nBefore (UTC):\n' + before
|
||||
'Command: docker compose restart api postgres redis nginx\nBefore (UTC):\n' + before
|
||||
+ 'After (UTC):\n' + after, encoding='utf-8')
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
from ops import compose, rolling
|
||||
|
||||
if __name__ == '__main__':
|
||||
compose('build', 'api')
|
||||
compose('run', '--rm', '--no-deps', 'migrate')
|
||||
rolling()
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\stop-one.py
|
||||
@@ -0,0 +1,5 @@
|
||||
import os
|
||||
from ops import stop_instance
|
||||
|
||||
if __name__ == '__main__':
|
||||
stop_instance(os.environ['TARGET_INSTANCE_ID'])
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Repeatable grade-4 experiments. Run on host with Docker access and UTF-8 mode."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from ops import ROOT, compose, restore, rolling, save, snapshot, stop_instance, wait_api
|
||||
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
|
||||
|
||||
def command(name, args):
|
||||
result = subprocess.run(args, cwd=ROOT, text=True, encoding='utf-8',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180)
|
||||
(ROOT / 'evidence' / name).write_text(result.stdout, encoding='utf-8')
|
||||
print(result.stdout, flush=True)
|
||||
check(result.returncode == 0, 'Command failed: ' + name)
|
||||
|
||||
|
||||
def concurrency(base):
|
||||
c = Client(base)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
key = uuid4().hex
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def send(_):
|
||||
client = Client(base)
|
||||
barrier.wait(timeout=10)
|
||||
return client.message(alice, chat, 'one concurrent message', key=key)
|
||||
|
||||
responses = []
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
for _ in range(3):
|
||||
responses.extend(executor.map(send, range(8)))
|
||||
first = responses[0]['json']
|
||||
check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replies differ')
|
||||
instances = sorted({r['headers']['X-Instance-Id'] for r in responses})
|
||||
check(len(instances) >= 2, 'Concurrent test did not reach both replicas')
|
||||
check(history(c, alice, chat) == [first], 'Concurrent retry duplicated message')
|
||||
c.call('POST', path, 409, actor=alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', path, 401, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', path, 404, actor=eve, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
|
||||
for invalid in (None, '', 'bad key', 'x' * 129):
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'invalid key'},
|
||||
headers={} if invalid is None else {'Idempotency-Key': invalid})
|
||||
other = c.message(bob, chat, 'independent user', key=key)['json']
|
||||
check(other['id'] != first['id'], 'Key scope must include actor')
|
||||
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Scope', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
other = c.message(alice, chat2, 'independent chat', key=key)['json']
|
||||
check(other['id'] != first['id'], 'Key scope must include chat')
|
||||
save('idempotency.json', {'requests': 24, 'concurrency': 8, 'instances': instances,
|
||||
'unique_messages': 1, 'same_response': True,
|
||||
'conflict_auth_access_key_scope': 'PASS'})
|
||||
return c, alice, chat, key, first
|
||||
|
||||
|
||||
def experiment(base, name, action, target=None):
|
||||
c = Client(base, timeout=9)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
confirmed_before = c.message(alice, chat, 'before ' + name)['json']
|
||||
events = []
|
||||
stop = threading.Event()
|
||||
started = time.monotonic()
|
||||
|
||||
def worker():
|
||||
client = Client(base, timeout=9)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
begin = time.monotonic()
|
||||
try:
|
||||
response = client.message(alice, chat, key, key=key)
|
||||
msg = response['json']
|
||||
events.append({'kind': 'write', 'time': time.monotonic() - started,
|
||||
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id'],
|
||||
'message': msg})
|
||||
key = uuid4().hex
|
||||
except Failure as exc:
|
||||
events.append({'kind': 'error', 'operation': 'write', 'time': time.monotonic() - started,
|
||||
'latency': time.monotonic() - begin, 'error': str(exc)})
|
||||
continue # Keep original key/body after an uncertain write.
|
||||
begin = time.monotonic()
|
||||
try:
|
||||
response = client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')
|
||||
events.append({'kind': 'read', 'time': time.monotonic() - started,
|
||||
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id']})
|
||||
except Failure as exc:
|
||||
events.append({'kind': 'error', 'operation': 'read', 'time': time.monotonic() - started,
|
||||
'latency': time.monotonic() - begin, 'error': str(exc)})
|
||||
time.sleep(0.02)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [executor.submit(worker) for _ in range(2)]
|
||||
try:
|
||||
time.sleep(3)
|
||||
action_started = time.monotonic() - started
|
||||
effect = action()
|
||||
action_finished = time.monotonic() - started
|
||||
time.sleep(max(5, 20 - (time.monotonic() - started)))
|
||||
finally:
|
||||
stop.set()
|
||||
for future in futures:
|
||||
future.result()
|
||||
duration = time.monotonic() - started
|
||||
successes = [e for e in events if e['kind'] != 'error']
|
||||
errors = [e for e in events if e['kind'] == 'error']
|
||||
writes = [e for e in successes if e['kind'] == 'write']
|
||||
recovery = {}
|
||||
for kind in ('read', 'write'):
|
||||
after = [e for e in successes if e['kind'] == kind and e['time'] >= action_finished]
|
||||
check(bool(after), 'No ' + kind + ' after action')
|
||||
recovery[kind] = round(min(e['time'] for e in after) - action_started, 3)
|
||||
if target:
|
||||
check(all(e['instance'] != target for e in after), 'Stopped target still serves')
|
||||
check(recovery[kind] <= 10, 'Recovery exceeded 10 seconds')
|
||||
messages = history(c, bob, chat)
|
||||
by_id = {m['id']: m for m in messages}
|
||||
check(by_id.get(confirmed_before['id']) == confirmed_before, 'Pre-failure message lost')
|
||||
check(all(by_id.get(e['message']['id']) == e['message'] for e in writes), 'Confirmed write lost')
|
||||
check(len({m['text'] for m in messages}) == len(messages), 'Retry duplicated a message')
|
||||
# Retry a pre-failure key through the surviving/new processes.
|
||||
if writes:
|
||||
item = writes[0]['message']
|
||||
check(c.message(alice, chat, item['text'], key=item['text'])['json'] == item, 'Replay changed after action')
|
||||
latencies = sorted(e['latency'] for e in successes)
|
||||
gaps = {}
|
||||
for kind in ('read', 'write'):
|
||||
times = sorted(e['time'] for e in successes if e['kind'] == kind)
|
||||
gaps[kind] = round(max((b - a for a, b in zip(times, times[1:])), default=0), 3)
|
||||
summary = {'duration_seconds': round(duration, 3), 'concurrency': 2,
|
||||
'successful_http_rps': round(len(successes) / duration, 2),
|
||||
'successful_write_rps': round(len(writes) / duration, 2),
|
||||
'successful_requests': len(successes), 'errors': len(errors),
|
||||
'action_started_seconds': round(action_started, 3),
|
||||
'action_finished_seconds': round(action_finished, 3),
|
||||
'first_read_write_after_action_seconds_from_start': recovery,
|
||||
'max_success_gap_seconds': gaps,
|
||||
'latency_ms': {str(p): round(latencies[min(int(len(latencies)*p/100), len(latencies)-1)]*1000, 2)
|
||||
for p in (50, 95, 99)},
|
||||
'confirmed_messages_preserved': True, 'duplicate_messages': 0,
|
||||
'instances': sorted({e['instance'] for e in successes}), 'target': target}
|
||||
save(name + '.json', {'summary': summary, 'containers': effect, 'errors': errors})
|
||||
print(name + ': ' + json.dumps(summary), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
args = parser.parse_args()
|
||||
os.environ['PYTHONUTF8'] = '1'
|
||||
base = args.base_url
|
||||
wait_api()
|
||||
command('check.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
|
||||
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
|
||||
c, actor, chat, key, original = concurrency(base)
|
||||
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
|
||||
command('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence',
|
||||
'--base-url', base, '--action-script', hook])
|
||||
wait_api()
|
||||
check(c.message(actor, chat, original['text'], key=key)['json'] == original,
|
||||
'Idempotency/session not preserved across full restart')
|
||||
save('idempotency-restart.json', {'same_token': True, 'same_201_message': True})
|
||||
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
|
||||
for index in range(2):
|
||||
try:
|
||||
command('public-failover-' + str(index+1) + '.txt',
|
||||
[sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover',
|
||||
'--base-url', base, '--action-script', hook])
|
||||
finally:
|
||||
restore()
|
||||
results = {}
|
||||
for index, row in enumerate(snapshot('api'), start=1):
|
||||
target = row['instance']
|
||||
try:
|
||||
results['failover-' + str(index)] = experiment(base, 'failover-' + str(index),
|
||||
lambda: stop_instance(target), target)
|
||||
finally:
|
||||
restore()
|
||||
# Build one image first; apply compatible migrations once before replacing replicas.
|
||||
compose('build', 'api')
|
||||
compose('run', '--rm', '--no-deps', 'migrate')
|
||||
results['rolling'] = experiment(base, 'rolling-load', rolling)
|
||||
save('grade4-summary.json', results)
|
||||
save('final-containers.json', snapshot())
|
||||
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
|
||||
print('PASS: all grade-4 scenarios', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user