This commit is contained in:
2026-10-23 15:36:38 +03:00
parent cd9c3402fa
commit 1d85938fb4
35 changed files with 1262 additions and 70 deletions
+79
View File
@@ -0,0 +1,79 @@
"""Project-scoped Docker operations; never read/export container environments."""
import json
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, timeout=120).strip()
def compose(*args):
return docker('compose', *args)
def snapshot(service=None):
ids = compose('ps', '-a', '-q', *([service] if service else [])).split()
if not ids:
return []
# Select fields before returning; no secrets in evidence.
template = ('{"id":{{json .Id}},"name":{{json .Name}},"instance":{{json .Config.Hostname}},'
'"image":{{json .Image}},"state":{{json .State.Status}},'
'"started":{{json .State.StartedAt}},'
'"health":{{with index .State "Health"}}{{json .Status}}{{else}}null{{end}},'
'"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},'
'"ports":{{json .HostConfig.PortBindings}}}')
return [json.loads(line) for line in docker('inspect', '--format', template, *ids).splitlines()]
def wait_api(count=2, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
rows = snapshot('api')
if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows):
time.sleep(3) # Docker DNS validity 2s, plus resolver budget 1s.
return rows
time.sleep(0.5)
raise RuntimeError('API replicas did not become healthy')
def save(name, value):
(ROOT / 'evidence').mkdir(exist_ok=True)
(ROOT / 'evidence' / name).write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
def stop_instance(instance):
before = snapshot('api')
matches = [r for r in before if r['instance'] == instance and r['state'] == 'running']
if len(matches) != 1 or sum(r['state'] == 'running' for r in before) < 2:
raise RuntimeError('Target must identify one running API with a running peer')
docker('stop', '--time', '8', matches[0]['id'])
result = {'target': instance, 'before': before, 'after': snapshot('api')}
if next(r for r in result['after'] if r['instance'] == instance)['state'] != 'exited':
raise RuntimeError('Target was not stopped')
return result
def restore():
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
return wait_api()
def rolling():
before = wait_api()
if len(before) != 2:
raise RuntimeError('This rolling scenario requires exactly two replicas')
steps = []
for old in before:
stop_instance(old['instance'])
# Only remove the selected, already stopped API; never remove a volume.
docker('rm', old['id'])
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api')
steps.append({'replaced': old['id'], 'ready': wait_api()})
result = {'before': before, 'steps': steps, 'after': snapshot('api')}
if {r['id'] for r in before} & {r['id'] for r in result['after']}:
raise RuntimeError('Not all original API containers were replaced')
return result
+2 -2
View File
@@ -15,12 +15,12 @@ def snapshot():
def main():
before = snapshot()
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 'nginx'],
cwd=ROOT, check=True, timeout=60)
after = snapshot()
(ROOT / 'evidence').mkdir(exist_ok=True)
(ROOT / 'evidence' / 'restart.txt').write_text(
'Command: docker compose restart api postgres redis\nBefore (UTC):\n' + before
'Command: docker compose restart api postgres redis nginx\nBefore (UTC):\n' + before
+ 'After (UTC):\n' + after, encoding='utf-8')
+6
View File
@@ -0,0 +1,6 @@
from ops import compose, rolling
if __name__ == '__main__':
compose('build', 'api')
compose('run', '--rm', '--no-deps', 'migrate')
rolling()
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
python scripts\stop-one.py
+5
View File
@@ -0,0 +1,5 @@
import os
from ops import stop_instance
if __name__ == '__main__':
stop_instance(os.environ['TARGET_INSTANCE_ID'])
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+199
View File
@@ -0,0 +1,199 @@
"""Repeatable grade-4 experiments. Run on host with Docker access and UTF-8 mode."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from ops import ROOT, compose, restore, rolling, save, snapshot, stop_instance, wait_api
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
from resilience import history
def command(name, args):
result = subprocess.run(args, cwd=ROOT, text=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180)
(ROOT / 'evidence' / name).write_text(result.stdout, encoding='utf-8')
print(result.stdout, flush=True)
check(result.returncode == 0, 'Command failed: ' + name)
def concurrency(base):
c = Client(base)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
key = uuid4().hex
barrier = threading.Barrier(8)
def send(_):
client = Client(base)
barrier.wait(timeout=10)
return client.message(alice, chat, 'one concurrent message', key=key)
responses = []
with ThreadPoolExecutor(max_workers=8) as executor:
for _ in range(3):
responses.extend(executor.map(send, range(8)))
first = responses[0]['json']
check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replies differ')
instances = sorted({r['headers']['X-Instance-Id'] for r in responses})
check(len(instances) >= 2, 'Concurrent test did not reach both replicas')
check(history(c, alice, chat) == [first], 'Concurrent retry duplicated message')
c.call('POST', path, 409, actor=alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
c.call('POST', path, 401, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
c.call('POST', path, 404, actor=eve, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
for invalid in (None, '', 'bad key', 'x' * 129):
c.call('POST', path, 400, actor=alice, body={'text': 'invalid key'},
headers={} if invalid is None else {'Idempotency-Key': invalid})
other = c.message(bob, chat, 'independent user', key=key)['json']
check(other['id'] != first['id'], 'Key scope must include actor')
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Scope', 'member_ids': [alice['id']]}, schema='Chat')['json']
other = c.message(alice, chat2, 'independent chat', key=key)['json']
check(other['id'] != first['id'], 'Key scope must include chat')
save('idempotency.json', {'requests': 24, 'concurrency': 8, 'instances': instances,
'unique_messages': 1, 'same_response': True,
'conflict_auth_access_key_scope': 'PASS'})
return c, alice, chat, key, first
def experiment(base, name, action, target=None):
c = Client(base, timeout=9)
alice, bob, _, chat = c.bootstrap()
confirmed_before = c.message(alice, chat, 'before ' + name)['json']
events = []
stop = threading.Event()
started = time.monotonic()
def worker():
client = Client(base, timeout=9)
key = uuid4().hex
while not stop.is_set():
begin = time.monotonic()
try:
response = client.message(alice, chat, key, key=key)
msg = response['json']
events.append({'kind': 'write', 'time': time.monotonic() - started,
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id'],
'message': msg})
key = uuid4().hex
except Failure as exc:
events.append({'kind': 'error', 'operation': 'write', 'time': time.monotonic() - started,
'latency': time.monotonic() - begin, 'error': str(exc)})
continue # Keep original key/body after an uncertain write.
begin = time.monotonic()
try:
response = client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')
events.append({'kind': 'read', 'time': time.monotonic() - started,
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id']})
except Failure as exc:
events.append({'kind': 'error', 'operation': 'read', 'time': time.monotonic() - started,
'latency': time.monotonic() - begin, 'error': str(exc)})
time.sleep(0.02)
with ThreadPoolExecutor(max_workers=2) as executor:
futures = [executor.submit(worker) for _ in range(2)]
try:
time.sleep(3)
action_started = time.monotonic() - started
effect = action()
action_finished = time.monotonic() - started
time.sleep(max(5, 20 - (time.monotonic() - started)))
finally:
stop.set()
for future in futures:
future.result()
duration = time.monotonic() - started
successes = [e for e in events if e['kind'] != 'error']
errors = [e for e in events if e['kind'] == 'error']
writes = [e for e in successes if e['kind'] == 'write']
recovery = {}
for kind in ('read', 'write'):
after = [e for e in successes if e['kind'] == kind and e['time'] >= action_finished]
check(bool(after), 'No ' + kind + ' after action')
recovery[kind] = round(min(e['time'] for e in after) - action_started, 3)
if target:
check(all(e['instance'] != target for e in after), 'Stopped target still serves')
check(recovery[kind] <= 10, 'Recovery exceeded 10 seconds')
messages = history(c, bob, chat)
by_id = {m['id']: m for m in messages}
check(by_id.get(confirmed_before['id']) == confirmed_before, 'Pre-failure message lost')
check(all(by_id.get(e['message']['id']) == e['message'] for e in writes), 'Confirmed write lost')
check(len({m['text'] for m in messages}) == len(messages), 'Retry duplicated a message')
# Retry a pre-failure key through the surviving/new processes.
if writes:
item = writes[0]['message']
check(c.message(alice, chat, item['text'], key=item['text'])['json'] == item, 'Replay changed after action')
latencies = sorted(e['latency'] for e in successes)
gaps = {}
for kind in ('read', 'write'):
times = sorted(e['time'] for e in successes if e['kind'] == kind)
gaps[kind] = round(max((b - a for a, b in zip(times, times[1:])), default=0), 3)
summary = {'duration_seconds': round(duration, 3), 'concurrency': 2,
'successful_http_rps': round(len(successes) / duration, 2),
'successful_write_rps': round(len(writes) / duration, 2),
'successful_requests': len(successes), 'errors': len(errors),
'action_started_seconds': round(action_started, 3),
'action_finished_seconds': round(action_finished, 3),
'first_read_write_after_action_seconds_from_start': recovery,
'max_success_gap_seconds': gaps,
'latency_ms': {str(p): round(latencies[min(int(len(latencies)*p/100), len(latencies)-1)]*1000, 2)
for p in (50, 95, 99)},
'confirmed_messages_preserved': True, 'duplicate_messages': 0,
'instances': sorted({e['instance'] for e in successes}), 'target': target}
save(name + '.json', {'summary': summary, 'containers': effect, 'errors': errors})
print(name + ': ' + json.dumps(summary), flush=True)
return summary
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
args = parser.parse_args()
os.environ['PYTHONUTF8'] = '1'
base = args.base_url
wait_api()
command('check.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
c, actor, chat, key, original = concurrency(base)
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
command('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence',
'--base-url', base, '--action-script', hook])
wait_api()
check(c.message(actor, chat, original['text'], key=key)['json'] == original,
'Idempotency/session not preserved across full restart')
save('idempotency-restart.json', {'same_token': True, 'same_201_message': True})
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
for index in range(2):
try:
command('public-failover-' + str(index+1) + '.txt',
[sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover',
'--base-url', base, '--action-script', hook])
finally:
restore()
results = {}
for index, row in enumerate(snapshot('api'), start=1):
target = row['instance']
try:
results['failover-' + str(index)] = experiment(base, 'failover-' + str(index),
lambda: stop_instance(target), target)
finally:
restore()
# Build one image first; apply compatible migrations once before replacing replicas.
compose('build', 'api')
compose('run', '--rm', '--no-deps', 'migrate')
results['rolling'] = experiment(base, 'rolling-load', rolling)
save('grade4-summary.json', results)
save('final-containers.json', snapshot())
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
print('PASS: all grade-4 scenarios', flush=True)
if __name__ == '__main__':
main()