test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = {'backend', 'postgres', 'redis'}
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = subprocess.check_output(
|
||||
['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
).split()
|
||||
if len(ids) != len(SERVICES):
|
||||
raise RuntimeError('Start backend, postgres and redis before this test')
|
||||
template = (
|
||||
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
|
||||
'"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"running":{{json .State.Running}},"mounts":{{json .Mounts}}}'
|
||||
)
|
||||
output = subprocess.check_output(
|
||||
['docker', 'inspect', '--format', template, *ids],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
)
|
||||
result = {}
|
||||
for line in output.splitlines():
|
||||
entry = json.loads(line)
|
||||
service = entry.pop('service')
|
||||
entry['volumes'] = [
|
||||
{'name': mount['Name'], 'destination': mount['Destination']}
|
||||
for mount in entry.pop('mounts') if mount['Type'] == 'volume'
|
||||
]
|
||||
result[service] = entry
|
||||
if set(result) != SERVICES or not all(v['running'] for v in result.values()):
|
||||
raise RuntimeError('All three services must be running')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json')
|
||||
args = parser.parse_args()
|
||||
before = snapshot()
|
||||
hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh')
|
||||
if args.scenario == 'public':
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence']
|
||||
else:
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')]
|
||||
command += ['--action-script', str(hook), '--base-url', args.base_url]
|
||||
if args.ca_file:
|
||||
command += ['--ca-file', args.ca_file]
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
|
||||
print(proc.stdout, end='')
|
||||
print(proc.stderr, end='', file=sys.stderr)
|
||||
if proc.returncode:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps({
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'result': 'FAIL', 'exit_code': proc.returncode,
|
||||
'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(),
|
||||
'before': before,
|
||||
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
return proc.returncode
|
||||
after = snapshot()
|
||||
for service in sorted(SERVICES):
|
||||
old, new = before[service], after[service]
|
||||
if old['id'] != new['id'] or old['started_at'] == new['started_at']:
|
||||
raise RuntimeError(f'{service}: expected a restart of the same container')
|
||||
if not old['volumes'] or old['volumes'] != new['volumes']:
|
||||
raise RuntimeError(f'{service}: persistent volumes changed or are missing')
|
||||
evidence = {
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'elapsed_seconds': round(time.monotonic() - started, 3),
|
||||
'result': 'PASS', 'test_output': proc.stdout.strip(),
|
||||
'before': before, 'after': after,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: backend, postgres and redis restarted; named volumes preserved.')
|
||||
print(f'Evidence: {args.output}')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user