test: cover persistence failover and graceful draining

Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
2026-10-29 11:47:46 +03:00
parent 676b38c18b
commit 5c8bcf0f80
37 changed files with 5538 additions and 0 deletions
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Run lab3 checks and retain one compact result without request logs or secrets."""
import argparse
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
ROOT = Path(__file__).resolve().parents[1]
def capture(command, timeout=180):
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=timeout)
def snapshot():
result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30)
result.check_returncode()
ids = result.stdout.split()
inspected = capture(['docker', 'inspect', *ids], 30)
inspected.check_returncode()
rows = []
for c in json.loads(inspected.stdout):
service = c['Config']['Labels'].get('com.docker.compose.service')
if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}:
continue
pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None)
rows.append({'service': service, 'id': c['Id'], 'image': c['Image'],
'started_at': c['State']['StartedAt'], 'status': c['State']['Status'],
'health': c['State'].get('Health', {}).get('Status'),
'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9,
'memory_mib': c['HostConfig']['Memory'] / 1024**2,
'pool_max': pool, 'command': c['Config']['Cmd'],
'ports': c['HostConfig'].get('PortBindings'),
'volumes': [{'name': m['Name'], 'destination': m['Destination']}
for m in c['Mounts'] if m['Type'] == 'volume']})
return rows
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json')
parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts')
args = parser.parse_args()
args.output.parent.mkdir(parents=True, exist_ok=True)
if args.retry_failed:
report = json.loads(args.output.read_text(encoding='utf-8'))
if report.get('source_sha256'):
report['source_before_recheck_sha256'] = report['source_sha256']
selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'}
report['recheck_started'] = datetime.now(timezone.utc).isoformat()
else:
previous = (json.loads(args.output.read_text(encoding='utf-8'))
if args.output.exists() else None)
report = {'recorded_at': datetime.now(timezone.utc).isoformat(),
'base_url': args.base_url, 'checks': [], 'initial': snapshot()}
if previous:
report['previous_validation'] = {
'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'),
'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}),
'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c}
for c in previous['checks']],
}
selected = None
report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
for name in ('src/main.ts', 'src/app.controller.ts',
'src/common/http/fastify-adapter.ts',
'infra/start-nginx.sh', 'scripts/stop-one.sh',
'scripts/restart.sh', 'test/check_shutdown.py')}
def save():
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
def run(name, command, timeout=180, details=None, parse_json=False):
if selected is not None and name not in selected and name != 'final-smoke':
return None
print(f'START {name}', flush=True)
started = time.monotonic()
item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()}
try:
result = capture(command, timeout)
item['exit_code'] = result.returncode
item['status'] = 'PASS' if result.returncode == 0 else 'FAIL'
lines = (result.stdout + '\n' + result.stderr).splitlines()
item['summary'] = [line for line in lines if line.startswith(
('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:]
if details is not None and details.exists():
item['details'] = json.loads(details.read_text(encoding='utf-8'))
elif parse_json:
for pos in (i for i, char in enumerate(result.stdout) if char == '{'):
try:
item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:])
break
except json.JSONDecodeError:
continue
except (OSError, subprocess.TimeoutExpired) as exc:
item.update(status='FAIL', error=type(exc).__name__)
item['seconds'] = round(time.monotonic() - started, 3)
previous = next((c for c in report['checks'] if c['name'] == name), None)
if previous is not None:
item['previous_attempts'] = previous.get('previous_attempts', []) + [
{key: value for key, value in previous.items() if key != 'previous_attempts'}]
report['checks'][report['checks'].index(previous)] = item
else:
report['checks'].append(item)
save()
print(f"{item['status']} {name} ({item['seconds']}s)", flush=True)
for line in item.get('summary', [])[-3:]:
print(line, flush=True)
return item
def restore():
result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml',
'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180)
result.check_returncode()
time.sleep(6)
py = [sys.executable, '-X', 'utf8']
pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm'
hook_suffix = 'cmd' if os.name == 'nt' else 'sh'
save()
with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp:
temp = Path(tmp)
run('contract', py + ['scripts/check_contract.py'])
run('lint', [pnpm, 'lint'])
run('build', [pnpm, 'build'], 240)
run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120)
for name, script in (('nginx-readiness', 'nginx-readiness.sh'),
('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')):
run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh',
'-v', f'{ROOT / "infra"}:/source:ro',
'-v', f'{ROOT / "test" / script}:/test.sh:ro',
'nginx:1.30.6-alpine', '/test.sh'], 60)
run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
'-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro',
'api', 'node', '--test', 'test/idempotency.mjs'], 180)
if selected is None or selected & {'persistence', 'public-failover'}:
try:
if selected is None or 'persistence' in selected:
before = snapshot()
item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence',
'--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240)
after = snapshot()
item['before'] = before
item['after'] = after
restarted = {c['id']: c for c in before if c['service'] != 'migrate'}
same = {c['id']: c for c in after if c['service'] != 'migrate'}
item['containers_restarted'] = restarted.keys() == same.keys() and all(
c['started_at'] != same[key]['started_at'] for key, c in restarted.items())
item['volumes_preserved'] = restarted.keys() == same.keys() and all(
c['volumes'] == same[key]['volumes'] for key, c in restarted.items())
if not item['containers_restarted'] or not item['volumes_preserved']:
item['status'] = 'FAIL'
save()
run('public-failover', py + ['tests/resilience.py', '--mode', 'failover',
'--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120,
parse_json=True)
finally:
restore()
for scenario in ('resources', 'failover', 'rolling'):
result_file = temp / f'grade4-{scenario}.json'
run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario,
'--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10',
'--output', str(result_file)], 300, details=result_file)
run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True)
images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest',
'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest']
if selected is None or 'migration-compatibility' in selected:
available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30)
if available.returncode == 0:
run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True)
else:
report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN',
'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'})
run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
report['final'] = snapshot()
report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks'])
report['finished'] = datetime.now(timezone.utc).isoformat()
save()
print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True)
if not report['all_executed_checks_pass']:
raise SystemExit(1)
if __name__ == '__main__':
main()
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
import argparse
import datetime as dt
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
SERVICES = {'backend', 'postgres', 'redis'}
def snapshot():
ids = subprocess.check_output(
['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)],
cwd=ROOT, text=True, timeout=30,
).split()
if len(ids) != len(SERVICES):
raise RuntimeError('Start backend, postgres and redis before this test')
template = (
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
'"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"running":{{json .State.Running}},"mounts":{{json .Mounts}}}'
)
output = subprocess.check_output(
['docker', 'inspect', '--format', template, *ids],
cwd=ROOT, text=True, timeout=30,
)
result = {}
for line in output.splitlines():
entry = json.loads(line)
service = entry.pop('service')
entry['volumes'] = [
{'name': mount['Name'], 'destination': mount['Destination']}
for mount in entry.pop('mounts') if mount['Type'] == 'volume'
]
result[service] = entry
if set(result) != SERVICES or not all(v['running'] for v in result.values()):
raise RuntimeError('All three services must be running')
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
parser.add_argument('--ca-file')
parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json')
args = parser.parse_args()
before = snapshot()
hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh')
if args.scenario == 'public':
command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence']
else:
command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')]
command += ['--action-script', str(hook), '--base-url', args.base_url]
if args.ca_file:
command += ['--ca-file', args.ca_file]
started = time.monotonic()
proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
print(proc.stdout, end='')
print(proc.stderr, end='', file=sys.stderr)
if proc.returncode:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps({
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'scenario': args.scenario,
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
'hook': hook.relative_to(ROOT).as_posix(),
'result': 'FAIL', 'exit_code': proc.returncode,
'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(),
'before': before,
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
return proc.returncode
after = snapshot()
for service in sorted(SERVICES):
old, new = before[service], after[service]
if old['id'] != new['id'] or old['started_at'] == new['started_at']:
raise RuntimeError(f'{service}: expected a restart of the same container')
if not old['volumes'] or old['volumes'] != new['volumes']:
raise RuntimeError(f'{service}: persistent volumes changed or are missing')
evidence = {
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'scenario': args.scenario,
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
'hook': hook.relative_to(ROOT).as_posix(),
'elapsed_seconds': round(time.monotonic() - started, 3),
'result': 'PASS', 'test_output': proc.stdout.strip(),
'before': before, 'after': after,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print('PASS: backend, postgres and redis restarted; named volumes preserved.')
print(f'Evidence: {args.output}')
return 0
if __name__ == '__main__':
sys.exit(main())