test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run lab3 checks and retain one compact result without request logs or secrets."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def capture(command, timeout=180):
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=timeout)
|
||||
|
||||
|
||||
def snapshot():
|
||||
result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30)
|
||||
result.check_returncode()
|
||||
ids = result.stdout.split()
|
||||
inspected = capture(['docker', 'inspect', *ids], 30)
|
||||
inspected.check_returncode()
|
||||
rows = []
|
||||
for c in json.loads(inspected.stdout):
|
||||
service = c['Config']['Labels'].get('com.docker.compose.service')
|
||||
if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}:
|
||||
continue
|
||||
pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None)
|
||||
rows.append({'service': service, 'id': c['Id'], 'image': c['Image'],
|
||||
'started_at': c['State']['StartedAt'], 'status': c['State']['Status'],
|
||||
'health': c['State'].get('Health', {}).get('Status'),
|
||||
'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_mib': c['HostConfig']['Memory'] / 1024**2,
|
||||
'pool_max': pool, 'command': c['Config']['Cmd'],
|
||||
'ports': c['HostConfig'].get('PortBindings'),
|
||||
'volumes': [{'name': m['Name'], 'destination': m['Destination']}
|
||||
for m in c['Mounts'] if m['Type'] == 'volume']})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json')
|
||||
parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts')
|
||||
args = parser.parse_args()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if args.retry_failed:
|
||||
report = json.loads(args.output.read_text(encoding='utf-8'))
|
||||
if report.get('source_sha256'):
|
||||
report['source_before_recheck_sha256'] = report['source_sha256']
|
||||
selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'}
|
||||
report['recheck_started'] = datetime.now(timezone.utc).isoformat()
|
||||
else:
|
||||
previous = (json.loads(args.output.read_text(encoding='utf-8'))
|
||||
if args.output.exists() else None)
|
||||
report = {'recorded_at': datetime.now(timezone.utc).isoformat(),
|
||||
'base_url': args.base_url, 'checks': [], 'initial': snapshot()}
|
||||
if previous:
|
||||
report['previous_validation'] = {
|
||||
'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'),
|
||||
'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}),
|
||||
'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c}
|
||||
for c in previous['checks']],
|
||||
}
|
||||
selected = None
|
||||
report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
||||
for name in ('src/main.ts', 'src/app.controller.ts',
|
||||
'src/common/http/fastify-adapter.ts',
|
||||
'infra/start-nginx.sh', 'scripts/stop-one.sh',
|
||||
'scripts/restart.sh', 'test/check_shutdown.py')}
|
||||
|
||||
def save():
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
def run(name, command, timeout=180, details=None, parse_json=False):
|
||||
if selected is not None and name not in selected and name != 'final-smoke':
|
||||
return None
|
||||
print(f'START {name}', flush=True)
|
||||
started = time.monotonic()
|
||||
item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()}
|
||||
try:
|
||||
result = capture(command, timeout)
|
||||
item['exit_code'] = result.returncode
|
||||
item['status'] = 'PASS' if result.returncode == 0 else 'FAIL'
|
||||
lines = (result.stdout + '\n' + result.stderr).splitlines()
|
||||
item['summary'] = [line for line in lines if line.startswith(
|
||||
('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:]
|
||||
if details is not None and details.exists():
|
||||
item['details'] = json.loads(details.read_text(encoding='utf-8'))
|
||||
elif parse_json:
|
||||
for pos in (i for i, char in enumerate(result.stdout) if char == '{'):
|
||||
try:
|
||||
item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:])
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
item.update(status='FAIL', error=type(exc).__name__)
|
||||
item['seconds'] = round(time.monotonic() - started, 3)
|
||||
previous = next((c for c in report['checks'] if c['name'] == name), None)
|
||||
if previous is not None:
|
||||
item['previous_attempts'] = previous.get('previous_attempts', []) + [
|
||||
{key: value for key, value in previous.items() if key != 'previous_attempts'}]
|
||||
report['checks'][report['checks'].index(previous)] = item
|
||||
else:
|
||||
report['checks'].append(item)
|
||||
save()
|
||||
print(f"{item['status']} {name} ({item['seconds']}s)", flush=True)
|
||||
for line in item.get('summary', [])[-3:]:
|
||||
print(line, flush=True)
|
||||
return item
|
||||
|
||||
def restore():
|
||||
result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml',
|
||||
'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180)
|
||||
result.check_returncode()
|
||||
time.sleep(6)
|
||||
|
||||
py = [sys.executable, '-X', 'utf8']
|
||||
pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm'
|
||||
hook_suffix = 'cmd' if os.name == 'nt' else 'sh'
|
||||
save()
|
||||
with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp:
|
||||
temp = Path(tmp)
|
||||
run('contract', py + ['scripts/check_contract.py'])
|
||||
run('lint', [pnpm, 'lint'])
|
||||
run('build', [pnpm, 'build'], 240)
|
||||
run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120)
|
||||
for name, script in (('nginx-readiness', 'nginx-readiness.sh'),
|
||||
('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')):
|
||||
run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh',
|
||||
'-v', f'{ROOT / "infra"}:/source:ro',
|
||||
'-v', f'{ROOT / "test" / script}:/test.sh:ro',
|
||||
'nginx:1.30.6-alpine', '/test.sh'], 60)
|
||||
run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
||||
run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
|
||||
'-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro',
|
||||
'api', 'node', '--test', 'test/idempotency.mjs'], 180)
|
||||
if selected is None or selected & {'persistence', 'public-failover'}:
|
||||
try:
|
||||
if selected is None or 'persistence' in selected:
|
||||
before = snapshot()
|
||||
item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence',
|
||||
'--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240)
|
||||
after = snapshot()
|
||||
item['before'] = before
|
||||
item['after'] = after
|
||||
restarted = {c['id']: c for c in before if c['service'] != 'migrate'}
|
||||
same = {c['id']: c for c in after if c['service'] != 'migrate'}
|
||||
item['containers_restarted'] = restarted.keys() == same.keys() and all(
|
||||
c['started_at'] != same[key]['started_at'] for key, c in restarted.items())
|
||||
item['volumes_preserved'] = restarted.keys() == same.keys() and all(
|
||||
c['volumes'] == same[key]['volumes'] for key, c in restarted.items())
|
||||
if not item['containers_restarted'] or not item['volumes_preserved']:
|
||||
item['status'] = 'FAIL'
|
||||
save()
|
||||
run('public-failover', py + ['tests/resilience.py', '--mode', 'failover',
|
||||
'--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120,
|
||||
parse_json=True)
|
||||
finally:
|
||||
restore()
|
||||
for scenario in ('resources', 'failover', 'rolling'):
|
||||
result_file = temp / f'grade4-{scenario}.json'
|
||||
run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario,
|
||||
'--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10',
|
||||
'--output', str(result_file)], 300, details=result_file)
|
||||
run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True)
|
||||
images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest',
|
||||
'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest']
|
||||
if selected is None or 'migration-compatibility' in selected:
|
||||
available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30)
|
||||
if available.returncode == 0:
|
||||
run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True)
|
||||
else:
|
||||
report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN',
|
||||
'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'})
|
||||
run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
||||
report['final'] = snapshot()
|
||||
report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks'])
|
||||
report['finished'] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True)
|
||||
if not report['all_executed_checks_pass']:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = {'backend', 'postgres', 'redis'}
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = subprocess.check_output(
|
||||
['docker', 'compose', 'ps', '--all', '--quiet', *sorted(SERVICES)],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
).split()
|
||||
if len(ids) != len(SERVICES):
|
||||
raise RuntimeError('Start backend, postgres and redis before this test')
|
||||
template = (
|
||||
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
|
||||
'"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"running":{{json .State.Running}},"mounts":{{json .Mounts}}}'
|
||||
)
|
||||
output = subprocess.check_output(
|
||||
['docker', 'inspect', '--format', template, *ids],
|
||||
cwd=ROOT, text=True, timeout=30,
|
||||
)
|
||||
result = {}
|
||||
for line in output.splitlines():
|
||||
entry = json.loads(line)
|
||||
service = entry.pop('service')
|
||||
entry['volumes'] = [
|
||||
{'name': mount['Name'], 'destination': mount['Destination']}
|
||||
for mount in entry.pop('mounts') if mount['Type'] == 'volume'
|
||||
]
|
||||
result[service] = entry
|
||||
if set(result) != SERVICES or not all(v['running'] for v in result.values()):
|
||||
raise RuntimeError('All three services must be running')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario', choices=['public', 'full-restart'], default='public')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence' / 'persistence.json')
|
||||
args = parser.parse_args()
|
||||
before = snapshot()
|
||||
hook = ROOT / 'scripts' / ('restart.cmd' if os.name == 'nt' else 'restart.sh')
|
||||
if args.scenario == 'public':
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'tests' / 'resilience.py'), '--mode', 'persistence']
|
||||
else:
|
||||
command = [sys.executable, '-X', 'utf8', str(ROOT / 'test' / 'full-restart.py')]
|
||||
command += ['--action-script', str(hook), '--base-url', args.base_url]
|
||||
if args.ca_file:
|
||||
command += ['--ca-file', args.ca_file]
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(command, cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
|
||||
print(proc.stdout, end='')
|
||||
print(proc.stderr, end='', file=sys.stderr)
|
||||
if proc.returncode:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps({
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'result': 'FAIL', 'exit_code': proc.returncode,
|
||||
'test_output': proc.stdout.strip(), 'test_error': proc.stderr.strip(),
|
||||
'before': before,
|
||||
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
return proc.returncode
|
||||
after = snapshot()
|
||||
for service in sorted(SERVICES):
|
||||
old, new = before[service], after[service]
|
||||
if old['id'] != new['id'] or old['started_at'] == new['started_at']:
|
||||
raise RuntimeError(f'{service}: expected a restart of the same container')
|
||||
if not old['volumes'] or old['volumes'] != new['volumes']:
|
||||
raise RuntimeError(f'{service}: persistent volumes changed or are missing')
|
||||
evidence = {
|
||||
'recorded_at': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'scenario': args.scenario,
|
||||
'command': f'python scripts/test-persistence.py --scenario {args.scenario} --base-url {args.base_url}',
|
||||
'hook': hook.relative_to(ROOT).as_posix(),
|
||||
'elapsed_seconds': round(time.monotonic() - started, 3),
|
||||
'result': 'PASS', 'test_output': proc.stdout.strip(),
|
||||
'before': before, 'after': after,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: backend, postgres and redis restarted; named volumes preserved.')
|
||||
print(f'Evidence: {args.output}')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user