forked from mtusi2026/mtusi2026lab3
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
195 lines
11 KiB
Python
195 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Run lab3 checks and retain one compact result without request logs or secrets."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def capture(command, timeout=180):
|
|
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace', timeout=timeout)
|
|
|
|
|
|
def snapshot():
|
|
result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30)
|
|
result.check_returncode()
|
|
ids = result.stdout.split()
|
|
inspected = capture(['docker', 'inspect', *ids], 30)
|
|
inspected.check_returncode()
|
|
rows = []
|
|
for c in json.loads(inspected.stdout):
|
|
service = c['Config']['Labels'].get('com.docker.compose.service')
|
|
if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}:
|
|
continue
|
|
pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None)
|
|
rows.append({'service': service, 'id': c['Id'], 'image': c['Image'],
|
|
'started_at': c['State']['StartedAt'], 'status': c['State']['Status'],
|
|
'health': c['State'].get('Health', {}).get('Status'),
|
|
'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9,
|
|
'memory_mib': c['HostConfig']['Memory'] / 1024**2,
|
|
'pool_max': pool, 'command': c['Config']['Cmd'],
|
|
'ports': c['HostConfig'].get('PortBindings'),
|
|
'volumes': [{'name': m['Name'], 'destination': m['Destination']}
|
|
for m in c['Mounts'] if m['Type'] == 'volume']})
|
|
return rows
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
|
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json')
|
|
parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts')
|
|
args = parser.parse_args()
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
if args.retry_failed:
|
|
report = json.loads(args.output.read_text(encoding='utf-8'))
|
|
if report.get('source_sha256'):
|
|
report['source_before_recheck_sha256'] = report['source_sha256']
|
|
selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'}
|
|
report['recheck_started'] = datetime.now(timezone.utc).isoformat()
|
|
else:
|
|
previous = (json.loads(args.output.read_text(encoding='utf-8'))
|
|
if args.output.exists() else None)
|
|
report = {'recorded_at': datetime.now(timezone.utc).isoformat(),
|
|
'base_url': args.base_url, 'checks': [], 'initial': snapshot()}
|
|
if previous:
|
|
report['previous_validation'] = {
|
|
'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'),
|
|
'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}),
|
|
'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c}
|
|
for c in previous['checks']],
|
|
}
|
|
selected = None
|
|
report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
|
for name in ('src/main.ts', 'src/app.controller.ts',
|
|
'src/common/http/fastify-adapter.ts',
|
|
'infra/start-nginx.sh', 'scripts/stop-one.sh',
|
|
'scripts/restart.sh', 'test/check_shutdown.py')}
|
|
|
|
def save():
|
|
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
|
|
def run(name, command, timeout=180, details=None, parse_json=False):
|
|
if selected is not None and name not in selected and name != 'final-smoke':
|
|
return None
|
|
print(f'START {name}', flush=True)
|
|
started = time.monotonic()
|
|
item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()}
|
|
try:
|
|
result = capture(command, timeout)
|
|
item['exit_code'] = result.returncode
|
|
item['status'] = 'PASS' if result.returncode == 0 else 'FAIL'
|
|
lines = (result.stdout + '\n' + result.stderr).splitlines()
|
|
item['summary'] = [line for line in lines if line.startswith(
|
|
('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:]
|
|
if details is not None and details.exists():
|
|
item['details'] = json.loads(details.read_text(encoding='utf-8'))
|
|
elif parse_json:
|
|
for pos in (i for i, char in enumerate(result.stdout) if char == '{'):
|
|
try:
|
|
item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:])
|
|
break
|
|
except json.JSONDecodeError:
|
|
continue
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
item.update(status='FAIL', error=type(exc).__name__)
|
|
item['seconds'] = round(time.monotonic() - started, 3)
|
|
previous = next((c for c in report['checks'] if c['name'] == name), None)
|
|
if previous is not None:
|
|
item['previous_attempts'] = previous.get('previous_attempts', []) + [
|
|
{key: value for key, value in previous.items() if key != 'previous_attempts'}]
|
|
report['checks'][report['checks'].index(previous)] = item
|
|
else:
|
|
report['checks'].append(item)
|
|
save()
|
|
print(f"{item['status']} {name} ({item['seconds']}s)", flush=True)
|
|
for line in item.get('summary', [])[-3:]:
|
|
print(line, flush=True)
|
|
return item
|
|
|
|
def restore():
|
|
result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml',
|
|
'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180)
|
|
result.check_returncode()
|
|
time.sleep(6)
|
|
|
|
py = [sys.executable, '-X', 'utf8']
|
|
pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm'
|
|
hook_suffix = 'cmd' if os.name == 'nt' else 'sh'
|
|
save()
|
|
with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp:
|
|
temp = Path(tmp)
|
|
run('contract', py + ['scripts/check_contract.py'])
|
|
run('lint', [pnpm, 'lint'])
|
|
run('build', [pnpm, 'build'], 240)
|
|
run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120)
|
|
for name, script in (('nginx-readiness', 'nginx-readiness.sh'),
|
|
('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')):
|
|
run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh',
|
|
'-v', f'{ROOT / "infra"}:/source:ro',
|
|
'-v', f'{ROOT / "test" / script}:/test.sh:ro',
|
|
'nginx:1.30.6-alpine', '/test.sh'], 60)
|
|
run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
|
run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
|
|
'-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro',
|
|
'api', 'node', '--test', 'test/idempotency.mjs'], 180)
|
|
if selected is None or selected & {'persistence', 'public-failover'}:
|
|
try:
|
|
if selected is None or 'persistence' in selected:
|
|
before = snapshot()
|
|
item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence',
|
|
'--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240)
|
|
after = snapshot()
|
|
item['before'] = before
|
|
item['after'] = after
|
|
restarted = {c['id']: c for c in before if c['service'] != 'migrate'}
|
|
same = {c['id']: c for c in after if c['service'] != 'migrate'}
|
|
item['containers_restarted'] = restarted.keys() == same.keys() and all(
|
|
c['started_at'] != same[key]['started_at'] for key, c in restarted.items())
|
|
item['volumes_preserved'] = restarted.keys() == same.keys() and all(
|
|
c['volumes'] == same[key]['volumes'] for key, c in restarted.items())
|
|
if not item['containers_restarted'] or not item['volumes_preserved']:
|
|
item['status'] = 'FAIL'
|
|
save()
|
|
run('public-failover', py + ['tests/resilience.py', '--mode', 'failover',
|
|
'--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120,
|
|
parse_json=True)
|
|
finally:
|
|
restore()
|
|
for scenario in ('resources', 'failover', 'rolling'):
|
|
result_file = temp / f'grade4-{scenario}.json'
|
|
run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario,
|
|
'--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10',
|
|
'--output', str(result_file)], 300, details=result_file)
|
|
run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True)
|
|
images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest',
|
|
'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest']
|
|
if selected is None or 'migration-compatibility' in selected:
|
|
available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30)
|
|
if available.returncode == 0:
|
|
run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True)
|
|
else:
|
|
report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN',
|
|
'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'})
|
|
run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
|
|
report['final'] = snapshot()
|
|
report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks'])
|
|
report['finished'] = datetime.now(timezone.utc).isoformat()
|
|
save()
|
|
print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True)
|
|
if not report['all_executed_checks_pass']:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|