Files
mtusi2026lab3-fork/scripts/collect-report-checks.py
qrtx 5c8bcf0f80 test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
2026-10-29 11:47:46 +03:00

195 lines
11 KiB
Python

#!/usr/bin/env python3
"""Run lab3 checks and retain one compact result without request logs or secrets."""
import argparse
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
ROOT = Path(__file__).resolve().parents[1]
def capture(command, timeout=180):
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=timeout)
def snapshot():
result = capture(['docker', 'compose', 'ps', '--all', '-q'], 30)
result.check_returncode()
ids = result.stdout.split()
inspected = capture(['docker', 'inspect', *ids], 30)
inspected.check_returncode()
rows = []
for c in json.loads(inspected.stdout):
service = c['Config']['Labels'].get('com.docker.compose.service')
if service not in {'api', 'nginx', 'postgres', 'redis', 'migrate'}:
continue
pool = next((v.split('=', 1)[1] for v in c['Config'].get('Env', []) if v.startswith('DB_POOL_MAX=')), None)
rows.append({'service': service, 'id': c['Id'], 'image': c['Image'],
'started_at': c['State']['StartedAt'], 'status': c['State']['Status'],
'health': c['State'].get('Health', {}).get('Status'),
'exit_code': c['State']['ExitCode'], 'cpu': c['HostConfig']['NanoCpus'] / 1e9,
'memory_mib': c['HostConfig']['Memory'] / 1024**2,
'pool_max': pool, 'command': c['Config']['Cmd'],
'ports': c['HostConfig'].get('PortBindings'),
'volumes': [{'name': m['Name'], 'destination': m['Destination']}
for m in c['Mounts'] if m['Type'] == 'volume']})
return rows
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/checks.json')
parser.add_argument('--retry-failed', action='store_true', help='Rerun failed checks and retain previous attempts')
args = parser.parse_args()
args.output.parent.mkdir(parents=True, exist_ok=True)
if args.retry_failed:
report = json.loads(args.output.read_text(encoding='utf-8'))
if report.get('source_sha256'):
report['source_before_recheck_sha256'] = report['source_sha256']
selected = {c['name'] for c in report['checks'] if c['status'] == 'FAIL'}
report['recheck_started'] = datetime.now(timezone.utc).isoformat()
else:
previous = (json.loads(args.output.read_text(encoding='utf-8'))
if args.output.exists() else None)
report = {'recorded_at': datetime.now(timezone.utc).isoformat(),
'base_url': args.base_url, 'checks': [], 'initial': snapshot()}
if previous:
report['previous_validation'] = {
'recorded_at': previous['recorded_at'], 'finished': previous.get('finished'),
'api_images': sorted({c['image'] for c in previous.get('final', []) if c['service'] == 'api'}),
'checks': [{k: c[k] for k in ('name', 'status', 'seconds', 'previous_attempts') if k in c}
for c in previous['checks']],
}
selected = None
report['source_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
for name in ('src/main.ts', 'src/app.controller.ts',
'src/common/http/fastify-adapter.ts',
'infra/start-nginx.sh', 'scripts/stop-one.sh',
'scripts/restart.sh', 'test/check_shutdown.py')}
def save():
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
def run(name, command, timeout=180, details=None, parse_json=False):
if selected is not None and name not in selected and name != 'final-smoke':
return None
print(f'START {name}', flush=True)
started = time.monotonic()
item = {'name': name, 'command': command, 'started': datetime.now(timezone.utc).isoformat()}
try:
result = capture(command, timeout)
item['exit_code'] = result.returncode
item['status'] = 'PASS' if result.returncode == 0 else 'FAIL'
lines = (result.stdout + '\n' + result.stderr).splitlines()
item['summary'] = [line for line in lines if line.startswith(
('PASS:', 'FAIL:', 'ok ', 'not ok ', '# tests', '# pass', '# fail', '# cancelled'))][-40:]
if details is not None and details.exists():
item['details'] = json.loads(details.read_text(encoding='utf-8'))
elif parse_json:
for pos in (i for i, char in enumerate(result.stdout) if char == '{'):
try:
item['details'], _ = json.JSONDecoder().raw_decode(result.stdout[pos:])
break
except json.JSONDecodeError:
continue
except (OSError, subprocess.TimeoutExpired) as exc:
item.update(status='FAIL', error=type(exc).__name__)
item['seconds'] = round(time.monotonic() - started, 3)
previous = next((c for c in report['checks'] if c['name'] == name), None)
if previous is not None:
item['previous_attempts'] = previous.get('previous_attempts', []) + [
{key: value for key, value in previous.items() if key != 'previous_attempts'}]
report['checks'][report['checks'].index(previous)] = item
else:
report['checks'].append(item)
save()
print(f"{item['status']} {name} ({item['seconds']}s)", flush=True)
for line in item.get('summary', [])[-3:]:
print(line, flush=True)
return item
def restore():
result = capture(['docker', 'compose', '-f', 'compose.yaml', '-f', 'configs/two.yaml',
'up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], 180)
result.check_returncode()
time.sleep(6)
py = [sys.executable, '-X', 'utf8']
pnpm = 'pnpm.cmd' if os.name == 'nt' else 'pnpm'
hook_suffix = 'cmd' if os.name == 'nt' else 'sh'
save()
with tempfile.TemporaryDirectory(prefix='lab3-report-') as tmp:
temp = Path(tmp)
run('contract', py + ['scripts/check_contract.py'])
run('lint', [pnpm, 'lint'])
run('build', [pnpm, 'build'], 240)
run('draining', ['node', '--test', '--test-reporter=tap', 'test/draining.mjs'], 120)
for name, script in (('nginx-readiness', 'nginx-readiness.sh'),
('nginx-readiness-parallel', 'nginx-readiness-parallel.sh')):
run(name, ['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/bin/sh',
'-v', f'{ROOT / "infra"}:/source:ro',
'-v', f'{ROOT / "test" / script}:/test.sh:ro',
'nginx:1.30.6-alpine', '/test.sh'], 60)
run('smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
run('idempotency', ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
'-v', f'{ROOT / "test/idempotency.mjs"}:/app/test/idempotency.mjs:ro',
'api', 'node', '--test', 'test/idempotency.mjs'], 180)
if selected is None or selected & {'persistence', 'public-failover'}:
try:
if selected is None or 'persistence' in selected:
before = snapshot()
item = run('persistence', py + ['tests/resilience.py', '--mode', 'persistence',
'--action-script', f'scripts/restart.{hook_suffix}', '--base-url', args.base_url], 240)
after = snapshot()
item['before'] = before
item['after'] = after
restarted = {c['id']: c for c in before if c['service'] != 'migrate'}
same = {c['id']: c for c in after if c['service'] != 'migrate'}
item['containers_restarted'] = restarted.keys() == same.keys() and all(
c['started_at'] != same[key]['started_at'] for key, c in restarted.items())
item['volumes_preserved'] = restarted.keys() == same.keys() and all(
c['volumes'] == same[key]['volumes'] for key, c in restarted.items())
if not item['containers_restarted'] or not item['volumes_preserved']:
item['status'] = 'FAIL'
save()
run('public-failover', py + ['tests/resilience.py', '--mode', 'failover',
'--action-script', f'scripts/stop-one.{hook_suffix}', '--base-url', args.base_url], 120,
parse_json=True)
finally:
restore()
for scenario in ('resources', 'failover', 'rolling'):
result_file = temp / f'grade4-{scenario}.json'
run(f'grade4-{scenario}', py + ['test/grade4.py', '--scenario', scenario,
'--base-url', args.base_url, '--concurrency', '4', '--observe-seconds', '10',
'--output', str(result_file)], 300, details=result_file)
run('shutdown', py + ['test/check_shutdown.py', '--base-url', args.base_url], 180, parse_json=True)
images = ['mtusi2026lab2-fork-backend:latest', 'mtusi2026lab2-fork-migrate:latest',
'mtusi2026lab3-fork-api:latest', 'mtusi2026lab3-fork-migrate:latest']
if selected is None or 'migration-compatibility' in selected:
available = capture(['docker', 'image', 'inspect', '--format', '{{.Id}}', *images], 30)
if available.returncode == 0:
run('migration-compatibility', py + ['test/check_migration_compat.py'], 360, parse_json=True)
else:
report['checks'].append({'name': 'migration-compatibility', 'status': 'NOT_RUN',
'reason': 'Required local lab2/lab3 images are not all available; no images pulled.'})
run('final-smoke', py + ['tests/smoke.py', '--base-url', args.base_url])
report['final'] = snapshot()
report['all_executed_checks_pass'] = all(c['status'] != 'FAIL' for c in report['checks'])
report['finished'] = datetime.now(timezone.utc).isoformat()
save()
print(f"Saved {len(report['checks'])} checks to {args.output}", flush=True)
if not report['all_executed_checks_pass']:
raise SystemExit(1)
if __name__ == '__main__':
main()