forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
163 lines
8.0 KiB
Python
163 lines
8.0 KiB
Python
"""Run the existing regression suites sequentially and restore the local Compose stack."""
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from queue import Queue, Empty
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
from direct_client import ROOT, Client, Failure, check, docker
|
|
|
|
RUNTIME = {'api', 'nginx', 'postgres', 'redis', 's3', 'reconciler'}
|
|
|
|
|
|
def fingerprint():
|
|
paths = {ROOT / name for name in ['Dockerfile', 'compose.yaml', 'package.json', 'pnpm-lock.yaml',
|
|
'tsconfig.json', 'tsconfig.build.json', 'nest-cli.json', 'prisma.config.ts']}
|
|
for directory in ['src', 'prisma', 'test', 'tests', 'scripts', 'contracts', 'infra']:
|
|
paths.update(path for path in (ROOT / directory).rglob('*') if path.is_file()
|
|
and 'generated' not in path.parts and '__pycache__' not in path.parts)
|
|
digest = hashlib.sha256()
|
|
for path in sorted(paths):
|
|
digest.update(path.relative_to(ROOT).as_posix().encode() + b'\0' + path.read_bytes() + b'\0')
|
|
return digest.hexdigest()
|
|
|
|
|
|
def states():
|
|
raw = docker('compose', 'ps', '--all', '--format', 'json').strip()
|
|
records = json.loads(raw) if raw.startswith('[') else [json.loads(line) for line in raw.splitlines()]
|
|
return [{'service': r['Service'], 'name': r['Name'], 'id': r['ID'],
|
|
'state': r['State'], 'health': r.get('Health', ''), 'image': r['Image']}
|
|
for r in records if r['Service'] in RUNTIME]
|
|
|
|
|
|
def healthy(records):
|
|
return (len(records) == 7 and sum(r['service'] == 'api' for r in records) == 2
|
|
and {r['service'] for r in records} == RUNTIME
|
|
and all(r['state'] == 'running' and r['health'] == 'healthy' for r in records))
|
|
|
|
|
|
def wait_ready():
|
|
client = Client('http://127.0.0.1:8080', timeout=5)
|
|
routed = set()
|
|
deadline = time.monotonic() + 120
|
|
while time.monotonic() < deadline:
|
|
current = states()
|
|
if healthy(current):
|
|
try:
|
|
routed.add(client.call('GET', '/health/ready')['headers']['X-Instance-Id'])
|
|
if len(routed) == 2: return current
|
|
except Failure: pass
|
|
time.sleep(1)
|
|
raise Failure('Complete stack and both Nginx upstreams did not become ready')
|
|
|
|
|
|
def main():
|
|
report = {'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(),
|
|
'sourceSha256': fingerprint(), 'steps': [],
|
|
'scope': 'Existing suites, sequential execution, no load benchmark rerun. IPv4 public endpoint. All runtime services restored without deleting volumes.'}
|
|
output = ROOT / 'evidence/final-regression.json'
|
|
def save(): output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
|
|
def step(name, command, timeout):
|
|
entry = {'name': name, 'command': command, 'status': 'running', 'output': []}
|
|
report['steps'].append(entry)
|
|
save()
|
|
print('START: ' + name, flush=True)
|
|
started = time.monotonic()
|
|
process = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
text=True, encoding='utf-8', errors='replace', bufsize=1)
|
|
messages = Queue()
|
|
def read():
|
|
try:
|
|
for line in process.stdout: messages.put(line.rstrip())
|
|
finally: messages.put(None)
|
|
threading.Thread(target=read, daemon=True).start()
|
|
try:
|
|
while True:
|
|
if time.monotonic() - started > timeout:
|
|
process.kill()
|
|
process.wait(timeout=15)
|
|
raise Failure('Regression step timed out: ' + name)
|
|
try: line = messages.get(timeout=1)
|
|
except Empty: continue
|
|
if line is None: break
|
|
# Keep deliberately emitted test results only; no request/env dumps or stacks.
|
|
if line.startswith(('PASS:', 'FAIL:', 'Test Suites:', 'Tests:')):
|
|
entry['output'].append(line)
|
|
print(line, flush=True)
|
|
entry['exitCode'] = process.wait(timeout=15)
|
|
check(entry['exitCode'] == 0, f'Regression step failed: {name} (exit {entry["exitCode"]})')
|
|
entry['status'] = 'passed'
|
|
print('PASS: regression step ' + name, flush=True)
|
|
except BaseException:
|
|
entry['status'] = 'failed'
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait(timeout=15)
|
|
raise
|
|
finally:
|
|
process.stdout.close()
|
|
entry['elapsedSeconds'] = time.monotonic() - started
|
|
save()
|
|
py = [sys.executable, '-X', 'utf8', '-u']
|
|
pnpm = ['cmd.exe', '/d', '/c', 'pnpm.cmd'] if os.name == 'nt' else ['pnpm']
|
|
try:
|
|
report['before'] = wait_ready()
|
|
api_ids = docker('compose', 'ps', '-q', 'api').split()
|
|
report['apiImages'] = docker('inspect', '--format', '{{.Config.Hostname}} {{.Image}}', *api_ids).splitlines()
|
|
for container in api_ids:
|
|
check(docker('exec', container, 'node', '-p', "process.env.UPLOAD_MEMORY_PROFILE || '0'").strip() == '0', 'Disable diagnostic profiling')
|
|
step('offline_contract', py + ['scripts/check_contract.py'], 90)
|
|
step('unit', pnpm + ['test', '--runInBand'], 240)
|
|
step('lint', pnpm + ['lint'], 180)
|
|
step('public_smoke_before', py + ['tests/smoke.py', '--base-url', 'http://127.0.0.1:8080'], 120)
|
|
step('oversized_upload_transport', py + ['test/oversized-uploads.py'], 180)
|
|
step('attachments_s3_outage_persistence', py + ['test/attachments.py', '--base-url', 'http://127.0.0.1:8080', '--lifecycle'], 600)
|
|
wait_ready()
|
|
step('reconciliation_crash_and_database_outage', py + ['test/run-reconcile.py'], 180)
|
|
step('public_persistence_and_each_api_failover', py + ['test/hooks.py'], 1200)
|
|
wait_ready()
|
|
step('swagger_and_attachment_failover', py + ['test/replicas.py', '--base-url', 'http://127.0.0.1:8080'], 240)
|
|
wait_ready()
|
|
step('interrupted_upload_and_receive_deadline', py + ['test/upload-interruption.py'], 240)
|
|
step('presigned_positive_and_negative', py + ['test/direct-uploads.py'], 240)
|
|
for name, script in [('s3_restricted_roles', 's3-permissions.mjs'),
|
|
('s3_conditional_write_checksum_copy', 's3-direct-capabilities.mjs')]:
|
|
step(name, ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
|
|
'-v', f'{ROOT / "test"}:/app/test:ro', '--entrypoint', 'node', 's3-init', 'test/' + script], 180)
|
|
step('public_smoke_after', py + ['tests/smoke.py', '--base-url', 'http://127.0.0.1:8080'], 120)
|
|
report['sourceUnchanged'] = fingerprint() == report['sourceSha256']
|
|
check(report['sourceUnchanged'], 'Source changed during regression')
|
|
report['status'] = 'passed'
|
|
except BaseException as error:
|
|
report['status'] = 'failed'
|
|
report['errorType'] = type(error).__name__
|
|
if isinstance(error, Failure): report['failure'] = str(error)
|
|
raise
|
|
finally:
|
|
try:
|
|
if not healthy(states()):
|
|
docker('compose', 'up', '-d', '--no-build', '--wait', '--wait-timeout', '180')
|
|
report['after'] = wait_ready()
|
|
report['stackRestored'] = True
|
|
except Exception as error:
|
|
report['stackRestored'] = False
|
|
report['restoreErrorType'] = type(error).__name__
|
|
report['status'] = 'failed'
|
|
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
|
|
save()
|
|
check(report['status'] == 'passed' and report['stackRestored'], 'Final regression or stack restoration failed')
|
|
print('PASS: final regression; all suites passed; complete stack ready', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (Failure, OSError, subprocess.SubprocessError) as error:
|
|
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
|
|
sys.exit(1)
|