Files
mtusi2026lab4/test/final-regression.py
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

163 lines
8.0 KiB
Python

"""Run the existing regression suites sequentially and restore the local Compose stack."""
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
from queue import Queue, Empty
import subprocess
import sys
import threading
import time
from direct_client import ROOT, Client, Failure, check, docker
RUNTIME = {'api', 'nginx', 'postgres', 'redis', 's3', 'reconciler'}
def fingerprint():
paths = {ROOT / name for name in ['Dockerfile', 'compose.yaml', 'package.json', 'pnpm-lock.yaml',
'tsconfig.json', 'tsconfig.build.json', 'nest-cli.json', 'prisma.config.ts']}
for directory in ['src', 'prisma', 'test', 'tests', 'scripts', 'contracts', 'infra']:
paths.update(path for path in (ROOT / directory).rglob('*') if path.is_file()
and 'generated' not in path.parts and '__pycache__' not in path.parts)
digest = hashlib.sha256()
for path in sorted(paths):
digest.update(path.relative_to(ROOT).as_posix().encode() + b'\0' + path.read_bytes() + b'\0')
return digest.hexdigest()
def states():
raw = docker('compose', 'ps', '--all', '--format', 'json').strip()
records = json.loads(raw) if raw.startswith('[') else [json.loads(line) for line in raw.splitlines()]
return [{'service': r['Service'], 'name': r['Name'], 'id': r['ID'],
'state': r['State'], 'health': r.get('Health', ''), 'image': r['Image']}
for r in records if r['Service'] in RUNTIME]
def healthy(records):
return (len(records) == 7 and sum(r['service'] == 'api' for r in records) == 2
and {r['service'] for r in records} == RUNTIME
and all(r['state'] == 'running' and r['health'] == 'healthy' for r in records))
def wait_ready():
client = Client('http://127.0.0.1:8080', timeout=5)
routed = set()
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
current = states()
if healthy(current):
try:
routed.add(client.call('GET', '/health/ready')['headers']['X-Instance-Id'])
if len(routed) == 2: return current
except Failure: pass
time.sleep(1)
raise Failure('Complete stack and both Nginx upstreams did not become ready')
def main():
report = {'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(),
'sourceSha256': fingerprint(), 'steps': [],
'scope': 'Existing suites, sequential execution, no load benchmark rerun. IPv4 public endpoint. All runtime services restored without deleting volumes.'}
output = ROOT / 'evidence/final-regression.json'
def save(): output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
def step(name, command, timeout):
entry = {'name': name, 'command': command, 'status': 'running', 'output': []}
report['steps'].append(entry)
save()
print('START: ' + name, flush=True)
started = time.monotonic()
process = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, encoding='utf-8', errors='replace', bufsize=1)
messages = Queue()
def read():
try:
for line in process.stdout: messages.put(line.rstrip())
finally: messages.put(None)
threading.Thread(target=read, daemon=True).start()
try:
while True:
if time.monotonic() - started > timeout:
process.kill()
process.wait(timeout=15)
raise Failure('Regression step timed out: ' + name)
try: line = messages.get(timeout=1)
except Empty: continue
if line is None: break
# Keep deliberately emitted test results only; no request/env dumps or stacks.
if line.startswith(('PASS:', 'FAIL:', 'Test Suites:', 'Tests:')):
entry['output'].append(line)
print(line, flush=True)
entry['exitCode'] = process.wait(timeout=15)
check(entry['exitCode'] == 0, f'Regression step failed: {name} (exit {entry["exitCode"]})')
entry['status'] = 'passed'
print('PASS: regression step ' + name, flush=True)
except BaseException:
entry['status'] = 'failed'
if process.poll() is None:
process.kill()
process.wait(timeout=15)
raise
finally:
process.stdout.close()
entry['elapsedSeconds'] = time.monotonic() - started
save()
py = [sys.executable, '-X', 'utf8', '-u']
pnpm = ['cmd.exe', '/d', '/c', 'pnpm.cmd'] if os.name == 'nt' else ['pnpm']
try:
report['before'] = wait_ready()
api_ids = docker('compose', 'ps', '-q', 'api').split()
report['apiImages'] = docker('inspect', '--format', '{{.Config.Hostname}} {{.Image}}', *api_ids).splitlines()
for container in api_ids:
check(docker('exec', container, 'node', '-p', "process.env.UPLOAD_MEMORY_PROFILE || '0'").strip() == '0', 'Disable diagnostic profiling')
step('offline_contract', py + ['scripts/check_contract.py'], 90)
step('unit', pnpm + ['test', '--runInBand'], 240)
step('lint', pnpm + ['lint'], 180)
step('public_smoke_before', py + ['tests/smoke.py', '--base-url', 'http://127.0.0.1:8080'], 120)
step('oversized_upload_transport', py + ['test/oversized-uploads.py'], 180)
step('attachments_s3_outage_persistence', py + ['test/attachments.py', '--base-url', 'http://127.0.0.1:8080', '--lifecycle'], 600)
wait_ready()
step('reconciliation_crash_and_database_outage', py + ['test/run-reconcile.py'], 180)
step('public_persistence_and_each_api_failover', py + ['test/hooks.py'], 1200)
wait_ready()
step('swagger_and_attachment_failover', py + ['test/replicas.py', '--base-url', 'http://127.0.0.1:8080'], 240)
wait_ready()
step('interrupted_upload_and_receive_deadline', py + ['test/upload-interruption.py'], 240)
step('presigned_positive_and_negative', py + ['test/direct-uploads.py'], 240)
for name, script in [('s3_restricted_roles', 's3-permissions.mjs'),
('s3_conditional_write_checksum_copy', 's3-direct-capabilities.mjs')]:
step(name, ['docker', 'compose', 'run', '--rm', '--no-deps', '-T',
'-v', f'{ROOT / "test"}:/app/test:ro', '--entrypoint', 'node', 's3-init', 'test/' + script], 180)
step('public_smoke_after', py + ['tests/smoke.py', '--base-url', 'http://127.0.0.1:8080'], 120)
report['sourceUnchanged'] = fingerprint() == report['sourceSha256']
check(report['sourceUnchanged'], 'Source changed during regression')
report['status'] = 'passed'
except BaseException as error:
report['status'] = 'failed'
report['errorType'] = type(error).__name__
if isinstance(error, Failure): report['failure'] = str(error)
raise
finally:
try:
if not healthy(states()):
docker('compose', 'up', '-d', '--no-build', '--wait', '--wait-timeout', '180')
report['after'] = wait_ready()
report['stackRestored'] = True
except Exception as error:
report['stackRestored'] = False
report['restoreErrorType'] = type(error).__name__
report['status'] = 'failed'
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
save()
check(report['status'] == 'passed' and report['stackRestored'], 'Final regression or stack restoration failed')
print('PASS: final regression; all suites passed; complete stack ready', flush=True)
if __name__ == '__main__':
try:
main()
except (Failure, OSError, subprocess.SubprocessError) as error:
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
sys.exit(1)