205 lines
11 KiB
Python
205 lines
11 KiB
Python
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
|
|
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
from importlib.metadata import version
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
from tempfile import TemporaryDirectory
|
|
from zoneinfo import ZoneInfo
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROJECT = 'mtusi-messenger-lab5-test'
|
|
EVIDENCE = ROOT / 'evidence'
|
|
ENVIRONMENT = {
|
|
**os.environ,
|
|
'COMPOSE_PROJECT_NAME': PROJECT,
|
|
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
|
|
'LB_HOST_PORT': '28080',
|
|
}
|
|
COMPOSE = ['docker', 'compose', '--env-file', '.env']
|
|
NONSECRET_SETTINGS = {
|
|
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
|
|
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
|
|
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
|
|
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
|
|
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
|
|
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
|
|
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
|
|
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
|
|
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
|
|
'S3_CLEANUP_INTERVAL_SECONDS',
|
|
}
|
|
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
|
|
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
|
|
|
|
|
|
def timestamp():
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def invoke(command):
|
|
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
|
|
|
|
def read_command(command):
|
|
result = invoke(command)
|
|
if result.returncode:
|
|
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
|
|
return result.stdout.strip()
|
|
|
|
|
|
def source_manifest():
|
|
names = {
|
|
'.dockerignore', '.env.example',
|
|
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
|
|
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
|
|
}
|
|
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
|
|
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
|
|
if path.is_file() and '__pycache__' not in path.parts
|
|
and path.suffix != '.md' and not path.name.startswith('.'))
|
|
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
|
|
|
|
|
|
def sanitize(text, secrets, staging):
|
|
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
|
|
for secret in secrets:
|
|
clean = clean.replace(secret, '<redacted>')
|
|
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
|
|
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
|
|
return clean
|
|
|
|
|
|
def container_snapshot(images):
|
|
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
|
rows = []
|
|
if ids:
|
|
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
|
|
rows = [json.loads(line) for line in output.splitlines()]
|
|
for row in rows:
|
|
image = row['image_id']
|
|
if image not in images:
|
|
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
|
|
return rows
|
|
|
|
|
|
def remaining_resources():
|
|
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
|
|
f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
|
for resource in ('container', 'network', 'volume')}
|
|
|
|
|
|
def extract_metrics(output, upload_file):
|
|
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
|
|
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
|
|
if pytest is None:
|
|
raise RuntimeError('В выводе нет успешного итогового результата pytest')
|
|
return {
|
|
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
|
|
'uploads': json.loads(upload_file.read_text()),
|
|
'public_failover': [row for row in rows if 'observed_instances_before' in row],
|
|
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
|
|
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
|
|
}
|
|
|
|
|
|
def main():
|
|
if any(remaining_resources().values()):
|
|
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
|
|
started = timestamp()
|
|
manifest = source_manifest()
|
|
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
|
|
secrets = [value for value in secrets if value]
|
|
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
|
|
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
|
|
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
|
|
services = {name: {
|
|
'image': service.get('image'),
|
|
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
|
|
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
|
|
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
|
|
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
|
|
'tmpfs': service.get('tmpfs', []),
|
|
} for name, service in config['services'].items()}
|
|
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
|
|
staging = Path(folder)
|
|
upload_file = staging / 'uploads.json'
|
|
|
|
def execute(command):
|
|
begin = timestamp()
|
|
result = invoke(command)
|
|
output = sanitize(result.stdout, secrets, staging)
|
|
display = sanitize(shlex.join(command), secrets, staging)
|
|
commands.append({'command': display, 'started_at_utc': begin,
|
|
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
|
|
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
|
|
if result.returncode:
|
|
print(output[-5000:], flush=True)
|
|
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
|
|
return f'$ {display}\n{output}\n'
|
|
|
|
cleanup_output = ''
|
|
try:
|
|
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
|
|
snapshots['started'] = container_snapshot(images)
|
|
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
|
|
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
|
|
snapshots['before_restart'] = container_snapshot(images)
|
|
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
|
|
snapshots['after_restart'] = container_snapshot(images)
|
|
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
|
|
snapshots['after_failover'] = container_snapshot(images)
|
|
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults']))
|
|
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)]))
|
|
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080']))
|
|
snapshots['after_update_and_failures'] = container_snapshot(images)
|
|
nginx_output = execute(['make', 'test-nginx'])
|
|
finally:
|
|
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
|
|
remaining = remaining_resources()
|
|
if any(remaining.values()):
|
|
raise RuntimeError('После очистки остались ресурсы тестового проекта')
|
|
if source_manifest() != manifest:
|
|
raise RuntimeError('Во время прогона изменились проверяемые исходники')
|
|
output = ''.join(combined)
|
|
metrics = extract_metrics(output, upload_file)
|
|
run = {
|
|
'started_at_utc': started, 'finished_at_utc': timestamp(),
|
|
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
|
|
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
|
|
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
|
|
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
|
|
'project': PROJECT, 'runtime': {'python': platform.python_version(),
|
|
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
|
|
'compose': read_command(['docker', 'compose', 'version'])},
|
|
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
|
|
'services': services, 'container_snapshots': snapshots, 'images': images,
|
|
'commands': commands, 'remaining_test_resources': remaining,
|
|
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
|
|
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
|
|
}
|
|
artifacts = {
|
|
'test-output.txt': output, 'nginx-output.txt': nginx_output,
|
|
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
|
|
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
|
|
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
|
|
}
|
|
for name, content in artifacts.items():
|
|
if any(secret in content for secret in secrets):
|
|
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
for name, content in artifacts.items():
|
|
(EVIDENCE / name).write_text(content)
|
|
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|