Files
mtusi2026lab5-fork/scripts/collect_evidence.py
T
2026-11-07 16:10:32 +03:00

205 lines
11 KiB
Python

"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
from datetime import datetime, timezone
import hashlib
from importlib.metadata import version
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
from tempfile import TemporaryDirectory
from zoneinfo import ZoneInfo
ROOT = Path(__file__).resolve().parents[1]
PROJECT = 'mtusi-messenger-lab5-test'
EVIDENCE = ROOT / 'evidence'
ENVIRONMENT = {
**os.environ,
'COMPOSE_PROJECT_NAME': PROJECT,
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
'LB_HOST_PORT': '28080',
}
COMPOSE = ['docker', 'compose', '--env-file', '.env']
NONSECRET_SETTINGS = {
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
'S3_CLEANUP_INTERVAL_SECONDS',
}
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
def timestamp():
return datetime.now(timezone.utc).isoformat()
def invoke(command):
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
def read_command(command):
result = invoke(command)
if result.returncode:
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
return result.stdout.strip()
def source_manifest():
names = {
'.dockerignore', '.env.example',
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
}
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
if path.is_file() and '__pycache__' not in path.parts
and path.suffix != '.md' and not path.name.startswith('.'))
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
def sanitize(text, secrets, staging):
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
for secret in secrets:
clean = clean.replace(secret, '<redacted>')
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
return clean
def container_snapshot(images):
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
rows = []
if ids:
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
rows = [json.loads(line) for line in output.splitlines()]
for row in rows:
image = row['image_id']
if image not in images:
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
return rows
def remaining_resources():
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
f'label=com.docker.compose.project={PROJECT}']).splitlines()
for resource in ('container', 'network', 'volume')}
def extract_metrics(output, upload_file):
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
if pytest is None:
raise RuntimeError('В выводе нет успешного итогового результата pytest')
return {
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
'uploads': json.loads(upload_file.read_text()),
'public_failover': [row for row in rows if 'observed_instances_before' in row],
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
}
def main():
if any(remaining_resources().values()):
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
started = timestamp()
manifest = source_manifest()
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
secrets = [value for value in secrets if value]
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
services = {name: {
'image': service.get('image'),
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
'tmpfs': service.get('tmpfs', []),
} for name, service in config['services'].items()}
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
staging = Path(folder)
upload_file = staging / 'uploads.json'
def execute(command):
begin = timestamp()
result = invoke(command)
output = sanitize(result.stdout, secrets, staging)
display = sanitize(shlex.join(command), secrets, staging)
commands.append({'command': display, 'started_at_utc': begin,
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
if result.returncode:
print(output[-5000:], flush=True)
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
return f'$ {display}\n{output}\n'
cleanup_output = ''
try:
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
snapshots['started'] = container_snapshot(images)
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
snapshots['before_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
snapshots['after_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
snapshots['after_failover'] = container_snapshot(images)
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults']))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)]))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080']))
snapshots['after_update_and_failures'] = container_snapshot(images)
nginx_output = execute(['make', 'test-nginx'])
finally:
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
remaining = remaining_resources()
if any(remaining.values()):
raise RuntimeError('После очистки остались ресурсы тестового проекта')
if source_manifest() != manifest:
raise RuntimeError('Во время прогона изменились проверяемые исходники')
output = ''.join(combined)
metrics = extract_metrics(output, upload_file)
run = {
'started_at_utc': started, 'finished_at_utc': timestamp(),
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
'project': PROJECT, 'runtime': {'python': platform.python_version(),
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
'compose': read_command(['docker', 'compose', 'version'])},
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
'services': services, 'container_snapshots': snapshots, 'images': images,
'commands': commands, 'remaining_test_resources': remaining,
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
}
artifacts = {
'test-output.txt': output, 'nginx-output.txt': nginx_output,
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
}
for name, content in artifacts.items():
if any(secret in content for secret in secrets):
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
EVIDENCE.mkdir(exist_ok=True)
for name, content in artifacts.items():
(EVIDENCE / name).write_text(content)
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
if __name__ == '__main__':
main()