forked from mtusi2026/mtusi2026lab7
241 lines
11 KiB
Python
241 lines
11 KiB
Python
"""Общие операции с репликами текущего Compose-проекта."""
|
|
|
|
from dataclasses import dataclass
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Replica:
|
|
"""Состояние контейнера API."""
|
|
|
|
id: str
|
|
image: str
|
|
instance_id: str
|
|
running: bool
|
|
health: str | None
|
|
|
|
|
|
class ComposeProject:
|
|
"""Работает только с выбранным Compose-проектом."""
|
|
|
|
def __init__(self) -> None:
|
|
self.command = ["docker", "compose", "--env-file", ".env"]
|
|
self.config = json.loads(self.run("config", "--format", "json"))
|
|
self.name = self.config["name"]
|
|
|
|
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
|
|
"""Выполняет Docker-команду с ограничением времени."""
|
|
result = subprocess.run(
|
|
["docker", *args],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
env={**os.environ, **(env or {})},
|
|
)
|
|
if result.returncode:
|
|
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
|
|
return result.stdout.strip()
|
|
|
|
def run(self, *args: str, timeout: int = 90, env=None) -> str:
|
|
"""Выполняет команду Compose с текущими overrides."""
|
|
return self.docker(*self.command[1:], *args, timeout=timeout, env=env)
|
|
|
|
def replicas(self) -> list[Replica]:
|
|
"""Читает состояние API без окружения контейнеров."""
|
|
ids = self.run("ps", "--all", "--quiet", "api").splitlines()
|
|
if not ids:
|
|
return []
|
|
template = (
|
|
'{"id":{{json .Id}},"image":{{json .Image}},'
|
|
'"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},'
|
|
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
|
|
)
|
|
rows = self.docker("inspect", "--format", template, *ids).splitlines()
|
|
configured = str(
|
|
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
|
|
or ""
|
|
).strip()
|
|
return [
|
|
Replica(
|
|
row["id"],
|
|
row["image"],
|
|
configured or row["hostname"],
|
|
row["running"],
|
|
row["health"],
|
|
)
|
|
for row in map(json.loads, rows)
|
|
]
|
|
|
|
|
|
def cleanup_docker(*args, allow_failure=False):
|
|
result = subprocess.run(['docker',*args],cwd=ROOT,text=True,capture_output=True,timeout=120)
|
|
if result.returncode and not allow_failure:
|
|
raise RuntimeError('Docker cleanup failed: ' + ' '.join(args[:2]))
|
|
return result.stdout.strip()
|
|
|
|
|
|
def inventory(projects):
|
|
resources = {'containers':[], 'networks':[], 'volumes':[]}
|
|
for project in projects:
|
|
label = 'label=com.docker.compose.project=' + project
|
|
resources['containers'].extend(cleanup_docker('ps','-aq','--filter',label).splitlines())
|
|
resources['networks'].extend(cleanup_docker('network','ls','-q','--filter',label).splitlines())
|
|
resources['volumes'].extend(cleanup_docker('volume','ls','-q','--filter',label).splitlines())
|
|
return resources
|
|
|
|
|
|
|
|
def clean_stands(*, apply=False):
|
|
config = json.loads(cleanup_docker('compose','--env-file','.env','config','--format','json'))
|
|
projects = sorted({config['name'], os.getenv('MAIN_PROJECT', 'messenger-lab7'), os.getenv('TEST_PROJECT', 'messenger-lab7-test')})
|
|
resources = inventory(projects)
|
|
names = {service['image'] for service in config['services'].values() if 'build' in service}
|
|
image_tags = []
|
|
image_ids = set()
|
|
rows = cleanup_docker('image', 'ls', '--format', '{{.Repository}} {{.Tag}} {{.ID}}')
|
|
for row in rows.splitlines():
|
|
repository, tag, image_id = row.split()
|
|
name = f'{repository}:{tag}'
|
|
if tag != '<none>' and (
|
|
name in names or repository == 'messenger'
|
|
or repository.startswith(('messenger-', 'mns-'))
|
|
):
|
|
image_tags.append(name)
|
|
image_ids.add(image_id)
|
|
plan = {'projects': projects, 'resources': resources,
|
|
'application_images': sorted(image_tags), 'application_image_ids': sorted(image_ids),
|
|
'local_settings_preserved': True}
|
|
if not apply:
|
|
return plan
|
|
if resources['containers']:
|
|
cleanup_docker('rm','-f','-v',*resources['containers'])
|
|
if resources['networks']:
|
|
cleanup_docker('network','rm',*resources['networks'])
|
|
if resources['volumes']:
|
|
cleanup_docker('volume','rm',*resources['volumes'])
|
|
if image_tags:
|
|
cleanup_docker('image','rm',*image_tags)
|
|
remaining = inventory(projects)
|
|
if any(remaining.values()):
|
|
raise RuntimeError('Cleanup incomplete')
|
|
plan['remaining'] = remaining
|
|
plan['application_images_remaining'] = [name for name in image_tags
|
|
if cleanup_docker('image', 'ls', '-q', name)]
|
|
if plan['application_images_remaining']:
|
|
raise RuntimeError('Application images remain')
|
|
plan['result'] = 'PASS'
|
|
return plan
|
|
|
|
|
|
def verify(*, keep_stand=False):
|
|
"""Один последовательный прогон на одном стенде с обязательной очисткой."""
|
|
from datetime import datetime, timezone
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
project_name = os.getenv('TEST_PROJECT', 'messenger-lab7-test')
|
|
if any(inventory([project_name]).values()):
|
|
raise RuntimeError(f'Choose an unused TEST_PROJECT; {project_name} already has resources')
|
|
bao_port = os.getenv("TEST_BAO_PORT", "28200")
|
|
os.environ.update(COMPOSE_PROJECT_NAME=project_name,
|
|
COMPOSE_FILE='compose.yaml:compose.test.yaml',
|
|
GRAFANA_HOST_PORT='23000', LB_HOST_PORT='28080', BAO_HOST_PORT=bao_port,
|
|
CA_FILE=str(ROOT / 'secrets/tls/ca.crt'),
|
|
DOCKER_TESTS='1', OTEL_ENABLED='true')
|
|
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'checks': [],
|
|
'profile': {'single_stack': True, 'load_duration_seconds': 10,
|
|
'load_requests_per_second': 10, 'load_concurrency': 2}}
|
|
destination = ROOT / 'evidence/verification.json'
|
|
destination.parent.mkdir(exist_ok=True)
|
|
|
|
def run(name, command, timeout=600, *, env=None):
|
|
print('RUN: ' + name, flush=True)
|
|
started = time.monotonic()
|
|
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=timeout,
|
|
env={**os.environ, **(env or {})})
|
|
executable = Path(command[0])
|
|
display_command = [str(executable.relative_to(ROOT)) if executable.is_relative_to(ROOT)
|
|
else command[0], *command[1:]]
|
|
entry = {'name': name, 'command': display_command, 'exit_code': result.returncode,
|
|
'duration_seconds': round(time.monotonic()-started, 3),
|
|
'result': 'PASS' if result.returncode == 0 else 'FAIL'}
|
|
if name == 'regression':
|
|
summary = re.search(r'\d+ passed[^\n]*', result.stdout)
|
|
if summary:
|
|
entry['summary'] = summary.group(0)
|
|
if name == 'security' and not result.returncode:
|
|
entry['summary'] = result.stdout.strip().splitlines()
|
|
print(result.stdout.strip(), flush=True)
|
|
if result.returncode:
|
|
# Полные диагностические логи локальны; в evidence нет путей/credentials.
|
|
(ROOT / '.verification-error.log').write_text(result.stdout + result.stderr)
|
|
entry['diagnostics'] = '.verification-error.log (local, not committed)'
|
|
report['checks'].append(entry)
|
|
print(entry['result'] + ': ' + name, flush=True)
|
|
if result.returncode:
|
|
raise RuntimeError(name + ' failed; inspect local diagnostic log')
|
|
|
|
try:
|
|
run('contract', [sys.executable, 'scripts/check_contract.py'])
|
|
run('openapi', [sys.executable, 'scripts/validate_openapi.py'])
|
|
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
|
|
run('startup', ['make', '--no-print-directory', 'test-stand-up'], timeout=600)
|
|
# Локальные ASGI fixtures не находятся в сети Compose. Проверки SDK
|
|
# используют exporters в памяти, интеграционный стенд экспортирует OTLP.
|
|
run('regression', ['make', '--no-print-directory', 'test-unit', 'OTEL_ENABLED=false'])
|
|
run('api', [sys.executable, 'tests/smoke.py', '--base-url', 'https://localhost:28080'])
|
|
run('observability', [sys.executable, '-m', 'scripts.check_observability',
|
|
'--base-url', 'https://localhost:28080',
|
|
'--grafana-url', 'https://localhost:23000',
|
|
'--incidents', '--restart-telemetry'])
|
|
run('persistence', [sys.executable, 'tests/resilience.py', '--mode', 'persistence',
|
|
'--action-script', 'scripts/restart.sh', '--base-url', 'https://localhost:28080'])
|
|
run('external-tls', [sys.executable, 'tests/security.py', '--base-url', 'https://localhost:28080'])
|
|
run('security', [sys.executable, '-m', 'scripts.check_security',
|
|
'--base-url', 'https://localhost:28080', '--bao-port', bao_port])
|
|
run('failover', [sys.executable, 'tests/resilience.py', '--mode', 'failover',
|
|
'--action-script', 'scripts/stop-one.sh', '--base-url', 'https://localhost:28080'])
|
|
report['result'] = 'PASS'
|
|
except BaseException:
|
|
report['result'] = 'FAIL'
|
|
raise
|
|
finally:
|
|
try:
|
|
if keep_stand:
|
|
report['cleanup'] = {'project': project_name, 'result': 'DEFERRED'}
|
|
else:
|
|
print('RUN: cleanup', flush=True)
|
|
ComposeProject().run('down', '--volumes', '--remove-orphans', timeout=120)
|
|
remaining = inventory([project_name])
|
|
if any(remaining.values()):
|
|
raise RuntimeError('Verification project cleanup incomplete')
|
|
report['cleanup'] = {'project': project_name, 'remaining': remaining, 'result': 'PASS'}
|
|
print('PASS: cleanup', flush=True)
|
|
finally:
|
|
report['finished_at_utc'] = datetime.now(timezone.utc).isoformat()
|
|
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description='Проверка и очистка Compose-стендов')
|
|
parser.add_argument('command', choices=['verify', 'clean'])
|
|
parser.add_argument('--keep-stand', action='store_true')
|
|
parser.add_argument('--apply', action='store_true', help='Подтвердить удаление ресурсов')
|
|
args = parser.parse_args()
|
|
if args.command == 'verify':
|
|
verify(keep_stand=args.keep_stand)
|
|
else:
|
|
print(json.dumps(clean_stands(apply=args.apply), indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|