"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена.""" import argparse import json import os from pathlib import Path import subprocess import time import urllib.request ROOT = Path(__file__).resolve().parents[1] def command(args, env=None, timeout=120, input=None): result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, input=input) if result.returncode: raise RuntimeError(result.stderr[-2000:]) return result.stdout.strip() def containers(service=None): args = ['docker', 'compose', 'ps', '-a', '-q'] if service: args.append(service) ids = command(args).split() if not ids: return [] # Сохраняем только нужные поля, без environment с возможными секретами. rows = json.loads(command(['docker', 'inspect', *ids])) return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'], service=r['Config']['Labels']['com.docker.compose.service'], running=r['State']['Running'], started_at=r['State']['StartedAt'], restart_count=r.get('RestartCount',0), cpu_limit=r['HostConfig']['NanoCpus'] / 1e9, memory_limit_bytes=r['HostConfig']['Memory'], published_ports=r['NetworkSettings']['Ports']) for r in rows] def save(name, data): path = ROOT / 'evidence' / name path.parent.mkdir(exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') def wait(count, base=None): base = base or os.getenv('BASE_URL', 'http://localhost:8080') deadline = time.monotonic() + 75 while time.monotonic() < deadline: observed = set() for _ in range(max(12, count * 8)): try: with urllib.request.urlopen(base + '/health/ready', timeout=3) as response: observed.add(response.headers['X-Instance-Id']) except (OSError, ValueError): break if len(observed) == count and None not in observed: return sorted(observed) time.sleep(.5) raise RuntimeError(f'Не дождались {count} готовых API через LB') def stop(instance): hostname = instance.split(':')[0] found = [r for r in containers('api') if r['hostname'] == hostname and r['running']] if len(found) != 1: raise RuntimeError('Instance alias не соответствует одной работающей API-реплике') before = containers() command(['docker', 'stop', '--time', '3', found[0]['id']]) result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers()) save('last_stop.json', result) return result def scale(count, cpu, memory, pool): env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool)) command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate', '--scale', f'api={count}', 'api'], env) instances = wait(count) rows = containers('api') if len(rows) != count or not all(r['running'] for r in rows): raise RuntimeError('Фактическое число контейнеров отличается от запрошенного') for row in rows: if abs(row['cpu_limit'] - cpu) > .001: raise RuntimeError('Docker не применил CPU limit') if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024: raise RuntimeError('Docker не применил RAM limit') return dict(instances=instances, containers=rows) def rolling(): old = [r['id'] for r in containers('api') if r['running']] if len(old) != 2: raise RuntimeError('Rolling проверяется на двух репликах обычного стенда') before = containers() command(['docker', 'compose', 'build', 'api']) steps = [] for container in old: # Не пересоздаём одновременно старые контейнеры при изменении image. command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api']) wait(3) command(['docker', 'stop', '--time', '10', container]) command(['docker', 'rm', container]) wait(2) steps.append(containers('api')) save('rolling_containers.json', dict(before=before, steps=steps, after=containers())) def main(): p = argparse.ArgumentParser() p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling']) p.add_argument('--count', type=int, default=2) p.add_argument('--instance') p.add_argument('--output', default='containers.json') args = p.parse_args() if args.action == 'wait': print('API ready:', ', '.join(wait(args.count))) elif args.action == 'snapshot': save(args.output, containers()) print('Container identities and limits saved') elif args.action == 'stop': if not args.instance: p.error('--instance required') stop(args.instance) print('Selected API stopped') else: rolling() print('API replaced one at a time') if __name__ == '__main__': main()