forked from mtusi2026/mtusi2026lab5
130 lines
5.3 KiB
Python
130 lines
5.3 KiB
Python
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def command(args, env=None, timeout=120, input=None):
|
|
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, input=input)
|
|
if result.returncode:
|
|
raise RuntimeError(result.stderr[-2000:])
|
|
return result.stdout.strip()
|
|
|
|
|
|
def containers(service=None):
|
|
args = ['docker', 'compose', 'ps', '-a', '-q']
|
|
if service:
|
|
args.append(service)
|
|
ids = command(args).split()
|
|
if not ids:
|
|
return []
|
|
# Сохраняем только нужные поля, без environment с возможными секретами.
|
|
rows = json.loads(command(['docker', 'inspect', *ids]))
|
|
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
|
|
service=r['Config']['Labels']['com.docker.compose.service'],
|
|
running=r['State']['Running'], started_at=r['State']['StartedAt'], restart_count=r.get('RestartCount',0),
|
|
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
|
|
memory_limit_bytes=r['HostConfig']['Memory'],
|
|
published_ports=r['NetworkSettings']['Ports']) for r in rows]
|
|
|
|
|
|
def save(name, data):
|
|
path = ROOT / 'evidence' / name
|
|
path.parent.mkdir(exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
|
|
|
|
|
def wait(count, base=None):
|
|
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
|
|
deadline = time.monotonic() + 75
|
|
while time.monotonic() < deadline:
|
|
observed = set()
|
|
for _ in range(max(12, count * 8)):
|
|
try:
|
|
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
|
|
observed.add(response.headers['X-Instance-Id'])
|
|
except (OSError, ValueError):
|
|
break
|
|
if len(observed) == count and None not in observed:
|
|
return sorted(observed)
|
|
time.sleep(.5)
|
|
raise RuntimeError(f'Не дождались {count} готовых API через LB')
|
|
|
|
|
|
def stop(instance):
|
|
hostname = instance.split(':')[0]
|
|
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
|
|
if len(found) != 1:
|
|
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
|
|
before = containers()
|
|
command(['docker', 'stop', '--time', '3', found[0]['id']])
|
|
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
|
|
save('last_stop.json', result)
|
|
return result
|
|
|
|
|
|
def scale(count, cpu, memory, pool):
|
|
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
|
|
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
|
|
'--scale', f'api={count}', 'api'], env)
|
|
instances = wait(count)
|
|
rows = containers('api')
|
|
if len(rows) != count or not all(r['running'] for r in rows):
|
|
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
|
|
for row in rows:
|
|
if abs(row['cpu_limit'] - cpu) > .001:
|
|
raise RuntimeError('Docker не применил CPU limit')
|
|
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
|
|
raise RuntimeError('Docker не применил RAM limit')
|
|
return dict(instances=instances, containers=rows)
|
|
|
|
|
|
def rolling():
|
|
old = [r['id'] for r in containers('api') if r['running']]
|
|
if len(old) != 2:
|
|
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
|
|
before = containers()
|
|
command(['docker', 'compose', 'build', 'api'])
|
|
steps = []
|
|
for container in old:
|
|
# Не пересоздаём одновременно старые контейнеры при изменении image.
|
|
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
|
|
wait(3)
|
|
command(['docker', 'stop', '--time', '10', container])
|
|
command(['docker', 'rm', container])
|
|
wait(2)
|
|
steps.append(containers('api'))
|
|
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
|
|
p.add_argument('--count', type=int, default=2)
|
|
p.add_argument('--instance')
|
|
p.add_argument('--output', default='containers.json')
|
|
args = p.parse_args()
|
|
if args.action == 'wait':
|
|
print('API ready:', ', '.join(wait(args.count)))
|
|
elif args.action == 'snapshot':
|
|
save(args.output, containers())
|
|
print('Container identities and limits saved')
|
|
elif args.action == 'stop':
|
|
if not args.instance:
|
|
p.error('--instance required')
|
|
stop(args.instance)
|
|
print('Selected API stopped')
|
|
else:
|
|
rolling()
|
|
print('API replaced one at a time')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|