lab5
This commit is contained in:
@@ -0,0 +1,129 @@
|
||||
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, env=None, timeout=120, input=None):
|
||||
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, input=input)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr[-2000:])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def containers(service=None):
|
||||
args = ['docker', 'compose', 'ps', '-a', '-q']
|
||||
if service:
|
||||
args.append(service)
|
||||
ids = command(args).split()
|
||||
if not ids:
|
||||
return []
|
||||
# Сохраняем только нужные поля, без environment с возможными секретами.
|
||||
rows = json.loads(command(['docker', 'inspect', *ids]))
|
||||
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
|
||||
service=r['Config']['Labels']['com.docker.compose.service'],
|
||||
running=r['State']['Running'], started_at=r['State']['StartedAt'], restart_count=r.get('RestartCount',0),
|
||||
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
|
||||
memory_limit_bytes=r['HostConfig']['Memory'],
|
||||
published_ports=r['NetworkSettings']['Ports']) for r in rows]
|
||||
|
||||
|
||||
def save(name, data):
|
||||
path = ROOT / 'evidence' / name
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
def wait(count, base=None):
|
||||
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
|
||||
deadline = time.monotonic() + 75
|
||||
while time.monotonic() < deadline:
|
||||
observed = set()
|
||||
for _ in range(max(12, count * 8)):
|
||||
try:
|
||||
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
|
||||
observed.add(response.headers['X-Instance-Id'])
|
||||
except (OSError, ValueError):
|
||||
break
|
||||
if len(observed) == count and None not in observed:
|
||||
return sorted(observed)
|
||||
time.sleep(.5)
|
||||
raise RuntimeError(f'Не дождались {count} готовых API через LB')
|
||||
|
||||
|
||||
def stop(instance):
|
||||
hostname = instance.split(':')[0]
|
||||
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
|
||||
if len(found) != 1:
|
||||
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
|
||||
before = containers()
|
||||
command(['docker', 'stop', '--time', '3', found[0]['id']])
|
||||
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
|
||||
save('last_stop.json', result)
|
||||
return result
|
||||
|
||||
|
||||
def scale(count, cpu, memory, pool):
|
||||
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
|
||||
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
|
||||
'--scale', f'api={count}', 'api'], env)
|
||||
instances = wait(count)
|
||||
rows = containers('api')
|
||||
if len(rows) != count or not all(r['running'] for r in rows):
|
||||
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
|
||||
for row in rows:
|
||||
if abs(row['cpu_limit'] - cpu) > .001:
|
||||
raise RuntimeError('Docker не применил CPU limit')
|
||||
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
|
||||
raise RuntimeError('Docker не применил RAM limit')
|
||||
return dict(instances=instances, containers=rows)
|
||||
|
||||
|
||||
def rolling():
|
||||
old = [r['id'] for r in containers('api') if r['running']]
|
||||
if len(old) != 2:
|
||||
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
|
||||
before = containers()
|
||||
command(['docker', 'compose', 'build', 'api'])
|
||||
steps = []
|
||||
for container in old:
|
||||
# Не пересоздаём одновременно старые контейнеры при изменении image.
|
||||
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
|
||||
wait(3)
|
||||
command(['docker', 'stop', '--time', '10', container])
|
||||
command(['docker', 'rm', container])
|
||||
wait(2)
|
||||
steps.append(containers('api'))
|
||||
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
|
||||
p.add_argument('--count', type=int, default=2)
|
||||
p.add_argument('--instance')
|
||||
p.add_argument('--output', default='containers.json')
|
||||
args = p.parse_args()
|
||||
if args.action == 'wait':
|
||||
print('API ready:', ', '.join(wait(args.count)))
|
||||
elif args.action == 'snapshot':
|
||||
save(args.output, containers())
|
||||
print('Container identities and limits saved')
|
||||
elif args.action == 'stop':
|
||||
if not args.instance:
|
||||
p.error('--instance required')
|
||||
stop(args.instance)
|
||||
print('Selected API stopped')
|
||||
else:
|
||||
rolling()
|
||||
print('API replaced one at a time')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user