This commit is contained in:
Максим Шептунов
2026-10-13 19:30:32 +03:00
parent 0ab3a341aa
commit 08d70c462c
65 changed files with 148882 additions and 18 deletions
+129
View File
@@ -0,0 +1,129 @@
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
def command(args, env=None):
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=120)
if result.returncode:
raise RuntimeError(result.stderr[-2000:])
return result.stdout.strip()
def containers(service=None):
args = ['docker', 'compose', 'ps', '-a', '-q']
if service:
args.append(service)
ids = command(args).split()
if not ids:
return []
# Сохраняем только нужные поля, без environment с возможными секретами.
rows = json.loads(command(['docker', 'inspect', *ids]))
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
service=r['Config']['Labels']['com.docker.compose.service'],
running=r['State']['Running'], started_at=r['State']['StartedAt'],
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
memory_limit_bytes=r['HostConfig']['Memory'],
published_ports=r['NetworkSettings']['Ports']) for r in rows]
def save(name, data):
path = ROOT / 'evidence' / name
path.parent.mkdir(exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
def wait(count, base=None):
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
deadline = time.monotonic() + 75
while time.monotonic() < deadline:
observed = set()
for _ in range(max(12, count * 8)):
try:
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
observed.add(response.headers['X-Instance-Id'])
except (OSError, ValueError):
break
if len(observed) == count and None not in observed:
return sorted(observed)
time.sleep(.5)
raise RuntimeError(f'Не дождались {count} готовых API через LB')
def stop(instance):
hostname = instance.split(':')[0]
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
if len(found) != 1:
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
before = containers()
command(['docker', 'stop', '--time', '3', found[0]['id']])
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
save('last_stop.json', result)
return result
def scale(count, cpu, memory, pool):
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
'--scale', f'api={count}', 'api'], env)
instances = wait(count)
rows = containers('api')
if len(rows) != count or not all(r['running'] for r in rows):
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
for row in rows:
if abs(row['cpu_limit'] - cpu) > .001:
raise RuntimeError('Docker не применил CPU limit')
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
raise RuntimeError('Docker не применил RAM limit')
return dict(instances=instances, containers=rows)
def rolling():
old = [r['id'] for r in containers('api') if r['running']]
if len(old) != 2:
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
before = containers()
command(['docker', 'compose', 'build', 'api'])
steps = []
for container in old:
# Не пересоздаём одновременно старые контейнеры при изменении image.
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
wait(3)
command(['docker', 'stop', '--time', '10', container])
command(['docker', 'rm', container])
wait(2)
steps.append(containers('api'))
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
def main():
p = argparse.ArgumentParser()
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
p.add_argument('--count', type=int, default=2)
p.add_argument('--instance')
p.add_argument('--output', default='containers.json')
args = p.parse_args()
if args.action == 'wait':
print('API ready:', ', '.join(wait(args.count)))
elif args.action == 'snapshot':
save(args.output, containers())
print('Container identities and limits saved')
elif args.action == 'stop':
if not args.instance:
p.error('--instance required')
stop(args.instance)
print('Selected API stopped')
else:
rolling()
print('API replaced one at a time')
if __name__ == '__main__':
main()