314 lines
13 KiB
Python
314 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Bootstrap и эксплуатация стенда lab7 (операторский инструмент, не часть приложения).
|
||
|
||
python3 scripts/bootstrap.py all PKI, operator secrets, OpenBao (init/unseal/
|
||
настройка), сервисные credentials → PostgreSQL,
|
||
MinIO, Redis, RabbitMQ; запуск стенда
|
||
python3 scripts/bootstrap.py unseal распечатать OpenBao после рестарта (2 доли из 3)
|
||
python3 scripts/bootstrap.py status seal-статус, версии секретов (значения не печатаются)
|
||
python3 scripts/bootstrap.py rotate-db <svc> новый пароль роли БД сервиса (ALTER ROLE + KV)
|
||
python3 scripts/bootstrap.py rotate-s3 <svc> новый secret key S3 сервиса (MinIO + KV)
|
||
python3 scripts/bootstrap.py worker-identity <gN> новая identity worker (cert CN worker-gN):
|
||
OpenBao cert role, пользователь RabbitMQ; старая
|
||
identity отзывается (см. student-tests/compromise.py)
|
||
python3 scripts/bootstrap.py revoke-tokens <svc> отозвать все токены OpenBao, выданные сервису
|
||
|
||
Материалы оператора — secrets/operator (каталог 700, вне Git): bao-init.json (root token и
|
||
3 доли unseal; в эксплуатации доли раздаются разным людям и здесь не лежат), redis ACL,
|
||
список ключей MinIO для s3-init. Значения никогда не печатаются.
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import secrets as rnd
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OP = ROOT / 'secrets' / 'operator'
|
||
SERVICES = ('api', 'worker', 'relay', 'reconciler')
|
||
DB_ROLE = {s: 'app_' + s for s in SERVICES}
|
||
CERT_CN = {'api': 'api', 'worker': 'worker-g1', 'relay': 'relay-g1', 'reconciler': 'reconciler'}
|
||
TOKEN_TTL, TOKEN_MAX_TTL = os.environ.get('BAO_TOKEN_TTL', '15m'), os.environ.get('BAO_TOKEN_MAX_TTL', '1h')
|
||
|
||
|
||
def run(*args, input=None, check=True, env=None, timeout=300):
|
||
r = subprocess.run(list(args), cwd=ROOT, input=input, capture_output=True, text=True, timeout=timeout,
|
||
env=dict(os.environ, **(env or {})))
|
||
if check and r.returncode != 0:
|
||
# stderr может содержать значения? bao/psql/mc не печатают переданные секреты в ошибках.
|
||
raise SystemExit('FAILED: %s: %s' % (' '.join(args[:6]), (r.stderr or r.stdout)[-600:]))
|
||
return r
|
||
|
||
|
||
def compose(*args, **kw):
|
||
return run('docker', 'compose', *args, **kw)
|
||
|
||
|
||
# ---------------- OpenBao ----------------
|
||
|
||
def init_json():
|
||
p = OP / 'bao-init.json'
|
||
return json.loads(p.read_text()) if p.exists() else None
|
||
|
||
|
||
def bao(*args, token=True, input=None, check=True):
|
||
env = ['-e', 'BAO_ADDR=https://127.0.0.1:8200', '-e', 'BAO_CACERT=/run/secrets/ca.crt']
|
||
if token:
|
||
env += ['-e', 'BAO_TOKEN=' + init_json()['root_token']]
|
||
return compose('exec', '-T', *env, 'openbao', 'bao', *args, input=input, check=check)
|
||
|
||
|
||
def bao_status():
|
||
r = bao('status', '-format=json', token=False, check=False)
|
||
try:
|
||
return json.loads(r.stdout)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def wait_bao(timeout=90):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
st = bao_status()
|
||
if st is not None:
|
||
return st
|
||
time.sleep(2)
|
||
raise SystemExit('openbao not reachable')
|
||
|
||
|
||
def bao_init_unseal():
|
||
OP.mkdir(parents=True, exist_ok=True)
|
||
os.chmod(OP, 0o700)
|
||
st = wait_bao()
|
||
if not st['initialized']:
|
||
r = bao('operator', 'init', '-key-shares=3', '-key-threshold=2', '-format=json', token=False)
|
||
p = OP / 'bao-init.json'
|
||
p.write_text(r.stdout)
|
||
os.chmod(p, 0o600)
|
||
print('openbao initialized (3 shares, threshold 2; stored in secrets/operator, not printed)')
|
||
unseal()
|
||
|
||
|
||
def unseal():
|
||
st = wait_bao()
|
||
if not st['sealed']:
|
||
print('openbao unsealed')
|
||
return
|
||
keys = init_json()['unseal_keys_b64']
|
||
for k in keys[:2]:
|
||
# Доля unseal — через stdin (key=-), не аргументом командной строки.
|
||
bao('write', '-format=json', 'sys/unseal', 'key=-', token=False, input=k)
|
||
st = bao_status()
|
||
print('openbao sealed=%s' % st['sealed'])
|
||
|
||
|
||
def bao_configure():
|
||
out = bao('secrets', 'list', '-format=json').stdout
|
||
if 'secret/' not in json.loads(out):
|
||
bao('secrets', 'enable', '-path=secret', '-version=2', 'kv')
|
||
auths = json.loads(bao('auth', 'list', '-format=json').stdout)
|
||
if 'cert/' not in auths:
|
||
bao('auth', 'enable', 'cert')
|
||
audits = json.loads(bao('audit', 'list', '-format=json').stdout or '{}')
|
||
if 'file/' not in audits:
|
||
# Аудит всех запросов (значения секретов — HMAC, не открытым текстом).
|
||
bao('audit', 'enable', 'file', 'file_path=/openbao/file/audit.log')
|
||
for s in SERVICES:
|
||
bao('policy', 'write', 'messenger-' + s, '/openbao/policies/messenger-%s.hcl' % s)
|
||
set_cert_role(s, CERT_CN[s])
|
||
print('openbao configured: kv-v2 secret/, auth/cert roles %s, policies messenger-*' % ', '.join(SERVICES))
|
||
|
||
|
||
def set_cert_role(svc, cn, ttl=None, max_ttl=None):
|
||
bao('write', 'auth/cert/certs/' + svc, 'display_name=' + svc, 'certificate=@/run/secrets/ca.crt',
|
||
'allowed_common_names=' + cn, 'token_policies=messenger-' + svc,
|
||
'token_ttl=' + (ttl or TOKEN_TTL), 'token_max_ttl=' + (max_ttl or TOKEN_MAX_TTL))
|
||
|
||
|
||
def kv_get(svc):
|
||
r = bao('kv', 'get', '-format=json', 'secret/messenger/' + svc, check=False)
|
||
if r.returncode != 0:
|
||
return None
|
||
return json.loads(r.stdout)['data']['data']
|
||
|
||
|
||
def kv_put(svc, data):
|
||
# Значения передаются через stdin (JSON), не аргументами командной строки.
|
||
bao('kv', 'put', 'secret/messenger/' + svc, '-', input=json.dumps(data))
|
||
|
||
|
||
def ensure_creds():
|
||
creds = {}
|
||
for s in SERVICES:
|
||
cur = kv_get(s)
|
||
if not cur:
|
||
cur = {'db_user': DB_ROLE[s], 'db_password': rnd.token_hex(24),
|
||
's3_access_key': 'svc-' + s, 's3_secret_key': rnd.token_hex(24)}
|
||
if s == 'api':
|
||
cur.update({'redis_user': 'api', 'redis_password': rnd.token_hex(24)})
|
||
kv_put(s, cur)
|
||
creds[s] = cur
|
||
return creds
|
||
|
||
|
||
# ---------------- backends ----------------
|
||
|
||
def write_backend_files(creds):
|
||
acl = OP / 'redis_users.acl'
|
||
h = hashlib.sha256(creds['api']['redis_password'].encode()).hexdigest()
|
||
# default отключён; api — только свои ключи и нужные команды; healthcheck — только PING.
|
||
acl.write_text(
|
||
'user default off\n'
|
||
'user healthcheck on nopass resetkeys resetchannels -@all +ping +hello +auth\n'
|
||
'user api on #%s resetchannels ~session:* ~messages:* ~rl:* -@all +@connection '
|
||
'+get +set +del +ttl +scan +incr +expire +multi +exec +discard +ping\n' % h)
|
||
s3 = OP / 's3_users.txt'
|
||
s3.write_text(''.join('%s %s %s\n' % (s, creds[s]['s3_access_key'], creds[s]['s3_secret_key']) for s in SERVICES))
|
||
for p in (acl, s3):
|
||
os.chmod(p, 0o644) # читают контейнеры (uid ≠ оператора); каталог 700 закрывает доступ с хоста
|
||
|
||
|
||
def pg_apply(creds, services=SERVICES):
|
||
sql = ''.join("ALTER ROLE %s LOGIN PASSWORD '%s';\n" % (creds[s]['db_user'], creds[s]['db_password']) for s in services)
|
||
compose('exec', '-T', 'postgres', 'psql', '-q', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1',
|
||
input=sql)
|
||
|
||
|
||
def rabbit_users(worker_cn='worker-g1', relay_cn='relay-g1', drop=()):
|
||
def ctl(*a, check=True):
|
||
return compose('exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', *a, check=check)
|
||
users = ctl('list_users', '--formatter', 'json').stdout
|
||
names = {u['user'] for u in json.loads(users or '[]')}
|
||
if 'guest' in names:
|
||
ctl('delete_user', 'guest')
|
||
perms = {relay_cn: (r'^(attachments|attachments\.dlx|attachment\.process|attachment\.dead)$',
|
||
r'^(attachments|attachments\.dlx|attachment\.process|attachment\.dead)$',
|
||
r'^(attachments|attachments\.dlx)$'),
|
||
worker_cn: ('^$', '^$', r'^attachment\.process$')}
|
||
for u, (conf, write, read) in perms.items():
|
||
if u not in names:
|
||
# Пароль случайный и сразу удаляется: вход только SASL EXTERNAL (сертификат).
|
||
ctl('add_user', u, rnd.token_hex(16))
|
||
ctl('clear_password', u)
|
||
ctl('set_permissions', '-p', '/', u, conf, write, read)
|
||
for u in drop:
|
||
if u in names:
|
||
ctl('delete_user', u) # закрывает и его текущие соединения
|
||
|
||
|
||
def s3_init():
|
||
compose('run', '--rm', '--no-deps', '-T', 's3-init')
|
||
|
||
|
||
# ---------------- команды ----------------
|
||
|
||
def cmd_all():
|
||
run('sh', 'scripts/init-secrets.sh')
|
||
run('sh', 'scripts/pki.sh', 'init')
|
||
OP.mkdir(parents=True, exist_ok=True)
|
||
os.chmod(OP, 0o700)
|
||
for f in ('redis_users.acl', 's3_users.txt'):
|
||
if not (OP / f).exists():
|
||
(OP / f).write_text('')
|
||
os.chmod(OP / f, 0o644)
|
||
compose('up', '-d', 'openbao')
|
||
bao_init_unseal()
|
||
bao_configure()
|
||
creds = ensure_creds()
|
||
write_backend_files(creds)
|
||
compose('up', '-d', '--wait', 'postgres', 'redis', 's3', 'rabbitmq', timeout=600)
|
||
compose('restart', 'redis') # перечитать ACL-файл (первый запуск мог быть с пустым)
|
||
compose('up', '-d', '--wait', 'redis', timeout=300)
|
||
compose('run', '--rm', 'migrate', timeout=300)
|
||
pg_apply(creds)
|
||
s3_init()
|
||
rabbit_users()
|
||
compose('up', '-d', '--wait', timeout=900)
|
||
print('bootstrap complete')
|
||
|
||
|
||
def cmd_status():
|
||
st = bao_status()
|
||
print(json.dumps({'initialized': st and st['initialized'], 'sealed': st and st['sealed']}))
|
||
for s in SERVICES:
|
||
r = bao('kv', 'metadata', 'get', '-format=json', 'secret/messenger/' + s, check=False)
|
||
if r.returncode == 0:
|
||
print(s, 'version', json.loads(r.stdout)['data']['current_version'])
|
||
|
||
|
||
def cmd_rotate_db(svc):
|
||
creds = kv_get(svc)
|
||
old = creds['db_password']
|
||
creds['db_password'] = rnd.token_hex(24)
|
||
pg_apply({svc: creds}, [svc])
|
||
kv_put(svc, creds)
|
||
print('rotated db password of %s (role %s); new connections use it after refresh (≤ SECRETS_REFRESH)'
|
||
% (svc, creds['db_user']))
|
||
return old
|
||
|
||
|
||
def cmd_rotate_s3(svc):
|
||
creds = kv_get(svc)
|
||
creds['s3_secret_key'] = rnd.token_hex(24)
|
||
all_creds = {s: kv_get(s) for s in SERVICES}
|
||
all_creds[svc] = creds
|
||
write_backend_files(all_creds)
|
||
s3_init() # mc admin user add с тем же access key обновляет secret
|
||
kv_put(svc, creds)
|
||
print('rotated s3 secret of %s' % svc)
|
||
|
||
|
||
def cmd_revoke_tokens(svc):
|
||
"""Отозвать все токены OpenBao, выданные cert-роли сервиса (по метаданным accessor)."""
|
||
acc = bao('list', '-format=json', 'auth/token/accessors', check=False)
|
||
n = 0
|
||
for a in json.loads(acc.stdout or '[]'):
|
||
r = bao('write', '-format=json', 'auth/token/lookup-accessor', 'accessor=' + a, check=False)
|
||
if r.returncode != 0:
|
||
continue
|
||
meta = (json.loads(r.stdout)['data'].get('meta') or {})
|
||
if meta.get('cert_name') == svc:
|
||
bao('write', 'auth/token/revoke-accessor', 'accessor=' + a)
|
||
n += 1
|
||
print('revoked %d openbao tokens of %s' % (n, svc))
|
||
return n
|
||
|
||
|
||
def cmd_worker_identity(gen):
|
||
cn = 'worker-' + gen
|
||
set_cert_role('worker', cn) # OpenBao: только новый CN
|
||
rabbit_users(worker_cn=cn, drop=[u for u in ('worker-g1', 'worker-g2', 'worker-g3') if u != cn])
|
||
print('worker identity is now %s' % cn)
|
||
|
||
|
||
def main():
|
||
cmd = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||
if cmd == 'all':
|
||
cmd_all()
|
||
elif cmd == 'unseal':
|
||
unseal()
|
||
elif cmd == 'status':
|
||
cmd_status()
|
||
elif cmd == 'rotate-db':
|
||
cmd_rotate_db(sys.argv[2])
|
||
elif cmd == 'rotate-s3':
|
||
cmd_rotate_s3(sys.argv[2])
|
||
elif cmd == 'revoke-tokens':
|
||
cmd_revoke_tokens(sys.argv[2])
|
||
elif cmd == 'worker-identity':
|
||
cmd_worker_identity(sys.argv[2])
|
||
elif cmd == 'cert-role': # cert-role <svc> <cn> [ttl] [max_ttl] — тест истечения токенов
|
||
set_cert_role(sys.argv[2], sys.argv[3], *(sys.argv[4:6]))
|
||
elif cmd == 'seal':
|
||
bao('operator', 'seal')
|
||
print('openbao sealed')
|
||
else:
|
||
raise SystemExit(__doc__)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|