forked from mtusi2026/mtusi2026lab7
324 lines
17 KiB
Python
324 lines
17 KiB
Python
"""lab7, уровень 4: жизненный цикл секретов и сертификатов ПОД НАГРУЗКОЙ, без пересборки образов.
|
||
|
||
python student-tests/lifecycle.py rotate-secret | rotate-leaf | token-expiry | sealed | all
|
||
|
||
Нужны: запущенный стенд, Python с cryptography на хосте (requirements-ops.txt) и файлы оператора
|
||
(secrets/operator/*). Всё время сценария фоновая нагрузка (вход + чтение + отправка сообщений через LB)
|
||
считает ответы; итог — evidence/lifecycle-<сценарий>.json. Секреты не печатаются.
|
||
|
||
rotate-secret Пароль ACL-пользователя api в Redis (рабочий секрет API):
|
||
1) Redis: ACL SETUSER api >new — временно действуют оба пароля;
|
||
2) OpenBao: новая версия messenger/api.redis_password (bao.py kv-rotate, check-and-set);
|
||
3) ждём, пока КАЖДАЯ реплика API прочитала новую версию (лог secrets.rotated);
|
||
4) Redis: ACL SETUSER api resetpass >new — старый пароль больше не действует; CLIENT KILL USER api
|
||
рвёт старые соединения, пул API переподключается уже с новым паролем; ACL SAVE.
|
||
rotate-leaf Клиентский сертификат LB (identity CN=lb для mTLS LB -> API):
|
||
1) pki.py rotate lb (старый leaf — в lb/previous); 2) reload HAProxy (USR2, без обрыва);
|
||
3) pki.py revoke lb --previous + bao.py sync-crl; reload lb и otel-gate (их CRL);
|
||
4) API перечитывает CRL сам (CRL_RELOAD_SECONDS) и закрывает открытые соединения со старым leaf;
|
||
5) зонд: старый leaf -> отказ (revoked), новый -> 200.
|
||
token-expiry Короткий TTL токенов роли api (20 с / max 40 с) + отзыв всех токенов cert auth: сервисы
|
||
перелогиниваются сертификатом, renew/relogin видны в логах, запросы не падают. TTL возвращается.
|
||
sealed OpenBao запечатан: работающие реплики обслуживают запросы (секреты в памяти), перезапущенная
|
||
реплика НЕ становится готовой (fail closed, LB её не использует); после unseal — готова.
|
||
"""
|
||
import datetime as dt
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import secrets
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
from collections import Counter
|
||
|
||
from common import (EVIDENCE, ROOT, api_containers, check, compose, compose_output, docker, health, new_chat,
|
||
new_key, new_user, try_call, wait_balanced)
|
||
|
||
PY = sys.executable
|
||
RESULTS = {}
|
||
|
||
|
||
# ---------------- фоновая нагрузка ----------------
|
||
|
||
class Load:
|
||
"""2 потока: GET /users/me + POST message (Idempotency-Key) и периодический новый вход (новый Redis SET)."""
|
||
|
||
def __init__(self):
|
||
self.user = new_user("lifecycle")
|
||
self.chat = new_chat(self.user)
|
||
self.stop = threading.Event()
|
||
self.codes = Counter()
|
||
self.errors = []
|
||
self.threads = [threading.Thread(target=self.reader, daemon=True),
|
||
threading.Thread(target=self.login_loop, daemon=True)]
|
||
|
||
def note(self, what, reply):
|
||
code = reply.status if reply is not None else "conn_error"
|
||
self.codes[f"{what}:{code}"] += 1
|
||
if reply is None or reply.status >= 500:
|
||
self.errors.append({"t": time.time(), "what": what, "code": code})
|
||
|
||
def reader(self):
|
||
while not self.stop.is_set():
|
||
self.note("me", try_call("GET", "/api/v1/users/me", token=self.user["token"], timeout=5))
|
||
self.note("send", try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", token=self.user["token"],
|
||
body={"text": "load"}, headers={"Idempotency-Key": new_key()}, timeout=5))
|
||
time.sleep(0.05)
|
||
|
||
def login_loop(self):
|
||
users = [new_user("lifecycle_login") for _ in range(4)] # лимит 10 входов/мин на username
|
||
i = 0
|
||
while not self.stop.is_set():
|
||
u = users[i % len(users)]
|
||
i += 1
|
||
import base64
|
||
auth = "Basic " + base64.b64encode(f"{u['username']}:{u['password']}".encode()).decode()
|
||
self.note("login", try_call("POST", "/api/v1/auth/sessions", headers={"Authorization": auth}, timeout=5))
|
||
time.sleep(2)
|
||
|
||
def __enter__(self):
|
||
self.started = time.time()
|
||
for t in self.threads:
|
||
t.start()
|
||
return self
|
||
|
||
def __exit__(self, *exc):
|
||
self.stop.set()
|
||
for t in self.threads:
|
||
t.join(timeout=10)
|
||
self.seconds = round(time.time() - self.started, 1)
|
||
return False
|
||
|
||
def summary(self):
|
||
return {"seconds": self.seconds, "codes": dict(self.codes), "errors_5xx_or_conn": len(self.errors),
|
||
"errors": self.errors[:10]}
|
||
|
||
|
||
# ---------------- помощники оператора ----------------
|
||
|
||
def operator(*args):
|
||
result = subprocess.run([PY, *args], cwd=ROOT, capture_output=True, text=True, check=False)
|
||
check(result.returncode == 0, f"{' '.join(args[:3])}: {result.stderr.strip()[-300:]}")
|
||
return result.stdout.strip()
|
||
|
||
|
||
def redis_admin(commands):
|
||
"""Команды redis-cli от admin: пароль admin и сами команды идут через stdin (не в argv, не в выводе)."""
|
||
password = (ROOT / "secrets" / "operator" / "redis_admin_password").read_text(encoding="utf-8").strip()
|
||
script = ("read -r REDISCLI_AUTH; export REDISCLI_AUTH; exec redis-cli --tls --cacert /run/tls-ca/ca.crt "
|
||
"--cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis --user admin --no-auth-warning")
|
||
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", script], cwd=ROOT,
|
||
input=(password + "\n" + "\n".join(commands) + "\n").encode(), capture_output=True,
|
||
check=False)
|
||
check(result.returncode == 0, f"redis-cli failed: {result.stderr.decode()[-200:]}")
|
||
return result.stdout.decode()
|
||
|
||
|
||
def events_since(service, since, names):
|
||
"""События из логов сервиса (JSON-строки) после момента since: {event: count}."""
|
||
counts = Counter()
|
||
for line in compose_output("logs", "--no-log-prefix", service).splitlines():
|
||
start = line.find("{")
|
||
if start < 0:
|
||
continue
|
||
try:
|
||
item = json.loads(line[start:])
|
||
except ValueError:
|
||
continue
|
||
ts = item.get("ts", "")
|
||
try:
|
||
moment = dt.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
|
||
except ValueError:
|
||
continue
|
||
if moment >= since and item.get("event") in names:
|
||
counts[item["event"]] += 1
|
||
if item["event"] == "secrets.rotated":
|
||
counts[f"rotated:{item.get('message', '')[-20:]}"] += 1
|
||
return counts
|
||
|
||
|
||
def save(name, data):
|
||
RESULTS[name] = data
|
||
path = EVIDENCE / f"lifecycle-{name}.json"
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print(f"saved {path}")
|
||
|
||
|
||
# ---------------- сценарии ----------------
|
||
|
||
def rotate_secret():
|
||
print("--- rotate-secret: пароль Redis ACL-пользователя api")
|
||
replicas = len(api_containers())
|
||
version_before = json.loads(operator("scripts/bao.py", "kv-version", "api"))["version"]
|
||
new_password = secrets.token_hex(24)
|
||
digest = hashlib.sha256(new_password.encode()).hexdigest()
|
||
started = time.time()
|
||
with Load() as load:
|
||
time.sleep(3)
|
||
redis_admin([f"ACL SETUSER api >{new_password}"])
|
||
both = sum(1 for line in redis_admin(["ACL GETUSER api"]).splitlines() if len(line.strip()) == 64)
|
||
with tempfile.NamedTemporaryFile("w", delete=False, dir=ROOT / "secrets" / "operator") as tmp:
|
||
tmp.write(new_password)
|
||
try:
|
||
os.chmod(tmp.name, 0o600)
|
||
operator("scripts/bao.py", "kv-rotate", "api", "redis_password", "--value-file", tmp.name)
|
||
finally:
|
||
os.unlink(tmp.name)
|
||
version_after = json.loads(operator("scripts/bao.py", "kv-version", "api"))["version"]
|
||
t_rotated = time.time()
|
||
deadline = time.time() + 90
|
||
while True:
|
||
seen = events_since("api", started, {"secrets.rotated"})
|
||
if seen.get("secrets.rotated", 0) >= replicas:
|
||
break
|
||
check(time.time() < deadline, f"replicas did not pick up the new version: {dict(seen)}")
|
||
time.sleep(1)
|
||
picked_up = round(time.time() - t_rotated, 1)
|
||
redis_admin([f"ACL SETUSER api resetpass >{new_password}", "CLIENT KILL USER api", "ACL SAVE"])
|
||
acl = redis_admin(["ACL GETUSER api"])
|
||
time.sleep(8) # новые соединения пула — уже с новым паролем
|
||
passwords = [line.strip() for line in acl.splitlines() if len(line.strip()) == 64
|
||
and all(c in "0123456789abcdef" for c in line.strip())]
|
||
check(passwords == [digest], "old password still active or new one missing")
|
||
data = {"kv_version": [version_before, version_after], "replicas": replicas,
|
||
"active_passwords_during_overlap": both,
|
||
"all_replicas_picked_up_after_s": picked_up, "active_password_hashes_after": len(passwords),
|
||
"new_password_hash_matches": digest in acl, "load": load.summary()}
|
||
check(len(passwords) == 1, f"expected exactly one active password, got {len(passwords)}")
|
||
check(load.summary()["errors_5xx_or_conn"] == 0, f"errors during rotation: {load.errors[:5]}")
|
||
print(json.dumps(data, ensure_ascii=False))
|
||
print("PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors")
|
||
save("rotate-secret", data)
|
||
|
||
|
||
def leaf_serial(path):
|
||
from cryptography import x509
|
||
return format(x509.load_pem_x509_certificate(path.read_bytes()).serial_number, "x")
|
||
|
||
|
||
def probe_lb():
|
||
result = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "probe",
|
||
"python", "/probe/tls_probe.py", "lb-api"], cwd=ROOT, capture_output=True, check=False)
|
||
rows = [json.loads(line) for line in result.stdout.decode().splitlines() if line.startswith("{")]
|
||
return {r["case"]: r["observed"] for r in rows if "case" in r}, [r for r in rows if "case" in r and not r["pass"]]
|
||
|
||
|
||
def rotate_leaf():
|
||
print("--- rotate-leaf: клиентский сертификат LB (CN=lb)")
|
||
lb_dir = ROOT / "secrets" / "pki" / "lb"
|
||
old_serial = leaf_serial(lb_dir / "tls.crt")
|
||
with Load() as load:
|
||
time.sleep(3)
|
||
operator("scripts/pki.py", "rotate", "lb")
|
||
new_serial = leaf_serial(lb_dir / "tls.crt")
|
||
compose("kill", "-s", "USR2", "lb") # master-worker reload: новый leaf для новых соединений
|
||
time.sleep(5)
|
||
operator("scripts/pki.py", "revoke", "lb", "--previous")
|
||
operator("scripts/bao.py", "sync-crl")
|
||
compose("kill", "-s", "USR2", "lb") # HAProxy и otel-gate перечитывают CRL при reload
|
||
compose("kill", "-s", "USR2", "otel-gate")
|
||
t_revoked = time.time()
|
||
time.sleep(12) # > CRL_RELOAD_SECONDS: API перечитал CRL
|
||
observed, failed = probe_lb()
|
||
closed = events_since("api", t_revoked - 1, {"tls.crl_loaded", "tls.revoked_connections_closed",
|
||
"tls.client_rejected"})
|
||
data = {"old_serial": old_serial[:16], "new_serial": new_serial[:16],
|
||
"probe": {k: v for k, v in observed.items() if "LB" in k or "lb" in k},
|
||
"api_events_after_revoke": dict(closed), "load": load.summary()}
|
||
print(json.dumps(data, ensure_ascii=False))
|
||
check(observed.get("previous (rotated, revoked) LB leaf") in ("server_rejected_client_cert",
|
||
"connection_closed_after_handshake"),
|
||
"old leaf is still accepted")
|
||
check(observed.get("valid LB identity (CN=lb)") == "http_200", "new leaf is not accepted")
|
||
check(not failed, f"probe failures: {failed}")
|
||
check(load.summary()["errors_5xx_or_conn"] == 0, f"errors during rotation: {load.errors[:5]}")
|
||
print("PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors")
|
||
save("rotate-leaf", data)
|
||
|
||
|
||
def token_expiry():
|
||
print("--- token-expiry: короткий TTL токенов OpenBao роли api")
|
||
started = time.time()
|
||
try:
|
||
with Load() as load:
|
||
operator("scripts/bao.py", "role-ttl", "api", "20s", "40s")
|
||
operator("scripts/bao.py", "revoke-service-tokens")
|
||
time.sleep(100)
|
||
finally:
|
||
operator("scripts/bao.py", "role-ttl", "api", "15m", "1h")
|
||
operator("scripts/bao.py", "revoke-service-tokens") # вернуть обычные токены
|
||
events = events_since("api", started, {"secrets.login", "secrets.renewed", "secrets.relogin",
|
||
"secrets.refresh_failed", "secrets.renew_failed"})
|
||
data = {"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": dict(events),
|
||
"load": load.summary()}
|
||
print(json.dumps(data, ensure_ascii=False))
|
||
# Отзыв замечает то, что раньше обратится к OpenBao с отозванным токеном: чтение KV (refresh_failed)
|
||
# или продление (renew_failed -> сразу новый вход). Оба пути — реакция именно на revoke.
|
||
check(events.get("secrets.refresh_failed", 0) + events.get("secrets.renew_failed", 0) >= 1,
|
||
"revoked token was not noticed")
|
||
check(events.get("secrets.login", 0) + events.get("secrets.relogin", 0) + events.get("secrets.renew_failed", 0) >= 2,
|
||
"no re-login after expiry")
|
||
check(load.summary()["errors_5xx_or_conn"] == 0, f"errors: {load.errors[:5]}")
|
||
print("PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected")
|
||
save("token-expiry", data)
|
||
|
||
|
||
def sealed():
|
||
print("--- sealed: OpenBao запечатан")
|
||
containers = api_containers()
|
||
check(len(containers) >= 2, "need 2 api replicas")
|
||
victim = containers[0]
|
||
started = time.time()
|
||
with Load() as load:
|
||
time.sleep(3)
|
||
operator("scripts/bao.py", "seal")
|
||
t_sealed = time.time()
|
||
time.sleep(40) # > SECRETS_REFRESH_SECONDS: отказ обновления замечен
|
||
docker("stop", "--time", "15", victim)
|
||
docker("start", victim)
|
||
time.sleep(30)
|
||
victim_health = health(victim)
|
||
serving = len({try_call("GET", "/health/ready").instance for _ in range(6)} - {None})
|
||
operator("scripts/bao.py", "unseal")
|
||
t_unsealed = time.time()
|
||
deadline = time.time() + 120
|
||
while health(victim) != "healthy":
|
||
check(time.time() < deadline, "replica did not recover after unseal")
|
||
time.sleep(1)
|
||
recovered = round(time.time() - t_unsealed, 1)
|
||
time.sleep(3)
|
||
events = events_since("api", started, {"secrets.refresh_failed", "secrets.waiting", "secrets.unavailable",
|
||
"secrets.login"})
|
||
data = {"sealed_seconds": round(t_unsealed - t_sealed, 1), "restarted_replica_health_while_sealed": victim_health,
|
||
"replicas_answering_while_sealed": serving, "restarted_replica_ready_after_unseal_s": recovered,
|
||
"api_events": dict(events), "load": load.summary()}
|
||
print(json.dumps(data, ensure_ascii=False))
|
||
check(victim_health != "healthy", "a replica became ready without secrets")
|
||
check(events.get("secrets.refresh_failed", 0) >= 1, "sealed OpenBao was not noticed")
|
||
errors = [e for e in load.errors if e["code"] != 503]
|
||
check(not errors, f"unexpected errors while sealed: {errors[:5]}")
|
||
print("PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready "
|
||
"(fail closed) until unseal")
|
||
save("sealed", data)
|
||
wait_balanced(2, 60)
|
||
|
||
|
||
SCENARIOS = {"rotate-secret": rotate_secret, "rotate-leaf": rotate_leaf, "token-expiry": token_expiry,
|
||
"sealed": sealed}
|
||
|
||
if __name__ == "__main__":
|
||
names = sys.argv[1:] or ["all"]
|
||
if names == ["all"]:
|
||
names = list(SCENARIOS)
|
||
failed = 0
|
||
for name in names:
|
||
try:
|
||
SCENARIOS[name]()
|
||
except AssertionError as exc:
|
||
failed += 1
|
||
print(f"FAIL: {name}: {exc}")
|
||
raise SystemExit(1 if failed else 0)
|