Files
mtusi2026lab7/student-tests/lifecycle.py

324 lines
17 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""lab7, уровень 4: жизненный цикл секретов и сертификатов ПОД НАГРУЗКОЙ, без пересборки образов.
python student-tests/lifecycle.py rotate-secret | rotate-leaf | token-expiry | sealed | all
Нужны: запущенный стенд, Python с cryptography на хосте (requirements-ops.txt) и файлы оператора
(secrets/operator/*). Всё время сценария фоновая нагрузка (вход + чтение + отправка сообщений через LB)
считает ответы; итог — evidence/lifecycle-<сценарий>.json. Секреты не печатаются.
rotate-secret Пароль ACL-пользователя api в Redis (рабочий секрет API):
1) Redis: ACL SETUSER api >new — временно действуют оба пароля;
2) OpenBao: новая версия messenger/api.redis_password (bao.py kv-rotate, check-and-set);
3) ждём, пока КАЖДАЯ реплика API прочитала новую версию (лог secrets.rotated);
4) Redis: ACL SETUSER api resetpass >new — старый пароль больше не действует; CLIENT KILL USER api
рвёт старые соединения, пул API переподключается уже с новым паролем; ACL SAVE.
rotate-leaf Клиентский сертификат LB (identity CN=lb для mTLS LB -> API):
1) pki.py rotate lb (старый leaf — в lb/previous); 2) reload HAProxy (USR2, без обрыва);
3) pki.py revoke lb --previous + bao.py sync-crl; reload lb и otel-gate (их CRL);
4) API перечитывает CRL сам (CRL_RELOAD_SECONDS) и закрывает открытые соединения со старым leaf;
5) зонд: старый leaf -> отказ (revoked), новый -> 200.
token-expiry Короткий TTL токенов роли api (20 с / max 40 с) + отзыв всех токенов cert auth: сервисы
перелогиниваются сертификатом, renew/relogin видны в логах, запросы не падают. TTL возвращается.
sealed OpenBao запечатан: работающие реплики обслуживают запросы (секреты в памяти), перезапущенная
реплика НЕ становится готовой (fail closed, LB её не использует); после unseal — готова.
"""
import datetime as dt
import hashlib
import json
import os
import secrets
import subprocess
import sys
import tempfile
import threading
import time
from collections import Counter
from common import (EVIDENCE, ROOT, api_containers, check, compose, compose_output, docker, health, new_chat,
new_key, new_user, try_call, wait_balanced)
PY = sys.executable
RESULTS = {}
# ---------------- фоновая нагрузка ----------------
class Load:
"""2 потока: GET /users/me + POST message (Idempotency-Key) и периодический новый вход (новый Redis SET)."""
def __init__(self):
self.user = new_user("lifecycle")
self.chat = new_chat(self.user)
self.stop = threading.Event()
self.codes = Counter()
self.errors = []
self.threads = [threading.Thread(target=self.reader, daemon=True),
threading.Thread(target=self.login_loop, daemon=True)]
def note(self, what, reply):
code = reply.status if reply is not None else "conn_error"
self.codes[f"{what}:{code}"] += 1
if reply is None or reply.status >= 500:
self.errors.append({"t": time.time(), "what": what, "code": code})
def reader(self):
while not self.stop.is_set():
self.note("me", try_call("GET", "/api/v1/users/me", token=self.user["token"], timeout=5))
self.note("send", try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", token=self.user["token"],
body={"text": "load"}, headers={"Idempotency-Key": new_key()}, timeout=5))
time.sleep(0.05)
def login_loop(self):
users = [new_user("lifecycle_login") for _ in range(4)] # лимит 10 входов/мин на username
i = 0
while not self.stop.is_set():
u = users[i % len(users)]
i += 1
import base64
auth = "Basic " + base64.b64encode(f"{u['username']}:{u['password']}".encode()).decode()
self.note("login", try_call("POST", "/api/v1/auth/sessions", headers={"Authorization": auth}, timeout=5))
time.sleep(2)
def __enter__(self):
self.started = time.time()
for t in self.threads:
t.start()
return self
def __exit__(self, *exc):
self.stop.set()
for t in self.threads:
t.join(timeout=10)
self.seconds = round(time.time() - self.started, 1)
return False
def summary(self):
return {"seconds": self.seconds, "codes": dict(self.codes), "errors_5xx_or_conn": len(self.errors),
"errors": self.errors[:10]}
# ---------------- помощники оператора ----------------
def operator(*args):
result = subprocess.run([PY, *args], cwd=ROOT, capture_output=True, text=True, check=False)
check(result.returncode == 0, f"{' '.join(args[:3])}: {result.stderr.strip()[-300:]}")
return result.stdout.strip()
def redis_admin(commands):
"""Команды redis-cli от admin: пароль admin и сами команды идут через stdin (не в argv, не в выводе)."""
password = (ROOT / "secrets" / "operator" / "redis_admin_password").read_text(encoding="utf-8").strip()
script = ("read -r REDISCLI_AUTH; export REDISCLI_AUTH; exec redis-cli --tls --cacert /run/tls-ca/ca.crt "
"--cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis --user admin --no-auth-warning")
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", script], cwd=ROOT,
input=(password + "\n" + "\n".join(commands) + "\n").encode(), capture_output=True,
check=False)
check(result.returncode == 0, f"redis-cli failed: {result.stderr.decode()[-200:]}")
return result.stdout.decode()
def events_since(service, since, names):
"""События из логов сервиса (JSON-строки) после момента since: {event: count}."""
counts = Counter()
for line in compose_output("logs", "--no-log-prefix", service).splitlines():
start = line.find("{")
if start < 0:
continue
try:
item = json.loads(line[start:])
except ValueError:
continue
ts = item.get("ts", "")
try:
moment = dt.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
except ValueError:
continue
if moment >= since and item.get("event") in names:
counts[item["event"]] += 1
if item["event"] == "secrets.rotated":
counts[f"rotated:{item.get('message', '')[-20:]}"] += 1
return counts
def save(name, data):
RESULTS[name] = data
path = EVIDENCE / f"lifecycle-{name}.json"
path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
print(f"saved {path}")
# ---------------- сценарии ----------------
def rotate_secret():
print("--- rotate-secret: пароль Redis ACL-пользователя api")
replicas = len(api_containers())
version_before = json.loads(operator("scripts/bao.py", "kv-version", "api"))["version"]
new_password = secrets.token_hex(24)
digest = hashlib.sha256(new_password.encode()).hexdigest()
started = time.time()
with Load() as load:
time.sleep(3)
redis_admin([f"ACL SETUSER api >{new_password}"])
both = sum(1 for line in redis_admin(["ACL GETUSER api"]).splitlines() if len(line.strip()) == 64)
with tempfile.NamedTemporaryFile("w", delete=False, dir=ROOT / "secrets" / "operator") as tmp:
tmp.write(new_password)
try:
os.chmod(tmp.name, 0o600)
operator("scripts/bao.py", "kv-rotate", "api", "redis_password", "--value-file", tmp.name)
finally:
os.unlink(tmp.name)
version_after = json.loads(operator("scripts/bao.py", "kv-version", "api"))["version"]
t_rotated = time.time()
deadline = time.time() + 90
while True:
seen = events_since("api", started, {"secrets.rotated"})
if seen.get("secrets.rotated", 0) >= replicas:
break
check(time.time() < deadline, f"replicas did not pick up the new version: {dict(seen)}")
time.sleep(1)
picked_up = round(time.time() - t_rotated, 1)
redis_admin([f"ACL SETUSER api resetpass >{new_password}", "CLIENT KILL USER api", "ACL SAVE"])
acl = redis_admin(["ACL GETUSER api"])
time.sleep(8) # новые соединения пула — уже с новым паролем
passwords = [line.strip() for line in acl.splitlines() if len(line.strip()) == 64
and all(c in "0123456789abcdef" for c in line.strip())]
check(passwords == [digest], "old password still active or new one missing")
data = {"kv_version": [version_before, version_after], "replicas": replicas,
"active_passwords_during_overlap": both,
"all_replicas_picked_up_after_s": picked_up, "active_password_hashes_after": len(passwords),
"new_password_hash_matches": digest in acl, "load": load.summary()}
check(len(passwords) == 1, f"expected exactly one active password, got {len(passwords)}")
check(load.summary()["errors_5xx_or_conn"] == 0, f"errors during rotation: {load.errors[:5]}")
print(json.dumps(data, ensure_ascii=False))
print("PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors")
save("rotate-secret", data)
def leaf_serial(path):
from cryptography import x509
return format(x509.load_pem_x509_certificate(path.read_bytes()).serial_number, "x")
def probe_lb():
result = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "probe",
"python", "/probe/tls_probe.py", "lb-api"], cwd=ROOT, capture_output=True, check=False)
rows = [json.loads(line) for line in result.stdout.decode().splitlines() if line.startswith("{")]
return {r["case"]: r["observed"] for r in rows if "case" in r}, [r for r in rows if "case" in r and not r["pass"]]
def rotate_leaf():
print("--- rotate-leaf: клиентский сертификат LB (CN=lb)")
lb_dir = ROOT / "secrets" / "pki" / "lb"
old_serial = leaf_serial(lb_dir / "tls.crt")
with Load() as load:
time.sleep(3)
operator("scripts/pki.py", "rotate", "lb")
new_serial = leaf_serial(lb_dir / "tls.crt")
compose("kill", "-s", "USR2", "lb") # master-worker reload: новый leaf для новых соединений
time.sleep(5)
operator("scripts/pki.py", "revoke", "lb", "--previous")
operator("scripts/bao.py", "sync-crl")
compose("kill", "-s", "USR2", "lb") # HAProxy и otel-gate перечитывают CRL при reload
compose("kill", "-s", "USR2", "otel-gate")
t_revoked = time.time()
time.sleep(12) # > CRL_RELOAD_SECONDS: API перечитал CRL
observed, failed = probe_lb()
closed = events_since("api", t_revoked - 1, {"tls.crl_loaded", "tls.revoked_connections_closed",
"tls.client_rejected"})
data = {"old_serial": old_serial[:16], "new_serial": new_serial[:16],
"probe": {k: v for k, v in observed.items() if "LB" in k or "lb" in k},
"api_events_after_revoke": dict(closed), "load": load.summary()}
print(json.dumps(data, ensure_ascii=False))
check(observed.get("previous (rotated, revoked) LB leaf") in ("server_rejected_client_cert",
"connection_closed_after_handshake"),
"old leaf is still accepted")
check(observed.get("valid LB identity (CN=lb)") == "http_200", "new leaf is not accepted")
check(not failed, f"probe failures: {failed}")
check(load.summary()["errors_5xx_or_conn"] == 0, f"errors during rotation: {load.errors[:5]}")
print("PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors")
save("rotate-leaf", data)
def token_expiry():
print("--- token-expiry: короткий TTL токенов OpenBao роли api")
started = time.time()
try:
with Load() as load:
operator("scripts/bao.py", "role-ttl", "api", "20s", "40s")
operator("scripts/bao.py", "revoke-service-tokens")
time.sleep(100)
finally:
operator("scripts/bao.py", "role-ttl", "api", "15m", "1h")
operator("scripts/bao.py", "revoke-service-tokens") # вернуть обычные токены
events = events_since("api", started, {"secrets.login", "secrets.renewed", "secrets.relogin",
"secrets.refresh_failed", "secrets.renew_failed"})
data = {"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": dict(events),
"load": load.summary()}
print(json.dumps(data, ensure_ascii=False))
# Отзыв замечает то, что раньше обратится к OpenBao с отозванным токеном: чтение KV (refresh_failed)
# или продление (renew_failed -> сразу новый вход). Оба пути — реакция именно на revoke.
check(events.get("secrets.refresh_failed", 0) + events.get("secrets.renew_failed", 0) >= 1,
"revoked token was not noticed")
check(events.get("secrets.login", 0) + events.get("secrets.relogin", 0) + events.get("secrets.renew_failed", 0) >= 2,
"no re-login after expiry")
check(load.summary()["errors_5xx_or_conn"] == 0, f"errors: {load.errors[:5]}")
print("PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected")
save("token-expiry", data)
def sealed():
print("--- sealed: OpenBao запечатан")
containers = api_containers()
check(len(containers) >= 2, "need 2 api replicas")
victim = containers[0]
started = time.time()
with Load() as load:
time.sleep(3)
operator("scripts/bao.py", "seal")
t_sealed = time.time()
time.sleep(40) # > SECRETS_REFRESH_SECONDS: отказ обновления замечен
docker("stop", "--time", "15", victim)
docker("start", victim)
time.sleep(30)
victim_health = health(victim)
serving = len({try_call("GET", "/health/ready").instance for _ in range(6)} - {None})
operator("scripts/bao.py", "unseal")
t_unsealed = time.time()
deadline = time.time() + 120
while health(victim) != "healthy":
check(time.time() < deadline, "replica did not recover after unseal")
time.sleep(1)
recovered = round(time.time() - t_unsealed, 1)
time.sleep(3)
events = events_since("api", started, {"secrets.refresh_failed", "secrets.waiting", "secrets.unavailable",
"secrets.login"})
data = {"sealed_seconds": round(t_unsealed - t_sealed, 1), "restarted_replica_health_while_sealed": victim_health,
"replicas_answering_while_sealed": serving, "restarted_replica_ready_after_unseal_s": recovered,
"api_events": dict(events), "load": load.summary()}
print(json.dumps(data, ensure_ascii=False))
check(victim_health != "healthy", "a replica became ready without secrets")
check(events.get("secrets.refresh_failed", 0) >= 1, "sealed OpenBao was not noticed")
errors = [e for e in load.errors if e["code"] != 503]
check(not errors, f"unexpected errors while sealed: {errors[:5]}")
print("PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready "
"(fail closed) until unseal")
save("sealed", data)
wait_balanced(2, 60)
SCENARIOS = {"rotate-secret": rotate_secret, "rotate-leaf": rotate_leaf, "token-expiry": token_expiry,
"sealed": sealed}
if __name__ == "__main__":
names = sys.argv[1:] or ["all"]
if names == ["all"]:
names = list(SCENARIOS)
failed = 0
for name in names:
try:
SCENARIOS[name]()
except AssertionError as exc:
failed += 1
print(f"FAIL: {name}: {exc}")
raise SystemExit(1 if failed else 0)