"""lab7, уровень 4: жизненный цикл секретов и сертификатов ПОД НАГРУЗКОЙ, без пересборки образов. python student-tests/lifecycle.py rotate-secret | rotate-leaf | token-expiry | sealed | all Нужны: запущенный стенд, Python с cryptography на хосте (requirements-ops.txt) и файлы оператора (secrets/operator/*). Всё время сценария фоновая нагрузка (вход + чтение + отправка сообщений через LB) считает ответы; итог — evidence/lifecycle-<сценарий>.json. Секреты не печатаются. rotate-secret Пароль ACL-пользователя api в Redis (рабочий секрет API): 1) Redis: ACL SETUSER api >new — временно действуют оба пароля; 2) OpenBao: новая версия messenger/api.redis_password (bao.py kv-rotate, check-and-set); 3) ждём, пока КАЖДАЯ реплика API прочитала новую версию (лог secrets.rotated); 4) Redis: ACL SETUSER api resetpass >new — старый пароль больше не действует; CLIENT KILL USER api рвёт старые соединения, пул API переподключается уже с новым паролем; ACL SAVE. rotate-leaf Клиентский сертификат LB (identity CN=lb для mTLS LB -> API): 1) pki.py rotate lb (старый leaf — в lb/previous); 2) reload HAProxy (USR2, без обрыва); 3) pki.py revoke lb --previous + bao.py sync-crl; reload lb и otel-gate (их CRL); 4) API перечитывает CRL сам (CRL_RELOAD_SECONDS) и закрывает открытые соединения со старым leaf; 5) зонд: старый leaf -> отказ (revoked), новый -> 200. token-expiry Короткий TTL токенов роли api (20 с / max 40 с) + отзыв всех токенов cert auth: сервисы перелогиниваются сертификатом, renew/relogin видны в логах, запросы не падают. TTL возвращается. sealed OpenBao запечатан: работающие реплики обслуживают запросы (секреты в памяти), перезапущенная реплика НЕ становится готовой (fail closed, LB её не использует); после unseal — готова. """ import datetime as dt import hashlib import json import os import secrets import subprocess import sys import tempfile import threading import time from collections import Counter from common import (EVIDENCE, ROOT, api_containers, check, compose, compose_output, docker, health, new_chat, new_key, new_user, try_call, wait_balanced) PY = sys.executable RESULTS = {} # ---------------- фоновая нагрузка ---------------- class Load: """2 потока: GET /users/me + POST message (Idempotency-Key) и периодический новый вход (новый Redis SET).""" def __init__(self): self.user = new_user("lifecycle") self.chat = new_chat(self.user) self.stop = threading.Event() self.codes = Counter() self.errors = [] self.threads = [threading.Thread(target=self.reader, daemon=True), threading.Thread(target=self.login_loop, daemon=True)] def note(self, what, reply): code = reply.status if reply is not None else "conn_error" self.codes[f"{what}:{code}"] += 1 if reply is None or reply.status >= 500: self.errors.append({"t": time.time(), "what": what, "code": code}) def reader(self): while not self.stop.is_set(): self.note("me", try_call("GET", "/api/v1/users/me", token=self.user["token"], timeout=5)) self.note("send", try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", token=self.user["token"], body={"text": "load"}, headers={"Idempotency-Key": new_key()}, timeout=5)) time.sleep(0.05) def login_loop(self): users = [new_user("lifecycle_login") for _ in range(4)] # лимит 10 входов/мин на username i = 0 while not self.stop.is_set(): u = users[i % len(users)] i += 1 import base64 auth = "Basic " + base64.b64encode(f"{u['username']}:{u['password']}".encode()).decode() self.note("login", try_call("POST", "/api/v1/auth/sessions", headers={"Authorization": auth}, timeout=5)) time.sleep(2) def __enter__(self): self.started = time.time() for t in self.threads: t.start() return self def __exit__(self, *exc): self.stop.set() for t in self.threads: t.join(timeout=10) self.seconds = round(time.time() - self.started, 1) return False def summary(self): return {"seconds": self.seconds, "codes": dict(self.codes), "errors_5xx_or_conn": len(self.errors), "errors": self.errors[:10]} # ---------------- помощники оператора ---------------- def operator(*args): result = subprocess.run([PY, *args], cwd=ROOT, capture_output=True, text=True, check=False) check(result.returncode == 0, f"{' '.join(args[:3])}: {result.stderr.strip()[-300:]}") return result.stdout.strip() def redis_admin(commands): """Команды redis-cli от admin: пароль admin и сами команды идут через stdin (не в argv, не в выводе).""" password = (ROOT / "secrets" / "operator" / "redis_admin_password").read_text(encoding="utf-8").strip() script = ("read -r REDISCLI_AUTH; export REDISCLI_AUTH; exec redis-cli --tls --cacert /run/tls-ca/ca.crt " "--cert /run/tls/tls.crt --key /run/tls/tls.key -h 127.0.0.1 --sni redis --user admin --no-auth-warning") result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", script], cwd=ROOT, input=(password + "\n" + "\n".join(commands) + "\n").encode(), capture_output=True, check=False) check(result.returncode == 0, f"redis-cli failed: {result.stderr.decode()[-200:]}") return result.stdout.decode() def events_since(service, since, names): """События из логов сервиса (JSON-строки) после момента since: {event: count}.""" counts = Counter() for line in compose_output("logs", "--no-log-prefix", service).splitlines(): start = line.find("{") if start < 0: continue try: item = json.loads(line[start:]) except ValueError: continue ts = item.get("ts", "") try: moment = dt.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp() except ValueError: continue if moment >= since and item.get("event") in names: counts[item["event"]] += 1 if item["event"] == "secrets.rotated": counts[f"rotated:{item.get('message', '')[-20:]}"] += 1 return counts def save(name, data): RESULTS[name] = data path = EVIDENCE / f"lifecycle-{name}.json" path.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8") print(f"saved {path}") # ---------------- сценарии ---------------- def rotate_secret(): print("--- rotate-secret: пароль Redis ACL-пользователя api") replicas = len(api_containers()) version_before = json.loads(operator("scripts/bao.py", "kv-version", "api"))["version"] new_password = secrets.token_hex(24) digest = hashlib.sha256(new_password.encode()).hexdigest() started = time.time() with Load() as load: time.sleep(3) redis_admin([f"ACL SETUSER api >{new_password}"]) both = sum(1 for line in redis_admin(["ACL GETUSER api"]).splitlines() if len(line.strip()) == 64) with tempfile.NamedTemporaryFile("w", delete=False, dir=ROOT / "secrets" / "operator") as tmp: tmp.write(new_password) try: os.chmod(tmp.name, 0o600) operator("scripts/bao.py", "kv-rotate", "api", "redis_password", "--value-file", tmp.name) finally: os.unlink(tmp.name) version_after = json.loads(operator("scripts/bao.py", "kv-version", "api"))["version"] t_rotated = time.time() deadline = time.time() + 90 while True: seen = events_since("api", started, {"secrets.rotated"}) if seen.get("secrets.rotated", 0) >= replicas: break check(time.time() < deadline, f"replicas did not pick up the new version: {dict(seen)}") time.sleep(1) picked_up = round(time.time() - t_rotated, 1) redis_admin([f"ACL SETUSER api resetpass >{new_password}", "CLIENT KILL USER api", "ACL SAVE"]) acl = redis_admin(["ACL GETUSER api"]) time.sleep(8) # новые соединения пула — уже с новым паролем passwords = [line.strip() for line in acl.splitlines() if len(line.strip()) == 64 and all(c in "0123456789abcdef" for c in line.strip())] check(passwords == [digest], "old password still active or new one missing") data = {"kv_version": [version_before, version_after], "replicas": replicas, "active_passwords_during_overlap": both, "all_replicas_picked_up_after_s": picked_up, "active_password_hashes_after": len(passwords), "new_password_hash_matches": digest in acl, "load": load.summary()} check(len(passwords) == 1, f"expected exactly one active password, got {len(passwords)}") check(load.summary()["errors_5xx_or_conn"] == 0, f"errors during rotation: {load.errors[:5]}") print(json.dumps(data, ensure_ascii=False)) print("PASS: Redis password rotated via OpenBao without restarts; old password revoked; no errors") save("rotate-secret", data) def leaf_serial(path): from cryptography import x509 return format(x509.load_pem_x509_certificate(path.read_bytes()).serial_number, "x") def probe_lb(): result = subprocess.run(["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "probe", "python", "/probe/tls_probe.py", "lb-api"], cwd=ROOT, capture_output=True, check=False) rows = [json.loads(line) for line in result.stdout.decode().splitlines() if line.startswith("{")] return {r["case"]: r["observed"] for r in rows if "case" in r}, [r for r in rows if "case" in r and not r["pass"]] def rotate_leaf(): print("--- rotate-leaf: клиентский сертификат LB (CN=lb)") lb_dir = ROOT / "secrets" / "pki" / "lb" old_serial = leaf_serial(lb_dir / "tls.crt") with Load() as load: time.sleep(3) operator("scripts/pki.py", "rotate", "lb") new_serial = leaf_serial(lb_dir / "tls.crt") compose("kill", "-s", "USR2", "lb") # master-worker reload: новый leaf для новых соединений time.sleep(5) operator("scripts/pki.py", "revoke", "lb", "--previous") operator("scripts/bao.py", "sync-crl") compose("kill", "-s", "USR2", "lb") # HAProxy и otel-gate перечитывают CRL при reload compose("kill", "-s", "USR2", "otel-gate") t_revoked = time.time() time.sleep(12) # > CRL_RELOAD_SECONDS: API перечитал CRL observed, failed = probe_lb() closed = events_since("api", t_revoked - 1, {"tls.crl_loaded", "tls.revoked_connections_closed", "tls.client_rejected"}) data = {"old_serial": old_serial[:16], "new_serial": new_serial[:16], "probe": {k: v for k, v in observed.items() if "LB" in k or "lb" in k}, "api_events_after_revoke": dict(closed), "load": load.summary()} print(json.dumps(data, ensure_ascii=False)) check(observed.get("previous (rotated, revoked) LB leaf") in ("server_rejected_client_cert", "connection_closed_after_handshake"), "old leaf is still accepted") check(observed.get("valid LB identity (CN=lb)") == "http_200", "new leaf is not accepted") check(not failed, f"probe failures: {failed}") check(load.summary()["errors_5xx_or_conn"] == 0, f"errors during rotation: {load.errors[:5]}") print("PASS: LB leaf rotated without rebuild/restart; old leaf revoked by CRL and rejected; no errors") save("rotate-leaf", data) def token_expiry(): print("--- token-expiry: короткий TTL токенов OpenBao роли api") started = time.time() try: with Load() as load: operator("scripts/bao.py", "role-ttl", "api", "20s", "40s") operator("scripts/bao.py", "revoke-service-tokens") time.sleep(100) finally: operator("scripts/bao.py", "role-ttl", "api", "15m", "1h") operator("scripts/bao.py", "revoke-service-tokens") # вернуть обычные токены events = events_since("api", started, {"secrets.login", "secrets.renewed", "secrets.relogin", "secrets.refresh_failed", "secrets.renew_failed"}) data = {"role_ttl": "20s / max 40s", "observed_seconds": 100, "api_events": dict(events), "load": load.summary()} print(json.dumps(data, ensure_ascii=False)) # Отзыв замечает то, что раньше обратится к OpenBao с отозванным токеном: чтение KV (refresh_failed) # или продление (renew_failed -> сразу новый вход). Оба пути — реакция именно на revoke. check(events.get("secrets.refresh_failed", 0) + events.get("secrets.renew_failed", 0) >= 1, "revoked token was not noticed") check(events.get("secrets.login", 0) + events.get("secrets.relogin", 0) + events.get("secrets.renew_failed", 0) >= 2, "no re-login after expiry") check(load.summary()["errors_5xx_or_conn"] == 0, f"errors: {load.errors[:5]}") print("PASS: expired/revoked tokens -> services re-login with their certificate; requests unaffected") save("token-expiry", data) def sealed(): print("--- sealed: OpenBao запечатан") containers = api_containers() check(len(containers) >= 2, "need 2 api replicas") victim = containers[0] started = time.time() with Load() as load: time.sleep(3) operator("scripts/bao.py", "seal") t_sealed = time.time() time.sleep(40) # > SECRETS_REFRESH_SECONDS: отказ обновления замечен docker("stop", "--time", "15", victim) docker("start", victim) time.sleep(30) victim_health = health(victim) serving = len({try_call("GET", "/health/ready").instance for _ in range(6)} - {None}) operator("scripts/bao.py", "unseal") t_unsealed = time.time() deadline = time.time() + 120 while health(victim) != "healthy": check(time.time() < deadline, "replica did not recover after unseal") time.sleep(1) recovered = round(time.time() - t_unsealed, 1) time.sleep(3) events = events_since("api", started, {"secrets.refresh_failed", "secrets.waiting", "secrets.unavailable", "secrets.login"}) data = {"sealed_seconds": round(t_unsealed - t_sealed, 1), "restarted_replica_health_while_sealed": victim_health, "replicas_answering_while_sealed": serving, "restarted_replica_ready_after_unseal_s": recovered, "api_events": dict(events), "load": load.summary()} print(json.dumps(data, ensure_ascii=False)) check(victim_health != "healthy", "a replica became ready without secrets") check(events.get("secrets.refresh_failed", 0) >= 1, "sealed OpenBao was not noticed") errors = [e for e in load.errors if e["code"] != 503] check(not errors, f"unexpected errors while sealed: {errors[:5]}") print("PASS: sealed OpenBao -> running replicas keep serving from memory, a restarted replica stays not ready " "(fail closed) until unseal") save("sealed", data) wait_balanced(2, 60) SCENARIOS = {"rotate-secret": rotate_secret, "rotate-leaf": rotate_leaf, "token-expiry": token_expiry, "sealed": sealed} if __name__ == "__main__": names = sys.argv[1:] or ["all"] if names == ["all"]: names = list(SCENARIOS) failed = 0 for name in names: try: SCENARIOS[name]() except AssertionError as exc: failed += 1 print(f"FAIL: {name}: {exc}") raise SystemExit(1 if failed else 0)