#!/usr/bin/env python3 """Операции над compose-стендом lab7. Python 3.9+, только стандартная библиотека. Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает `docker compose` и `docker` без shell-интерполяции. Значения секретов не печатает. Стенд: python scripts/ops.py up # bootstrap с нуля: secrets -> образ -> PKI -> OpenBao init/unseal/configure -> стенд -> ожидание python scripts/ops.py wait # дождаться N здоровых реплик, UP в LB и HTTPS /health/ready python scripts/ops.py ps # реплики: alias (X-Instance-Id), контейнер, health, статус в LB python scripts/ops.py down | destroy # остановить (volumes целы) | НАМЕРЕННО удалить данные стенда Hooks публичных тестов: python scripts/ops.py stop-one # TARGET_INSTANCE_ID=; реплика остаётся остановленной python scripts/ops.py restore # вернуть остановленные реплики python scripts/ops.py restart # persistence: API, worker, PostgreSQL, Redis, broker(+gate), S3(+gate), OpenBao(+unseal), LB lab7 — безопасность (результаты дописываются в evidence/): python scripts/ops.py tls-audit # вся trust-matrix: TLS/mTLS/identity/ACL отрицательные и положительные проверки python scripts/ops.py security-http # ACL/вложения, X-User-Id, logout, 429, 413, утечки в ошибках (через LB) python scripts/ops.py leak-check # отсутствие паролей/токенов/секретов в Loki, Tempo, Prometheus python scripts/ops.py hardening # docker inspect: user, read_only, caps, no-new-privileges, limits, порты, docker.sock python scripts/ops.py bao-policy # OpenBao: allow/deny политик реальными токенами AppRole python scripts/ops.py rehash-demo # Argon2id: старый hash -> успешный вход -> пересчёт под текущие параметры python scripts/ops.py rotate-db-password [api|worker] # ротация рабочего секрета без пересборки/рестарта python scripts/ops.py rotate-lb-cert # ротация leaf LB->API + отзыв старого (CRL), проверка старым/новым python scripts/ops.py rotate-secret-id [api|worker] # новый SecretID агента, старый уничтожен python scripts/ops.py revoke-identity [api|worker] # отзыв AppRole identity -> агент без доступа -> восстановление python scripts/ops.py bao-sealed # OpenBao sealed: API продолжает работать на отрисованных секретах, unseal python scripts/ops.py login-budget # бюджет login DoS: Argon2id на стенде + поток входов python scripts/ops.py scan # pip-audit + trivy образов -> evidence/scan/ lab3–6 (как раньше): rolling-update, in-network storage|expiry|consistency, wait-s3, s3-check, reconcile [--apply], api-fs, worker-stop|worker-kill|worker-start, api-restart, broker-restart, jobs ..., pipeline-faults Переменные: COMPOSE_PROJECT_NAME (messenger-lab7), ENV_FILE (.env), API_REPLICAS (2), HTTPS_PORT (8443), STOP_MODE (stop|kill), CA_FILE (secrets/pki/ca.crt). """ import csv import io import json import os import secrets as pysecrets import ssl import subprocess import sys import time import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] EVIDENCE = ROOT / "evidence" OPS_LOG = EVIDENCE / "ops.log" SECRETS = ROOT / "secrets" try: # консоль Windows может быть не в UTF-8 sys.stdout.reconfigure(errors="replace") sys.stderr.reconfigure(errors="replace") except AttributeError: pass # ---------------------------------------------------------------- settings def _dotenv(): path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env"))) values = {} if path.is_file(): for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if line and not line.startswith("#") and "=" in line: k, v = line.split("=", 1) values[k.strip()] = v.strip().strip('"').strip("'") return path if path.is_file() else None, values ENV_PATH, DOTENV = _dotenv() def setting(name, default): return os.environ.get(name) or DOTENV.get(name) or default PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab7") os.environ.setdefault("COMPOSE_PROJECT_NAME", PROJECT) REPLICAS = int(setting("API_REPLICAS", "2")) HTTPS_PORT = int(setting("HTTPS_PORT", "8443")) LB_URL = "https://127.0.0.1:%d" % HTTPS_PORT CA_FILE = setting("CA_FILE", str(SECRETS / "pki" / "ca.crt")) def tls_context(): return ssl.create_default_context(cafile=CA_FILE if Path(CA_FILE).exists() else None) def now_iso(): return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" def log(msg): line = "%s %s" % (now_iso(), msg) print(line, flush=True) try: EVIDENCE.mkdir(exist_ok=True) with OPS_LOG.open("a", encoding="utf-8") as f: f.write(line + "\n") except OSError: pass # ---------------------------------------------------------------- docker def compose_cmd(*args): cmd = ["docker", "compose", "-p", PROJECT] if ENV_PATH: cmd += ["--env-file", str(ENV_PATH)] return cmd + list(args) def run(cmd, check=True, capture=False, timeout=300, show=False, env=None, stdin=None): """capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо.""" out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL) result = subprocess.run(cmd, cwd=ROOT, check=False, timeout=timeout, stdout=out, stderr=out, text=True, env=env, input=stdin) if check and result.returncode != 0: detail = (result.stderr or "").strip()[-400:] if capture else "" raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:5]), detail)) return result def compose(*args, timeout=300): return run(compose_cmd(*args), timeout=timeout, capture=True) def py(): return sys.executable or "python" def service_ids(service, include_stopped=False): cmd = ["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, "--filter", "label=com.docker.compose.service=" + service, "--filter", "label=com.docker.compose.oneoff=False"] if include_stopped: cmd.insert(2, "-a") return run(cmd, capture=True).stdout.split() def instance_alias(container_id): """X-Instance-Id реплики: случайный alias, который процесс записал в /tmp/instance-id.""" res = run(["docker", "exec", container_id, "cat", "/tmp/instance-id"], check=False, capture=True, timeout=10) return res.stdout.strip() if res.returncode == 0 else "" def api_containers(include_stopped=True, with_alias=True): cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}", "--filter", "label=com.docker.compose.project=" + PROJECT, "--filter", "label=com.docker.compose.service=api", "--filter", "label=com.docker.compose.oneoff=False"] if include_stopped: cmd.insert(2, "-a") ids = [x for x in run(cmd, capture=True).stdout.split() if x] if not ids: return [] result = [] for item in json.loads(run(["docker", "inspect"] + ids, capture=True).stdout): state = item["State"] nets = item.get("NetworkSettings", {}).get("Networks") or {} ingress = [n.get("IPAddress") for k, n in nets.items() if k.endswith("_ingress") and n.get("IPAddress")] running_ = state.get("Status") == "running" result.append({ "id": item["Id"], "short": item["Id"][:12], "name": item["Name"].lstrip("/"), "alias": instance_alias(item["Id"]) if (with_alias and running_) else "", "state": state.get("Status"), "health": (state.get("Health") or {}).get("Status", "-"), "image": item["Config"].get("Image"), "ip": ingress[0] if ingress else "", "created": item.get("Created", "")[:19], }) result.sort(key=lambda c: c["name"]) return result def running(containers=None): return [c for c in (containers if containers is not None else api_containers(with_alias=False)) if c["state"] == "running"] # ---------------------------------------------------------------- load balancer def lb_servers(): """Статистика HAProxy (порт 8404 только внутри контейнера lb) через docker exec.""" out = run(compose_cmd("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv"), capture=True, check=False, timeout=15).stdout or "" servers = [] for r in csv.DictReader(io.StringIO(out.lstrip("# "))): if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"): continue addr = r.get("addr") or "" status = r.get("status", "") if status.startswith("MAINT") and not addr: continue servers.append({"name": r["svname"], "status": status, "addr": addr}) return servers def lb_up_count(): return sum(1 for s in lb_servers() if s["status"].startswith("UP")) def lb_ready(): try: with urllib.request.urlopen(LB_URL + "/health/ready", timeout=3, context=tls_context()) as r: return r.status == 200 except (urllib.error.URLError, OSError): return False def wait_until(predicate, timeout, what, interval=0.5): deadline = time.monotonic() + timeout last_exc = None while time.monotonic() < deadline: try: if predicate(): return True except (urllib.error.URLError, OSError, ValueError, KeyError) as exc: last_exc = exc time.sleep(interval) raise SystemExit("timeout after %ss waiting for %s%s" % ( timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else "")) def wait_healthy(ids, timeout=90): def ok(): states = {c["id"]: c for c in api_containers(with_alias=False)} return all(states.get(i, {}).get("health") == "healthy" for i in ids) wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids)) def wait_stack(replicas, timeout=180): wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas, timeout, "%d healthy api replicas" % replicas) wait_until(lambda: lb_up_count() >= replicas, 60, "%d replicas UP in HAProxy (mTLS checks)" % replicas) wait_until(lb_ready, 60, "LB https /health/ready = 200 (CA %s)" % CA_FILE) def wait_service_healthy(service, timeout): def ok(): ids = service_ids(service) if not ids: return False out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip() return out == "healthy" wait_until(ok, timeout, service + " healthy") def container_started(service): ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT, "--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split() if not ids: return "-" return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23] # ---------------------------------------------------------------- bootstrap def pki(*args): """scripts/pki.py: на хосте, если есть cryptography, иначе в контейнере tools (сеть none).""" try: import cryptography # noqa: F401 run([py(), "scripts/pki.py"] + list(args), show=True, timeout=120) except ImportError: run(compose_cmd("--profile", "tools", "run", "--rm", "-T", "tools", "scripts/pki.py") + list(args), show=True, timeout=300) def bao(*args, check=True): return run([py(), "scripts/bao.py"] + list(args), show=True, timeout=300, check=check) def cmd_up(_args): t0 = time.monotonic() if not ENV_PATH and (ROOT / ".env.example").exists(): (ROOT / ".env").write_text((ROOT / ".env.example").read_text(encoding="utf-8"), encoding="utf-8") log("up: .env created from .env.example") run([py(), "scripts/init_secrets.py"], show=True) log("up: building image (api/worker/migrate/s3-init/tools)") run(compose_cmd("build", "api"), show=True, timeout=1800) pki("init") compose("up", "-d", "openbao", timeout=180) bao("init") # init (однократно) + unseal; ключи -> operator-only/ bao("configure") # kv, approle, политики, секреты -> KV, RoleID/SecretID -> secrets/bao/ log("up: starting the stand") run(compose_cmd("up", "-d"), show=True, timeout=1200) wait_stack(REPLICAS) wait_service_healthy("worker", 120) log("up: ready in %.0fs -> %s (CA: %s); Grafana https://127.0.0.1:%s (admin / secrets/grafana_admin_password)" % (time.monotonic() - t0, LB_URL, Path(CA_FILE).relative_to(ROOT) if Path(CA_FILE).is_relative_to(ROOT) else CA_FILE, setting("GRAFANA_PORT", "3000"))) def cmd_down(_args): run(compose_cmd("--profile", "ttl", "--profile", "faults", "--profile", "audit", "down"), show=True, timeout=600) def cmd_destroy(_args): """Намеренное удаление учебных данных: volumes (БД, S3, broker, OpenBao, telemetry).""" run(compose_cmd("--profile", "ttl", "--profile", "faults", "--profile", "audit", "down", "-v"), show=True, timeout=600) log("destroy: volumes removed; secrets/ and operator-only/ kept (OpenBao keys now refer to a deleted storage:" " remove operator-only/openbao-init.json and secrets/bao/* before the next `up`)") # ---------------------------------------------------------------- stand state def cmd_wait(_args): t0 = time.monotonic() wait_stack(REPLICAS) print("ready: %d api replicas healthy and UP in LB, HTTPS ready (%.1fs)" % (REPLICAS, time.monotonic() - t0)) def cmd_ps(_args): containers = api_containers() try: by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()} except OSError: by_addr = {} print("%-14s %-28s %-9s %-10s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "lb", "lb_status")) for c in containers: s = by_addr.get(c["ip"], {}) print("%-14s %-28s %-9s %-10s %-6s %s" % (c["alias"] or "-", c["name"], c["state"], c["health"], s.get("name", "-"), s.get("status", "-"))) def find_target(target): for c in api_containers(): if target and (c["alias"] == target or c["id"].startswith(target)): return c return None def cmd_stop_one(_args): target = os.environ.get("TARGET_INSTANCE_ID", "").strip() if not target: raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)") mode = os.environ.get("STOP_MODE", "stop").strip().lower() c = find_target(target) if c is None: raise SystemExit("no api container of project %s has X-Instance-Id %s" % (PROJECT, target)) if c["state"] != "running": raise SystemExit("container %s is already %s" % (c["name"], c["state"])) t0 = time.monotonic() log("stop-one: mode=%s alias=%s container=%s" % (mode, target, c["name"])) if mode == "kill": run(["docker", "update", "--restart", "no", c["id"]]) run(["docker", "kill", "--signal", "KILL", c["id"]]) else: run(["docker", "stop", "--time", "20", c["id"]], timeout=40) log("stop-one: container=%s stopped after %.2fs; left stopped (restore: python scripts/ops.py restore)" % (c["name"], time.monotonic() - t0)) def cmd_restore(_args): t0 = time.monotonic() stopped = [c for c in api_containers(with_alias=False) if c["state"] != "running"] for c in stopped: run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False) compose("up", "-d", "--no-deps", "--no-recreate", "api") wait_stack(REPLICAS) log("restore: started %s; %d replicas healthy and UP after %.1fs" % ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0)) def cmd_restart(_args): """Hook make test-persistence: перезапуск API, worker, всех хранилищ (PostgreSQL, Redis, broker, S3, OpenBao) и LB. Volumes не удаляются, данные не пересоздаются. OpenBao после рестарта запечатан: оператор распечатывает его (`bao.py unseal`, ключи из operator-only/). Пока он sealed, агенты держат уже отрисованные секреты (tmpfs volume), поэтому API/worker стартуют и без OpenBao. Gates делят сетевой namespace с broker / collector и пересоздаются после рестарта своего сервиса. """ t0 = time.monotonic() stores = ("postgres", "redis", "broker", "s3", "openbao") started_before = {svc: container_started(svc) for svc in stores} before = sorted(c["short"] for c in running()) compose("stop", "lb", "api", "worker", timeout=180) compose("restart", *stores, timeout=180) compose("up", "-d", "--no-deps", "--force-recreate", "broker-gate", "s3-gate", timeout=120) wait_service_healthy("broker", 60) wait_service_healthy("broker-gate", 60) bao("unseal") log("restart: openbao restarted -> sealed -> unsealed by operator (keys: operator-only/)") wait_service_healthy("s3", 60) wait_service_healthy("s3-gate", 60) t_warm = time.monotonic() warm = s3_warmup(70) log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm)) compose("start", "api", "worker", timeout=120) wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS, 90, "api healthy after restart") compose("start", "lb", timeout=60) log("restart: started_at %s; api %s restarted; worker restarted; lb restarted; %.1fs" % ("; ".join("%s %s -> %s" % (s, started_before[s], container_started(s)) for s in stores), before, time.monotonic() - t0)) def cmd_worker_stop(_args): t0 = time.monotonic() ids = service_ids("worker") compose("stop", "worker", timeout=120) log("worker-stop: stopped %s in %.1fs" % ([i[:12] for i in ids], time.monotonic() - t0)) def cmd_worker_kill(_args): ids = service_ids("worker") for i in ids: run(["docker", "update", "--restart", "no", i]) run(["docker", "kill", "--signal", "KILL", i]) log("worker-kill: SIGKILL %s (restart policy disabled until worker-start)" % [i[:12] for i in ids]) def cmd_worker_start(_args): t0 = time.monotonic() for i in service_ids("worker", include_stopped=True): run(["docker", "update", "--restart", "unless-stopped", i], check=False) compose("up", "-d", "--no-deps", "--no-recreate", "worker", timeout=120) wait_service_healthy("worker", 90) log("worker-start: worker healthy after %.1fs" % (time.monotonic() - t0)) def cmd_api_restart(_args): t0 = time.monotonic() before = [c["short"] for c in running()] compose("restart", "api", timeout=120) wait_stack(REPLICAS) log("api-restart: %s restarted, %d replicas UP after %.1fs" % (before, REPLICAS, time.monotonic() - t0)) def cmd_broker_restart(_args): t0 = time.monotonic() before = container_started("broker") compose("restart", "broker", timeout=60) compose("up", "-d", "--no-deps", "--force-recreate", "broker-gate", timeout=120) wait_service_healthy("broker", 60) wait_service_healthy("broker-gate", 60) log("broker-restart: started_at %s -> %s, healthy after %.1fs (volume brokerdata kept, gate recreated)" % (before, container_started("broker"), time.monotonic() - t0)) def cmd_jobs(args): run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.jobs_admin") + list(args), show=True, timeout=300) def s3_warmup(timeout=70): out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", "--wait-ready", str(timeout)), capture=True, timeout=timeout + 60, check=False) lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")] return lines[-1] if lines else "" def cmd_wait_s3(_args): wait_service_healthy("s3", 60) wait_service_healthy("s3-gate", 60) print(s3_warmup(90)) def cmd_s3_check(_args): """Приватный бакет вне API: анонимные запросы, права ключа API по префиксам, закрытые порты.""" run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", "--check-only", "--sample-from-db"), show=True, timeout=180) def cmd_api_fs(_args): for c in api_containers(with_alias=False): info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0] mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])] print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % ( c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none")) def cmd_reconcile(args): # lab7: обход бакета требует права List — оно есть только у identity worker. run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.reconcile") + list(args), show=True, timeout=600) def cmd_rolling_update(args): """Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.""" t_all = time.monotonic() if [c for c in api_containers(with_alias=False) if c["state"] != "running"]: raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first") old = running() n = len(old) if n < 1: raise SystemExit("no running api replicas") log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old])) if "--no-build" not in args: compose("build", "api", timeout=900) out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines() log("rolling-update: migrations once: %s" % (out[-1] if out else "")) for i, victim in enumerate(old, 1): before = {c["id"] for c in api_containers(with_alias=False)} t = time.monotonic() compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180) new = [c for c in api_containers(with_alias=False) if c["id"] not in before] if len(new) != 1: raise SystemExit("expected exactly one new container, got %d" % len(new)) fresh = new[0] wait_healthy([fresh["id"]], timeout=90) wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy") t_ready = time.monotonic() - t t = time.monotonic() run(["docker", "stop", "--time", "20", victim["id"]], timeout=40) run(["docker", "rm", victim["id"]]) log("rolling-update: step %d/%d new=%s ready+UP in %.1fs; old=%s stopped in %.1fs" % (i, n, fresh["short"], t_ready, victim["short"], time.monotonic() - t)) wait_stack(n) log("rolling-update: done in %.1fs" % (time.monotonic() - t_all)) # ---------------------------------------------------------------- in-network тесты (lab2-5) def _in_network_base(): """One-off контейнер образа API (сети api) + тесты + CA и identity lb для прямых вызовов реплик.""" return compose_cmd("run", "--rm", "-T", "--no-deps", "-v", "%s:/app/student-tests:ro" % (ROOT / "student-tests"), "-v", "%s:/app/tests/fixtures:ro" % (ROOT / "tests" / "fixtures"), "-v", "%s:/opt/test/lb.pem:ro" % (SECRETS / "pki" / "lb-client.pem"), "-e", "CA_FILE=/run/secrets/ca_crt", "-e", "CLIENT_CERT_FILE=/opt/test/lb.pem", "-e", "BASE_URL=https://api:8443") def cmd_in_network(args): which = args[0] if args else "" base = _in_network_base() if which == "storage": # White-box тесты хранилища применяют миграции (DDL) -> нужна identity администратора # (как у one-shot migrate); у роли messenger_api прав на DDL нет намеренно. admin = ["-v", "%s:/opt/test/admin_url:ro" % (SECRETS / "database_admin_url"), "-e", "DATABASE_URL_FILE=/opt/test/admin_url"] for pattern in ("test_storage_*.py", "test_session_cache.py"): run(base + admin + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"], show=True, timeout=600) elif which == "expiry": prof = compose_cmd("--profile", "ttl") run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300) env = ["-e", "EXPIRY_BASE_URL=https://api-short-ttl:8443"] try: run(base + env + ["api", "python", "student-tests/wait_url.py", "https://api-short-ttl:8443/health/ready"], show=True, timeout=120) run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_expiry.py", "-v"], show=True, timeout=300) finally: run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120) else: raise SystemExit("usage: python scripts/ops.py in-network storage|expiry") def faults_compose(*args): return run(["docker", "compose", "-p", PROJECT] + (["--env-file", str(ENV_PATH)] if ENV_PATH else []) + ["-f", "compose.yaml", "-f", "compose.faults.yaml"] + list(args), capture=True, timeout=300) def cmd_pipeline_faults(_args): """Отказы worker'а (маркеры в filename, compose.faults.yaml) — сценарии c..j lab5.""" log("pipeline-faults: recreating worker with compose.faults.yaml") faults_compose("up", "-d", "--no-deps", "--force-recreate", "worker") wait_service_healthy("worker", 90) try: run(_in_network_base() + ["-e", "BASE_URL=https://api:8443", "api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_pipeline_faults.py", "-v"], show=True, timeout=900, check=False) finally: compose("up", "-d", "--no-deps", "--force-recreate", "worker", timeout=180) wait_service_healthy("worker", 90) log("pipeline-faults: worker restored to normal configuration") # ---------------------------------------------------------------- lab7: безопасность def save(name, text): if not text.strip(): log("evidence/%s not saved: empty output (see the error above)" % name) return EVIDENCE.mkdir(exist_ok=True) (EVIDENCE / name).write_text(text, encoding="utf-8") log("saved evidence/%s" % name) def audit_run(script, *args, extra=None, timeout=600): cmd = compose_cmd("--profile", "audit", "run", "--rm", "-T", "--no-deps") + list(extra or []) \ + ["audit", "student-tests/security/" + script] + list(args) res = run(cmd, capture=True, check=False, timeout=timeout) sys.stdout.write(res.stdout or "") if res.returncode != 0: sys.stderr.write((res.stderr or "")[-2000:]) return res def cmd_tls_audit(_args): res = audit_run("tls_audit.py") save("tls_audit.json", res.stdout or "") if res.returncode != 0: raise SystemExit("tls-audit: FAILED checks above") def cmd_leak_check(_args): res = audit_run("leak_check.py", timeout=300) save("leak_check.json", res.stdout or "") if res.returncode != 0: raise SystemExit("leak-check: FAILED") def cmd_security_http(_args): res = run([py(), "student-tests/security/security_http.py", "--base-url", LB_URL, "--ca-file", CA_FILE], capture=True, check=False, timeout=600) sys.stdout.write(res.stdout or "") sys.stderr.write((res.stderr or "")[-2000:]) save("security_http.txt", (res.stdout or "") + (res.stderr or "")) if res.returncode != 0: raise SystemExit("security-http: FAILED") def cmd_hardening(_args): res = run([py(), "student-tests/security/hardening_audit.py", "--project", PROJECT], capture=True, check=False) sys.stdout.write(res.stdout or "") sys.stderr.write(res.stderr or "") save("hardening.txt", res.stdout or "") if res.returncode != 0: raise SystemExit("hardening: FAILED") def cmd_bao_policy(_args): res = run([py(), "scripts/bao.py", "policy-test"], capture=True, check=False) sys.stdout.write(res.stdout or "") save("bao_policy.json", res.stdout or "") if res.returncode != 0: raise SystemExit("bao-policy: FAILED") def psql_admin(sql, timeout=60): """SQL от имени администратора внутри контейнера postgres (unix socket, пароль из secret).""" return run(compose_cmd("exec", "-T", "postgres", "sh", "-c", 'PGPASSWORD="$(cat /run/secrets/postgres_password)" psql -v ON_ERROR_STOP=1 -qAt ' '-U messenger -d messenger -h /var/run/postgresql -f -'), capture=True, timeout=timeout, stdin=sql) def pg_login_ok(role, password): """Новое TLS-соединение роли с паролем (из контейнера postgres, hostssl 127.0.0.1).""" env = dict(os.environ, PGCHECK_PW=password) res = run(["docker", "exec", "-i", "-e", "PGCHECK_PW", service_ids("postgres")[0], "sh", "-c", 'PGPASSWORD="$PGCHECK_PW" psql "host=127.0.0.1 dbname=messenger user=%s sslmode=require" -qAtc "select 1"' % role], capture=True, check=False, env=env, timeout=30) return res.returncode == 0, (res.stderr or "").strip().splitlines()[-1:] if res.returncode else "" def rendered_hash(role, name="database_url"): ids = service_ids("bao-agent-" + role) res = run(["docker", "exec", ids[0], "sha256sum", "/run/bao/" + name], capture=True, check=False, timeout=15) return res.stdout.split()[0] if res.returncode == 0 and res.stdout else "" def cmd_rotate_db_password(args): """Ротация рабочего секрета (пароль роли БД) без пересборки образа и рестарта API. 1) ALTER ROLE ... PASSWORD (админ, внутри контейнера postgres); 2) OpenBao KV patch (root token оператора); 3) файл оператора secrets/db__password; 4) агент перерисовывает /run/bao/database_url (<= 5 с + backoff); 5) старый пароль отклоняется на НОВОМ соединении, новый принимается; 6) существующие соединения роли разрываются (pg_terminate_backend) — пул открывает новые уже с новым паролем; затем публичный smoke проходит. Окно: между шагами 1 и 4 НОВЫЕ соединения со старым паролем получают отказ (уже открытые соединения пула работают); измеренное окно пишется в лог. """ sys.path.insert(0, str(ROOT / "scripts")) import bao as baomod role = (args[0] if args else "api").strip() if role not in ("api", "worker"): raise SystemExit("usage: rotate-db-password [api|worker]") secret_file = SECRETS / ("db_%s_password" % role) old = secret_file.read_text(encoding="utf-8").strip() new = pysecrets.token_hex(24) before = rendered_hash(role) t0 = time.monotonic() psql_admin("ALTER ROLE messenger_%s PASSWORD '%s';\n" % (role, new)) baomod.bao("kv", "patch", "-mount=kv", "messenger/" + role, "-", token=baomod.root_token(), stdin=json.dumps({"db_password": new})) with open(secret_file, "w", encoding="utf-8", newline="\n") as fh: fh.write(new + "\n") wait_until(lambda: rendered_hash(role) not in ("", before), 120, "agent re-rendered database_url", interval=1) window = time.monotonic() - t0 old_ok, old_err = pg_login_ok("messenger_" + role, old) new_ok, _ = pg_login_ok("messenger_" + role, new) killed = psql_admin("SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE usename = 'messenger_%s';\n" % role).stdout.strip() log("rotate-db-password: role=messenger_%s; agent re-rendered after %.1fs; old password accepted=%s (%s); " "new password accepted=%s; existing sessions terminated=%s" % (role, window, old_ok, old_err, new_ok, killed)) if old_ok or not new_ok: raise SystemExit("rotation check failed") time.sleep(2) if role == "api": res = run([py(), "tests/smoke.py", "--base-url", LB_URL, "--ca-file", CA_FILE], capture=True, check=False, timeout=300) log("rotate-db-password: smoke after rotation (new pool connections): %s" % (res.stdout.strip().splitlines()[-1:] or res.stderr.strip()[-200:])) if res.returncode != 0: raise SystemExit(1) else: wait_service_healthy("worker", 60) log("rotate-db-password: worker healthy after its sessions were terminated") def probe_api_cert(pem_host_path, label): """mTLS-проба реплики API заданным сертификатом из контейнера audit (сеть ingress).""" res = audit_run("mtls_probe.py", "--target", "api:8443", "--server-name", "api", "--cert", "/opt/probe/cert.pem", "--label", label, extra=["-v", "%s:/opt/probe/cert.pem:ro" % pem_host_path], timeout=120) return res.returncode == 0 def cmd_rotate_lb_cert(_args): """Ротация leaf LB->API без пересборки образов и отзыв старой identity через CRL.""" t0 = time.monotonic() pki("rotate", "lb-client") compose("restart", "lb", timeout=60) wait_service_healthy("lb", 60) wait_until(lambda: lb_up_count() >= REPLICAS, 60, "replicas UP with the new LB certificate") wait_until(lb_ready, 30, "LB ready") log("rotate-lb-cert: new lb-client in use (lb restarted, no image rebuild) after %.1fs" % (time.monotonic() - t0)) old_before = probe_api_cert(SECRETS / "pki" / "lb-client.prev.pem", "old-before-revoke") pki("revoke", "lb-client") time.sleep(7) # API перечитывает CRL не реже раза в 5 с old_after = probe_api_cert(SECRETS / "pki" / "lb-client.prev.pem", "old-after-revoke") new_after = probe_api_cert(SECRETS / "pki" / "lb-client.pem", "new-after-revoke") ok = lb_ready() log("rotate-lb-cert: old cert accepted before revoke=%s, after revoke=%s; new cert accepted=%s; LB ready=%s" % (old_before, old_after, new_after, ok)) if old_after or not new_after or not ok: raise SystemExit("rotation/revocation check failed") def cmd_rotate_secret_id(args): roles = args or ["api"] bao("issue-secret-ids", *roles) for role in roles: compose("restart", "bao-agent-" + role, timeout=60) wait_service_healthy("bao-agent-" + role, 120) log("rotate-secret-id: %s agent(s) re-authenticated with the new SecretID; old SecretID destroyed" % roles) def cmd_revoke_identity(args): role = (args[0] if args else "worker") bao("revoke-role", role) compose("restart", "bao-agent-" + role, timeout=60) time.sleep(15) logs = run(compose_cmd("logs", "--since", "30s", "bao-agent-" + role), capture=True, check=False).stdout or "" denied = "invalid" in logs.lower() or "permission denied" in logs.lower() or "error" in logs.lower() log("rotate/revoke: identity %s revoked; agent re-auth denied=%s; already rendered files remain -> %s keeps" " working until restart of the volume" % (role, denied, role)) ok = lb_ready() log("revoke-identity: LB ready while %s identity is revoked = %s" % (role, ok)) bao("issue-secret-ids", role) compose("restart", "bao-agent-" + role, timeout=60) wait_service_healthy("bao-agent-" + role, 120) log("revoke-identity: recovered with a new SecretID; agent healthy") def cmd_bao_sealed(_args): bao("seal") t0 = time.monotonic() time.sleep(10) res = run([py(), "tests/smoke.py", "--base-url", LB_URL, "--ca-file", CA_FILE], capture=True, check=False) log("bao-sealed: smoke while OpenBao sealed -> %s" % ("PASS" if res.returncode == 0 else "FAIL")) compose("restart", "api", timeout=120) wait_stack(REPLICAS) log("bao-sealed: api replicas restarted while sealed -> healthy (secrets from agent tmpfs volume)") bao("unseal") log("bao-sealed: unsealed after %.0fs; agents resume renew/render with backoff (<= 1 min)" % (time.monotonic() - t0)) if res.returncode != 0: raise SystemExit(1) def cmd_rehash_demo(_args): """Argon2id rehash: hash пользователя заменяется на старый (m=8 MiB, t=1), вход его пересчитывает.""" import base64 username = "rehash_" + pysecrets.token_hex(4) password = "rehash-demo-" + pysecrets.token_urlsafe(12) ctx = tls_context() def req(method, path, body=None, headers=None): r = urllib.request.Request(LB_URL + path, data=json.dumps(body).encode() if body else None, headers=dict({"Content-Type": "application/json"}, **(headers or {})), method=method) with urllib.request.urlopen(r, context=ctx, timeout=10) as resp: return resp.status req("POST", "/api/v1/users", {"username": username, "display_name": "Rehash", "password": password}) old_hash = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-c", "import sys; from argon2 import PasswordHasher as P; " "print(P(time_cost=1, memory_cost=8192, parallelism=1).hash(sys.stdin.read()))"), capture=True, stdin=password).stdout.strip().splitlines()[-1] psql_admin("UPDATE users SET password_hash = '%s' WHERE username = '%s';\n" % (old_hash, username)) q = "SELECT split_part(password_hash, '$', 4) FROM users WHERE username = '%s';\n" % username before = psql_admin(q).stdout.strip() token = base64.b64encode(("%s:%s" % (username, password)).encode()).decode() status = req("POST", "/api/v1/auth/sessions", None, {"Authorization": "Basic " + token}) after = psql_admin(q).stdout.strip() text = "rehash-demo: user=%s params before=%s -> login HTTP %d -> params after=%s" % (username, before, status, after) log(text) save("rehash.txt", text + "\n") if before == after: raise SystemExit("hash was not upgraded") def cmd_login_budget(_args): res = run(compose_cmd("run", "--rm", "-T", "--no-deps", "-v", "%s:/app/student-tests:ro" % (ROOT / "student-tests"), "api", "python", "student-tests/security/argon2_bench.py"), capture=True, check=False, timeout=600) bench = res.stdout or "" print(bench) res2 = run([py(), "student-tests/security/login_flood.py", "--base-url", LB_URL, "--ca-file", CA_FILE], capture=True, check=False, timeout=600) print(res2.stdout or "", res2.stderr[-1000:] if res2.stderr else "") save("login_budget.txt", bench + "\n" + (res2.stdout or "")) def cmd_scan(_args): """Сканирование зависимостей (pip-audit) и образов (trivy). Разбор — docs/security/SCAN_TRIAGE.md.""" out_dir = EVIDENCE / "scan" out_dir.mkdir(parents=True, exist_ok=True) image = "messenger-api:" + setting("APP_VERSION", "lab7") res = run(["docker", "run", "--rm", "-v", "%s:/req.txt:ro" % (ROOT / "requirements.txt"), "python:3.12.7-slim-bookworm", "sh", "-c", "pip install -q pip-audit==2.7.3 >/dev/null 2>&1 && pip-audit -r /req.txt --format columns"], capture=True, check=False, timeout=900) (out_dir / "pip-audit.txt").write_text((res.stdout or "") + (res.stderr or ""), encoding="utf-8") images = [image, "haproxy:2.8.30-alpine", "postgres:16.4-alpine", "redis:7.4.1-alpine", "openbao/openbao:" + setting("OPENBAO_VERSION", "2.1.0"), "chrislusf/seaweedfs:" + setting("SEAWEEDFS_VERSION", "4.50")] for img in images: res = run(["docker", "run", "--rm", "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", "aquasec/trivy:0.57.1", "image", "--quiet", "--scanners", "vuln", "--severity", "HIGH,CRITICAL", "--ignore-unfixed", img], capture=True, check=False, timeout=1800) name = img.replace("/", "_").replace(":", "_") (out_dir / ("trivy_%s.txt" % name)).write_text((res.stdout or "") + (res.stderr or ""), encoding="utf-8") log("scan: %s -> evidence/scan/trivy_%s.txt (exit %d)" % (img, name, res.returncode)) log("scan: done; triage in docs/security/SCAN_TRIAGE.md") COMMANDS = { "up": cmd_up, "down": cmd_down, "destroy": cmd_destroy, "wait": cmd_wait, "ps": cmd_ps, "stop-one": cmd_stop_one, "restore": cmd_restore, "restart": cmd_restart, "rolling-update": cmd_rolling_update, "in-network": cmd_in_network, "wait-s3": cmd_wait_s3, "s3-check": cmd_s3_check, "reconcile": cmd_reconcile, "api-fs": cmd_api_fs, "worker-stop": cmd_worker_stop, "worker-kill": cmd_worker_kill, "worker-start": cmd_worker_start, "api-restart": cmd_api_restart, "broker-restart": cmd_broker_restart, "jobs": cmd_jobs, "pipeline-faults": cmd_pipeline_faults, "tls-audit": cmd_tls_audit, "security-http": cmd_security_http, "leak-check": cmd_leak_check, "hardening": cmd_hardening, "bao-policy": cmd_bao_policy, "rehash-demo": cmd_rehash_demo, "rotate-db-password": cmd_rotate_db_password, "rotate-lb-cert": cmd_rotate_lb_cert, "rotate-secret-id": cmd_rotate_secret_id, "revoke-identity": cmd_revoke_identity, "bao-sealed": cmd_bao_sealed, "login-budget": cmd_login_budget, "scan": cmd_scan, } def main(): if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: print(__doc__) sys.exit(2) COMMANDS[sys.argv[1]](sys.argv[2:]) if __name__ == "__main__": main()