forked from mtusi2026/mtusi2026lab7
лр7: Docker-прогон, evidence, исправления
This commit is contained in:
+309
@@ -0,0 +1,309 @@
|
||||
"""Операторские действия с OpenBao (только stdlib). Запускает ОПЕРАТОР на хосте, OpenBao — https://127.0.0.1:8200.
|
||||
|
||||
python scripts/bao.py status # sealed? initialized? (без токена)
|
||||
python scripts/bao.py init # один раз: 3 доли ключа (порог 2) + root token
|
||||
python scripts/bao.py unseal [--prompt] # после КАЖДОГО старта OpenBao: 2 доли (из файла или ввод вручную)
|
||||
python scripts/bao.py configure # KV v2, cert auth, политики, роли, CRL, секреты приложений
|
||||
python scripts/bao.py sync-crl # загрузить свежий CRL (после pki.py revoke)
|
||||
python scripts/bao.py kv-rotate api redis_password --value-file F # новая версия одного ключа (cas)
|
||||
python scripts/bao.py role-ttl api 15m 1h # TTL токенов роли (для теста истечения — короче)
|
||||
python scripts/bao.py revoke-service-tokens # отозвать все токены cert auth: сервисы перелогинятся сертификатом,
|
||||
# отозванный сертификат войти уже не сможет
|
||||
python scripts/bao.py seal # запечатать (демонстрация sealed)
|
||||
|
||||
Файл оператора secrets/operator/openbao-init.json (0600, вне Git, ни в один контейнер не монтируется)
|
||||
хранит доли unseal и root token учебного стенда. В реальной эксплуатации доли раздаются разным людям
|
||||
(`init --print-shares-once` выводит их один раз и НЕ сохраняет, затем `unseal --prompt`), root token
|
||||
отзывается после настройки. Ни одна команда не печатает значения секретов, долей или токенов
|
||||
(кроме явного --print-shares-once).
|
||||
"""
|
||||
import argparse
|
||||
import getpass
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = Path(os.environ.get("SECRETS_DIR", ROOT / "secrets"))
|
||||
OPERATOR = SECRETS / "operator"
|
||||
INIT_FILE = OPERATOR / "openbao-init.json"
|
||||
APP_FILE = OPERATOR / "app-secrets.json"
|
||||
LOADED_MARKER = OPERATOR / "app-secrets.loaded"
|
||||
CA_FILE = Path(os.environ.get("BAO_CACERT", SECRETS / "pki" / "public" / "ca.crt"))
|
||||
CRL_FILE = SECRETS / "pki" / "public" / "crl.pem"
|
||||
POLICIES = ROOT / "infra" / "openbao" / "policies"
|
||||
ADDR = os.environ.get("BAO_ADDR", "https://127.0.0.1:8200").rstrip("/")
|
||||
KV_MOUNT = "messenger"
|
||||
|
||||
# Роли cert auth = identity (CN) сервиса. Токен: TTL 15 мин, renewable, не дольше 1 ч; дальше — новый вход.
|
||||
SERVICES = ["api", "worker", "dispatcher", "janitor", "migrate", "s3-init"]
|
||||
TOKEN_TTL, TOKEN_MAX_TTL = "15m", "1h"
|
||||
|
||||
|
||||
class BaoError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def request(method: str, path: str, body=None, token: str | None = None, ok=(200, 204)) -> dict:
|
||||
context = ssl.create_default_context(cafile=str(CA_FILE))
|
||||
data = json.dumps(body).encode("utf-8") if body is not None else None
|
||||
req = urllib.request.Request(ADDR + path, data=data, method=method, headers={"Content-Type": "application/json"})
|
||||
if token:
|
||||
req.add_header("X-Vault-Token", token)
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context))
|
||||
try:
|
||||
with opener.open(req, timeout=10) as reply:
|
||||
status, raw = reply.status, reply.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
status, raw = exc.code, exc.read()
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
raise BaoError(f"OpenBao at {ADDR} is unreachable: {getattr(exc, 'reason', exc)}") from None
|
||||
if status not in ok:
|
||||
try:
|
||||
errors = json.loads(raw).get("errors")
|
||||
except ValueError:
|
||||
errors = None
|
||||
raise BaoError(f"{method} {path} -> HTTP {status} {errors or ''}")
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def init_data() -> dict:
|
||||
if not INIT_FILE.exists():
|
||||
raise BaoError(f"{INIT_FILE} not found: run `python scripts/bao.py init` first (or use --prompt)")
|
||||
return json.loads(INIT_FILE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def root_token() -> str:
|
||||
token = os.environ.get("BAO_TOKEN") or init_data().get("root_token")
|
||||
if not token:
|
||||
raise BaoError("no operator token: set BAO_TOKEN or keep it in the init file")
|
||||
return token
|
||||
|
||||
|
||||
def seal_status() -> dict:
|
||||
return request("GET", "/v1/sys/seal-status")
|
||||
|
||||
|
||||
def cmd_status(_args) -> int:
|
||||
status = seal_status()
|
||||
print(json.dumps({k: status.get(k) for k in ("initialized", "sealed", "t", "n", "progress", "version",
|
||||
"storage_type")}))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_init(args) -> int:
|
||||
if request("GET", "/v1/sys/init").get("initialized"):
|
||||
print("openbao: already initialized, nothing to do")
|
||||
return 0
|
||||
reply = request("PUT", "/v1/sys/init", {"secret_shares": 3, "secret_threshold": 2})
|
||||
if args.print_shares_once:
|
||||
print("Unseal shares (hand each to a different operator, they are NOT saved):")
|
||||
for number, share in enumerate(reply["keys_base64"], 1):
|
||||
print(f" share {number}: {share}")
|
||||
print("Root token (revoke after configure in real life): " + reply["root_token"])
|
||||
return 0
|
||||
OPERATOR.mkdir(parents=True, exist_ok=True)
|
||||
OPERATOR.chmod(0o700)
|
||||
with open(INIT_FILE, "w", encoding="utf-8") as file:
|
||||
json.dump({"unseal_keys_b64": reply["keys_base64"], "root_token": reply["root_token"],
|
||||
"threshold": 2, "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, file)
|
||||
INIT_FILE.chmod(0o600)
|
||||
print(f"openbao: initialized (3 shares, threshold 2); shares and root token saved to {INIT_FILE} (not printed)")
|
||||
return 0
|
||||
|
||||
|
||||
def wait_reachable(seconds: float = 30) -> dict:
|
||||
"""После (ре)старта контейнера listener поднимается не сразу: ждём ответа /sys/seal-status."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
return seal_status()
|
||||
except BaoError:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def cmd_unseal(args) -> int:
|
||||
status = wait_reachable()
|
||||
if not status.get("sealed"):
|
||||
print("openbao: already unsealed")
|
||||
return 0
|
||||
if args.prompt:
|
||||
shares = [getpass.getpass(f"unseal share {i + 1}: ") for i in range(status.get("t", 2))]
|
||||
else:
|
||||
shares = init_data()["unseal_keys_b64"]
|
||||
for share in shares:
|
||||
status = request("PUT", "/v1/sys/unseal", {"key": share})
|
||||
if not status.get("sealed"):
|
||||
break
|
||||
if status.get("sealed"):
|
||||
print("openbao: still sealed, progress " + str(status.get("progress")))
|
||||
return 1
|
||||
print("openbao: unsealed")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_seal(_args) -> int:
|
||||
request("PUT", "/v1/sys/seal", {}, token=root_token())
|
||||
print("openbao: sealed (services keep cached secrets; new logins fail until unseal)")
|
||||
return 0
|
||||
|
||||
|
||||
def ensure_mounts(token: str) -> None:
|
||||
mounts = request("GET", "/v1/sys/mounts", token=token)
|
||||
mounts = mounts.get("data", mounts)
|
||||
if f"{KV_MOUNT}/" not in mounts:
|
||||
request("POST", f"/v1/sys/mounts/{KV_MOUNT}", {"type": "kv", "options": {"version": "2"},
|
||||
"description": "messenger application secrets"}, token=token)
|
||||
print(f"openbao: KV v2 mounted at {KV_MOUNT}/")
|
||||
auths = request("GET", "/v1/sys/auth", token=token)
|
||||
auths = auths.get("data", auths)
|
||||
if "cert/" not in auths:
|
||||
request("POST", "/v1/sys/auth/cert", {"type": "cert", "description": "service identities (mTLS leaf)"},
|
||||
token=token)
|
||||
print("openbao: cert auth enabled at cert/")
|
||||
|
||||
|
||||
def write_policies(token: str) -> None:
|
||||
for path in sorted(POLICIES.glob("*.hcl")):
|
||||
request("PUT", f"/v1/sys/policies/acl/{path.stem}", {"policy": path.read_text(encoding="utf-8")}, token=token)
|
||||
print("openbao: policies " + ", ".join(p.stem for p in sorted(POLICIES.glob("*.hcl"))))
|
||||
|
||||
|
||||
def write_roles(token: str, ttl: str = TOKEN_TTL, max_ttl: str = TOKEN_MAX_TTL, only: list | None = None) -> None:
|
||||
ca = CA_FILE.read_text(encoding="utf-8")
|
||||
for name in only or SERVICES:
|
||||
request("POST", f"/v1/auth/cert/certs/{name}", {
|
||||
"display_name": name,
|
||||
"certificate": ca, # доверяем только учебному CA ...
|
||||
"allowed_common_names": [name], # ... и только сертификату с CN = имя роли
|
||||
"token_policies": [name],
|
||||
"token_ttl": ttl,
|
||||
"token_max_ttl": max_ttl,
|
||||
"token_type": "service",
|
||||
}, token=token)
|
||||
print(f"openbao: cert roles {', '.join(only or SERVICES)} (ttl {ttl}, max {max_ttl}, CN must equal role)")
|
||||
|
||||
|
||||
def sync_crl(token: str) -> None:
|
||||
request("POST", "/v1/auth/cert/crls/messenger-ca", {"crl": CRL_FILE.read_text(encoding="utf-8")}, token=token)
|
||||
print("openbao: CRL uploaded to cert auth (revoked certificates cannot log in)")
|
||||
|
||||
|
||||
def kv_read(token: str, path: str) -> tuple[dict | None, int]:
|
||||
try:
|
||||
reply = request("GET", f"/v1/{KV_MOUNT}/data/{path}", token=token)
|
||||
except BaoError as exc:
|
||||
if "HTTP 404" in str(exc):
|
||||
return None, 0
|
||||
raise
|
||||
return reply["data"]["data"], reply["data"]["metadata"]["version"]
|
||||
|
||||
|
||||
def load_app_secrets(token: str, force: bool) -> None:
|
||||
if not APP_FILE.exists():
|
||||
if LOADED_MARKER.exists():
|
||||
print("openbao: application secrets already loaded earlier (source of truth is OpenBao)")
|
||||
return
|
||||
raise BaoError(f"{APP_FILE} not found: run python scripts/init_secrets.py")
|
||||
app = json.loads(APP_FILE.read_text(encoding="utf-8"))
|
||||
for path, values in app.items():
|
||||
existing, _version = kv_read(token, path)
|
||||
if existing is not None and not force:
|
||||
continue
|
||||
request("POST", f"/v1/{KV_MOUNT}/data/{path}", {"data": values}, token=token)
|
||||
LOADED_MARKER.write_text(time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + "\n", encoding="utf-8")
|
||||
APP_FILE.unlink()
|
||||
print(f"openbao: application secrets written to {KV_MOUNT}/{{{','.join(app)}}}; {APP_FILE.name} deleted")
|
||||
|
||||
|
||||
def cmd_configure(args) -> int:
|
||||
token = root_token()
|
||||
ensure_mounts(token)
|
||||
write_policies(token)
|
||||
write_roles(token)
|
||||
sync_crl(token)
|
||||
load_app_secrets(token, args.force)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync_crl(_args) -> int:
|
||||
sync_crl(root_token())
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_kv_rotate(args) -> int:
|
||||
"""Новая версия ОДНОГО ключа в пути сервиса (check-and-set: не затираем чужую параллельную запись)."""
|
||||
token = root_token()
|
||||
values, version = kv_read(token, args.path)
|
||||
if values is None:
|
||||
raise BaoError(f"{KV_MOUNT}/{args.path} does not exist")
|
||||
value = Path(args.value_file).read_text(encoding="utf-8").strip()
|
||||
values[args.key] = value
|
||||
request("POST", f"/v1/{KV_MOUNT}/data/{args.path}", {"options": {"cas": version}, "data": values}, token=token)
|
||||
print(f"openbao: {KV_MOUNT}/{args.path} key {args.key} -> version {version + 1} (value not printed)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_kv_version(args) -> int:
|
||||
_values, version = kv_read(root_token(), args.path)
|
||||
print(json.dumps({"path": f"{KV_MOUNT}/{args.path}", "version": version}))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_role_ttl(args) -> int:
|
||||
write_roles(root_token(), args.ttl, args.max_ttl, only=[args.role])
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_revoke_tokens(_args) -> int:
|
||||
request("PUT", "/v1/sys/leases/revoke-prefix/auth/cert/login", {}, token=root_token())
|
||||
print("openbao: all tokens issued by cert auth revoked; services re-login with their certificates")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("status").set_defaults(func=cmd_status)
|
||||
p = sub.add_parser("init")
|
||||
p.add_argument("--print-shares-once", action="store_true", help="print shares/root token instead of saving")
|
||||
p.set_defaults(func=cmd_init)
|
||||
p = sub.add_parser("unseal")
|
||||
p.add_argument("--prompt", action="store_true", help="type the shares instead of reading the operator file")
|
||||
p.set_defaults(func=cmd_unseal)
|
||||
sub.add_parser("seal").set_defaults(func=cmd_seal)
|
||||
p = sub.add_parser("configure")
|
||||
p.add_argument("--force", action="store_true", help="overwrite existing KV values from app-secrets.json")
|
||||
p.set_defaults(func=cmd_configure)
|
||||
sub.add_parser("sync-crl").set_defaults(func=cmd_sync_crl)
|
||||
p = sub.add_parser("kv-rotate")
|
||||
p.add_argument("path", choices=SERVICES)
|
||||
p.add_argument("key")
|
||||
p.add_argument("--value-file", required=True, help="file with the new value (not echoed)")
|
||||
p.set_defaults(func=cmd_kv_rotate)
|
||||
p = sub.add_parser("kv-version")
|
||||
p.add_argument("path", choices=SERVICES)
|
||||
p.set_defaults(func=cmd_kv_version)
|
||||
p = sub.add_parser("role-ttl")
|
||||
p.add_argument("role", choices=SERVICES)
|
||||
p.add_argument("ttl")
|
||||
p.add_argument("max_ttl")
|
||||
p.set_defaults(func=cmd_role_ttl)
|
||||
sub.add_parser("revoke-service-tokens").set_defaults(func=cmd_revoke_tokens)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return args.func(args)
|
||||
except BaoError as exc:
|
||||
print(f"bao.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git).
|
||||
|
||||
python scripts/build_dashboard.py
|
||||
|
||||
Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос
|
||||
против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой.
|
||||
Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "infra/grafana/dashboards/messenger.json"
|
||||
|
||||
PROM = {"type": "prometheus", "uid": "prometheus"}
|
||||
LOKI = {"type": "loki", "uid": "loki"}
|
||||
TEMPO = {"type": "tempo", "uid": "tempo"}
|
||||
|
||||
API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO)
|
||||
QUEUE = 'messaging_destination_name="attachments.process.v1"'
|
||||
|
||||
_panels = []
|
||||
_next_id = [1]
|
||||
_y = [0]
|
||||
|
||||
|
||||
def _id():
|
||||
_next_id[0] += 1
|
||||
return _next_id[0]
|
||||
|
||||
|
||||
def row(title):
|
||||
_panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []})
|
||||
_y[0] += 1
|
||||
|
||||
|
||||
class Line:
|
||||
"""Раскладка панелей в строку сетки Grafana (24 колонки)."""
|
||||
|
||||
def __init__(self, height):
|
||||
self.x, self.h = 0, height
|
||||
|
||||
def place(self, width):
|
||||
pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]}
|
||||
self.x += width
|
||||
return pos
|
||||
|
||||
def end(self):
|
||||
_y[0] += self.h
|
||||
|
||||
|
||||
def prom(expr, legend="", instant=False, fmt=None):
|
||||
target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "",
|
||||
"range": not instant, "instant": instant}
|
||||
if fmt:
|
||||
target["format"] = fmt
|
||||
return target
|
||||
|
||||
|
||||
def _with_refs(targets):
|
||||
for n, target in enumerate(targets):
|
||||
target["refId"] = chr(ord("A") + n)
|
||||
return targets
|
||||
|
||||
|
||||
def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None):
|
||||
defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never",
|
||||
"stacking": {"mode": "normal" if stack else "none"}}}
|
||||
if minimum is not None:
|
||||
defaults["min"] = minimum
|
||||
if thresholds:
|
||||
defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [
|
||||
{"color": color, "value": value} for value, color in thresholds]}
|
||||
defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"}
|
||||
_panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description,
|
||||
"datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets),
|
||||
"fieldConfig": {"defaults": defaults, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}}})
|
||||
|
||||
|
||||
def stat(line, width, title, description, expr, unit, steps, decimals=None):
|
||||
defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [
|
||||
{"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}}
|
||||
if decimals is not None:
|
||||
defaults["decimals"] = decimals
|
||||
_panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM,
|
||||
"gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]),
|
||||
"fieldConfig": {"defaults": defaults, "overrides": []},
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"colorMode": "background", "graphMode": "none", "textMode": "value"}})
|
||||
|
||||
|
||||
def quantiles(metric, selector, by="le"):
|
||||
return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))",
|
||||
f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)]
|
||||
|
||||
|
||||
def build():
|
||||
row("Обзор и SLO (окно SLO — 1 час)")
|
||||
line = Line(4)
|
||||
stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.",
|
||||
"messenger:http_requests:rate1m", "reqps", [(None, "blue")])
|
||||
stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.",
|
||||
"messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2)
|
||||
stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).",
|
||||
"messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")])
|
||||
stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.",
|
||||
f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")])
|
||||
stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.",
|
||||
"messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2)
|
||||
stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.",
|
||||
"messenger:slo_availability:error_budget_remaining_1h", "percentunit",
|
||||
[(None, "red"), (0, "orange"), (0.5, "green")], 0)
|
||||
stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.",
|
||||
"messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
|
||||
stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.",
|
||||
"messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
|
||||
line.end()
|
||||
line = Line(5)
|
||||
_panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)",
|
||||
"description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.",
|
||||
"datasource": PROM, "gridPos": line.place(24),
|
||||
"targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]),
|
||||
"transformations": [{"id": "organize", "options": {"excludeByName": {
|
||||
"Time": True, "Value": True, "__name__": True}}}],
|
||||
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
|
||||
line.end()
|
||||
|
||||
row("API: запросы, ошибки, задержка (RED)")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.",
|
||||
[prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
|
||||
"{{http_request_method}} {{http_route}}")], "reqps")
|
||||
timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.",
|
||||
[prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
|
||||
"{{http_response_status_class}}")], "reqps", stack=True)
|
||||
timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.",
|
||||
[prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) '
|
||||
f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")],
|
||||
"percentunit", thresholds=[(0.05, "red")], minimum=0)
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.",
|
||||
quantiles("http_server_request_duration_seconds", API), "s")
|
||||
timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.",
|
||||
[prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))",
|
||||
"{{http_route}}")], "s")
|
||||
timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.",
|
||||
[prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short")
|
||||
line.end()
|
||||
|
||||
row("Фоновая обработка: очередь, worker, время до ready")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).",
|
||||
[prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")],
|
||||
"ops", stack=True)
|
||||
timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.",
|
||||
quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s")
|
||||
timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).",
|
||||
quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s",
|
||||
thresholds=[(30, "red")])
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.",
|
||||
[prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"),
|
||||
prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short")
|
||||
timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.",
|
||||
[prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"),
|
||||
prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s",
|
||||
thresholds=[(30, "red")])
|
||||
timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.",
|
||||
[prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"),
|
||||
prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")],
|
||||
"ops")
|
||||
line.end()
|
||||
|
||||
row("Насыщение: пул БД, worker, CPU/RAM")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).",
|
||||
[prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"),
|
||||
prom("max(db_client_connection_max)", "max per replica"),
|
||||
prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short")
|
||||
timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).",
|
||||
[prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))",
|
||||
"p95 {{instance}}")], "s")
|
||||
timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.",
|
||||
[prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit",
|
||||
thresholds=[(0.8, "orange"), (1, "red")], minimum=0)
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.",
|
||||
[prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")],
|
||||
"short")
|
||||
timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.",
|
||||
[prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)",
|
||||
"{{job}} {{instance}}")], "percentunit")
|
||||
timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).",
|
||||
[prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"),
|
||||
prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes")
|
||||
line.end()
|
||||
|
||||
row("Конвейер телеметрии: Collector и SDK")
|
||||
line = Line(8)
|
||||
timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.",
|
||||
[prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"),
|
||||
prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"),
|
||||
prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")],
|
||||
"ops")
|
||||
timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.",
|
||||
[prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"),
|
||||
prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"),
|
||||
prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")],
|
||||
"ops")
|
||||
timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.",
|
||||
[prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")],
|
||||
"percentunit", minimum=0)
|
||||
line.end()
|
||||
line = Line(8)
|
||||
timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.",
|
||||
[prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes")
|
||||
timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).",
|
||||
[prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"),
|
||||
prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"),
|
||||
prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"),
|
||||
prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")],
|
||||
"ops")
|
||||
line.end()
|
||||
|
||||
row("Логи и трассы")
|
||||
line = Line(9)
|
||||
_panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)",
|
||||
"description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.",
|
||||
"datasource": LOKI, "gridPos": line.place(12),
|
||||
"targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range",
|
||||
"expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]),
|
||||
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending",
|
||||
"enableLogDetails": True}})
|
||||
_panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)",
|
||||
"description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.",
|
||||
"datasource": TEMPO, "gridPos": line.place(12),
|
||||
"targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql",
|
||||
"query": "{ status = error }", "limit": 20, "tableType": "traces"}]),
|
||||
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
|
||||
line.end()
|
||||
|
||||
return {
|
||||
"uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"],
|
||||
"timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1,
|
||||
"refresh": "10s", "time": {"from": "now-30m", "to": "now"},
|
||||
"templating": {"list": []}, "annotations": {"list": [{
|
||||
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
|
||||
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
|
||||
"panels": _panels,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels")
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Секреты учебного стенда lab7 (только stdlib). Запускает оператор ПОСЛЕ scripts/pki.py init.
|
||||
|
||||
Создаёт недостающее, существующее не трогает (иначе разошлось бы с данными в volumes):
|
||||
|
||||
Для серверов (монтируются в СВОЙ контейнер, приложение их не видит):
|
||||
secrets/postgres_password пароль владельца БД messenger (контейнер postgres при initdb)
|
||||
secrets/redis/users.acl начальный ACL Redis: default off, admin (оператор), api (приложение);
|
||||
в файле SHA-256 паролей, не сами пароли. Redis копирует его в свой volume
|
||||
при первом старте, дальше ACL меняется командами (ротация).
|
||||
secrets/s3_config.json ключи SeaweedFS: s3-init (admin), api, worker, janitor — права по prefix
|
||||
secrets/rabbitmq_definitions.json vhost, очереди, пользователи worker/dispatcher БЕЗ пароля (вход только
|
||||
по клиентскому сертификату, EXTERNAL) и их права
|
||||
secrets/grafana_admin_password вход в Grafana (admin)
|
||||
|
||||
Только оператору (никуда не монтируются):
|
||||
secrets/operator/app-secrets.json учётные данные приложений -> загружаются в OpenBao (scripts/bao.py configure),
|
||||
после загрузки файл удаляется; дальше источник истины — OpenBao
|
||||
secrets/operator/redis_admin_password, secrets/operator/postgres_owner_password — для ротации и диагностики
|
||||
|
||||
Значения не печатаются. Права файлов: каталоги 0700; файлы, которые читает контейнер под своим uid, — 0644
|
||||
(на Windows/Docker Desktop права bind mount не применяются, защита — сам каталог пользователя).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = Path(os.environ.get("SECRETS_DIR", ROOT / "secrets"))
|
||||
OPERATOR = SECRETS / "operator"
|
||||
|
||||
BROKER_VHOST = "messenger"
|
||||
# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей).
|
||||
EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1"
|
||||
DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq"
|
||||
|
||||
|
||||
def write(path: Path, value: str, private: bool = False) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8", newline="") as file:
|
||||
file.write(value)
|
||||
path.chmod(0o600 if private else 0o644)
|
||||
|
||||
|
||||
def read(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def token(n: int = 24) -> str:
|
||||
return secrets.token_hex(n)
|
||||
|
||||
|
||||
def bucket_name() -> str:
|
||||
env = ROOT / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("S3_BUCKET="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
return "messenger-attachments"
|
||||
|
||||
|
||||
def redis_acl(admin_password: str, api_password: str) -> str:
|
||||
"""ACL Redis 7. api: только ключи session:* и ratelimit:*, только нужные команды, без pub/sub."""
|
||||
sha = lambda value: hashlib.sha256(value.encode("utf-8")).hexdigest() # noqa: E731
|
||||
api_commands = " ".join("+" + c for c in (
|
||||
"get", "set", "del", "incr", "incrby", "pexpire", "pttl", "ping", "multi", "exec", "discard", "hello", "auth"))
|
||||
return (
|
||||
"user default off resetkeys resetchannels -@all\n"
|
||||
f"user admin on #{sha(admin_password)} ~* &* +@all\n"
|
||||
f"user api on #{sha(api_password)} resetkeys ~session:* ~ratelimit:* resetchannels -@all {api_commands}\n"
|
||||
)
|
||||
|
||||
|
||||
def s3_config(bucket: str, keys: dict) -> str:
|
||||
"""SeaweedFS identities. Действия с суффиксом :<bucket>/<prefix>* действуют только в этом prefix
|
||||
(проверено: api не пишет в attachments/, worker не пишет в staging/ чужими ключами и т.д.).
|
||||
Write включает PUT и DELETE; List — листинг bucket; Admin — создание bucket и пр. — только у s3-init."""
|
||||
b = bucket
|
||||
actions = {
|
||||
"s3-init": ["Admin", "Read", "Write", "List", "Tagging"],
|
||||
# API: принимает исходник в staging/ (PUT, откат DELETE), отдаёт готовые файлы (GET).
|
||||
"api": [f"Write:{b}/staging/*", f"Read:{b}/attachments/*", f"Read:{b}/originals/*"],
|
||||
# Worker: читает исходник (staging/ или lab4 originals/), пишет результат в attachments/ и может
|
||||
# читать/листать СВОЙ выходной prefix (проверка результата; исходники он и так читает — новой
|
||||
# конфиденциальной информации это не даёт), удаляет обработанный исходник. Пользователей, сессий,
|
||||
# чужих prefix-листингов и администрирования bucket у него нет.
|
||||
"worker": [f"Read:{b}/staging/*", f"Read:{b}/originals/*", f"Write:{b}/attachments/*",
|
||||
f"Read:{b}/attachments/*", f"List:{b}/attachments/*",
|
||||
f"Write:{b}/staging/*", f"Write:{b}/originals/*"],
|
||||
# Janitor: сверка bucket с БД — листинг и удаление сирот; читать содержимое ему не нужно.
|
||||
"janitor": [f"List:{b}", f"Write:{b}"],
|
||||
}
|
||||
identities = [{"name": name, "credentials": [{"accessKey": keys[name][0], "secretKey": keys[name][1]}],
|
||||
"actions": actions[name]} for name in actions]
|
||||
return json.dumps({"identities": identities}, indent=2)
|
||||
|
||||
|
||||
def rabbit_definitions() -> str:
|
||||
"""Топология и пользователи RabbitMQ. Пользователи БЕЗ пароля: вход только механизмом EXTERNAL по CN
|
||||
клиентского сертификата (infra/rabbitmq.conf). Права (configure / write / read, regex по именам):
|
||||
dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет;
|
||||
worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет.
|
||||
"""
|
||||
queue_args = {
|
||||
"x-queue-type": "quorum",
|
||||
"x-delivery-limit": 20,
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-dead-letter-routing-key": DLQ_ROUTING_KEY,
|
||||
"x-dead-letter-strategy": "at-least-once",
|
||||
"x-overflow": "reject-publish",
|
||||
"x-max-length": 100000,
|
||||
}
|
||||
definitions = {
|
||||
# password_hash "" — у пользователя нет пароля, PLAIN/AMQPLAIN для него невозможны.
|
||||
"users": [{"name": name, "password_hash": "", "hashing_algorithm": "rabbit_password_hashing_sha256",
|
||||
"tags": []} for name in ("dispatcher", "worker")],
|
||||
"vhosts": [{"name": BROKER_VHOST}],
|
||||
"permissions": [
|
||||
{"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"},
|
||||
{"user": "worker", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"},
|
||||
],
|
||||
"exchanges": [
|
||||
{"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
{"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
],
|
||||
"queues": [
|
||||
{"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args},
|
||||
{"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "classic"}},
|
||||
],
|
||||
"bindings": [
|
||||
{"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue",
|
||||
"routing_key": ROUTING_KEY, "arguments": {}},
|
||||
{"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue",
|
||||
"routing_key": DLQ_ROUTING_KEY, "arguments": {}},
|
||||
],
|
||||
}
|
||||
return json.dumps(definitions, indent=2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
OPERATOR.mkdir(exist_ok=True)
|
||||
for directory in (SECRETS, OPERATOR):
|
||||
directory.chmod(0o700)
|
||||
created = []
|
||||
app_file = OPERATOR / "app-secrets.json"
|
||||
loaded_marker = OPERATOR / "app-secrets.loaded"
|
||||
|
||||
if not (SECRETS / "postgres_password").exists():
|
||||
owner = token()
|
||||
write(SECRETS / "postgres_password", owner)
|
||||
write(OPERATOR / "postgres_owner_password", owner, private=True)
|
||||
created.append("postgres_password")
|
||||
|
||||
if not (SECRETS / "s3_config.json").exists():
|
||||
if loaded_marker.exists():
|
||||
raise SystemExit("secrets/s3_config.json is missing but app secrets were already loaded into OpenBao: "
|
||||
"restore it or reset the stand (see REPORT.md, «Сброс»)")
|
||||
app = {
|
||||
"api": {"db_password": token(), "redis_password": token()},
|
||||
"worker": {"db_password": token()},
|
||||
"dispatcher": {"db_password": token()},
|
||||
"janitor": {"db_password": token()},
|
||||
"migrate": {"db_owner_password": read(SECRETS / "postgres_password")},
|
||||
"s3-init": {},
|
||||
}
|
||||
s3_keys = {}
|
||||
for name in ("s3-init", "api", "worker", "janitor"):
|
||||
s3_keys[name] = (name.replace("-", "") + "-" + secrets.token_hex(8), token())
|
||||
app["s3-init"].update(s3_admin_access_key_id=s3_keys["s3-init"][0],
|
||||
s3_admin_secret_access_key=s3_keys["s3-init"][1])
|
||||
for name in ("api", "worker", "janitor"):
|
||||
app[name].update(s3_access_key_id=s3_keys[name][0], s3_secret_access_key=s3_keys[name][1])
|
||||
redis_admin = token()
|
||||
write(SECRETS / "s3_config.json", s3_config(bucket_name(), s3_keys))
|
||||
write(SECRETS / "redis" / "users.acl", redis_acl(redis_admin, app["api"]["redis_password"]))
|
||||
write(OPERATOR / "redis_admin_password", redis_admin, private=True)
|
||||
write(app_file, json.dumps(app, indent=2), private=True)
|
||||
created += ["s3_config.json", "redis/users.acl", "operator/app-secrets.json", "operator/redis_admin_password"]
|
||||
|
||||
if not (SECRETS / "rabbitmq_definitions.json").exists():
|
||||
write(SECRETS / "rabbitmq_definitions.json", rabbit_definitions())
|
||||
created.append("rabbitmq_definitions.json")
|
||||
if not (SECRETS / "grafana_admin_password").exists():
|
||||
write(SECRETS / "grafana_admin_password", secrets.token_urlsafe(18))
|
||||
created.append("grafana_admin_password")
|
||||
if (SECRETS / "redis").exists():
|
||||
(SECRETS / "redis").chmod(0o755)
|
||||
|
||||
if created:
|
||||
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
|
||||
else:
|
||||
print("secrets/ is complete, nothing changed.")
|
||||
if app_file.exists():
|
||||
print("Next: start openbao, then python scripts/bao.py init && python scripts/bao.py unseal "
|
||||
"&& python scripts/bao.py configure (loads operator/app-secrets.json into OpenBao and deletes it).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+334
@@ -0,0 +1,334 @@
|
||||
"""Локальный учебный PKI оператора (lab7): корневой CA, сертификаты сервисов, ротация, отзыв (CRL).
|
||||
|
||||
Запускает ОПЕРАТОР на своей машине (не контейнер приложения):
|
||||
python -m pip install -r requirements-ops.txt # cryptography
|
||||
python scripts/pki.py init # CA + все сертификаты сервисов + тестовые (идемпотентно)
|
||||
python scripts/pki.py status # сроки, серийные номера, отозванные
|
||||
python scripts/pki.py rotate lb # новый leaf той же identity, старый -> secrets/pki/lb/previous/
|
||||
python scripts/pki.py revoke lb --previous # отозвать предыдущий leaf (CRL), текущий остаётся
|
||||
python scripts/pki.py crl # перевыпустить CRL (nextUpdate через CRL_DAYS)
|
||||
|
||||
Раскладка (вся папка secrets/ в .gitignore, ничего отсюда не коммитится и не попадает в образ):
|
||||
secrets/pki/ca/ca.key закрытый ключ CA — ТОЛЬКО у оператора, ни в один контейнер не монтируется
|
||||
secrets/pki/ca/revoked.json серийные номера отозванных сертификатов (источник для CRL)
|
||||
secrets/pki/public/ca.crt корневой сертификат: его можно раздавать (клиенты, тесты, браузер)
|
||||
secrets/pki/public/crl.pem список отзыва, подписан CA; его читают API, HAProxy, OpenBao cert auth
|
||||
secrets/pki/<identity>/tls.crt leaf-сертификат сервиса, tls.key — его ключ, tls.pem — crt+key (HAProxy)
|
||||
secrets/pki/test/... сертификаты ТОЛЬКО для отрицательных тестов (чужой CA, чужая identity, истёкший)
|
||||
|
||||
Идентичность сервиса = Subject CN (и URI SAN spiffe://messenger.lab/<cn>). Её проверяют получатели:
|
||||
API пускает только CN=lb, otel-gate — только CN приложений, RabbitMQ сопоставляет CN с пользователем,
|
||||
OpenBao cert auth выдаёт policy по CN. «Подписан нашим CA» — необходимое, но не достаточное условие.
|
||||
|
||||
Ключи — ECDSA P-256. Leaf живёт LEAF_DAYS (по умолчанию 30 суток), CA — CA_DAYS (365).
|
||||
Значения ключей скрипт не печатает.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI = Path(os.environ.get("PKI_DIR", ROOT / "secrets" / "pki"))
|
||||
CA_DIR, PUBLIC, TEST = PKI / "ca", PKI / "public", PKI / "test"
|
||||
TRUST_DOMAIN = "messenger.lab"
|
||||
CA_DAYS = int(os.environ.get("CA_DAYS", "365"))
|
||||
LEAF_DAYS = int(os.environ.get("LEAF_DAYS", "30"))
|
||||
CRL_DAYS = int(os.environ.get("CRL_DAYS", "30"))
|
||||
SERVER, CLIENT = "server", "client"
|
||||
|
||||
# identity -> (DNS-имена сервера, роли). Имена DNS = имена сервисов в compose.yaml.
|
||||
LOCAL = ["localhost"]
|
||||
LOCAL_IP = ["127.0.0.1", "::1"]
|
||||
IDENTITIES = {
|
||||
# внешний вход: сертификат, который видит клиент (браузер/тесты) на https://localhost:8443
|
||||
"ingress": (["localhost", "lb"], LOCAL_IP, {SERVER}),
|
||||
# клиентский сертификат балансировщика для mTLS LB -> API (единственная разрешённая identity у API)
|
||||
"lb": ([], [], {CLIENT}),
|
||||
"api": (["api"], [], {SERVER, CLIENT}),
|
||||
"worker": ([], [], {CLIENT}),
|
||||
"dispatcher": ([], [], {CLIENT}),
|
||||
"janitor": ([], [], {CLIENT}),
|
||||
"migrate": ([], [], {CLIENT}),
|
||||
"s3-init": ([], [], {CLIENT}),
|
||||
"postgres": (["postgres"], [], {SERVER}),
|
||||
"redis": (["redis"], [], {SERVER, CLIENT}), # client: healthcheck redis-cli внутри контейнера
|
||||
"s3": (["s3"], [], {SERVER, CLIENT}), # client: внутренний gRPC SeaweedFS (security.toml)
|
||||
"rabbitmq": (["rabbitmq"], [], {SERVER}),
|
||||
"otel-collector": (["otel-collector", "otel-gate"], [], {SERVER, CLIENT}),
|
||||
"prometheus": (["prometheus"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
|
||||
"loki": (["loki"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
|
||||
"tempo": (["tempo"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
|
||||
"grafana": (["grafana"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
|
||||
"openbao": (["openbao"] + LOCAL, LOCAL_IP, {SERVER}),
|
||||
# оператор/скрипты доказательств: чтение Prometheus/Loki/Tempo, отрицательные тесты
|
||||
"operator": ([], [], {CLIENT}),
|
||||
}
|
||||
|
||||
|
||||
def now() -> dt.datetime:
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def write(path: Path, data: bytes, private: bool = False) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Запись в тот же inode (а не replace): файл может быть смонтирован в контейнер по отдельности.
|
||||
with open(path, "wb") as file:
|
||||
file.write(data)
|
||||
# 0644: ключ сервиса читает пользователь контейнера (не root, другой uid). От остальных пользователей
|
||||
# хоста ключи закрывает каталог secrets/ (0700, см. protect()).
|
||||
path.chmod(0o600 if private else 0o644)
|
||||
|
||||
|
||||
def protect() -> None:
|
||||
for directory in (PKI.parent, PKI, CA_DIR):
|
||||
if directory.exists():
|
||||
directory.chmod(0o700)
|
||||
if (CA_DIR / "ca.key").exists():
|
||||
(CA_DIR / "ca.key").chmod(0o600)
|
||||
|
||||
|
||||
def new_key() -> ec.EllipticCurvePrivateKey:
|
||||
return ec.generate_private_key(ec.SECP256R1())
|
||||
|
||||
|
||||
def key_pem(key) -> bytes:
|
||||
return key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption())
|
||||
|
||||
|
||||
def cert_pem(cert) -> bytes:
|
||||
return cert.public_bytes(serialization.Encoding.PEM)
|
||||
|
||||
|
||||
def load_ca():
|
||||
key = serialization.load_pem_private_key((CA_DIR / "ca.key").read_bytes(), None)
|
||||
cert = x509.load_pem_x509_certificate((PUBLIC / "ca.crt").read_bytes())
|
||||
return key, cert
|
||||
|
||||
|
||||
def make_ca(common_name: str, days: int):
|
||||
key = new_key()
|
||||
name = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI messenger lab"),
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, common_name)])
|
||||
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now() - dt.timedelta(minutes=5)).not_valid_after(now() + dt.timedelta(days=days))
|
||||
.add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True)
|
||||
.add_extension(x509.KeyUsage(digital_signature=False, content_commitment=False, key_encipherment=False,
|
||||
data_encipherment=False, key_agreement=False, key_cert_sign=True,
|
||||
crl_sign=True, encipher_only=False, decipher_only=False), critical=True)
|
||||
.add_extension(x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False)
|
||||
.sign(key, hashes.SHA256()))
|
||||
return key, cert
|
||||
|
||||
|
||||
def make_leaf(identity: str, dns: list, ips: list, roles: set, ca_key, ca_cert, *,
|
||||
not_before: dt.datetime | None = None, not_after: dt.datetime | None = None):
|
||||
key = new_key()
|
||||
start = not_before or (now() - dt.timedelta(minutes=5))
|
||||
end = not_after or (now() + dt.timedelta(days=LEAF_DAYS))
|
||||
sans = [x509.UniformResourceIdentifier(f"spiffe://{TRUST_DOMAIN}/{identity}")]
|
||||
sans += [x509.DNSName(name) for name in dns]
|
||||
sans += [x509.IPAddress(ipaddress.ip_address(ip)) for ip in ips]
|
||||
usages = []
|
||||
if SERVER in roles:
|
||||
usages.append(ExtendedKeyUsageOID.SERVER_AUTH)
|
||||
if CLIENT in roles:
|
||||
usages.append(ExtendedKeyUsageOID.CLIENT_AUTH)
|
||||
cert = (x509.CertificateBuilder()
|
||||
.subject_name(x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI messenger lab"),
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, identity)]))
|
||||
.issuer_name(ca_cert.subject).public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(start).not_valid_after(end)
|
||||
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
|
||||
.add_extension(x509.KeyUsage(digital_signature=True, content_commitment=False, key_encipherment=False,
|
||||
data_encipherment=False, key_agreement=True, key_cert_sign=False,
|
||||
crl_sign=False, encipher_only=False, decipher_only=False), critical=True)
|
||||
.add_extension(x509.ExtendedKeyUsage(usages), critical=False)
|
||||
.add_extension(x509.SubjectAlternativeName(sans), critical=False)
|
||||
.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_cert.public_key()), critical=False)
|
||||
.sign(ca_key, hashes.SHA256()))
|
||||
return key, cert
|
||||
|
||||
|
||||
def store_leaf(directory: Path, key, cert) -> None:
|
||||
write(directory / "tls.key", key_pem(key), private=False)
|
||||
write(directory / "tls.crt", cert_pem(cert))
|
||||
write(directory / "tls.pem", cert_pem(cert) + key_pem(key)) # HAProxy: сертификат и ключ одним файлом
|
||||
|
||||
|
||||
def issue(identity: str, ca_key, ca_cert, directory: Path | None = None) -> x509.Certificate:
|
||||
dns, ips, roles = IDENTITIES[identity]
|
||||
key, cert = make_leaf(identity, dns, ips, roles, ca_key, ca_cert)
|
||||
store_leaf(directory or PKI / identity, key, cert)
|
||||
return cert
|
||||
|
||||
|
||||
def revoked() -> dict:
|
||||
path = CA_DIR / "revoked.json"
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
||||
|
||||
|
||||
def write_crl(ca_key, ca_cert) -> None:
|
||||
builder = (x509.CertificateRevocationListBuilder().issuer_name(ca_cert.subject)
|
||||
.last_update(now() - dt.timedelta(minutes=5)).next_update(now() + dt.timedelta(days=CRL_DAYS))
|
||||
.add_extension(x509.CRLNumber(int(now().timestamp())), critical=False))
|
||||
for serial, info in revoked().items():
|
||||
builder = builder.add_revoked_certificate(
|
||||
x509.RevokedCertificateBuilder().serial_number(int(serial, 16))
|
||||
.revocation_date(dt.datetime.fromisoformat(info["revoked_at"])).build())
|
||||
write(PUBLIC / "crl.pem", builder.sign(ca_key, hashes.SHA256()).public_bytes(serialization.Encoding.PEM))
|
||||
# ca-bundle для сервисов, которым CRL передаётся вместе с CA одним файлом (не используется сейчас)
|
||||
|
||||
|
||||
def make_test_material(ca_key, ca_cert) -> list:
|
||||
"""Сертификаты ТОЛЬКО для отрицательных тестов. Ни один сервис их не принимает как свои."""
|
||||
made = []
|
||||
if not (TEST / "rogue-ca" / "ca.crt").exists():
|
||||
rogue_key, rogue_cert = make_ca("Rogue CA (negative tests)", 30)
|
||||
write(TEST / "rogue-ca" / "ca.key", key_pem(rogue_key))
|
||||
write(TEST / "rogue-ca" / "ca.crt", cert_pem(rogue_cert))
|
||||
made.append("test/rogue-ca")
|
||||
rogue_key = serialization.load_pem_private_key((TEST / "rogue-ca" / "ca.key").read_bytes(), None)
|
||||
rogue_cert = x509.load_pem_x509_certificate((TEST / "rogue-ca" / "ca.crt").read_bytes())
|
||||
plans = {
|
||||
# тот же CN, что у настоящего LB/сервисов, но подписан чужим CA
|
||||
"rogue-lb": lambda: make_leaf("lb", [], [], {CLIENT}, rogue_key, rogue_cert),
|
||||
"rogue-worker": lambda: make_leaf("worker", [], [], {CLIENT}, rogue_key, rogue_cert),
|
||||
# чужой сервер с правильным именем: клиент должен отвергнуть (не наш CA)
|
||||
"rogue-server": lambda: make_leaf("api", ["api", "localhost", "postgres", "redis", "s3", "rabbitmq",
|
||||
"otel-collector", "openbao"], LOCAL_IP, {SERVER},
|
||||
rogue_key, rogue_cert),
|
||||
# подписан НАШИМ CA, но identity, которой нигде ничего не разрешено
|
||||
"intruder": lambda: make_leaf("intruder", ["intruder"], [], {SERVER, CLIENT}, ca_key, ca_cert),
|
||||
# наш CA, CN=lb, но срок действия уже истёк
|
||||
"expired-lb": lambda: make_leaf("lb", [], [], {CLIENT}, ca_key, ca_cert,
|
||||
not_before=now() - dt.timedelta(days=10),
|
||||
not_after=now() - dt.timedelta(days=1)),
|
||||
}
|
||||
for name, plan in plans.items():
|
||||
if not (TEST / name / "tls.crt").exists():
|
||||
key, cert = plan()
|
||||
store_leaf(TEST / name, key, cert)
|
||||
made.append(f"test/{name}")
|
||||
return made
|
||||
|
||||
|
||||
def cmd_init(_args) -> int:
|
||||
created = []
|
||||
if not (CA_DIR / "ca.key").exists():
|
||||
ca_key, ca_cert = make_ca("MTUSI messenger lab root CA", CA_DAYS)
|
||||
write(CA_DIR / "ca.key", key_pem(ca_key), private=True)
|
||||
write(PUBLIC / "ca.crt", cert_pem(ca_cert))
|
||||
created.append("ca")
|
||||
ca_key, ca_cert = load_ca()
|
||||
for identity in IDENTITIES:
|
||||
if not (PKI / identity / "tls.crt").exists():
|
||||
issue(identity, ca_key, ca_cert)
|
||||
created.append(identity)
|
||||
created += make_test_material(ca_key, ca_cert)
|
||||
if not (PUBLIC / "crl.pem").exists() or created:
|
||||
write_crl(ca_key, ca_cert)
|
||||
protect()
|
||||
print("pki: created " + (", ".join(created) if created else "nothing (all present)")
|
||||
+ f"; CA certificate: {PUBLIC / 'ca.crt'} (keys are not printed)")
|
||||
return 0
|
||||
|
||||
|
||||
def describe(path: Path) -> str:
|
||||
cert = x509.load_pem_x509_certificate(path.read_bytes())
|
||||
left = (cert.not_valid_after_utc - now()).total_seconds() / 86400
|
||||
serial = format(cert.serial_number, "x")
|
||||
state = "REVOKED" if serial in revoked() else ("EXPIRED" if left < 0 else f"{left:.1f} d left")
|
||||
cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
|
||||
return f"CN={cn:<15} serial={serial[:16]}… not_after={cert.not_valid_after_utc:%Y-%m-%d %H:%M}Z {state}"
|
||||
|
||||
|
||||
def cmd_status(_args) -> int:
|
||||
print(f"CA: {describe(PUBLIC / 'ca.crt')}")
|
||||
for identity in IDENTITIES:
|
||||
path = PKI / identity / "tls.crt"
|
||||
if path.exists():
|
||||
print(f" {identity:<15} {describe(path)}")
|
||||
previous = PKI / identity / "previous" / "tls.crt"
|
||||
if previous.exists():
|
||||
print(f" {identity + ' (prev)':<15} {describe(previous)}")
|
||||
crl = x509.load_pem_x509_crl((PUBLIC / "crl.pem").read_bytes())
|
||||
print(f"CRL: {len(list(crl))} revoked, next update {crl.next_update_utc:%Y-%m-%d}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_rotate(args) -> int:
|
||||
"""Новый leaf той же identity. Старый остаётся действительным до явного revoke (плавная замена)."""
|
||||
ca_key, ca_cert = load_ca()
|
||||
directory = PKI / args.identity
|
||||
if (directory / "tls.crt").exists():
|
||||
previous = directory / "previous"
|
||||
previous.mkdir(exist_ok=True)
|
||||
for name in ("tls.crt", "tls.key", "tls.pem"):
|
||||
shutil.copyfile(directory / name, previous / name)
|
||||
cert = issue(args.identity, ca_key, ca_cert)
|
||||
protect()
|
||||
print(f"pki: {args.identity} rotated, new serial {format(cert.serial_number, 'x')[:16]}…, "
|
||||
f"previous kept in {directory / 'previous'} until revoke")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_revoke(args) -> int:
|
||||
ca_key, ca_cert = load_ca()
|
||||
if args.serial:
|
||||
serial = args.serial.lower()
|
||||
name = args.identity or "manual"
|
||||
else:
|
||||
path = PKI / args.identity / ("previous" if args.previous else "") / "tls.crt"
|
||||
cert = x509.load_pem_x509_certificate(path.read_bytes())
|
||||
serial, name = format(cert.serial_number, "x"), args.identity
|
||||
data = revoked()
|
||||
data[serial] = {"identity": name, "revoked_at": now().isoformat()}
|
||||
(CA_DIR / "revoked.json").write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
write_crl(ca_key, ca_cert)
|
||||
protect()
|
||||
print(f"pki: revoked {name} serial {serial[:16]}…; CRL updated ({len(data)} revoked)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_crl(_args) -> int:
|
||||
ca_key, ca_cert = load_ca()
|
||||
write_crl(ca_key, ca_cert)
|
||||
print(f"pki: CRL re-signed, next update in {CRL_DAYS} d")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("init").set_defaults(func=cmd_init)
|
||||
sub.add_parser("status").set_defaults(func=cmd_status)
|
||||
sub.add_parser("crl").set_defaults(func=cmd_crl)
|
||||
rotate = sub.add_parser("rotate")
|
||||
rotate.add_argument("identity", choices=sorted(IDENTITIES))
|
||||
rotate.set_defaults(func=cmd_rotate)
|
||||
revoke = sub.add_parser("revoke")
|
||||
revoke.add_argument("identity", nargs="?", choices=sorted(IDENTITIES))
|
||||
revoke.add_argument("--previous", action="store_true", help="revoke the leaf kept by the last rotate")
|
||||
revoke.add_argument("--serial", help="hex serial instead of identity")
|
||||
revoke.set_defaults(func=cmd_revoke)
|
||||
args = parser.parse_args()
|
||||
if args.command == "revoke" and not (args.identity or args.serial):
|
||||
parser.error("revoke needs an identity or --serial")
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,19 @@
|
||||
@echo off
|
||||
rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh), lab7.
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, OpenBao, Prometheus, Loki, Tempo), then the operator
|
||||
rem unseals OpenBao (scripts\bao.py unseal, shares from secrets\operator, nothing is printed), then the
|
||||
rem collector gate, the application and the balancer. Volumes are kept.
|
||||
cd /d "%~dp0.."
|
||||
rem OpenBao first and alone, unsealed right away: while "restart" waits for the slow stores, a sealed OpenBao
|
||||
rem fails 3 health checks, becomes "unhealthy" and "up --wait" aborts although it is unsealed a moment later.
|
||||
docker compose restart openbao || exit /b 1
|
||||
python scripts\bao.py unseal >nul || exit /b 1
|
||||
docker compose restart postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq openbao prometheus loki tempo || exit /b 1
|
||||
rem otel-gate alone first: otel-collector lives in its network namespace and cannot start while the gate is stopped
|
||||
docker compose restart otel-gate || exit /b 1
|
||||
docker compose restart otel-collector api janitor dispatcher worker grafana || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps otel-gate api lb dispatcher worker grafana || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
# Restart hook для tests/resilience.py --mode persistence (Linux/macOS), lab7.
|
||||
# Перезапускает хранилища (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, OpenBao; телеметрия: Prometheus, Loki,
|
||||
# Tempo), затем OpenBao распечатывается ОПЕРАТОРОМ (scripts/bao.py unseal — доли из secrets/operator, ничего
|
||||
# не печатается), затем Collector/gate, приложение и балансировщик. Volumes сохраняются (без down -v), ничего
|
||||
# не пересоздаётся. "restart" не ждёт healthcheck — ждём явно (up --wait --no-recreate).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY="${PYTHON:-python3}"
|
||||
# OpenBao — первым и отдельно, распечатка сразу: пока restart ждёт медленные хранилища, sealed OpenBao проваливает
|
||||
# 3 healthcheck подряд, становится unhealthy, и up --wait падает, хотя через секунду он уже распечатан.
|
||||
docker compose restart openbao
|
||||
"$PY" scripts/bao.py unseal >/dev/null
|
||||
docker compose restart postgres redis s3 rabbitmq prometheus loki tempo
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq openbao prometheus loki tempo
|
||||
# otel-gate — отдельно и первым: Collector живёт в его network namespace и не стартует, пока gate остановлен
|
||||
docker compose restart otel-gate
|
||||
docker compose restart otel-collector api janitor dispatcher worker grafana
|
||||
docker compose restart lb
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps otel-gate api lb dispatcher worker grafana
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
"""Скан зависимостей и образов стенда с разбором применимости (lab7). Только stdlib + Docker.
|
||||
|
||||
python scripts/scan.py # все образы из compose.yaml + образ приложения + pip-audit
|
||||
python scripts/scan.py --only app # только образ приложения и его Python-зависимости
|
||||
|
||||
Что делает:
|
||||
1. pip-audit по requirements.txt (база уязвимостей PyPI/OSV) — в одноразовом контейнере python:3.11.13-slim.
|
||||
2. Trivy (aquasec/trivy) по каждому образу: ОС-пакеты и Python-пакеты образа, severity HIGH/CRITICAL.
|
||||
3. Черновой разбор применимости для каждой находки:
|
||||
applicable — Python-зависимость приложения или библиотека, которую загружает процесс
|
||||
(libc, OpenSSL, zlib, expat, libffi ...): нужен апдейт/обоснование;
|
||||
not-reachable — пакет есть в образе, но процесс его не исполняет (perl, apt, login, util-linux ...):
|
||||
корень read-only, shell не вызывается, no-new-privileges, нет capabilities;
|
||||
third-party-image — находка в образе стороннего сервиса: применимость — по его роли и сети (internal).
|
||||
и есть ли исправленная версия. Итог — evidence/scan/*.json и evidence/scan/triage.md: это ЧЕРНОВИК для
|
||||
ручной проверки, а не вердикт — в отчёт идёт только просмотренная таблица.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "scan"
|
||||
TRIVY = "aquasec/trivy:0.76.2"
|
||||
PIP_AUDIT = "pip-audit==2.10.1"
|
||||
APP_IMAGE = "messenger-lab7-api"
|
||||
|
||||
# ОС-библиотеки, которые реально загружает процесс CPython приложения (ssl, zlib, json/expat, ctypes, uuid ...)
|
||||
RUNTIME_LIBS = {"libc6", "libc-bin", "libssl3", "libssl3t64", "openssl", "zlib1g", "libexpat1", "libffi8",
|
||||
"libbz2-1.0", "liblzma5", "libuuid1", "libgcc-s1", "libstdc++6", "libcrypt1", "libzstd1", "tzdata",
|
||||
"ca-certificates", "libsqlite3-0", "libgdbm6", "libgdbm6t64", "libncursesw6", "libreadline8",
|
||||
"libreadline8t64", "libtinfo6", "libnsl2", "libtirpc3", "libtirpc3t64"}
|
||||
|
||||
|
||||
def run(cmd, **kw):
|
||||
return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, **kw)
|
||||
|
||||
|
||||
def images(only_app):
|
||||
config = json.loads(run(["docker", "compose", "config", "--format", "json"], check=True).stdout)
|
||||
found = {APP_IMAGE}
|
||||
if not only_app:
|
||||
found |= {svc["image"] for svc in config["services"].values() if svc.get("image")}
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def pip_audit():
|
||||
cmd = ["docker", "run", "--rm", "-v", f"{ROOT / 'requirements.txt'}:/r.txt:ro", "python:3.11.13-slim", "sh", "-c",
|
||||
f"pip install -q {PIP_AUDIT} >/dev/null 2>&1 && pip-audit -r /r.txt --no-deps --disable-pip -f json"]
|
||||
result = run(cmd)
|
||||
data = json.loads(result.stdout or "{}")
|
||||
(OUT / "pip-audit.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
return data
|
||||
|
||||
|
||||
def trivy(image):
|
||||
cache = Path.home() / ".cache" / "trivy-lab7" # база Trivy — вне репозитория
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
cmd = ["docker", "run", "--rm", "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", "-v",
|
||||
f"{cache}:/root/.cache/trivy", TRIVY, "image", "--quiet", "--format", "json", "--scanners", "vuln",
|
||||
"--severity", "HIGH,CRITICAL", image]
|
||||
# Docker socket получает только одноразовый контейнер сканера на хосте оператора, не стенд.
|
||||
result = run(cmd)
|
||||
if result.returncode != 0:
|
||||
return {"error": result.stderr[-500:]}
|
||||
data = json.loads(result.stdout)
|
||||
name = image.replace("/", "_").replace(":", "_")
|
||||
(OUT / f"trivy-{name}.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
return data
|
||||
|
||||
|
||||
def classify(image, result_class, pkg):
|
||||
if image != APP_IMAGE:
|
||||
return "third-party-image"
|
||||
if result_class == "lang-pkgs":
|
||||
return "applicable"
|
||||
return "applicable" if pkg in RUNTIME_LIBS else "not-reachable"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--only", choices=["app"])
|
||||
args = parser.parse_args()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
lines = ["# Скан зависимостей и образов — черновой разбор (проверить вручную)", ""]
|
||||
audit = pip_audit()
|
||||
vulns = [(d["name"], d["version"], v["id"], ",".join(v.get("fix_versions", [])))
|
||||
for d in audit.get("dependencies", []) for v in d.get("vulns", [])]
|
||||
lines += [f"## pip-audit requirements.txt: {len(vulns)} находок", ""]
|
||||
lines += [f"- {n} {ver}: {vid}, исправлено в {fix or '—'}" for n, ver, vid, fix in vulns] or ["- нет"]
|
||||
lines.append("")
|
||||
rows = []
|
||||
for image in images(args.only == "app"):
|
||||
print(f"trivy {image} ...", flush=True)
|
||||
data = trivy(image)
|
||||
if "error" in data:
|
||||
lines.append(f"## {image}: ошибка сканера: {data['error'][:200]}")
|
||||
continue
|
||||
for result in data.get("Results", []):
|
||||
for v in result.get("Vulnerabilities", []) or []:
|
||||
rows.append({"image": image, "target": result.get("Target"), "class": result.get("Class"),
|
||||
"pkg": v["PkgName"], "installed": v.get("InstalledVersion"),
|
||||
"fixed": v.get("FixedVersion") or "", "id": v["VulnerabilityID"],
|
||||
"severity": v.get("Severity"), "title": (v.get("Title") or "")[:90],
|
||||
"applicability": classify(image, result.get("Class"), v["PkgName"])})
|
||||
(OUT / "trivy-findings.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False), encoding="utf-8")
|
||||
lines += ["## Trivy (HIGH, CRITICAL)", "", "| Образ | Пакет | Версия | Исправлено в | CVE | Severity | Применимость |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- |"]
|
||||
for r in sorted(rows, key=lambda r: (r["image"] != APP_IMAGE, r["applicability"], r["image"], r["pkg"])):
|
||||
lines.append(f"| {r['image']} | {r['pkg']} | {r['installed']} | {r['fixed'] or '—'} | {r['id']} | "
|
||||
f"{r['severity']} | {r['applicability']} |")
|
||||
(OUT / "triage.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
by = {}
|
||||
for r in rows:
|
||||
by.setdefault((r["image"], r["applicability"]), 0)
|
||||
by[(r["image"], r["applicability"])] += 1
|
||||
for (image, kind), n in sorted(by.items()):
|
||||
print(f"{image:<45} {kind:<18} {n}")
|
||||
print(f"pip-audit: {len(vulns)} finding(s); draft triage: {OUT / 'triage.md'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py.
|
||||
python "%~dp0stop_one.py"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID.
|
||||
# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd.
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/stop_one.py"
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Failover hook (tests/resilience.py --mode failover): остановить ОДНУ реплику API по X-Instance-Id.
|
||||
|
||||
lab7: X-Instance-Id — случайный alias процесса, а не ID контейнера/hostname. Каждая реплика пишет его в
|
||||
tmpfs-файл /tmp/instance-id (settings.INSTANCE_ID_FILE); hook читает файл в каждом запущенном контейнере api
|
||||
и останавливает тот, чей alias совпал. Реплика остаётся остановленной (восстановление — отдельно).
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STOP_TIMEOUT_SECONDS = 8
|
||||
READ_ALIAS = "import settings; print(open(settings.INSTANCE_ID_FILE).read().strip())"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{8,64}", target):
|
||||
print("TARGET_INSTANCE_ID must be the X-Instance-Id alias of an api replica", file=sys.stderr)
|
||||
return 2
|
||||
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout.split()
|
||||
matches = []
|
||||
for container in running:
|
||||
alias = subprocess.run(["docker", "exec", container, "python", "-c", READ_ALIAS], cwd=ROOT,
|
||||
capture_output=True, text=True, check=False).stdout.strip()
|
||||
if alias == target:
|
||||
matches.append(container)
|
||||
if len(matches) != 1:
|
||||
print(f"no running api container with alias {target}", file=sys.stderr)
|
||||
return 1
|
||||
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
print(f"stopped api container {matches[0][:12]} (alias {target})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept).
|
||||
cd /d "%~dp0.."
|
||||
if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% )
|
||||
if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% )
|
||||
if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% )
|
||||
echo usage: %0 stop^|start^|status
|
||||
exit /b 2
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Demo hook for the defense: stop or start the worker without touching volumes.
|
||||
# scripts/worker.sh stop | start | status
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${1:-}" in
|
||||
stop) docker compose stop worker ;;
|
||||
start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;;
|
||||
status) docker compose ps worker ;;
|
||||
*) echo "usage: $0 stop|start|status" >&2; exit 2 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user