лр7: Docker-прогон, evidence, исправления

This commit is contained in:
kalibr456
2026-11-07 20:17:11 +03:00
parent 20afc2c790
commit cb38fd5231
249 changed files with 114500 additions and 82 deletions
+309
View File
@@ -0,0 +1,309 @@
"""Операторские действия с OpenBao (только stdlib). Запускает ОПЕРАТОР на хосте, OpenBao — https://127.0.0.1:8200.
python scripts/bao.py status # sealed? initialized? (без токена)
python scripts/bao.py init # один раз: 3 доли ключа (порог 2) + root token
python scripts/bao.py unseal [--prompt] # после КАЖДОГО старта OpenBao: 2 доли (из файла или ввод вручную)
python scripts/bao.py configure # KV v2, cert auth, политики, роли, CRL, секреты приложений
python scripts/bao.py sync-crl # загрузить свежий CRL (после pki.py revoke)
python scripts/bao.py kv-rotate api redis_password --value-file F # новая версия одного ключа (cas)
python scripts/bao.py role-ttl api 15m 1h # TTL токенов роли (для теста истечения — короче)
python scripts/bao.py revoke-service-tokens # отозвать все токены cert auth: сервисы перелогинятся сертификатом,
# отозванный сертификат войти уже не сможет
python scripts/bao.py seal # запечатать (демонстрация sealed)
Файл оператора secrets/operator/openbao-init.json (0600, вне Git, ни в один контейнер не монтируется)
хранит доли unseal и root token учебного стенда. В реальной эксплуатации доли раздаются разным людям
(`init --print-shares-once` выводит их один раз и НЕ сохраняет, затем `unseal --prompt`), root token
отзывается после настройки. Ни одна команда не печатает значения секретов, долей или токенов
(кроме явного --print-shares-once).
"""
import argparse
import getpass
import json
import os
import ssl
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = Path(os.environ.get("SECRETS_DIR", ROOT / "secrets"))
OPERATOR = SECRETS / "operator"
INIT_FILE = OPERATOR / "openbao-init.json"
APP_FILE = OPERATOR / "app-secrets.json"
LOADED_MARKER = OPERATOR / "app-secrets.loaded"
CA_FILE = Path(os.environ.get("BAO_CACERT", SECRETS / "pki" / "public" / "ca.crt"))
CRL_FILE = SECRETS / "pki" / "public" / "crl.pem"
POLICIES = ROOT / "infra" / "openbao" / "policies"
ADDR = os.environ.get("BAO_ADDR", "https://127.0.0.1:8200").rstrip("/")
KV_MOUNT = "messenger"
# Роли cert auth = identity (CN) сервиса. Токен: TTL 15 мин, renewable, не дольше 1 ч; дальше — новый вход.
SERVICES = ["api", "worker", "dispatcher", "janitor", "migrate", "s3-init"]
TOKEN_TTL, TOKEN_MAX_TTL = "15m", "1h"
class BaoError(Exception):
pass
def request(method: str, path: str, body=None, token: str | None = None, ok=(200, 204)) -> dict:
context = ssl.create_default_context(cafile=str(CA_FILE))
data = json.dumps(body).encode("utf-8") if body is not None else None
req = urllib.request.Request(ADDR + path, data=data, method=method, headers={"Content-Type": "application/json"})
if token:
req.add_header("X-Vault-Token", token)
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context))
try:
with opener.open(req, timeout=10) as reply:
status, raw = reply.status, reply.read()
except urllib.error.HTTPError as exc:
status, raw = exc.code, exc.read()
except (urllib.error.URLError, OSError) as exc:
raise BaoError(f"OpenBao at {ADDR} is unreachable: {getattr(exc, 'reason', exc)}") from None
if status not in ok:
try:
errors = json.loads(raw).get("errors")
except ValueError:
errors = None
raise BaoError(f"{method} {path} -> HTTP {status} {errors or ''}")
return json.loads(raw) if raw else {}
def init_data() -> dict:
if not INIT_FILE.exists():
raise BaoError(f"{INIT_FILE} not found: run `python scripts/bao.py init` first (or use --prompt)")
return json.loads(INIT_FILE.read_text(encoding="utf-8"))
def root_token() -> str:
token = os.environ.get("BAO_TOKEN") or init_data().get("root_token")
if not token:
raise BaoError("no operator token: set BAO_TOKEN or keep it in the init file")
return token
def seal_status() -> dict:
return request("GET", "/v1/sys/seal-status")
def cmd_status(_args) -> int:
status = seal_status()
print(json.dumps({k: status.get(k) for k in ("initialized", "sealed", "t", "n", "progress", "version",
"storage_type")}))
return 0
def cmd_init(args) -> int:
if request("GET", "/v1/sys/init").get("initialized"):
print("openbao: already initialized, nothing to do")
return 0
reply = request("PUT", "/v1/sys/init", {"secret_shares": 3, "secret_threshold": 2})
if args.print_shares_once:
print("Unseal shares (hand each to a different operator, they are NOT saved):")
for number, share in enumerate(reply["keys_base64"], 1):
print(f" share {number}: {share}")
print("Root token (revoke after configure in real life): " + reply["root_token"])
return 0
OPERATOR.mkdir(parents=True, exist_ok=True)
OPERATOR.chmod(0o700)
with open(INIT_FILE, "w", encoding="utf-8") as file:
json.dump({"unseal_keys_b64": reply["keys_base64"], "root_token": reply["root_token"],
"threshold": 2, "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, file)
INIT_FILE.chmod(0o600)
print(f"openbao: initialized (3 shares, threshold 2); shares and root token saved to {INIT_FILE} (not printed)")
return 0
def wait_reachable(seconds: float = 30) -> dict:
"""После (ре)старта контейнера listener поднимается не сразу: ждём ответа /sys/seal-status."""
deadline = time.monotonic() + seconds
while True:
try:
return seal_status()
except BaoError:
if time.monotonic() > deadline:
raise
time.sleep(0.5)
def cmd_unseal(args) -> int:
status = wait_reachable()
if not status.get("sealed"):
print("openbao: already unsealed")
return 0
if args.prompt:
shares = [getpass.getpass(f"unseal share {i + 1}: ") for i in range(status.get("t", 2))]
else:
shares = init_data()["unseal_keys_b64"]
for share in shares:
status = request("PUT", "/v1/sys/unseal", {"key": share})
if not status.get("sealed"):
break
if status.get("sealed"):
print("openbao: still sealed, progress " + str(status.get("progress")))
return 1
print("openbao: unsealed")
return 0
def cmd_seal(_args) -> int:
request("PUT", "/v1/sys/seal", {}, token=root_token())
print("openbao: sealed (services keep cached secrets; new logins fail until unseal)")
return 0
def ensure_mounts(token: str) -> None:
mounts = request("GET", "/v1/sys/mounts", token=token)
mounts = mounts.get("data", mounts)
if f"{KV_MOUNT}/" not in mounts:
request("POST", f"/v1/sys/mounts/{KV_MOUNT}", {"type": "kv", "options": {"version": "2"},
"description": "messenger application secrets"}, token=token)
print(f"openbao: KV v2 mounted at {KV_MOUNT}/")
auths = request("GET", "/v1/sys/auth", token=token)
auths = auths.get("data", auths)
if "cert/" not in auths:
request("POST", "/v1/sys/auth/cert", {"type": "cert", "description": "service identities (mTLS leaf)"},
token=token)
print("openbao: cert auth enabled at cert/")
def write_policies(token: str) -> None:
for path in sorted(POLICIES.glob("*.hcl")):
request("PUT", f"/v1/sys/policies/acl/{path.stem}", {"policy": path.read_text(encoding="utf-8")}, token=token)
print("openbao: policies " + ", ".join(p.stem for p in sorted(POLICIES.glob("*.hcl"))))
def write_roles(token: str, ttl: str = TOKEN_TTL, max_ttl: str = TOKEN_MAX_TTL, only: list | None = None) -> None:
ca = CA_FILE.read_text(encoding="utf-8")
for name in only or SERVICES:
request("POST", f"/v1/auth/cert/certs/{name}", {
"display_name": name,
"certificate": ca, # доверяем только учебному CA ...
"allowed_common_names": [name], # ... и только сертификату с CN = имя роли
"token_policies": [name],
"token_ttl": ttl,
"token_max_ttl": max_ttl,
"token_type": "service",
}, token=token)
print(f"openbao: cert roles {', '.join(only or SERVICES)} (ttl {ttl}, max {max_ttl}, CN must equal role)")
def sync_crl(token: str) -> None:
request("POST", "/v1/auth/cert/crls/messenger-ca", {"crl": CRL_FILE.read_text(encoding="utf-8")}, token=token)
print("openbao: CRL uploaded to cert auth (revoked certificates cannot log in)")
def kv_read(token: str, path: str) -> tuple[dict | None, int]:
try:
reply = request("GET", f"/v1/{KV_MOUNT}/data/{path}", token=token)
except BaoError as exc:
if "HTTP 404" in str(exc):
return None, 0
raise
return reply["data"]["data"], reply["data"]["metadata"]["version"]
def load_app_secrets(token: str, force: bool) -> None:
if not APP_FILE.exists():
if LOADED_MARKER.exists():
print("openbao: application secrets already loaded earlier (source of truth is OpenBao)")
return
raise BaoError(f"{APP_FILE} not found: run python scripts/init_secrets.py")
app = json.loads(APP_FILE.read_text(encoding="utf-8"))
for path, values in app.items():
existing, _version = kv_read(token, path)
if existing is not None and not force:
continue
request("POST", f"/v1/{KV_MOUNT}/data/{path}", {"data": values}, token=token)
LOADED_MARKER.write_text(time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + "\n", encoding="utf-8")
APP_FILE.unlink()
print(f"openbao: application secrets written to {KV_MOUNT}/{{{','.join(app)}}}; {APP_FILE.name} deleted")
def cmd_configure(args) -> int:
token = root_token()
ensure_mounts(token)
write_policies(token)
write_roles(token)
sync_crl(token)
load_app_secrets(token, args.force)
return 0
def cmd_sync_crl(_args) -> int:
sync_crl(root_token())
return 0
def cmd_kv_rotate(args) -> int:
"""Новая версия ОДНОГО ключа в пути сервиса (check-and-set: не затираем чужую параллельную запись)."""
token = root_token()
values, version = kv_read(token, args.path)
if values is None:
raise BaoError(f"{KV_MOUNT}/{args.path} does not exist")
value = Path(args.value_file).read_text(encoding="utf-8").strip()
values[args.key] = value
request("POST", f"/v1/{KV_MOUNT}/data/{args.path}", {"options": {"cas": version}, "data": values}, token=token)
print(f"openbao: {KV_MOUNT}/{args.path} key {args.key} -> version {version + 1} (value not printed)")
return 0
def cmd_kv_version(args) -> int:
_values, version = kv_read(root_token(), args.path)
print(json.dumps({"path": f"{KV_MOUNT}/{args.path}", "version": version}))
return 0
def cmd_role_ttl(args) -> int:
write_roles(root_token(), args.ttl, args.max_ttl, only=[args.role])
return 0
def cmd_revoke_tokens(_args) -> int:
request("PUT", "/v1/sys/leases/revoke-prefix/auth/cert/login", {}, token=root_token())
print("openbao: all tokens issued by cert auth revoked; services re-login with their certificates")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status").set_defaults(func=cmd_status)
p = sub.add_parser("init")
p.add_argument("--print-shares-once", action="store_true", help="print shares/root token instead of saving")
p.set_defaults(func=cmd_init)
p = sub.add_parser("unseal")
p.add_argument("--prompt", action="store_true", help="type the shares instead of reading the operator file")
p.set_defaults(func=cmd_unseal)
sub.add_parser("seal").set_defaults(func=cmd_seal)
p = sub.add_parser("configure")
p.add_argument("--force", action="store_true", help="overwrite existing KV values from app-secrets.json")
p.set_defaults(func=cmd_configure)
sub.add_parser("sync-crl").set_defaults(func=cmd_sync_crl)
p = sub.add_parser("kv-rotate")
p.add_argument("path", choices=SERVICES)
p.add_argument("key")
p.add_argument("--value-file", required=True, help="file with the new value (not echoed)")
p.set_defaults(func=cmd_kv_rotate)
p = sub.add_parser("kv-version")
p.add_argument("path", choices=SERVICES)
p.set_defaults(func=cmd_kv_version)
p = sub.add_parser("role-ttl")
p.add_argument("role", choices=SERVICES)
p.add_argument("ttl")
p.add_argument("max_ttl")
p.set_defaults(func=cmd_role_ttl)
sub.add_parser("revoke-service-tokens").set_defaults(func=cmd_revoke_tokens)
args = parser.parse_args()
try:
return args.func(args)
except BaoError as exc:
print(f"bao.py: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())
+265
View File
@@ -0,0 +1,265 @@
"""Генерирует infra/grafana/dashboards/messenger.json (импортируемый dashboard, provisioning из Git).
python scripts/build_dashboard.py
Все запросы панелей — здесь, в одном месте. student-tests/check_dashboard.py выполняет каждый запрос
против живых Prometheus/Loki/Tempo и падает, если имя метрики/метки не существует или запрос с ошибкой.
Единицы: секунды (s) для длительностей, reqps/ops для частот, percentunit для долей, bytes для памяти.
"""
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'job="messenger-api", http_route!~"/health/.*"' # бизнес-запросы API (знаменатель SLO)
QUEUE = 'messaging_destination_name="attachments.process.v1"'
_panels = []
_next_id = [1]
_y = [0]
def _id():
_next_id[0] += 1
return _next_id[0]
def row(title):
_panels.append({"type": "row", "title": title, "id": _id(), "collapsed": False,
"gridPos": {"h": 1, "w": 24, "x": 0, "y": _y[0]}, "panels": []})
_y[0] += 1
class Line:
"""Раскладка панелей в строку сетки Grafana (24 колонки)."""
def __init__(self, height):
self.x, self.h = 0, height
def place(self, width):
pos = {"h": self.h, "w": width, "x": self.x, "y": _y[0]}
self.x += width
return pos
def end(self):
_y[0] += self.h
def prom(expr, legend="", instant=False, fmt=None):
target = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": "",
"range": not instant, "instant": instant}
if fmt:
target["format"] = fmt
return target
def _with_refs(targets):
for n, target in enumerate(targets):
target["refId"] = chr(ord("A") + n)
return targets
def timeseries(line, width, title, description, targets, unit, thresholds=None, stack=False, minimum=None):
defaults = {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10, "showPoints": "never",
"stacking": {"mode": "normal" if stack else "none"}}}
if minimum is not None:
defaults["min"] = minimum
if thresholds:
defaults["thresholds"] = {"mode": "absolute", "steps": [{"color": "green", "value": None}] + [
{"color": color, "value": value} for value, color in thresholds]}
defaults["custom"]["thresholdsStyle"] = {"mode": "line+area"}
_panels.append({"type": "timeseries", "id": _id(), "title": title, "description": description,
"datasource": PROM, "gridPos": line.place(width), "targets": _with_refs(targets),
"fieldConfig": {"defaults": defaults, "overrides": []},
"options": {"legend": {"displayMode": "list", "placement": "bottom"},
"tooltip": {"mode": "multi", "sort": "desc"}}})
def stat(line, width, title, description, expr, unit, steps, decimals=None):
defaults = {"unit": unit, "thresholds": {"mode": "absolute", "steps": [
{"color": color, "value": value} for value, color in steps]}, "color": {"mode": "thresholds"}}
if decimals is not None:
defaults["decimals"] = decimals
_panels.append({"type": "stat", "id": _id(), "title": title, "description": description, "datasource": PROM,
"gridPos": line.place(width), "targets": _with_refs([prom(expr, instant=True)]),
"fieldConfig": {"defaults": defaults, "overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "background", "graphMode": "none", "textMode": "value"}})
def quantiles(metric, selector, by="le"):
return [prom(f"histogram_quantile({q}, sum by ({by}) (rate({metric}_bucket{{{selector}}}[$__rate_interval])))",
f"p{int(q * 100)}") for q in (0.5, 0.95, 0.99)]
def build():
row("Обзор и SLO (окно SLO — 1 час)")
line = Line(4)
stat(line, 3, "RPS API", "Бизнес-запросы API в секунду (без /health), все реплики.",
"messenger:http_requests:rate1m", "reqps", [(None, "blue")])
stat(line, 3, "Доля 5xx (1m)", "5xx / все бизнес-запросы за 1 минуту. Порог алерта 5%.",
"messenger:http_error_ratio:rate1m", "percentunit", [(None, "green"), (0.01, "orange"), (0.05, "red")], 2)
stat(line, 3, "p95 API (5m)", "95-й перцентиль из buckets гистограммы всех реплик (не среднее p95).",
"messenger:http_request_duration_seconds:p95_5m", "s", [(None, "green"), (0.25, "orange"), (1, "red")])
stat(line, 3, "Старейшее задание", "Возраст старейшего готового к обработке задания (queue lag). Порог алерта 30 с.",
f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "s", [(None, "green"), (10, "orange"), (30, "red")])
stat(line, 3, "SLO-1 доступность", "Доля бизнес-запросов без 5xx за 1 час. Цель 99%.",
"messenger:slo_availability:ratio_1h", "percentunit", [(None, "red"), (0.99, "green")], 2)
stat(line, 3, "Бюджет ошибок SLO-1", "Остаток бюджета ошибок (1% запросов) за 1 час; < 0 — SLO нарушен.",
"messenger:slo_availability:error_budget_remaining_1h", "percentunit",
[(None, "red"), (0, "orange"), (0.5, "green")], 0)
stat(line, 3, "SLO-2 POST < 250 мс", "Доля успешных POST /chats/{chat_id}/messages быстрее 250 мс за 1 час. Цель 95%.",
"messenger:slo_latency_messages:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
stat(line, 3, "SLO-3 ready ≤ 30 с", "Доля вложений, ставших ready не позже 30 с после 202, за 1 час. Цель 95%.",
"messenger:slo_time_to_ready:ratio_1h", "percentunit", [(None, "red"), (0.95, "green")], 2)
line.end()
line = Line(5)
_panels.append({"type": "table", "id": _id(), "title": "Алерты Prometheus (pending/firing)",
"description": "ALERTS из Prometheus: имя, состояние, severity. Пусто — все правила inactive.",
"datasource": PROM, "gridPos": line.place(24),
"targets": _with_refs([prom('ALERTS{alertstate=~"pending|firing"}', instant=True, fmt="table")]),
"transformations": [{"id": "organize", "options": {"excludeByName": {
"Time": True, "Value": True, "__name__": True}}}],
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
line.end()
row("API: запросы, ошибки, задержка (RED)")
line = Line(8)
timeseries(line, 8, "RPS по маршрутам", "Шаблон маршрута, а не URL: метка ограниченной кардинальности.",
[prom(f"sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
"{{http_request_method}} {{http_route}}")], "reqps")
timeseries(line, 8, "RPS по классу статуса", "2xx/4xx/5xx. 4xx — ошибки клиента, в SLO не входят.",
[prom(f"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))",
"{{http_response_status_class}}")], "reqps", stack=True)
timeseries(line, 8, "Доля 5xx", "Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы. Линия — порог алерта.",
[prom(f'(sum(rate(http_server_request_duration_seconds_count{{{API}, http_response_status_class="5xx"}}[$__rate_interval])) or on() vector(0)) '
f"/ sum(rate(http_server_request_duration_seconds_count{{{API}}}[$__rate_interval]))", "5xx ratio")],
"percentunit", thresholds=[(0.05, "red")], minimum=0)
line.end()
line = Line(8)
timeseries(line, 8, "Задержка API p50 / p95 / p99", "histogram_quantile по сумме buckets всех реплик.",
quantiles("http_server_request_duration_seconds", API), "s")
timeseries(line, 8, "p95 по маршрутам", "Где именно медленно.",
[prom(f"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{{{API}}}[$__rate_interval])))",
"{{http_route}}")], "s")
timeseries(line, 8, "Запросы в обработке (по репликам)", "http.server.active_requests: насыщение реплик API.",
[prom('sum by (instance) (http_server_active_requests{job="messenger-api"})', "{{instance}}")], "short")
line.end()
row("Фоновая обработка: очередь, worker, время до ready")
line = Line(8)
timeseries(line, 8, "Попытки заданий в секунду по исходу", "ready / retry / failed / stale / lease_lost (messenger.job.duration count).",
[prom("sum by (outcome) (rate(messenger_job_duration_seconds_count[$__rate_interval]))", "{{outcome}}")],
"ops", stack=True)
timeseries(line, 8, "Длительность попытки p50 / p95 / p99 (ready)", "От claim до COMMIT ready в worker.",
quantiles("messenger_job_duration_seconds", 'outcome="ready"'), "s")
timeseries(line, 8, "Время до ready p50 / p95 / p99", "От устойчивого приёма (COMMIT задания в API) до COMMIT ready. Линия — SLO-3 (30 с).",
quantiles("messenger_attachment_time_to_ready_seconds", 'job_type="attachment.process.v1"'), "s",
thresholds=[(30, "red")])
line.end()
line = Line(8)
timeseries(line, 8, "Глубина очереди и backlog", "Сообщения в RabbitMQ (очередь и DLQ) и незавершённые задания в PostgreSQL.",
[prom("max by (messaging_destination_name) (messenger_queue_depth)", "broker {{messaging_destination_name}}"),
prom("max by (status) (messenger_jobs_backlog)", "jobs {{status}}")], "short")
timeseries(line, 8, "Возраст старейшего задания и outbox", "Queue lag (worker не успевает) и publish lag (dispatcher/брокер). Линия — порог алерта.",
[prom(f"max(messenger_queue_oldest_age_seconds{{{QUEUE}}})", "oldest queued job"),
prom("max(messenger_outbox_oldest_age_seconds)", "oldest unpublished outbox row")], "s",
thresholds=[(30, "red")])
timeseries(line, 8, "Доставки и публикации", "Worker: processed / noop / dead_lettered / requeued; dispatcher: ok / error.",
[prom("sum by (outcome) (rate(messenger_messages_consumed_total[$__rate_interval]))", "consumed {{outcome}}"),
prom("sum by (outcome) (rate(messenger_messages_published_total[$__rate_interval]))", "published {{outcome}}")],
"ops")
line.end()
row("Насыщение: пул БД, worker, CPU/RAM")
line = Line(8)
timeseries(line, 8, "Пул PostgreSQL API: занято / максимум / ждут", "used растёт до max и появляются ждущие — запросы ждут соединение (медленная БД или блокировки).",
[prom('sum by (instance) (db_client_connection_count{db_client_connection_state="used"})', "used {{instance}}"),
prom("max(db_client_connection_max)", "max per replica"),
prom("sum by (instance) (db_client_connection_pending_requests)", "waiting {{instance}}")], "short")
timeseries(line, 8, "Ожидание соединения из пула p95", "Время получения соединения из пула API (не время запроса).",
[prom("histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[$__rate_interval])))",
"p95 {{instance}}")], "s")
timeseries(line, 8, "Worker: занятость (busy / capacity)", "Доля занятых потоков-потребителей. 100% — worker насыщен, очередь растёт.",
[prom("sum(messenger_worker_busy) / sum(messenger_worker_capacity)", "saturation")], "percentunit",
thresholds=[(0.8, "orange"), (1, "red")], minimum=0)
line.end()
line = Line(8)
timeseries(line, 8, "CPU процессов (ядра)", "rate(process.cpu.time): ядер на процесс.",
[prom("sum by (job, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))", "{{job}} {{instance}}")],
"short")
timeseries(line, 8, "CPU контейнеров / лимит", "cgroup v2 (только в Docker): доля лимита CPU контейнера, включая дочерние процессы worker.",
[prom("sum by (job, instance) (rate(container_cpu_time_seconds_total[$__rate_interval])) / on (job, instance) max by (job, instance) (container_cpu_limit)",
"{{job}} {{instance}}")], "percentunit")
timeseries(line, 8, "Память: RSS процесса и контейнер", "process.memory.usage (RSS) и container.memory.usage (cgroup).",
[prom("max by (job, instance) (process_memory_usage_bytes)", "rss {{job}} {{instance}}"),
prom("max by (job, instance) (container_memory_usage_bytes)", "cgroup {{job}} {{instance}}")], "bytes")
line.end()
row("Конвейер телеметрии: Collector и SDK")
line = Line(8)
timeseries(line, 8, "Collector: принято и отправлено", "Спаны/записи логов/точки метрик в секунду.",
[prom("sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "accepted spans"),
prom("sum(rate(otelcol_receiver_accepted_log_records[$__rate_interval]))", "accepted logs"),
prom("sum(rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", "accepted metric points"),
prom("sum by (exporter) (rate(otelcol_exporter_sent_spans[$__rate_interval]))", "sent spans {{exporter}}"),
prom("sum by (exporter) (rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "sent logs {{exporter}}")],
"ops")
timeseries(line, 8, "Collector: потери и отказы", "send_failed — истёк повтор; enqueue_failed — очередь полна; refused — memory_limiter.",
[prom("sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", "send failed spans {{exporter}}"),
prom("sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", "send failed logs {{exporter}}"),
prom("sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))", "queue full spans {{exporter}}"),
prom("sum(rate(otelcol_receiver_refused_spans[$__rate_interval]))", "refused spans (memory_limiter)")],
"ops")
timeseries(line, 8, "Очереди exporter и RSS Collector", "Заполнение sending_queue (доля ёмкости) и память процесса Collector.",
[prom("max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity)", "queue {{exporter}}")],
"percentunit", minimum=0)
line.end()
line = Line(8)
timeseries(line, 12, "Collector RSS", "Ограничено memory_limiter (200 MiB) при недоступном хранилище.",
[prom("max(otelcol_process_memory_rss)", "collector rss")], "bytes")
timeseries(line, 12, "SDK приложений: экспорт и потери", "Потери в процессе приложения: processor error.type=queue_full (очередь 2048 полна), exporter error.type (Collector не принял пакет).",
[prom('sum by (job) (rate(otel_sdk_exporter_span_exported_total{error_type=""}[$__rate_interval]))', "exported spans {{job}}"),
prom('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[$__rate_interval]))', "export failed spans {{job}} {{error_type}}"),
prom('sum by (job, error_type) (rate(otel_sdk_exporter_log_exported_total{error_type!=""}[$__rate_interval]))', "export failed logs {{job}} {{error_type}}"),
prom('sum by (job) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[$__rate_interval]))', "queue full spans {{job}}")],
"ops")
line.end()
row("Логи и трассы")
line = Line(9)
_panels.append({"type": "logs", "id": _id(), "title": "Предупреждения и ошибки (Loki)",
"description": "Структурные логи всех сервисов; trace_id в записи ведёт в Tempo.",
"datasource": LOKI, "gridPos": line.place(12),
"targets": _with_refs([{"datasource": LOKI, "refId": "", "queryType": "range",
"expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|error"'}]),
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending",
"enableLogDetails": True}})
_panels.append({"type": "table", "id": _id(), "title": "Трассы с ошибками (Tempo, TraceQL)",
"description": "Последние трассы со статусом ERROR: 5xx API, неудачные попытки заданий.",
"datasource": TEMPO, "gridPos": line.place(12),
"targets": _with_refs([{"datasource": TEMPO, "refId": "", "queryType": "traceql",
"query": "{ status = error }", "limit": 20, "tableType": "traces"}]),
"fieldConfig": {"defaults": {}, "overrides": []}, "options": {"showHeader": True}})
line.end()
return {
"uid": "messenger-lab6", "title": "Messenger — lab6 observability", "tags": ["messenger", "lab6"],
"timezone": "browser", "editable": False, "graphTooltip": 1, "schemaVersion": 39, "version": 1,
"refresh": "10s", "time": {"from": "now-30m", "to": "now"},
"templating": {"list": []}, "annotations": {"list": [{
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
"panels": _panels,
}
if __name__ == "__main__":
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f"written {OUT.relative_to(ROOT)}: {len(_panels)} panels")
+206
View File
@@ -0,0 +1,206 @@
"""Секреты учебного стенда lab7 (только stdlib). Запускает оператор ПОСЛЕ scripts/pki.py init.
Создаёт недостающее, существующее не трогает (иначе разошлось бы с данными в volumes):
Для серверов (монтируются в СВОЙ контейнер, приложение их не видит):
secrets/postgres_password пароль владельца БД messenger (контейнер postgres при initdb)
secrets/redis/users.acl начальный ACL Redis: default off, admin (оператор), api (приложение);
в файле SHA-256 паролей, не сами пароли. Redis копирует его в свой volume
при первом старте, дальше ACL меняется командами (ротация).
secrets/s3_config.json ключи SeaweedFS: s3-init (admin), api, worker, janitor — права по prefix
secrets/rabbitmq_definitions.json vhost, очереди, пользователи worker/dispatcher БЕЗ пароля (вход только
по клиентскому сертификату, EXTERNAL) и их права
secrets/grafana_admin_password вход в Grafana (admin)
Только оператору (никуда не монтируются):
secrets/operator/app-secrets.json учётные данные приложений -> загружаются в OpenBao (scripts/bao.py configure),
после загрузки файл удаляется; дальше источник истины — OpenBao
secrets/operator/redis_admin_password, secrets/operator/postgres_owner_password — для ротации и диагностики
Значения не печатаются. Права файлов: каталоги 0700; файлы, которые читает контейнер под своим uid, — 0644
(на Windows/Docker Desktop права bind mount не применяются, защита — сам каталог пользователя).
"""
import hashlib
import json
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = Path(os.environ.get("SECRETS_DIR", ROOT / "secrets"))
OPERATOR = SECRETS / "operator"
BROKER_VHOST = "messenger"
# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей).
EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1"
DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq"
def write(path: Path, value: str, private: bool = False) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8", newline="") as file:
file.write(value)
path.chmod(0o600 if private else 0o644)
def read(path: Path) -> str:
return path.read_text(encoding="utf-8").strip()
def token(n: int = 24) -> str:
return secrets.token_hex(n)
def bucket_name() -> str:
env = ROOT / ".env"
if env.exists():
for line in env.read_text(encoding="utf-8").splitlines():
if line.strip().startswith("S3_BUCKET="):
return line.split("=", 1)[1].strip()
return "messenger-attachments"
def redis_acl(admin_password: str, api_password: str) -> str:
"""ACL Redis 7. api: только ключи session:* и ratelimit:*, только нужные команды, без pub/sub."""
sha = lambda value: hashlib.sha256(value.encode("utf-8")).hexdigest() # noqa: E731
api_commands = " ".join("+" + c for c in (
"get", "set", "del", "incr", "incrby", "pexpire", "pttl", "ping", "multi", "exec", "discard", "hello", "auth"))
return (
"user default off resetkeys resetchannels -@all\n"
f"user admin on #{sha(admin_password)} ~* &* +@all\n"
f"user api on #{sha(api_password)} resetkeys ~session:* ~ratelimit:* resetchannels -@all {api_commands}\n"
)
def s3_config(bucket: str, keys: dict) -> str:
"""SeaweedFS identities. Действия с суффиксом :<bucket>/<prefix>* действуют только в этом prefix
(проверено: api не пишет в attachments/, worker не пишет в staging/ чужими ключами и т.д.).
Write включает PUT и DELETE; List — листинг bucket; Admin — создание bucket и пр. — только у s3-init."""
b = bucket
actions = {
"s3-init": ["Admin", "Read", "Write", "List", "Tagging"],
# API: принимает исходник в staging/ (PUT, откат DELETE), отдаёт готовые файлы (GET).
"api": [f"Write:{b}/staging/*", f"Read:{b}/attachments/*", f"Read:{b}/originals/*"],
# Worker: читает исходник (staging/ или lab4 originals/), пишет результат в attachments/ и может
# читать/листать СВОЙ выходной prefix (проверка результата; исходники он и так читает — новой
# конфиденциальной информации это не даёт), удаляет обработанный исходник. Пользователей, сессий,
# чужих prefix-листингов и администрирования bucket у него нет.
"worker": [f"Read:{b}/staging/*", f"Read:{b}/originals/*", f"Write:{b}/attachments/*",
f"Read:{b}/attachments/*", f"List:{b}/attachments/*",
f"Write:{b}/staging/*", f"Write:{b}/originals/*"],
# Janitor: сверка bucket с БД — листинг и удаление сирот; читать содержимое ему не нужно.
"janitor": [f"List:{b}", f"Write:{b}"],
}
identities = [{"name": name, "credentials": [{"accessKey": keys[name][0], "secretKey": keys[name][1]}],
"actions": actions[name]} for name in actions]
return json.dumps({"identities": identities}, indent=2)
def rabbit_definitions() -> str:
"""Топология и пользователи RabbitMQ. Пользователи БЕЗ пароля: вход только механизмом EXTERNAL по CN
клиентского сертификата (infra/rabbitmq.conf). Права (configure / write / read, regex по именам):
dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет;
worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет.
"""
queue_args = {
"x-queue-type": "quorum",
"x-delivery-limit": 20,
"x-dead-letter-exchange": DLX,
"x-dead-letter-routing-key": DLQ_ROUTING_KEY,
"x-dead-letter-strategy": "at-least-once",
"x-overflow": "reject-publish",
"x-max-length": 100000,
}
definitions = {
# password_hash "" — у пользователя нет пароля, PLAIN/AMQPLAIN для него невозможны.
"users": [{"name": name, "password_hash": "", "hashing_algorithm": "rabbit_password_hashing_sha256",
"tags": []} for name in ("dispatcher", "worker")],
"vhosts": [{"name": BROKER_VHOST}],
"permissions": [
{"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$",
"write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"},
{"user": "worker", "vhost": BROKER_VHOST, "configure": "^$",
"write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"},
],
"exchanges": [
{"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
"internal": False, "arguments": {}},
{"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
"internal": False, "arguments": {}},
],
"queues": [
{"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args},
{"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False,
"arguments": {"x-queue-type": "classic"}},
],
"bindings": [
{"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue",
"routing_key": ROUTING_KEY, "arguments": {}},
{"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue",
"routing_key": DLQ_ROUTING_KEY, "arguments": {}},
],
}
return json.dumps(definitions, indent=2)
def main() -> None:
SECRETS.mkdir(exist_ok=True)
OPERATOR.mkdir(exist_ok=True)
for directory in (SECRETS, OPERATOR):
directory.chmod(0o700)
created = []
app_file = OPERATOR / "app-secrets.json"
loaded_marker = OPERATOR / "app-secrets.loaded"
if not (SECRETS / "postgres_password").exists():
owner = token()
write(SECRETS / "postgres_password", owner)
write(OPERATOR / "postgres_owner_password", owner, private=True)
created.append("postgres_password")
if not (SECRETS / "s3_config.json").exists():
if loaded_marker.exists():
raise SystemExit("secrets/s3_config.json is missing but app secrets were already loaded into OpenBao: "
"restore it or reset the stand (see REPORT.md, «Сброс»)")
app = {
"api": {"db_password": token(), "redis_password": token()},
"worker": {"db_password": token()},
"dispatcher": {"db_password": token()},
"janitor": {"db_password": token()},
"migrate": {"db_owner_password": read(SECRETS / "postgres_password")},
"s3-init": {},
}
s3_keys = {}
for name in ("s3-init", "api", "worker", "janitor"):
s3_keys[name] = (name.replace("-", "") + "-" + secrets.token_hex(8), token())
app["s3-init"].update(s3_admin_access_key_id=s3_keys["s3-init"][0],
s3_admin_secret_access_key=s3_keys["s3-init"][1])
for name in ("api", "worker", "janitor"):
app[name].update(s3_access_key_id=s3_keys[name][0], s3_secret_access_key=s3_keys[name][1])
redis_admin = token()
write(SECRETS / "s3_config.json", s3_config(bucket_name(), s3_keys))
write(SECRETS / "redis" / "users.acl", redis_acl(redis_admin, app["api"]["redis_password"]))
write(OPERATOR / "redis_admin_password", redis_admin, private=True)
write(app_file, json.dumps(app, indent=2), private=True)
created += ["s3_config.json", "redis/users.acl", "operator/app-secrets.json", "operator/redis_admin_password"]
if not (SECRETS / "rabbitmq_definitions.json").exists():
write(SECRETS / "rabbitmq_definitions.json", rabbit_definitions())
created.append("rabbitmq_definitions.json")
if not (SECRETS / "grafana_admin_password").exists():
write(SECRETS / "grafana_admin_password", secrets.token_urlsafe(18))
created.append("grafana_admin_password")
if (SECRETS / "redis").exists():
(SECRETS / "redis").chmod(0o755)
if created:
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
else:
print("secrets/ is complete, nothing changed.")
if app_file.exists():
print("Next: start openbao, then python scripts/bao.py init && python scripts/bao.py unseal "
"&& python scripts/bao.py configure (loads operator/app-secrets.json into OpenBao and deletes it).")
if __name__ == "__main__":
main()
+334
View File
@@ -0,0 +1,334 @@
"""Локальный учебный PKI оператора (lab7): корневой CA, сертификаты сервисов, ротация, отзыв (CRL).
Запускает ОПЕРАТОР на своей машине (не контейнер приложения):
python -m pip install -r requirements-ops.txt # cryptography
python scripts/pki.py init # CA + все сертификаты сервисов + тестовые (идемпотентно)
python scripts/pki.py status # сроки, серийные номера, отозванные
python scripts/pki.py rotate lb # новый leaf той же identity, старый -> secrets/pki/lb/previous/
python scripts/pki.py revoke lb --previous # отозвать предыдущий leaf (CRL), текущий остаётся
python scripts/pki.py crl # перевыпустить CRL (nextUpdate через CRL_DAYS)
Раскладка (вся папка secrets/ в .gitignore, ничего отсюда не коммитится и не попадает в образ):
secrets/pki/ca/ca.key закрытый ключ CA — ТОЛЬКО у оператора, ни в один контейнер не монтируется
secrets/pki/ca/revoked.json серийные номера отозванных сертификатов (источник для CRL)
secrets/pki/public/ca.crt корневой сертификат: его можно раздавать (клиенты, тесты, браузер)
secrets/pki/public/crl.pem список отзыва, подписан CA; его читают API, HAProxy, OpenBao cert auth
secrets/pki/<identity>/tls.crt leaf-сертификат сервиса, tls.key — его ключ, tls.pem — crt+key (HAProxy)
secrets/pki/test/... сертификаты ТОЛЬКО для отрицательных тестов (чужой CA, чужая identity, истёкший)
Идентичность сервиса = Subject CN (и URI SAN spiffe://messenger.lab/<cn>). Её проверяют получатели:
API пускает только CN=lb, otel-gate — только CN приложений, RabbitMQ сопоставляет CN с пользователем,
OpenBao cert auth выдаёт policy по CN. «Подписан нашим CA» — необходимое, но не достаточное условие.
Ключи — ECDSA P-256. Leaf живёт LEAF_DAYS (по умолчанию 30 суток), CA — CA_DAYS (365).
Значения ключей скрипт не печатает.
"""
import argparse
import datetime as dt
import ipaddress
import json
import os
import shutil
import sys
from pathlib import Path
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
ROOT = Path(__file__).resolve().parents[1]
PKI = Path(os.environ.get("PKI_DIR", ROOT / "secrets" / "pki"))
CA_DIR, PUBLIC, TEST = PKI / "ca", PKI / "public", PKI / "test"
TRUST_DOMAIN = "messenger.lab"
CA_DAYS = int(os.environ.get("CA_DAYS", "365"))
LEAF_DAYS = int(os.environ.get("LEAF_DAYS", "30"))
CRL_DAYS = int(os.environ.get("CRL_DAYS", "30"))
SERVER, CLIENT = "server", "client"
# identity -> (DNS-имена сервера, роли). Имена DNS = имена сервисов в compose.yaml.
LOCAL = ["localhost"]
LOCAL_IP = ["127.0.0.1", "::1"]
IDENTITIES = {
# внешний вход: сертификат, который видит клиент (браузер/тесты) на https://localhost:8443
"ingress": (["localhost", "lb"], LOCAL_IP, {SERVER}),
# клиентский сертификат балансировщика для mTLS LB -> API (единственная разрешённая identity у API)
"lb": ([], [], {CLIENT}),
"api": (["api"], [], {SERVER, CLIENT}),
"worker": ([], [], {CLIENT}),
"dispatcher": ([], [], {CLIENT}),
"janitor": ([], [], {CLIENT}),
"migrate": ([], [], {CLIENT}),
"s3-init": ([], [], {CLIENT}),
"postgres": (["postgres"], [], {SERVER}),
"redis": (["redis"], [], {SERVER, CLIENT}), # client: healthcheck redis-cli внутри контейнера
"s3": (["s3"], [], {SERVER, CLIENT}), # client: внутренний gRPC SeaweedFS (security.toml)
"rabbitmq": (["rabbitmq"], [], {SERVER}),
"otel-collector": (["otel-collector", "otel-gate"], [], {SERVER, CLIENT}),
"prometheus": (["prometheus"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
"loki": (["loki"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
"tempo": (["tempo"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
"grafana": (["grafana"] + LOCAL, LOCAL_IP, {SERVER, CLIENT}),
"openbao": (["openbao"] + LOCAL, LOCAL_IP, {SERVER}),
# оператор/скрипты доказательств: чтение Prometheus/Loki/Tempo, отрицательные тесты
"operator": ([], [], {CLIENT}),
}
def now() -> dt.datetime:
return dt.datetime.now(dt.timezone.utc)
def write(path: Path, data: bytes, private: bool = False) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
# Запись в тот же inode (а не replace): файл может быть смонтирован в контейнер по отдельности.
with open(path, "wb") as file:
file.write(data)
# 0644: ключ сервиса читает пользователь контейнера (не root, другой uid). От остальных пользователей
# хоста ключи закрывает каталог secrets/ (0700, см. protect()).
path.chmod(0o600 if private else 0o644)
def protect() -> None:
for directory in (PKI.parent, PKI, CA_DIR):
if directory.exists():
directory.chmod(0o700)
if (CA_DIR / "ca.key").exists():
(CA_DIR / "ca.key").chmod(0o600)
def new_key() -> ec.EllipticCurvePrivateKey:
return ec.generate_private_key(ec.SECP256R1())
def key_pem(key) -> bytes:
return key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
serialization.NoEncryption())
def cert_pem(cert) -> bytes:
return cert.public_bytes(serialization.Encoding.PEM)
def load_ca():
key = serialization.load_pem_private_key((CA_DIR / "ca.key").read_bytes(), None)
cert = x509.load_pem_x509_certificate((PUBLIC / "ca.crt").read_bytes())
return key, cert
def make_ca(common_name: str, days: int):
key = new_key()
name = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI messenger lab"),
x509.NameAttribute(NameOID.COMMON_NAME, common_name)])
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now() - dt.timedelta(minutes=5)).not_valid_after(now() + dt.timedelta(days=days))
.add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True)
.add_extension(x509.KeyUsage(digital_signature=False, content_commitment=False, key_encipherment=False,
data_encipherment=False, key_agreement=False, key_cert_sign=True,
crl_sign=True, encipher_only=False, decipher_only=False), critical=True)
.add_extension(x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False)
.sign(key, hashes.SHA256()))
return key, cert
def make_leaf(identity: str, dns: list, ips: list, roles: set, ca_key, ca_cert, *,
not_before: dt.datetime | None = None, not_after: dt.datetime | None = None):
key = new_key()
start = not_before or (now() - dt.timedelta(minutes=5))
end = not_after or (now() + dt.timedelta(days=LEAF_DAYS))
sans = [x509.UniformResourceIdentifier(f"spiffe://{TRUST_DOMAIN}/{identity}")]
sans += [x509.DNSName(name) for name in dns]
sans += [x509.IPAddress(ipaddress.ip_address(ip)) for ip in ips]
usages = []
if SERVER in roles:
usages.append(ExtendedKeyUsageOID.SERVER_AUTH)
if CLIENT in roles:
usages.append(ExtendedKeyUsageOID.CLIENT_AUTH)
cert = (x509.CertificateBuilder()
.subject_name(x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI messenger lab"),
x509.NameAttribute(NameOID.COMMON_NAME, identity)]))
.issuer_name(ca_cert.subject).public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(start).not_valid_after(end)
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
.add_extension(x509.KeyUsage(digital_signature=True, content_commitment=False, key_encipherment=False,
data_encipherment=False, key_agreement=True, key_cert_sign=False,
crl_sign=False, encipher_only=False, decipher_only=False), critical=True)
.add_extension(x509.ExtendedKeyUsage(usages), critical=False)
.add_extension(x509.SubjectAlternativeName(sans), critical=False)
.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_cert.public_key()), critical=False)
.sign(ca_key, hashes.SHA256()))
return key, cert
def store_leaf(directory: Path, key, cert) -> None:
write(directory / "tls.key", key_pem(key), private=False)
write(directory / "tls.crt", cert_pem(cert))
write(directory / "tls.pem", cert_pem(cert) + key_pem(key)) # HAProxy: сертификат и ключ одним файлом
def issue(identity: str, ca_key, ca_cert, directory: Path | None = None) -> x509.Certificate:
dns, ips, roles = IDENTITIES[identity]
key, cert = make_leaf(identity, dns, ips, roles, ca_key, ca_cert)
store_leaf(directory or PKI / identity, key, cert)
return cert
def revoked() -> dict:
path = CA_DIR / "revoked.json"
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
def write_crl(ca_key, ca_cert) -> None:
builder = (x509.CertificateRevocationListBuilder().issuer_name(ca_cert.subject)
.last_update(now() - dt.timedelta(minutes=5)).next_update(now() + dt.timedelta(days=CRL_DAYS))
.add_extension(x509.CRLNumber(int(now().timestamp())), critical=False))
for serial, info in revoked().items():
builder = builder.add_revoked_certificate(
x509.RevokedCertificateBuilder().serial_number(int(serial, 16))
.revocation_date(dt.datetime.fromisoformat(info["revoked_at"])).build())
write(PUBLIC / "crl.pem", builder.sign(ca_key, hashes.SHA256()).public_bytes(serialization.Encoding.PEM))
# ca-bundle для сервисов, которым CRL передаётся вместе с CA одним файлом (не используется сейчас)
def make_test_material(ca_key, ca_cert) -> list:
"""Сертификаты ТОЛЬКО для отрицательных тестов. Ни один сервис их не принимает как свои."""
made = []
if not (TEST / "rogue-ca" / "ca.crt").exists():
rogue_key, rogue_cert = make_ca("Rogue CA (negative tests)", 30)
write(TEST / "rogue-ca" / "ca.key", key_pem(rogue_key))
write(TEST / "rogue-ca" / "ca.crt", cert_pem(rogue_cert))
made.append("test/rogue-ca")
rogue_key = serialization.load_pem_private_key((TEST / "rogue-ca" / "ca.key").read_bytes(), None)
rogue_cert = x509.load_pem_x509_certificate((TEST / "rogue-ca" / "ca.crt").read_bytes())
plans = {
# тот же CN, что у настоящего LB/сервисов, но подписан чужим CA
"rogue-lb": lambda: make_leaf("lb", [], [], {CLIENT}, rogue_key, rogue_cert),
"rogue-worker": lambda: make_leaf("worker", [], [], {CLIENT}, rogue_key, rogue_cert),
# чужой сервер с правильным именем: клиент должен отвергнуть (не наш CA)
"rogue-server": lambda: make_leaf("api", ["api", "localhost", "postgres", "redis", "s3", "rabbitmq",
"otel-collector", "openbao"], LOCAL_IP, {SERVER},
rogue_key, rogue_cert),
# подписан НАШИМ CA, но identity, которой нигде ничего не разрешено
"intruder": lambda: make_leaf("intruder", ["intruder"], [], {SERVER, CLIENT}, ca_key, ca_cert),
# наш CA, CN=lb, но срок действия уже истёк
"expired-lb": lambda: make_leaf("lb", [], [], {CLIENT}, ca_key, ca_cert,
not_before=now() - dt.timedelta(days=10),
not_after=now() - dt.timedelta(days=1)),
}
for name, plan in plans.items():
if not (TEST / name / "tls.crt").exists():
key, cert = plan()
store_leaf(TEST / name, key, cert)
made.append(f"test/{name}")
return made
def cmd_init(_args) -> int:
created = []
if not (CA_DIR / "ca.key").exists():
ca_key, ca_cert = make_ca("MTUSI messenger lab root CA", CA_DAYS)
write(CA_DIR / "ca.key", key_pem(ca_key), private=True)
write(PUBLIC / "ca.crt", cert_pem(ca_cert))
created.append("ca")
ca_key, ca_cert = load_ca()
for identity in IDENTITIES:
if not (PKI / identity / "tls.crt").exists():
issue(identity, ca_key, ca_cert)
created.append(identity)
created += make_test_material(ca_key, ca_cert)
if not (PUBLIC / "crl.pem").exists() or created:
write_crl(ca_key, ca_cert)
protect()
print("pki: created " + (", ".join(created) if created else "nothing (all present)")
+ f"; CA certificate: {PUBLIC / 'ca.crt'} (keys are not printed)")
return 0
def describe(path: Path) -> str:
cert = x509.load_pem_x509_certificate(path.read_bytes())
left = (cert.not_valid_after_utc - now()).total_seconds() / 86400
serial = format(cert.serial_number, "x")
state = "REVOKED" if serial in revoked() else ("EXPIRED" if left < 0 else f"{left:.1f} d left")
cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value
return f"CN={cn:<15} serial={serial[:16]}… not_after={cert.not_valid_after_utc:%Y-%m-%d %H:%M}Z {state}"
def cmd_status(_args) -> int:
print(f"CA: {describe(PUBLIC / 'ca.crt')}")
for identity in IDENTITIES:
path = PKI / identity / "tls.crt"
if path.exists():
print(f" {identity:<15} {describe(path)}")
previous = PKI / identity / "previous" / "tls.crt"
if previous.exists():
print(f" {identity + ' (prev)':<15} {describe(previous)}")
crl = x509.load_pem_x509_crl((PUBLIC / "crl.pem").read_bytes())
print(f"CRL: {len(list(crl))} revoked, next update {crl.next_update_utc:%Y-%m-%d}")
return 0
def cmd_rotate(args) -> int:
"""Новый leaf той же identity. Старый остаётся действительным до явного revoke (плавная замена)."""
ca_key, ca_cert = load_ca()
directory = PKI / args.identity
if (directory / "tls.crt").exists():
previous = directory / "previous"
previous.mkdir(exist_ok=True)
for name in ("tls.crt", "tls.key", "tls.pem"):
shutil.copyfile(directory / name, previous / name)
cert = issue(args.identity, ca_key, ca_cert)
protect()
print(f"pki: {args.identity} rotated, new serial {format(cert.serial_number, 'x')[:16]}…, "
f"previous kept in {directory / 'previous'} until revoke")
return 0
def cmd_revoke(args) -> int:
ca_key, ca_cert = load_ca()
if args.serial:
serial = args.serial.lower()
name = args.identity or "manual"
else:
path = PKI / args.identity / ("previous" if args.previous else "") / "tls.crt"
cert = x509.load_pem_x509_certificate(path.read_bytes())
serial, name = format(cert.serial_number, "x"), args.identity
data = revoked()
data[serial] = {"identity": name, "revoked_at": now().isoformat()}
(CA_DIR / "revoked.json").write_text(json.dumps(data, indent=2), encoding="utf-8")
write_crl(ca_key, ca_cert)
protect()
print(f"pki: revoked {name} serial {serial[:16]}…; CRL updated ({len(data)} revoked)")
return 0
def cmd_crl(_args) -> int:
ca_key, ca_cert = load_ca()
write_crl(ca_key, ca_cert)
print(f"pki: CRL re-signed, next update in {CRL_DAYS} d")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("init").set_defaults(func=cmd_init)
sub.add_parser("status").set_defaults(func=cmd_status)
sub.add_parser("crl").set_defaults(func=cmd_crl)
rotate = sub.add_parser("rotate")
rotate.add_argument("identity", choices=sorted(IDENTITIES))
rotate.set_defaults(func=cmd_rotate)
revoke = sub.add_parser("revoke")
revoke.add_argument("identity", nargs="?", choices=sorted(IDENTITIES))
revoke.add_argument("--previous", action="store_true", help="revoke the leaf kept by the last rotate")
revoke.add_argument("--serial", help="hex serial instead of identity")
revoke.set_defaults(func=cmd_revoke)
args = parser.parse_args()
if args.command == "revoke" and not (args.identity or args.serial):
parser.error("revoke needs an identity or --serial")
return args.func(args)
if __name__ == "__main__":
sys.exit(main())
+19
View File
@@ -0,0 +1,19 @@
@echo off
rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh), lab7.
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
rem Stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, OpenBao, Prometheus, Loki, Tempo), then the operator
rem unseals OpenBao (scripts\bao.py unseal, shares from secrets\operator, nothing is printed), then the
rem collector gate, the application and the balancer. Volumes are kept.
cd /d "%~dp0.."
rem OpenBao first and alone, unsealed right away: while "restart" waits for the slow stores, a sealed OpenBao
rem fails 3 health checks, becomes "unhealthy" and "up --wait" aborts although it is unsealed a moment later.
docker compose restart openbao || exit /b 1
python scripts\bao.py unseal >nul || exit /b 1
docker compose restart postgres redis s3 rabbitmq prometheus loki tempo || exit /b 1
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq openbao prometheus loki tempo || exit /b 1
rem otel-gate alone first: otel-collector lives in its network namespace and cannot start while the gate is stopped
docker compose restart otel-gate || exit /b 1
docker compose restart otel-collector api janitor dispatcher worker grafana || exit /b 1
docker compose restart lb || exit /b 1
docker compose up -d --wait --no-recreate --no-build --no-deps otel-gate api lb dispatcher worker grafana || exit /b 1
exit /b 0
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
# Restart hook для tests/resilience.py --mode persistence (Linux/macOS), lab7.
# Перезапускает хранилища (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ, OpenBao; телеметрия: Prometheus, Loki,
# Tempo), затем OpenBao распечатывается ОПЕРАТОРОМ (scripts/bao.py unseal — доли из secrets/operator, ничего
# не печатается), затем Collector/gate, приложение и балансировщик. Volumes сохраняются (без down -v), ничего
# не пересоздаётся. "restart" не ждёт healthcheck — ждём явно (up --wait --no-recreate).
set -eu
cd "$(dirname "$0")/.."
PY="${PYTHON:-python3}"
# OpenBao — первым и отдельно, распечатка сразу: пока restart ждёт медленные хранилища, sealed OpenBao проваливает
# 3 healthcheck подряд, становится unhealthy, и up --wait падает, хотя через секунду он уже распечатан.
docker compose restart openbao
"$PY" scripts/bao.py unseal >/dev/null
docker compose restart postgres redis s3 rabbitmq prometheus loki tempo
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq openbao prometheus loki tempo
# otel-gate — отдельно и первым: Collector живёт в его network namespace и не стартует, пока gate остановлен
docker compose restart otel-gate
docker compose restart otel-collector api janitor dispatcher worker grafana
docker compose restart lb
docker compose up -d --wait --no-recreate --no-build --no-deps otel-gate api lb dispatcher worker grafana
+126
View File
@@ -0,0 +1,126 @@
"""Скан зависимостей и образов стенда с разбором применимости (lab7). Только stdlib + Docker.
python scripts/scan.py # все образы из compose.yaml + образ приложения + pip-audit
python scripts/scan.py --only app # только образ приложения и его Python-зависимости
Что делает:
1. pip-audit по requirements.txt (база уязвимостей PyPI/OSV) — в одноразовом контейнере python:3.11.13-slim.
2. Trivy (aquasec/trivy) по каждому образу: ОС-пакеты и Python-пакеты образа, severity HIGH/CRITICAL.
3. Черновой разбор применимости для каждой находки:
applicable — Python-зависимость приложения или библиотека, которую загружает процесс
(libc, OpenSSL, zlib, expat, libffi ...): нужен апдейт/обоснование;
not-reachable — пакет есть в образе, но процесс его не исполняет (perl, apt, login, util-linux ...):
корень read-only, shell не вызывается, no-new-privileges, нет capabilities;
third-party-image — находка в образе стороннего сервиса: применимость — по его роли и сети (internal).
и есть ли исправленная версия. Итог — evidence/scan/*.json и evidence/scan/triage.md: это ЧЕРНОВИК для
ручной проверки, а не вердикт — в отчёт идёт только просмотренная таблица.
"""
import argparse
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "scan"
TRIVY = "aquasec/trivy:0.76.2"
PIP_AUDIT = "pip-audit==2.10.1"
APP_IMAGE = "messenger-lab7-api"
# ОС-библиотеки, которые реально загружает процесс CPython приложения (ssl, zlib, json/expat, ctypes, uuid ...)
RUNTIME_LIBS = {"libc6", "libc-bin", "libssl3", "libssl3t64", "openssl", "zlib1g", "libexpat1", "libffi8",
"libbz2-1.0", "liblzma5", "libuuid1", "libgcc-s1", "libstdc++6", "libcrypt1", "libzstd1", "tzdata",
"ca-certificates", "libsqlite3-0", "libgdbm6", "libgdbm6t64", "libncursesw6", "libreadline8",
"libreadline8t64", "libtinfo6", "libnsl2", "libtirpc3", "libtirpc3t64"}
def run(cmd, **kw):
return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, **kw)
def images(only_app):
config = json.loads(run(["docker", "compose", "config", "--format", "json"], check=True).stdout)
found = {APP_IMAGE}
if not only_app:
found |= {svc["image"] for svc in config["services"].values() if svc.get("image")}
return sorted(found)
def pip_audit():
cmd = ["docker", "run", "--rm", "-v", f"{ROOT / 'requirements.txt'}:/r.txt:ro", "python:3.11.13-slim", "sh", "-c",
f"pip install -q {PIP_AUDIT} >/dev/null 2>&1 && pip-audit -r /r.txt --no-deps --disable-pip -f json"]
result = run(cmd)
data = json.loads(result.stdout or "{}")
(OUT / "pip-audit.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
return data
def trivy(image):
cache = Path.home() / ".cache" / "trivy-lab7" # база Trivy — вне репозитория
cache.mkdir(parents=True, exist_ok=True)
cmd = ["docker", "run", "--rm", "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", "-v",
f"{cache}:/root/.cache/trivy", TRIVY, "image", "--quiet", "--format", "json", "--scanners", "vuln",
"--severity", "HIGH,CRITICAL", image]
# Docker socket получает только одноразовый контейнер сканера на хосте оператора, не стенд.
result = run(cmd)
if result.returncode != 0:
return {"error": result.stderr[-500:]}
data = json.loads(result.stdout)
name = image.replace("/", "_").replace(":", "_")
(OUT / f"trivy-{name}.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
return data
def classify(image, result_class, pkg):
if image != APP_IMAGE:
return "third-party-image"
if result_class == "lang-pkgs":
return "applicable"
return "applicable" if pkg in RUNTIME_LIBS else "not-reachable"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--only", choices=["app"])
args = parser.parse_args()
OUT.mkdir(parents=True, exist_ok=True)
lines = ["# Скан зависимостей и образов — черновой разбор (проверить вручную)", ""]
audit = pip_audit()
vulns = [(d["name"], d["version"], v["id"], ",".join(v.get("fix_versions", [])))
for d in audit.get("dependencies", []) for v in d.get("vulns", [])]
lines += [f"## pip-audit requirements.txt: {len(vulns)} находок", ""]
lines += [f"- {n} {ver}: {vid}, исправлено в {fix or '—'}" for n, ver, vid, fix in vulns] or ["- нет"]
lines.append("")
rows = []
for image in images(args.only == "app"):
print(f"trivy {image} ...", flush=True)
data = trivy(image)
if "error" in data:
lines.append(f"## {image}: ошибка сканера: {data['error'][:200]}")
continue
for result in data.get("Results", []):
for v in result.get("Vulnerabilities", []) or []:
rows.append({"image": image, "target": result.get("Target"), "class": result.get("Class"),
"pkg": v["PkgName"], "installed": v.get("InstalledVersion"),
"fixed": v.get("FixedVersion") or "", "id": v["VulnerabilityID"],
"severity": v.get("Severity"), "title": (v.get("Title") or "")[:90],
"applicability": classify(image, result.get("Class"), v["PkgName"])})
(OUT / "trivy-findings.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False), encoding="utf-8")
lines += ["## Trivy (HIGH, CRITICAL)", "", "| Образ | Пакет | Версия | Исправлено в | CVE | Severity | Применимость |",
"| --- | --- | --- | --- | --- | --- | --- |"]
for r in sorted(rows, key=lambda r: (r["image"] != APP_IMAGE, r["applicability"], r["image"], r["pkg"])):
lines.append(f"| {r['image']} | {r['pkg']} | {r['installed']} | {r['fixed'] or '—'} | {r['id']} | "
f"{r['severity']} | {r['applicability']} |")
(OUT / "triage.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
by = {}
for r in rows:
by.setdefault((r["image"], r["applicability"]), 0)
by[(r["image"], r["applicability"])] += 1
for (image, kind), n in sorted(by.items()):
print(f"{image:<45} {kind:<18} {n}")
print(f"pip-audit: {len(vulns)} finding(s); draft triage: {OUT / 'triage.md'}")
return 0
if __name__ == "__main__":
sys.exit(main())
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh).
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py.
python "%~dp0stop_one.py"
exit /b %ERRORLEVEL%
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID.
# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd.
set -eu
exec python3 "$(dirname "$0")/stop_one.py"
+41
View File
@@ -0,0 +1,41 @@
"""Failover hook (tests/resilience.py --mode failover): остановить ОДНУ реплику API по X-Instance-Id.
lab7: X-Instance-Id — случайный alias процесса, а не ID контейнера/hostname. Каждая реплика пишет его в
tmpfs-файл /tmp/instance-id (settings.INSTANCE_ID_FILE); hook читает файл в каждом запущенном контейнере api
и останавливает тот, чей alias совпал. Реплика остаётся остановленной (восстановление — отдельно).
"""
import os
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
STOP_TIMEOUT_SECONDS = 8
READ_ALIAS = "import settings; print(open(settings.INSTANCE_ID_FILE).read().strip())"
def main() -> int:
target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower()
if not re.fullmatch(r"[0-9a-f]{8,64}", target):
print("TARGET_INSTANCE_ID must be the X-Instance-Id alias of an api replica", file=sys.stderr)
return 2
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
capture_output=True, text=True, check=True).stdout.split()
matches = []
for container in running:
alias = subprocess.run(["docker", "exec", container, "python", "-c", READ_ALIAS], cwd=ROOT,
capture_output=True, text=True, check=False).stdout.strip()
if alias == target:
matches.append(container)
if len(matches) != 1:
print(f"no running api container with alias {target}", file=sys.stderr)
return 1
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
print(f"stopped api container {matches[0][:12]} (alias {target})")
return 0
if __name__ == "__main__":
sys.exit(main())
+8
View File
@@ -0,0 +1,8 @@
@echo off
rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept).
cd /d "%~dp0.."
if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% )
if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% )
if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% )
echo usage: %0 stop^|start^|status
exit /b 2
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
# Demo hook for the defense: stop or start the worker without touching volumes.
# scripts/worker.sh stop | start | status
set -eu
cd "$(dirname "$0")/.."
case "${1:-}" in
stop) docker compose stop worker ;;
start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;;
status) docker compose ps worker ;;
*) echo "usage: $0 stop|start|status" >&2; exit 2 ;;
esac