forked from mtusi2026/mtusi2026lab7
ЛР 7 — БПИ2303 — Лазарев — на 3
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS. - Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2. - Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1. - TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert, mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape. - OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором, auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login. - Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и пользователи брокера выдаются из OpenBao. - Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе. - Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL, no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API. - Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao, hardening; evidence, в т.ч. прогон из чистого clone.
This commit is contained in:
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/bin/sh
|
||||
# ОПЕРАТОР, один раз на новом volume baodata (повторный запуск безопасен — синхронизирует
|
||||
# policies/роли/секреты):
|
||||
# 1) operator init: 3 unseal shares, порог 2; root token и shares -> $OPERATOR_DIR/openbao/init.json
|
||||
# (каталог оператора вне проекта, права 600). На экран ничего секретного не выводится;
|
||||
# 2) unseal (scripts/bao-unseal.sh);
|
||||
# 3) с root token (только в env этого процесса и `docker compose exec -e BAO_TOKEN`):
|
||||
# KV v2 на kv/, policies из infra/openbao/policies,
|
||||
# auth/cert с ролью на каждый сервис (allowed_dns_sans = имя сервиса, TTL токена 15 мин,
|
||||
# max 1 ч), запись сервисных секретов kv/messenger/<svc> из secrets/ (через stdin).
|
||||
# Приложениям root token, unseal shares и BAO_TOKEN не передаются никогда.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
OPERATOR_DIR=${OPERATOR_DIR:-$HOME/.mtusi2026lab7-operator}
|
||||
umask 077
|
||||
mkdir -p "$OPERATOR_DIR/openbao"
|
||||
chmod 700 "$OPERATOR_DIR" "$OPERATOR_DIR/openbao"
|
||||
INIT=$OPERATOR_DIR/openbao/init.json
|
||||
|
||||
docker compose up -d openbao >/dev/null
|
||||
i=0
|
||||
until docker compose exec -T openbao bao status -format=json >/dev/null 2>&1 || [ $? -eq 2 ]; do
|
||||
i=$((i+1)); [ $i -gt 60 ] && { echo "openbao does not respond" >&2; exit 3; }
|
||||
sleep 1
|
||||
done
|
||||
initialized=$(docker compose exec -T openbao bao status -format=json 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['initialized'])" || true)
|
||||
if [ "$initialized" != "True" ]; then
|
||||
[ -s "$INIT" ] && { echo "$INIT exists but OpenBao is not initialized: refusing to overwrite operator material" >&2; exit 5; }
|
||||
docker compose exec -T openbao bao operator init -key-shares=3 -key-threshold=2 -format=json > "$INIT.tmp"
|
||||
mv "$INIT.tmp" "$INIT"
|
||||
chmod 600 "$INIT"
|
||||
echo "openbao initialized; root token and 3 unseal shares saved to $INIT (operator only)"
|
||||
fi
|
||||
./scripts/bao-unseal.sh
|
||||
|
||||
BAO_TOKEN=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['root_token'])" "$INIT")
|
||||
export BAO_TOKEN
|
||||
bao() { docker compose exec -T -e BAO_TOKEN openbao bao "$@"; }
|
||||
|
||||
bao secrets list -format=json | grep -q '"kv/"' || bao secrets enable -path=kv -version=2 kv >/dev/null
|
||||
bao auth list -format=json | grep -q '"cert/"' || bao auth enable cert >/dev/null
|
||||
|
||||
SERVICES="api worker relay janitor migrate s3-init ops"
|
||||
for svc in $SERVICES; do
|
||||
bao policy write "messenger-$svc" - < "infra/openbao/policies/messenger-$svc.hcl" >/dev/null
|
||||
ttl=15m; max=1h
|
||||
case $svc in migrate|s3-init) ttl=5m; max=10m ;; esac
|
||||
# Роль cert auth: доверяем ТОЛЬКО internal-ca и ТОЛЬКО сертификату с SAN DNS = имя сервиса.
|
||||
bao write "auth/cert/certs/$svc" display_name="$svc" token_policies="messenger-$svc" \
|
||||
certificate=@/run/secrets/internal-ca.crt allowed_dns_sans="$svc" \
|
||||
token_ttl="$ttl" token_max_ttl="$max" token_type=service >/dev/null
|
||||
done
|
||||
|
||||
# Сервисные секреты -> KV. JSON собирается из secrets/ и идёт в `bao kv put ... -` через stdin.
|
||||
for svc in $SERVICES; do
|
||||
python3 - "$svc" <<'PY' | bao kv put -mount=kv "messenger/$svc" - >/dev/null
|
||||
import json, sys
|
||||
svc = sys.argv[1]
|
||||
r = lambda n: open("secrets/" + n).read().strip()
|
||||
amqps = lambda u: "amqps://%s:%s@broker:5671/messenger" % (u, r("broker_%s_password" % u))
|
||||
s3 = lambda who: {"S3_ACCESS_KEY_ID": r("s3_%s_access_key" % who), "S3_SECRET_ACCESS_KEY": r("s3_%s_secret_key" % who)}
|
||||
data = {
|
||||
"api": lambda: {"POSTGRES_PASSWORD": r("db_api_password"), **s3("api")},
|
||||
"worker": lambda: {"POSTGRES_PASSWORD": r("db_worker_password"), "BROKER_URL": amqps("worker"), **s3("worker")},
|
||||
"relay": lambda: {"POSTGRES_PASSWORD": r("db_relay_password"), "BROKER_URL": amqps("relay")},
|
||||
"janitor": lambda: {"POSTGRES_PASSWORD": r("db_janitor_password"), **s3("janitor")},
|
||||
"migrate": lambda: {"POSTGRES_PASSWORD": r("postgres_password"),
|
||||
**{"DB_%s_PASSWORD" % w.upper(): r("db_%s_password" % w)
|
||||
for w in ("api", "worker", "relay", "janitor", "ops")}},
|
||||
"s3-init": lambda: s3("admin"),
|
||||
"ops": lambda: {"POSTGRES_PASSWORD": r("db_ops_password"), "BROKER_URL": amqps("ops"), **s3("janitor")},
|
||||
}[svc]()
|
||||
json.dump(data, sys.stdout)
|
||||
PY
|
||||
done
|
||||
unset BAO_TOKEN
|
||||
echo "openbao configured: kv v2 'kv/', audit file->stdout, auth/cert roles: $SERVICES"
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# ОПЕРАТОР: unseal OpenBao после старта/рестарта контейнера (file storage -> процесс sealed).
|
||||
# Берёт ровно threshold (2 из 3) unseal shares из каталога оператора $OPERATOR_DIR/openbao
|
||||
# (вне проекта, 600, никогда не монтируется в контейнеры) и передаёт их через stdin — значения не
|
||||
# появляются ни в выводе, ни в аргументах процессов. Повторный вызов на unsealed OpenBao — no-op.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
OPERATOR_DIR=${OPERATOR_DIR:-$HOME/.mtusi2026lab7-operator}
|
||||
INIT=$OPERATOR_DIR/openbao/init.json
|
||||
[ -s "$INIT" ] || { echo "no $INIT: run scripts/bao-init.sh first" >&2; exit 2; }
|
||||
# Ждём, пока процесс OpenBao начнёт отвечать (после рестарта контейнера).
|
||||
i=0
|
||||
until docker compose exec -T openbao bao status -format=json >/dev/null 2>&1 || [ $? -eq 2 ]; do
|
||||
i=$((i+1)); [ $i -gt 60 ] && { echo "openbao does not respond" >&2; exit 3; }
|
||||
sleep 1
|
||||
done
|
||||
sealed() { docker compose exec -T openbao bao status -format=json 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['sealed'])"; }
|
||||
if [ "$(sealed)" = "False" ]; then
|
||||
echo "openbao already unsealed"; exit 0
|
||||
fi
|
||||
for idx in 0 1; do
|
||||
python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['unseal_keys_b64'][int(sys.argv[2])])" "$INIT" "$idx" \
|
||||
| docker compose exec -T openbao bao write -format=json sys/unseal key=- >/dev/null
|
||||
done
|
||||
[ "$(sealed)" = "False" ] && echo "openbao unsealed (2 of 3 shares)" || { echo "unseal failed" >&2; exit 4; }
|
||||
Executable
+323
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Генератор импортируемого dashboard Grafana: infra/grafana/dashboards/messenger.json.
|
||||
|
||||
JSON лежит в Git и подхватывается provisioning-ом; этот скрипт — воспроизводимый способ его
|
||||
изменить (python3 scripts/build_dashboard.py). Только стандартная библиотека.
|
||||
Все запросы ссылаются на реальные имена метрик (см. REPORT.md «Схема сигналов»).
|
||||
Перцентили — histogram_quantile по sum by (le) buckets всех реплик, не среднее p95.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
|
||||
PROM = {"type": "prometheus", "uid": "prometheus"}
|
||||
LOKI = {"type": "loki", "uid": "loki"}
|
||||
TEMPO = {"type": "tempo", "uid": "tempo"}
|
||||
API = 'job="messenger-api", instance=~"$instance"'
|
||||
BUSINESS = API + ', http_route!~"/health/.*|unmatched"'
|
||||
|
||||
_panels = []
|
||||
_id = [0]
|
||||
_pos = {"x": 0, "y": 0, "h": 0}
|
||||
|
||||
|
||||
def _next_id():
|
||||
_id[0] += 1
|
||||
return _id[0]
|
||||
|
||||
|
||||
def place(w, h):
|
||||
if _pos["x"] + w > 24:
|
||||
_pos["x"] = 0
|
||||
_pos["y"] += _pos["h"]
|
||||
_pos["h"] = 0
|
||||
g = {"x": _pos["x"], "y": _pos["y"], "w": w, "h": h}
|
||||
_pos["x"] += w
|
||||
_pos["h"] = max(_pos["h"], h)
|
||||
return g
|
||||
|
||||
|
||||
def row(title):
|
||||
_pos["x"] = 0
|
||||
_pos["y"] += _pos["h"]
|
||||
_pos["h"] = 0
|
||||
_panels.append({"type": "row", "id": _next_id(), "title": title, "collapsed": False,
|
||||
"gridPos": {"x": 0, "y": _pos["y"], "w": 24, "h": 1}, "panels": []})
|
||||
_pos["y"] += 1
|
||||
|
||||
|
||||
def target(expr, legend="", ref="A", exemplar=False, instant=False):
|
||||
t = {"datasource": PROM, "expr": expr, "legendFormat": legend or "__auto", "refId": ref,
|
||||
"range": not instant, "instant": instant}
|
||||
if exemplar:
|
||||
t["exemplar"] = True
|
||||
return t
|
||||
|
||||
|
||||
def ts(title, targets, unit="short", w=8, h=8, description="", stack=False, min0=True, thresholds=None):
|
||||
refs = "ABCDEFGHIJ"
|
||||
tg = [target(*t[:2], ref=refs[i], exemplar=(len(t) > 2 and t[2])) for i, t in enumerate(targets)]
|
||||
p = {"type": "timeseries", "id": _next_id(), "title": title, "description": description,
|
||||
"datasource": PROM, "gridPos": place(w, h), "targets": tg,
|
||||
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"stacking": {"mode": "normal" if stack else "none"}},
|
||||
**({"min": 0} if min0 else {})},
|
||||
"overrides": []},
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}}}
|
||||
if thresholds:
|
||||
p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds}
|
||||
p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line+area"}
|
||||
_panels.append(p)
|
||||
|
||||
|
||||
def stat(title, expr, unit="short", w=3, h=4, steps=None, description="", decimals=None):
|
||||
p = {"type": "stat", "id": _next_id(), "title": title, "description": description, "datasource": PROM,
|
||||
"gridPos": place(w, h), "targets": [target(expr, "", instant=True)],
|
||||
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps or [
|
||||
{"color": "green", "value": None}]}}, "overrides": []},
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"colorMode": "background", "graphMode": "area", "textMode": "value"}}
|
||||
if decimals is not None:
|
||||
p["fieldConfig"]["defaults"]["decimals"] = decimals
|
||||
_panels.append(p)
|
||||
|
||||
|
||||
def build():
|
||||
# ------------------------------------------------------------------ обзор
|
||||
row("Обзор и SLO")
|
||||
stat("RPS бизнес-запросов", "messenger:http_requests:rate2m", "reqps",
|
||||
description="sum(rate(http_server_request_duration_seconds_count{job=messenger-api, без /health}[2m]))")
|
||||
stat("5xx, доля (2m)", "messenger:http_error_ratio:rate2m", "percentunit", decimals=2,
|
||||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.01}, {"color": "red", "value": 0.05}],
|
||||
description="Числитель: 5xx бизнес-запросов; знаменатель: все бизнес-запросы (без /health, unmatched).")
|
||||
stat("p95 API (5m)", "messenger:http_latency_seconds:p95_5m", "s", decimals=3,
|
||||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 0.5}, {"color": "red", "value": 1}])
|
||||
stat("SLO доступности, 7d", "messenger:slo_availability:ratio_7d", "percentunit", decimals=3,
|
||||
steps=[{"color": "red", "value": None}, {"color": "green", "value": 0.995}],
|
||||
description="SLO: 99.5 % бизнес-запросов без 5xx за скользящие 7 дней.")
|
||||
stat("Остаток error budget, 7d", "messenger:slo_availability:error_budget_remaining_7d", "percentunit",
|
||||
decimals=1, steps=[{"color": "red", "value": None}, {"color": "orange", "value": 0.25},
|
||||
{"color": "green", "value": 0.5}])
|
||||
stat("Time to ready p95", "messenger:time_to_ready_seconds:p95_5m", "s", decimals=2,
|
||||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
|
||||
description="От durable acceptance (COMMIT enqueue) до COMMIT ready. SLO: 99 % не дольше 30 с.")
|
||||
stat("Ready lag очереди", "max(messenger_jobs_ready_lag_seconds) or vector(0)", "s", decimals=0,
|
||||
steps=[{"color": "green", "value": None}, {"color": "orange", "value": 10}, {"color": "red", "value": 30}],
|
||||
description="Сколько ждёт worker самое старое созревшее задание (PostgreSQL jobs).")
|
||||
stat("Alerts firing", 'count(ALERTS{alertstate="firing"}) or vector(0)', "short",
|
||||
steps=[{"color": "green", "value": None}, {"color": "red", "value": 1}])
|
||||
_panels.append({
|
||||
"type": "state-timeline", "id": _next_id(), "title": "История alerts (pending/firing -> resolved)",
|
||||
"datasource": PROM, "gridPos": place(24, 6),
|
||||
"targets": [target('max by (alertname, source, alertstate) (ALERTS)', "{{alertname}} {{source}} {{alertstate}}")],
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"},
|
||||
"thresholds": {"mode": "absolute", "steps": [
|
||||
{"color": "transparent", "value": None}, {"color": "red", "value": 1}]}},
|
||||
"overrides": []},
|
||||
"options": {"showValue": "never", "mergeValues": True, "rowHeight": 0.8,
|
||||
"legend": {"showLegend": False}}})
|
||||
|
||||
# ------------------------------------------------------------------ API
|
||||
row("API: трафик, ошибки, задержка")
|
||||
ts("RPS по маршруту", [("sum by (http_route) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % API,
|
||||
"{{http_route}}")], "reqps", stack=True)
|
||||
ts("Ответы по классу статуса", [(
|
||||
"sum by (http_response_status_class) (rate(http_server_request_duration_seconds_count{%s}[1m]))" % BUSINESS,
|
||||
"{{http_response_status_class}}")], "reqps", stack=True)
|
||||
ts("Доля 5xx: API и LB", [
|
||||
("messenger:http_error_ratio:rate2m", "API (без /health)"),
|
||||
("messenger:lb_error_ratio:rate2m", "HAProxy fe_api")], "percentunit",
|
||||
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 0.05}],
|
||||
description="Если БД недоступна, реплики не ready и 503 отдаёт LB: эти запросы видны только в метрике LB.")
|
||||
ts("Задержка API p50/p95/p99 (из histogram)", [
|
||||
("histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p50", True),
|
||||
("histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p95", True),
|
||||
("histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{%s}[1m])))" % BUSINESS, "p99", True)],
|
||||
"s", w=12, description="Точки-exemplars ведут в трассу Tempo конкретного запроса.")
|
||||
ts("p95 по маршруту", [(
|
||||
"histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{%s}[5m])))" % BUSINESS,
|
||||
"{{http_route}}")], "s", w=12)
|
||||
ts("5xx по причине (error.type)", [(
|
||||
'sum by (error_type, http_route) (rate(http_server_request_duration_seconds_count{%s, http_response_status_class="5xx"}[1m]))' % API,
|
||||
"{{error_type}} {{http_route}}")], "reqps", stack=True)
|
||||
ts("Запросы в обработке (saturation API)", [
|
||||
("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")], "short")
|
||||
ts("HAProxy: UP-реплики и очередь LB", [
|
||||
('haproxy_backend_active_servers{proxy="be_api"}', "UP реплик"),
|
||||
('haproxy_backend_current_queue{proxy="be_api"}', "очередь LB")], "short")
|
||||
|
||||
# ------------------------------------------------------------------ worker
|
||||
row("Worker и очередь заданий")
|
||||
ts("Задания по исходу (в минуту)", [(
|
||||
"sum by (messenger_job_outcome) (increase(messenger_jobs_processed_total[1m]))", "{{messenger_job_outcome}}")],
|
||||
"short", stack=True, description="ready / retry / failed / duplicate / lease_lost / rejected / requeued")
|
||||
ts("Длительность обработки job p50/p95/p99", [
|
||||
('histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p50", True),
|
||||
('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p95", True),
|
||||
('histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{messenger_job_outcome="ready"}[5m])))', "p99", True)],
|
||||
"s")
|
||||
ts("Time to ready p50/p95/p99", [
|
||||
('histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p50", True),
|
||||
('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p95", True),
|
||||
('histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{outcome="ready"}[5m])))', "p99", True)],
|
||||
"s", description="Backlog-ожидание + обработка: время от 202 (COMMIT) до ready.")
|
||||
ts("Глубина очереди: PostgreSQL и RabbitMQ", [
|
||||
('max by (messenger_job_status) (messenger_jobs_count{messenger_job_status=~"queued|processing"})', "jobs {{messenger_job_status}} (БД)"),
|
||||
('max(messenger_outbox_pending)', "outbox не опубликовано"),
|
||||
('sum(rabbitmq_detailed_queue_messages_ready{queue="attachments.process.v1"})', "RabbitMQ ready"),
|
||||
('sum(rabbitmq_detailed_queue_messages_unacked{queue="attachments.process.v1"})', "RabbitMQ unacked"),
|
||||
('sum(rabbitmq_detailed_queue_messages{queue="attachments.dlq"})', "DLQ")], "short")
|
||||
ts("Возраст старейшего задания и lag", [
|
||||
('max by (messenger_job_status) (messenger_jobs_oldest_age_seconds)', "oldest {{messenger_job_status}}"),
|
||||
("max(messenger_jobs_ready_lag_seconds)", "ready lag"),
|
||||
("max(messenger_outbox_oldest_age_seconds)", "outbox oldest")], "s",
|
||||
thresholds=[{"color": "transparent", "value": None}, {"color": "red", "value": 30}])
|
||||
ts("Ожидание в очереди p95 (до claim)", [
|
||||
("histogram_quantile(0.95, sum by (le) (rate(messenger_job_queue_wait_seconds_bucket[5m])))", "p95 queue wait"),
|
||||
("histogram_quantile(0.95, sum by (le) (rate(messenger_outbox_publish_delay_seconds_bucket[5m])))", "p95 outbox -> broker")],
|
||||
"s", description="Backlog (сколько ждут) отдельно от processing latency (сколько обрабатываются).")
|
||||
ts("Saturation worker: занятость потребителей", [
|
||||
("sum(rate(messenger_worker_busy_time_seconds_total[1m])) / sum(last_over_time(messenger_worker_capacity[30s]))", "utilization (busy_time/capacity)"),
|
||||
("sum(messenger_worker_busy) / sum(last_over_time(messenger_worker_capacity[30s]))", "busy/capacity (мгновенно)")],
|
||||
"percentunit", description="1.0 = все WORKER_CONCURRENCY потоков всех реплик заняты; очередь растёт быстрее обработки.")
|
||||
ts("Relay: публикации outbox", [
|
||||
("sum by (outcome) (rate(messenger_outbox_published_total[1m]))", "{{outcome}}")], "short")
|
||||
|
||||
# ------------------------------------------------------------------ DB / S3
|
||||
row("PostgreSQL и S3")
|
||||
ts("Пул соединений API: used / idle / max", [
|
||||
('sum by (instance, db_client_connection_state) (db_client_connection_count{%s})' % API,
|
||||
"{{instance}} {{db_client_connection_state}}"),
|
||||
('max(db_client_connection_max{%s})' % API, "max на реплику")], "short")
|
||||
ts("Ожидающие соединение и таймауты пула", [
|
||||
('sum by (instance) (db_client_connection_pending_requests{%s})' % API, "pending {{instance}}"),
|
||||
('sum by (instance) (increase(db_client_connection_timeouts_total{%s}[1m]))' % API, "timeouts/мин {{instance}}")],
|
||||
"short")
|
||||
ts("Ожидание соединения пула p95", [
|
||||
('histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket{%s}[1m])))' % API, "p95 wait")],
|
||||
"s")
|
||||
ts("Операции БД p95 по операции (API)", [
|
||||
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{%s}[1m])))' % API,
|
||||
"{{db_operation_name}}")], "s", w=12)
|
||||
ts("Операции БД worker/relay p95", [
|
||||
('histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{job=~"messenger-worker|messenger-relay"}[5m])))',
|
||||
"{{db_operation_name}}")], "s", w=12)
|
||||
ts("Ошибки операций БД", [
|
||||
('sum by (job, db_operation_name, error_type) (rate(db_client_operation_duration_seconds_count{error_type!=""}[1m]))',
|
||||
"{{job}} {{db_operation_name}} {{error_type}}")], "short")
|
||||
ts("S3 p95 по операции", [
|
||||
('histogram_quantile(0.95, sum by (le, messenger_s3_operation) (rate(messenger_s3_operation_duration_seconds_bucket[5m])))',
|
||||
"{{messenger_s3_operation}}")], "s")
|
||||
ts("S3 ошибки", [
|
||||
('sum by (job, messenger_s3_operation, outcome) (rate(messenger_s3_operation_duration_seconds_count{outcome!="ok"}[1m]))',
|
||||
"{{job}} {{messenger_s3_operation}} {{outcome}}")], "short")
|
||||
|
||||
# ------------------------------------------------------------------ ресурсы
|
||||
row("Ресурсы контейнеров (CPU/RAM)")
|
||||
ts("CPU, ядер (cgroup контейнера)", [
|
||||
("sum by (job, instance) (rate(container_cpu_time_seconds_total[1m]))", "{{job}} {{instance}}")], "short", w=12)
|
||||
ts("CPU saturation: доля лимита", [
|
||||
("rate(container_cpu_time_seconds_total[1m]) / on(job, instance) container_cpu_limit", "{{job}} {{instance}}")],
|
||||
"percentunit", w=12)
|
||||
ts("Память контейнеров приложений", [
|
||||
("container_memory_usage_bytes", "{{job}} {{instance}}")], "bytes", w=12)
|
||||
ts("Память: доля лимита", [
|
||||
("container_memory_usage_bytes / on(job, instance) container_memory_limit_bytes", "{{job}} {{instance}}")],
|
||||
"percentunit", w=12)
|
||||
ts("Память telemetry-стека (RSS)", [
|
||||
('process_resident_memory_bytes{job=~"prometheus|loki|tempo"}', "{{job}}"),
|
||||
('{__name__=~"otelcol_process_memory_rss(_bytes)?"}', "otel-collector RSS (вкл. страницы бинарника)"),
|
||||
('otelcol_process_runtime_heap_alloc_bytes', "otel-collector Go heap"),
|
||||
('rabbitmq_process_resident_memory_bytes', "rabbitmq")], "bytes", w=12)
|
||||
ts("CPU telemetry-стека", [
|
||||
('rate(process_cpu_seconds_total{job=~"prometheus|loki|tempo"}[1m])', "{{job}}"),
|
||||
('rate({__name__=~"otelcol_process_cpu_seconds(_total)?"}[1m])', "otel-collector")], "short", w=12)
|
||||
|
||||
# ------------------------------------------------------------------ telemetry
|
||||
row("Конвейер телеметрии (потери)")
|
||||
ts("Collector: принято / отправлено / потеряно (spans)", [
|
||||
('sum(rate({__name__=~"otelcol_receiver_accepted_spans(_total)?"}[1m]))', "принято"),
|
||||
('sum(rate({__name__=~"otelcol_exporter_sent_spans(_total)?"}[1m]))', "отправлено в Tempo"),
|
||||
('sum(rate({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[1m]))', "ошибка отправки"),
|
||||
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[1m]))', "очередь полна (потеря)"),
|
||||
('sum(rate({__name__=~"otelcol_receiver_refused_spans(_total)?"}[1m]))', "отказ приёма (memory_limiter)")], "short")
|
||||
ts("Collector: логи", [
|
||||
('sum(rate({__name__=~"otelcol_receiver_accepted_log_records(_total)?"}[1m]))', "принято"),
|
||||
('sum(rate({__name__=~"otelcol_exporter_sent_log_records(_total)?"}[1m]))', "отправлено в Loki"),
|
||||
('sum(rate({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[1m]))', "ошибка отправки"),
|
||||
('sum(rate({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?"}[1m]))', "очередь полна (потеря)")], "short")
|
||||
ts("Collector: очередь экспортёров", [
|
||||
("max by (exporter) (otelcol_exporter_queue_size)", "size {{exporter}}"),
|
||||
("max by (exporter) (otelcol_exporter_queue_capacity)", "capacity {{exporter}}")], "short")
|
||||
ts("SDK приложений: потери на стороне процесса", [
|
||||
('sum by (job, error_type) (rate(otel_sdk_processor_span_processed_total{error_type!=""}[1m]))', "spans dropped {{job}} {{error_type}}"),
|
||||
('sum by (job, error_type) (rate(otel_sdk_processor_log_processed_total{error_type!=""}[1m]))', "logs dropped {{job}} {{error_type}}"),
|
||||
('sum by (job, error_type) (rate(otel_sdk_exporter_span_exported_total{error_type!=""}[1m]))', "span export failed {{job}} {{error_type}}")],
|
||||
"short", w=12)
|
||||
ts("SDK: заполнение очереди BatchSpanProcessor", [
|
||||
("max by (job, instance) (otel_sdk_processor_span_queue_size)", "{{job}} {{instance}}"),
|
||||
("max(otel_sdk_processor_span_queue_capacity)", "capacity")], "short", w=12)
|
||||
|
||||
# ------------------------------------------------------------------ logs / traces
|
||||
row("Логи и трассы")
|
||||
_panels.append({
|
||||
"type": "logs", "id": _next_id(), "title": "Логи WARN/ERROR приложений (Loki)", "datasource": LOKI,
|
||||
"gridPos": place(24, 10),
|
||||
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
|
||||
"expr": '{service_name=~"messenger-.+"} | severity_text=~"WARN.*|ERROR.*|CRITICAL"'}],
|
||||
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending",
|
||||
"dedupStrategy": "none", "prettifyLogMessage": False}})
|
||||
_panels.append({
|
||||
"type": "logs", "id": _next_id(), "title": "HTTP 5xx (event=http.request)", "datasource": LOKI,
|
||||
"gridPos": place(12, 10),
|
||||
"targets": [{"datasource": LOKI, "refId": "A", "queryType": "range",
|
||||
"expr": '{service_name="messenger-api"} | event="http.request" | http_response_status_code >= 500'}],
|
||||
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}})
|
||||
_panels.append({
|
||||
"type": "table", "id": _next_id(), "title": "Последние трассы обработки заданий (Tempo)", "datasource": TEMPO,
|
||||
"gridPos": place(12, 10),
|
||||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
|
||||
"query": '{ resource.service.name = "messenger-worker" && name = "attachments.process.v1 process" }'}]})
|
||||
_panels.append({
|
||||
"type": "table", "id": _next_id(), "title": "Медленные запросы API > 1 s (Tempo)", "datasource": TEMPO,
|
||||
"gridPos": place(12, 10),
|
||||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
|
||||
"query": '{ resource.service.name = "messenger-api" && kind = server && duration > 1s }'}]})
|
||||
_panels.append({
|
||||
"type": "table", "id": _next_id(), "title": "Трассы с ошибкой (Tempo)", "datasource": TEMPO,
|
||||
"gridPos": place(12, 10),
|
||||
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20, "tableType": "traces",
|
||||
"query": '{ status = error }'}]})
|
||||
|
||||
return {
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger — наблюдаемость (lab6)",
|
||||
"description": "API, worker, очередь, PostgreSQL, S3, ресурсы, конвейер телеметрии. Provisioning из Git.",
|
||||
"tags": ["messenger", "lab6", "otel"],
|
||||
"timezone": "browser",
|
||||
"editable": False,
|
||||
"graphTooltip": 1,
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"time": {"from": "now-30m", "to": "now"},
|
||||
"templating": {"list": [{
|
||||
"name": "instance", "label": "API-реплика", "type": "query", "datasource": PROM,
|
||||
"query": {"query": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"},
|
||||
"definition": 'label_values(http_server_request_duration_seconds_count{job="messenger-api"}, instance)',
|
||||
"includeAll": True, "multi": True, "allValue": ".*", "current": {"text": "All", "value": "$__all"},
|
||||
"refresh": 2, "sort": 1}]},
|
||||
"annotations": {"list": [{
|
||||
"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": True, "hide": True,
|
||||
"iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard"}]},
|
||||
"panels": _panels,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(json.dumps(build(), ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||
print("written %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(_panels)))
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
# Офлайн-проверка конфигураций телеметрии теми же образами, что и стенд (сеть не нужна):
|
||||
# promtool check config + check rules + unit-тесты alert rules,
|
||||
# otelcol-contrib validate, loki -verify-config.
|
||||
# Использование: scripts/check_rules.sh (код возврата 0 — всё валидно)
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
img() { grep -E "^\s+image: $1" compose.yaml | head -1 | awk '{print $2}'; }
|
||||
PROM=$(img prom/prometheus)
|
||||
OTEL=$(img otel/opentelemetry-collector-contrib)
|
||||
LOKI=$(img grafana/loki)
|
||||
echo "== promtool check config ($PROM)"
|
||||
# Lab7: конфиги ссылаются на TLS-файлы /run/secrets/* — монтируем сертификаты стенда (secrets/pki).
|
||||
pki() { # pki <service> -> -v аргументы для CA и leaf
|
||||
echo "-v $PWD/secrets/pki/internal-ca.crt:/run/secrets/internal-ca.crt:ro -v $PWD/secrets/pki/$1.crt:/run/secrets/service.crt:ro -v $PWD/secrets/pki/$1.key:/run/secrets/service.key:ro"
|
||||
}
|
||||
docker run --rm $(pki prometheus) -v "$PWD/infra/prometheus:/etc/prometheus:ro" --entrypoint promtool "$PROM" check config /etc/prometheus/prometheus.yml
|
||||
echo "== promtool check web-config (TLS + обязательный client cert)"
|
||||
docker run --rm $(pki prometheus) -v "$PWD/infra/prometheus:/etc/prometheus:ro" --entrypoint promtool "$PROM" check web-config /etc/prometheus/web.yml
|
||||
echo "== promtool test rules"
|
||||
docker run --rm -v "$PWD/infra/prometheus:/w:ro" -w /w/tests --entrypoint promtool "$PROM" test rules messenger.test.yml
|
||||
echo "== otelcol validate ($OTEL)"
|
||||
docker run --rm $(pki otel-collector) -v "$PWD/infra/otel/collector.yaml:/c.yaml:ro" "$OTEL" validate --config=/c.yaml && echo "collector config OK"
|
||||
echo "== loki -verify-config ($LOKI)"
|
||||
docker run --rm $(pki loki) -v "$PWD/infra/loki/loki.yaml:/c.yaml:ro" "$LOKI" -config.file=/c.yaml -verify-config && echo "loki config OK"
|
||||
echo "== dashboard JSON is up to date with scripts/build_dashboard.py"
|
||||
tmp=$(mktemp)
|
||||
cp infra/grafana/dashboards/messenger.json "$tmp"
|
||||
python3 scripts/build_dashboard.py >/dev/null
|
||||
cmp -s "$tmp" infra/grafana/dashboards/messenger.json && echo "dashboard OK" || { echo "dashboard JSON differs from generator" >&2; exit 1; }
|
||||
Executable
+135
@@ -0,0 +1,135 @@
|
||||
#!/bin/sh
|
||||
# Lab7: локальные секреты, PKI и .env учебного стенда. Повторный запуск ничего не перезаписывает,
|
||||
# значения не печатаются. Ничего из созданного не попадает в Git (.gitignore: secrets/, *.key, *.pem)
|
||||
# и в образы (.dockerignore).
|
||||
#
|
||||
# Где что лежит:
|
||||
# $OPERATOR_DIR (по умолчанию ~/.mtusi2026lab7-operator, вне проекта, 700):
|
||||
# ca/*.key — закрытые ключи public-ca и internal-ca (не монтируются ни в один контейнер);
|
||||
# openbao/ — root token и unseal shares (создаёт scripts/bao-init.sh).
|
||||
# secrets/ (в проекте, вне Git, 700):
|
||||
# pki/*.crt|*.key — leaf-сертификаты сервисов (каждый контейнер получает ТОЛЬКО свой ключ);
|
||||
# pki/test/ — rogue CA и identity intruder для отрицательных тестов;
|
||||
# *_password, s3_* — исходные значения сервисных credentials. Приложения (api, worker,
|
||||
# relay, janitor, migrate, s3-init, ops) получают их НЕ из этих файлов,
|
||||
# а из OpenBao (scripts/bao-init.sh кладёт их в KV). Файлы нужны серверной
|
||||
# стороне: s3_identities.json (SeaweedFS), rabbitmq_definitions.json (только
|
||||
# хеши паролей), postgres_password (суперпользователь в db).
|
||||
#
|
||||
# S3 (SeaweedFS) — разные ключи и права (secrets/s3_identities.json):
|
||||
# admin — только job s3-init; api — Read+Write; worker — Read+Write; janitor — List+Write.
|
||||
# RabbitMQ — пользователи с минимальными правами в vhost messenger:
|
||||
# relay — только publish в exchange attachments; worker — только consume из attachments.process.v1;
|
||||
# ops — consume из DLQ и publish в attachments. Ни у кого нет configure.
|
||||
# PostgreSQL — отдельные роли api_svc, worker_svc, relay_svc, janitor_svc, ops_svc с грантами по
|
||||
# таблицам (migrations/grants.sql); суперпользователь postgres — только у job migrate.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
OPERATOR_DIR=${OPERATOR_DIR:-$HOME/.mtusi2026lab7-operator}
|
||||
mkdir -p secrets/pki "$OPERATOR_DIR"
|
||||
chmod 700 secrets "$OPERATOR_DIR"
|
||||
gen() { # gen <file> <bytes>
|
||||
if [ ! -s "secrets/$1" ]; then
|
||||
python3 -c "import secrets,sys; print(secrets.token_urlsafe(int(sys.argv[1])))" "$2" > "secrets/$1"
|
||||
echo "created secrets/$1"
|
||||
fi
|
||||
}
|
||||
gen postgres_password 24
|
||||
for who in api worker relay janitor ops; do
|
||||
gen "db_${who}_password" 24
|
||||
done
|
||||
gen grafana_admin_password 18
|
||||
for who in admin api worker janitor; do
|
||||
if [ ! -s "secrets/s3_${who}_access_key" ]; then
|
||||
python3 -c "import secrets; print('MTUSI' + secrets.token_hex(8).upper())" > "secrets/s3_${who}_access_key"
|
||||
echo "created secrets/s3_${who}_access_key"
|
||||
fi
|
||||
gen "s3_${who}_secret_key" 30
|
||||
done
|
||||
for who in relay worker ops; do
|
||||
gen "broker_${who}_password" 24
|
||||
done
|
||||
|
||||
# PKI: OpenSSL закреплённого образа (тот же, на котором собрано приложение).
|
||||
PKI_IMAGE=python:3.12.7-slim@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d
|
||||
docker run --rm --network none --user "$(id -u):$(id -g)" \
|
||||
-e PKI_CA_DAYS -e PKI_LEAF_DAYS \
|
||||
-v "$PWD/secrets/pki:/out" -v "$OPERATOR_DIR:/op" -v "$PWD/scripts/pki-issue.sh:/pki-issue.sh:ro" \
|
||||
"$PKI_IMAGE" sh /pki-issue.sh
|
||||
|
||||
BUCKET=$(grep -E '^S3_BUCKET=' .env 2>/dev/null | cut -d= -f2 || true)
|
||||
BUCKET=${BUCKET:-messenger-attachments}
|
||||
python3 - "$BUCKET" <<'PY'
|
||||
import base64, hashlib, json, os, sys
|
||||
b = sys.argv[1]
|
||||
def read(name):
|
||||
return open("secrets/" + name).read().strip()
|
||||
def ident(name, actions):
|
||||
return {"name": name, "credentials": [{"accessKey": read("s3_%s_access_key" % name),
|
||||
"secretKey": read("s3_%s_secret_key" % name)}], "actions": actions}
|
||||
cfg = {"identities": [
|
||||
ident("admin", ["Admin", "Read", "List", "Tagging", "Write"]),
|
||||
ident("api", ["Read:" + b, "Write:" + b]),
|
||||
ident("worker", ["Read:" + b, "Write:" + b]),
|
||||
ident("janitor", ["List:" + b, "Write:" + b]),
|
||||
]}
|
||||
with open("secrets/s3_identities.json", "w") as f:
|
||||
json.dump(cfg, f, indent=1)
|
||||
|
||||
# RabbitMQ: rabbit_password_hashing_sha256 = base64(salt4 + sha256(salt4 + password)).
|
||||
def rmq_hash(password):
|
||||
salt = os.urandom(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode()
|
||||
vhost = "messenger"
|
||||
users, perms = [], []
|
||||
rights = {
|
||||
"relay": {"configure": "^$", "write": "^attachments$", "read": "^$"},
|
||||
"worker": {"configure": "^$", "write": "^$", "read": r"^attachments\.process\.v1$"},
|
||||
"ops": {"configure": "^$", "write": "^attachments$", "read": r"^attachments\.dlq$"},
|
||||
}
|
||||
for user, r in rights.items():
|
||||
users.append({"name": user, "password_hash": rmq_hash(read("broker_%s_password" % user)),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []})
|
||||
perms.append(dict(user=user, vhost=vhost, **r))
|
||||
x = lambda name, kind: {"name": name, "vhost": vhost, "type": kind, "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}}
|
||||
definitions = {
|
||||
"users": users,
|
||||
"vhosts": [{"name": vhost}],
|
||||
"permissions": perms,
|
||||
"exchanges": [x("attachments", "direct"), x("attachments.dlx", "fanout")],
|
||||
"queues": [
|
||||
{"name": "attachments.process.v1", "vhost": vhost, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "quorum", "x-delivery-limit": 10,
|
||||
"x-dead-letter-exchange": "attachments.dlx",
|
||||
"x-dead-letter-strategy": "at-least-once", "x-overflow": "reject-publish",
|
||||
"x-max-length": 100000}},
|
||||
{"name": "attachments.dlq", "vhost": vhost, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "quorum"}},
|
||||
],
|
||||
"bindings": [
|
||||
{"source": "attachments", "vhost": vhost, "destination": "attachments.process.v1",
|
||||
"destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}},
|
||||
{"source": "attachments.dlx", "vhost": vhost, "destination": "attachments.dlq",
|
||||
"destination_type": "queue", "routing_key": "", "arguments": {}},
|
||||
],
|
||||
}
|
||||
if not os.path.exists("secrets/rabbitmq_definitions.json"):
|
||||
with open("secrets/rabbitmq_definitions.json", "w") as f:
|
||||
json.dump(definitions, f, indent=1)
|
||||
print("created secrets/rabbitmq_definitions.json")
|
||||
PY
|
||||
# Файлы монтируются в контейнеры как Docker secrets (bind mount): uid процесса внутри контейнера
|
||||
# (70 postgres, 10001 app, 100 rabbitmq/openbao, 99 haproxy, 472 grafana, 65534 prometheus, 10001 loki/tempo)
|
||||
# должен их читать. От других пользователей ХОСТА их закрывает каталог secrets/ (700). Каждый
|
||||
# контейнер получает только свои файлы (см. compose.yaml: secrets per service).
|
||||
chmod 644 secrets/*_password secrets/s3_* secrets/rabbitmq_definitions.json secrets/pki/*.crt secrets/pki/*.key \
|
||||
secrets/pki/*.pem secrets/pki/test/*
|
||||
chmod 700 secrets/pki secrets/pki/test
|
||||
if [ ! -f .env ]; then
|
||||
cp .env.example .env
|
||||
chmod 600 .env
|
||||
echo "created .env from .env.example"
|
||||
fi
|
||||
echo "operator material: $OPERATOR_DIR (CA keys; OpenBao init material after scripts/bao-init.sh)"
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/bin/sh
|
||||
# Выпуск учебного PKI lab7. Запускается init-secrets.sh ВНУТРИ закреплённого образа
|
||||
# python:3.12.7-slim (OpenSSL 3.0) — результат не зависит от openssl/LibreSSL хоста.
|
||||
# /op — каталог оператора (вне проекта): закрытые ключи CA, никогда не монтируется в стенд;
|
||||
# /out — secrets/pki проекта (вне Git): сертификаты CA и leaf-сертификаты/ключи сервисов.
|
||||
# Повторный запуск ничего не перевыпускает (кроме явно удалённых файлов); значения не печатаются.
|
||||
#
|
||||
# Два CA с разным назначением:
|
||||
# public-ca — внешний TLS (HAProxy :8443, Grafana :3000). Его сертификат раздаётся клиентам.
|
||||
# internal-ca — mTLS/TLS внутри стенда (LB->API, сервисы -> PostgreSQL/S3/RabbitMQ/Collector/
|
||||
# Prometheus/Loki/Tempo/OpenBao). Клиенты извне ему не доверяют.
|
||||
# Третий, rogue-ca, — ТОЛЬКО для отрицательных тестов (сертификат «lb» от чужого CA).
|
||||
set -eu
|
||||
umask 077
|
||||
OP=/op
|
||||
OUT=/out
|
||||
DAYS_CA=${PKI_CA_DAYS:-365}
|
||||
DAYS_LEAF=${PKI_LEAF_DAYS:-30}
|
||||
mkdir -p "$OP/ca" "$OUT/test"
|
||||
|
||||
ca() { # ca <name> <CN> <dir_for_key>
|
||||
name=$1; cn=$2; keydir=$3
|
||||
if [ ! -s "$keydir/$name.key" ] || [ ! -s "$OUT/$name.crt" ]; then
|
||||
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$keydir/$name.key" 2>/dev/null
|
||||
openssl req -x509 -new -key "$keydir/$name.key" -sha256 -days "$DAYS_CA" \
|
||||
-subj "/O=MTUSI lab7/CN=$cn" \
|
||||
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
||||
-addext "keyUsage=critical,keyCertSign,cRLSign" \
|
||||
-addext "subjectKeyIdentifier=hash" \
|
||||
-out "$OUT/$name.crt"
|
||||
echo "created CA $name (key in operator dir)"
|
||||
fi
|
||||
}
|
||||
|
||||
leaf() { # leaf <ca> <name> <eku> <san...>
|
||||
caname=$1; name=$2; eku=$3; shift 3
|
||||
dir=$OUT; [ "$caname" = rogue-ca ] && dir=$OUT/test
|
||||
[ "$name" = intruder ] && dir=$OUT/test
|
||||
if [ -s "$dir/$name.crt" ] && [ -s "$dir/$name.key" ]; then return 0; fi
|
||||
cakey=$OP/ca/$caname.key; [ "$caname" = rogue-ca ] && cakey=$OUT/test/rogue-ca.key
|
||||
cacrt=$OUT/$caname.crt; [ "$caname" = rogue-ca ] && cacrt=$OUT/test/rogue-ca.crt
|
||||
san=""
|
||||
for s in "$@"; do san="${san:+$san,}$s"; done
|
||||
ext=$(mktemp)
|
||||
cat > "$ext" <<EOF
|
||||
basicConstraints=critical,CA:FALSE
|
||||
keyUsage=critical,digitalSignature
|
||||
extendedKeyUsage=$eku
|
||||
subjectAltName=$san
|
||||
subjectKeyIdentifier=hash
|
||||
authorityKeyIdentifier=keyid
|
||||
EOF
|
||||
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/$name.key" 2>/dev/null
|
||||
openssl req -new -key "$dir/$name.key" -subj "/O=MTUSI lab7/CN=$name" -out "$dir/$name.csr"
|
||||
openssl x509 -req -in "$dir/$name.csr" -CA "$cacrt" -CAkey "$cakey" -CAcreateserial \
|
||||
-CAserial "$OP/ca/$caname.srl" -days "$DAYS_LEAF" -sha256 -extfile "$ext" -out "$dir/$name.crt" 2>/dev/null
|
||||
rm -f "$dir/$name.csr" "$ext"
|
||||
echo "issued $name ($caname, $DAYS_LEAF days)"
|
||||
}
|
||||
|
||||
ca public-ca "MTUSI lab7 Public Ingress CA" "$OP/ca"
|
||||
ca internal-ca "MTUSI lab7 Internal Services CA" "$OP/ca"
|
||||
# rogue CA — тестовый, его ключ не секретен для стенда (лежит в secrets/pki/test).
|
||||
if [ ! -s "$OUT/test/rogue-ca.key" ]; then
|
||||
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$OUT/test/rogue-ca.key" 2>/dev/null
|
||||
openssl req -x509 -new -key "$OUT/test/rogue-ca.key" -sha256 -days 30 -subj "/O=Attacker/CN=Rogue CA" \
|
||||
-addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign" \
|
||||
-out "$OUT/test/rogue-ca.crt"
|
||||
cp "$OP/ca/internal-ca.srl" "$OP/ca/rogue-ca.srl" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
BOTH=serverAuth,clientAuth
|
||||
# Внешние (public CA): клиенты проверяют hostname localhost / 127.0.0.1.
|
||||
leaf public-ca lb-public serverAuth DNS:localhost IP:127.0.0.1
|
||||
leaf public-ca grafana-public serverAuth DNS:localhost IP:127.0.0.1
|
||||
# Внутренние identities (internal CA). SAN DNS = имя сервиса Compose = проверяемая identity.
|
||||
# lb — клиентская identity LB для mTLS к API (и серверная для scrape :8404).
|
||||
leaf internal-ca lb "$BOTH" DNS:lb
|
||||
leaf internal-ca api "$BOTH" DNS:api
|
||||
for s in worker relay janitor migrate s3-init ops grafana; do
|
||||
leaf internal-ca "$s" clientAuth "DNS:$s"
|
||||
done
|
||||
leaf internal-ca db serverAuth DNS:db DNS:localhost
|
||||
for s in s3 broker otel-collector prometheus loki tempo openbao; do
|
||||
leaf internal-ca "$s" "$BOTH" "DNS:$s" DNS:localhost IP:127.0.0.1
|
||||
done
|
||||
# Отрицательные тесты: доверенный CA, но запрещённая identity; чужой CA с «правильным» SAN lb.
|
||||
leaf internal-ca intruder "$BOTH" DNS:intruder
|
||||
leaf rogue-ca rogue-lb "$BOTH" DNS:lb
|
||||
|
||||
# HAProxy читает сертификат и ключ одним PEM.
|
||||
cat "$OUT/lb-public.crt" "$OUT/lb-public.key" > "$OUT/lb-public.pem"
|
||||
cat "$OUT/lb.crt" "$OUT/lb.key" > "$OUT/lb.pem"
|
||||
chmod 600 "$OP/ca/"*
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск ВСЕХ хранилищ — данных (PostgreSQL, SeaweedFS S3, RabbitMQ), секретов (OpenBao) и
|
||||
# телеметрии (Prometheus, Loki, Tempo) — и всех процессов (API-реплики, relay, worker, janitor, LB,
|
||||
# Collector, Grafana) с сохранением volumes. Не выполняет down -v, не пересоздаёт БД/bucket/очереди.
|
||||
#
|
||||
# Lab7: OpenBao после рестарта sealed — это хук ОПЕРАТОРА, поэтому он выполняет unseal
|
||||
# (scripts/bao-unseal.sh, 2 из 3 shares из $OPERATOR_DIR вне проекта). Процессы приложения
|
||||
# заново проходят cert auth и читают свои секреты; старые токены не нужны.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart openbao db s3 broker prometheus loki tempo
|
||||
./scripts/bao-unseal.sh >/dev/null
|
||||
docker compose up -d --no-deps --no-recreate --wait openbao db s3 broker prometheus
|
||||
docker compose restart otel-collector api relay worker janitor lb grafana
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка перед commit (lab7): значения секретов не встречаются ни в одном файле репозитория
|
||||
вне secrets/ и .env. Значения не печатаются — только имена файлов.
|
||||
|
||||
Источники значений:
|
||||
* secrets/**: пароли, ключи S3, password_hash RabbitMQ, ЗАКРЫТЫЕ КЛЮЧИ leaf (тело PEM по строкам);
|
||||
* $OPERATOR_DIR (по умолчанию ~/.mtusi2026lab7-operator): закрытые ключи CA, root token и
|
||||
unseal shares OpenBao (init.json).
|
||||
Публичные сертификаты (*.crt) секретом не являются и не проверяются.
|
||||
python3 scripts/scan_secrets.py (код 0 — чисто)"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OPERATOR_DIR = Path(os.environ.get("OPERATOR_DIR", Path.home() / ".mtusi2026lab7-operator"))
|
||||
SKIP_DIRS = {".git", "secrets", ".venv", "__pycache__", "node_modules"}
|
||||
values = set()
|
||||
|
||||
|
||||
def add_pem(text):
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if len(line) >= 16 and not line.startswith("-----"):
|
||||
values.add(line)
|
||||
|
||||
|
||||
def walk(x):
|
||||
if isinstance(x, dict):
|
||||
for k, v in x.items():
|
||||
if k in ("accessKey", "secretKey", "password_hash", "root_token", "unseal_keys_b64", "unseal_keys_hex"):
|
||||
for item in (v if isinstance(v, list) else [v]):
|
||||
values.add(str(item))
|
||||
walk(v)
|
||||
elif isinstance(x, list):
|
||||
for v in x:
|
||||
walk(v)
|
||||
|
||||
|
||||
sources = [f for f in (ROOT / "secrets").rglob("*") if f.is_file() and f.suffix != ".crt"]
|
||||
sources += [f for f in OPERATOR_DIR.rglob("*") if f.is_file()] if OPERATOR_DIR.is_dir() else []
|
||||
for f in sources:
|
||||
text = f.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if f.suffix == ".json":
|
||||
walk(json.loads(text))
|
||||
elif "PRIVATE KEY" in text:
|
||||
add_pem(text)
|
||||
elif len(text) >= 8 and "\n" not in text:
|
||||
values.add(text)
|
||||
if "://" in text and "@" in text:
|
||||
values.add(text.split("://", 1)[1].split("@", 1)[0].split(":", 1)[-1])
|
||||
hits = []
|
||||
for p in ROOT.rglob("*"):
|
||||
if p.is_dir() or set(p.relative_to(ROOT).parts) & SKIP_DIRS or p.name == ".env":
|
||||
continue
|
||||
try:
|
||||
data = p.read_text(encoding="utf-8", errors="ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if any(v in data for v in values):
|
||||
hits.append(str(p.relative_to(ROOT)))
|
||||
print("checked %d secret values from %d files; files with leaks: %s" % (len(values), len(sources), hits or "none"))
|
||||
sys.exit(1 if hits else 0)
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Восстанавливает остановленные API-реплики и worker (после stop-one.sh / stop-worker.sh) и ждёт готовности.
|
||||
# --no-deps: хранилища и job-ы не трогаются; --no-recreate: те же контейнеры (тот же образ).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-deps --no-recreate --wait api worker
|
||||
docker compose ps --format '{{.Name}} {{.Status}}' api worker
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Запустить worker после stop-worker.sh (те же контейнеры) и дождаться готовности.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-deps --no-recreate --wait worker
|
||||
docker compose ps --format '{{.Name}} {{.Status}}' worker
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной.
|
||||
# Lab7: TARGET_INSTANCE_ID = X-Instance-Id = СЛУЧАЙНЫЙ alias процесса (r-<10 hex>), не hostname.
|
||||
# Реплика пишет свой alias в /tmp/instance-id (tmpfs) — сопоставляем через docker exec.
|
||||
# DB, LB и volumes не трогаются. Восстановление: scripts/start-all.sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
if ! printf '%s' "$TARGET_INSTANCE_ID" | grep -Eq '^r-[0-9a-f]{10}$'; then
|
||||
echo "unexpected instance id format" >&2
|
||||
exit 2
|
||||
fi
|
||||
target=""
|
||||
for cid in $(docker compose ps -q --status running api); do
|
||||
alias=$(docker exec "$cid" cat /tmp/instance-id 2>/dev/null || true)
|
||||
if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then
|
||||
target=$cid
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$target" ] || { echo "no running api container for $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
name=$(docker inspect -f '{{.Name}}' "$target")
|
||||
echo "$(date -u +%FT%TZ) stopping api container $name ($TARGET_INSTANCE_ID)" >&2
|
||||
docker stop -t 20 "$target" >/dev/null
|
||||
echo "$(date -u +%FT%TZ) stopped $name" >&2
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Сценарий защиты: остановить ВСЕ worker (SIGTERM, текущие задания дорабатываются).
|
||||
# API, relay и брокер продолжают работать: upload -> 202, задания копятся в очереди.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop worker
|
||||
docker compose ps --format '{{.Name}} {{.State}}' worker
|
||||
Reference in New Issue
Block a user