forked from mtusi2026/mtusi2026lab7
ЛР7
This commit is contained in:
+252
@@ -0,0 +1,252 @@
|
||||
"""OpenBao operator tool for the lab7 stand (Python standard library only).
|
||||
|
||||
Runs in the short-lived "ops" container (make bao-init / bao-unseal / bao-bootstrap / ...).
|
||||
It is the ONLY place that reads the operator material in secrets/operator/:
|
||||
unseal-key-1..3 Shamir shares (threshold 2). In real life: three different people/places.
|
||||
root-token initial root token (used only by these operator commands)
|
||||
The files are created with mode 600, never mounted into services, never printed.
|
||||
|
||||
Commands:
|
||||
status initialized / sealed / version (no token needed)
|
||||
init operator init (3 shares, threshold 2); refuses to run twice
|
||||
unseal submit shares until unsealed (needed after every OpenBao restart)
|
||||
seal seal OpenBao (incident drill: sealed OpenBao)
|
||||
bootstrap idempotent: kv-v2 at secret/, cert auth, audit device, policies,
|
||||
cert roles for api/worker, application secrets from secrets/*
|
||||
rotate-db new password for DB role messenger_api: ALTER ROLE + new KV version
|
||||
revoke-tokens revoke every token issued by the cert auth method (forces re-login)
|
||||
token-ttl ROLE TTL change token TTL of a cert role (e.g. expiry drill: api 30s)
|
||||
|
||||
Plain ASCII on purpose (scripts/check_contract.py reads scripts/*.py with the default encoding).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import secrets as rnd
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = Path(os.environ.get("SECRETS_DIR", ROOT / "secrets"))
|
||||
OPERATOR = SECRETS / "operator"
|
||||
POLICIES = Path(os.environ.get("BAO_POLICY_DIR", ROOT / "infra" / "openbao" / "policies"))
|
||||
ADDR = os.environ.get("BAO_ADDR", "https://openbao:8200").rstrip("/")
|
||||
CACERT = os.environ.get("BAO_CACERT", str(SECRETS / "tls" / "ca" / "ca.crt"))
|
||||
SHARES, THRESHOLD = 3, 2
|
||||
|
||||
# role -> (certificate CN, policy, secret path). One policy per service, nothing shared.
|
||||
ROLES = {"messenger-api": ("api", "messenger-api", "messenger/api"),
|
||||
"messenger-worker": ("worker", "messenger-worker", "messenger/worker")}
|
||||
TOKEN_TTL, TOKEN_MAX_TTL = os.environ.get("BAO_TOKEN_TTL", "15m"), os.environ.get("BAO_TOKEN_MAX_TTL", "1h")
|
||||
|
||||
CTX = ssl.create_default_context(cafile=CACERT) # server certificate and hostname are verified
|
||||
|
||||
|
||||
class BaoError(Exception):
|
||||
def __init__(self, status, errors):
|
||||
super().__init__(f"HTTP {status}: {'; '.join(errors) or 'no details'}")
|
||||
self.status = status
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, ok=(200, 204)):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
request = urllib.request.Request(ADDR + "/v1/" + path.lstrip("/"), data=data, method=method)
|
||||
request.add_header("Content-Type", "application/json")
|
||||
if token:
|
||||
request.add_header("X-Vault-Token", token)
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=CTX, timeout=10) as response:
|
||||
raw = response.read()
|
||||
status = response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw, status = exc.read(), exc.code
|
||||
payload = json.loads(raw) if raw.strip() else {}
|
||||
if status not in ok:
|
||||
# Error bodies of OpenBao contain no secret values; tokens are never echoed.
|
||||
raise BaoError(status, payload.get("errors", []))
|
||||
return payload
|
||||
|
||||
|
||||
def secret_file(name: str) -> str:
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def write_private(path: Path, value: str):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(path.parent, 0o700)
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(value)
|
||||
|
||||
|
||||
def root_token() -> str:
|
||||
path = OPERATOR / "root-token"
|
||||
if not path.exists():
|
||||
sys.exit("secrets/operator/root-token not found: run make bao-init first")
|
||||
return path.read_text().strip()
|
||||
|
||||
|
||||
def cmd_status(_args):
|
||||
health = call("GET", "sys/seal-status")
|
||||
print(f"initialized={health['initialized']} sealed={health['sealed']} "
|
||||
f"progress={health.get('progress', 0)}/{health.get('t', 0)} version={health.get('version')} "
|
||||
f"storage={health.get('storage_type')}")
|
||||
return 0 if not health["sealed"] else 2
|
||||
|
||||
|
||||
def cmd_init(_args):
|
||||
if call("GET", "sys/init")["initialized"]:
|
||||
print("OpenBao already initialized: nothing to do (shares and root token stay with the operator)")
|
||||
return 0
|
||||
result = call("PUT", "sys/init", {"secret_shares": SHARES, "secret_threshold": THRESHOLD})
|
||||
for number, share in enumerate(result["keys_base64"], start=1):
|
||||
write_private(OPERATOR / f"unseal-key-{number}", share)
|
||||
write_private(OPERATOR / "root-token", result["root_token"])
|
||||
print(f"initialized: {SHARES} unseal shares (threshold {THRESHOLD}) and root token saved to secrets/operator/ "
|
||||
"(mode 600, not printed). Keep the shares apart: in production every share goes to a different person.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_unseal(_args):
|
||||
status = call("GET", "sys/seal-status")
|
||||
if not status["initialized"]:
|
||||
sys.exit("not initialized: run make bao-init")
|
||||
if not status["sealed"]:
|
||||
print("already unsealed")
|
||||
return 0
|
||||
keys = sorted(OPERATOR.glob("unseal-key-*"))
|
||||
if len(keys) < THRESHOLD:
|
||||
sys.exit(f"need {THRESHOLD} shares in secrets/operator/unseal-key-N")
|
||||
for path in keys[:THRESHOLD]:
|
||||
status = call("PUT", "sys/unseal", {"key": path.read_text().strip()})
|
||||
print(f"sealed={status['sealed']} (used {THRESHOLD} of {len(keys)} shares)")
|
||||
return 0 if not status["sealed"] else 1
|
||||
|
||||
|
||||
def cmd_seal(_args):
|
||||
call("PUT", "sys/seal", token=root_token())
|
||||
print("sealed: services keep their cached secrets, new logins and reads fail until make bao-unseal")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_bootstrap(_args):
|
||||
token = root_token()
|
||||
if "secret/" not in call("GET", "sys/mounts", token=token)["data"]:
|
||||
call("POST", "sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}, token=token)
|
||||
print("enabled kv-v2 at secret/")
|
||||
if "cert/" not in call("GET", "sys/auth", token=token)["data"]:
|
||||
call("POST", "sys/auth/cert", {"type": "cert"}, token=token)
|
||||
print("enabled cert auth at auth/cert/")
|
||||
# Audit device is declared in infra/openbao/bao.hcl: OpenBao 2.4 refuses to enable it via the API.
|
||||
ca_pem = Path(CACERT).read_text()
|
||||
for role, (common_name, policy, secret_path) in ROLES.items():
|
||||
call("PUT", f"sys/policies/acl/{policy}",
|
||||
{"policy": (POLICIES / f"{policy}.hcl").read_text(encoding="utf-8")}, token=token)
|
||||
call("POST", f"auth/cert/certs/{role}", {
|
||||
"display_name": role,
|
||||
"certificate": ca_pem, # trust anchor: our CA only
|
||||
"allowed_common_names": [common_name], # and only this identity
|
||||
"token_policies": [policy],
|
||||
"token_ttl": TOKEN_TTL, "token_max_ttl": TOKEN_MAX_TTL,
|
||||
"token_no_default_policy": False, # default policy = lookup-self / renew-self only
|
||||
}, token=token)
|
||||
put_kv(token, "messenger/api", {
|
||||
"db_password": secret_file("db_api_password"),
|
||||
"s3_access_key_id": secret_file("s3_api_access_key_id"),
|
||||
"s3_secret_access_key": secret_file("s3_api_secret_access_key"),
|
||||
"cache_password": secret_file("valkey_api_password"),
|
||||
})
|
||||
put_kv(token, "messenger/worker", {
|
||||
"db_password": secret_file("db_worker_password"),
|
||||
"s3_access_key_id": secret_file("s3_worker_access_key_id"),
|
||||
"s3_secret_access_key": secret_file("s3_worker_secret_access_key"),
|
||||
})
|
||||
print(f"policies messenger-api, messenger-worker; cert roles (CN api / worker, token ttl {TOKEN_TTL}, "
|
||||
f"max {TOKEN_MAX_TTL}); secrets secret/messenger/api and secret/messenger/worker written (values not printed)")
|
||||
return 0
|
||||
|
||||
|
||||
def put_kv(token, path, data):
|
||||
current = None
|
||||
try:
|
||||
current = call("GET", f"secret/data/{path}", token=token)["data"]["data"]
|
||||
except BaoError as exc:
|
||||
if exc.status != 404:
|
||||
raise
|
||||
if current != data:
|
||||
call("POST", f"secret/data/{path}", {"data": data}, token=token)
|
||||
|
||||
|
||||
def cmd_rotate_db(_args):
|
||||
"""Rotate the password of DB role messenger_api without rebuilding images or restarting the API.
|
||||
|
||||
Order: 1) ALTER ROLE in PostgreSQL, 2) new KV version in OpenBao, 3) the operator copy.
|
||||
Between 1) and the next API refresh (BAO_REFRESH_SECONDS) NEW DB connections of the API fail
|
||||
and are retried; connections that are already open keep working (PostgreSQL checks the password
|
||||
only at login). The API drains its pool when it sees the new value, so every connection after
|
||||
that is a new login with the new password.
|
||||
"""
|
||||
import psycopg # available in the application image the ops container runs from
|
||||
from psycopg import sql
|
||||
token = root_token()
|
||||
new = rnd.token_hex(24)
|
||||
owner = secret_file("postgres_password")
|
||||
with psycopg.connect(host=os.environ.get("DB_HOST", "postgres"), port=int(os.environ.get("DB_PORT", "5432")),
|
||||
dbname="messenger", user="messenger", password=owner,
|
||||
sslmode="verify-full", sslrootcert=CACERT, connect_timeout=5, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("ALTER ROLE messenger_api PASSWORD {}").format(sql.Literal(new)))
|
||||
data = call("GET", "secret/data/messenger/api", token=token)["data"]["data"]
|
||||
data["db_password"] = new
|
||||
result = call("POST", "secret/data/messenger/api", {"data": data}, token=token)
|
||||
(SECRETS / "db_api_password").write_text(new)
|
||||
print(f"rotated DB password of messenger_api: PostgreSQL updated, KV secret/messenger/api now version "
|
||||
f"{result['data']['version']} (value not printed)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_revoke_tokens(_args):
|
||||
call("PUT", "sys/leases/revoke-prefix/auth/cert/login", token=root_token())
|
||||
print("revoked all tokens issued by auth/cert: services must log in again with their certificates")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_token_ttl(args):
|
||||
if len(args) != 2 or args[0] not in ("api", "worker"):
|
||||
sys.exit("usage: token-ttl api|worker TTL (e.g. 30s, 15m)")
|
||||
role = f"messenger-{args[0]}"
|
||||
token = root_token()
|
||||
current = call("GET", f"auth/cert/certs/{role}", token=token)["data"]
|
||||
max_ttl = args[1] if args[1].endswith("s") else TOKEN_MAX_TTL
|
||||
call("POST", f"auth/cert/certs/{role}", {
|
||||
"display_name": role, "certificate": current["certificate"],
|
||||
"allowed_common_names": current["allowed_common_names"], "token_policies": current["token_policies"],
|
||||
"token_ttl": args[1], "token_max_ttl": max_ttl}, token=token)
|
||||
print(f"{role}: token_ttl={args[1]} token_max_ttl={max_ttl} (applies to NEW logins)")
|
||||
return 0
|
||||
|
||||
|
||||
COMMANDS = {"status": cmd_status, "init": cmd_init, "unseal": cmd_unseal, "seal": cmd_seal,
|
||||
"bootstrap": cmd_bootstrap, "rotate-db": cmd_rotate_db, "revoke-tokens": cmd_revoke_tokens,
|
||||
"token-ttl": cmd_token_ttl}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
|
||||
sys.exit("usage: bao.py " + "|".join(COMMANDS))
|
||||
deadline = time.monotonic() + 60
|
||||
while True: # OpenBao may still be starting after docker compose up
|
||||
try:
|
||||
return COMMANDS[sys.argv[1]](sys.argv[2:])
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
sys.exit(f"OpenBao at {ADDR} is not reachable: {getattr(exc, 'reason', exc)}")
|
||||
time.sleep(2)
|
||||
except BaoError as exc:
|
||||
sys.exit(f"OpenBao refused: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные секреты стенда в каталоге secrets/ (он в .gitignore и в Git не попадает).
|
||||
# Запуск один раз: sh scripts/init-secrets.sh (затем сертификаты: make pki)
|
||||
# Повторный запуск безопасен: существующие файлы не меняются, недостающие досоздаются.
|
||||
#
|
||||
# Кто что получает (lab7):
|
||||
# оператор (этот каталог) — всё; root token и unseal shares OpenBao появятся в secrets/operator/ (make bao-init)
|
||||
# postgres — пароль владельца схемы и пароли ролей messenger_api/messenger_worker (только для
|
||||
# первой инициализации volume, infra/postgres/initdb/10-roles.sh)
|
||||
# migrate — пароль владельца схемы messenger (одноразовый сервис, не приложение)
|
||||
# valkey — ACL-файл (пароли пользователей)
|
||||
# s3 (SeaweedFS) — s3_config.json (пользователи и права); s3-init — ключи администратора
|
||||
# grafana — пароль администратора
|
||||
# api / worker — НИЧЕГО из этого каталога: свои пароли и ключи они читают из OpenBao
|
||||
# (make bao-bootstrap переносит их туда), а входят в OpenBao сертификатом.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
umask 077 # каталог secrets/ доступен только владельцу
|
||||
mkdir -p secrets
|
||||
|
||||
# 24 случайных байта -> 48 hex-символов. /dev/urandom есть и в Linux/macOS, и в Git Bash.
|
||||
random_hex() { od -An -tx1 -N24 /dev/urandom | tr -d ' \n'; }
|
||||
# Файл создаётся, только если его ещё нет (printf без перевода строки: в файле только значение).
|
||||
make_secret() { [ -e "secrets/$1" ] || printf '%s%s' "$2" "$(random_hex | cut -c1-"$3")" > "secrets/$1"; }
|
||||
|
||||
# PostgreSQL: владелец схемы (migrate, оператор) и две роли приложения (с lab7 — раздельные).
|
||||
make_secret postgres_password "" 48
|
||||
make_secret db_api_password "" 48
|
||||
make_secret db_worker_password "" 48
|
||||
|
||||
# Valkey: пользователь messenger (API). health — без пароля, только ping/info/dbsize.
|
||||
make_secret valkey_api_password "" 48
|
||||
printf '%s\n' \
|
||||
'user default off' \
|
||||
'user health on nopass -@all +ping +info +dbsize' \
|
||||
"user messenger on >$(cat secrets/valkey_api_password) ~sess:* ~rl:* resetchannels -@all +get +mget +set +del +ping +incrby +ttl +multi +exec" \
|
||||
> secrets/valkey_users.acl
|
||||
|
||||
# S3 (SeaweedFS): администратор (только s3-init) и РАЗНЫЕ пользователи для API и worker.
|
||||
make_secret s3_root_user admin 12
|
||||
make_secret s3_root_password "" 32
|
||||
make_secret s3_api_access_key_id api 16
|
||||
make_secret s3_api_secret_access_key "" 32
|
||||
make_secret s3_worker_access_key_id wrk 16
|
||||
make_secret s3_worker_secret_access_key "" 32
|
||||
|
||||
# Права по префиксам (с lab7):
|
||||
# API — пишет только staging/ (принятый файл), читает только originals/ и variants/ (скачивание);
|
||||
# не может подделать оригинал или вариант и не видит список объектов;
|
||||
# worker — читает staging/ и originals/, пишет и удаляет в bucket (оригиналы, варианты, уборка),
|
||||
# видит список (уборщик); не читает variants/ и не управляет bucket.
|
||||
BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env 2>/dev/null | tr -d '\r')
|
||||
BUCKET=${BUCKET:-messenger-attachments}
|
||||
printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","List","Tagging","Write"]},{"name":"messenger-api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Write:%s/staging/*","Read:%s/originals/*","Read:%s/variants/*"]},{"name":"messenger-worker","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s/staging/*","Read:%s/originals/*","Write:%s","List:%s"]}]}\n' \
|
||||
"$(cat secrets/s3_root_user)" "$(cat secrets/s3_root_password)" \
|
||||
"$(cat secrets/s3_api_access_key_id)" "$(cat secrets/s3_api_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" \
|
||||
"$(cat secrets/s3_worker_access_key_id)" "$(cat secrets/s3_worker_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" "$BUCKET" \
|
||||
> secrets/s3_config.json
|
||||
|
||||
# Grafana: пароль администратора (https://127.0.0.1:3000, пользователь admin).
|
||||
make_secret grafana_admin_password "" 24
|
||||
|
||||
# С lab7 у брокера паролей нет: вход по клиентским сертификатам (make pki).
|
||||
# Файлы читают процессы в контейнерах под своими пользователями (postgres, valkey, seaweedfs, grafana),
|
||||
# поэтому сами файлы доступны на чтение; посторонних на хосте не пускает каталог (700).
|
||||
chmod 644 secrets/*.json secrets/*.acl secrets/*_password secrets/*_key_id secrets/*_access_key secrets/s3_root_* 2>/dev/null || true
|
||||
echo "Готово: secrets/ (PostgreSQL x3, Valkey, S3 admin/API/worker, Grafana; значения не выводятся). Дальше: make pki"
|
||||
+284
@@ -0,0 +1,284 @@
|
||||
"""Local PKI of the lab7 stand: one CA, one leaf certificate per service identity, CRL.
|
||||
|
||||
Operator tool. Runs in the "pki" container (docker compose --profile tools run --rm pki ...),
|
||||
writes ONLY into ./secrets (in .gitignore), never prints private keys.
|
||||
|
||||
Layout:
|
||||
secrets/ca/ca.key CA private key: only this tool reads it (never mounted into services)
|
||||
secrets/ca/state.json issued serials and revoked serials (no secrets inside)
|
||||
secrets/tls/ca/ca.crt CA certificate: public, mounted into every service as /run/ca/ca.crt
|
||||
secrets/tls/ca/crl.pem CRL signed by the CA: revoked leaf serials (API checks it for LB identity)
|
||||
secrets/tls/<id>/tls.crt leaf certificate of identity <id> (CN = SAN DNS = <id>)
|
||||
secrets/tls/<id>/tls.key its private key (mounted only into the container of that service)
|
||||
secrets/tls/<id>/tls.pem cert + key in one file (HAProxy format)
|
||||
secrets/tls-test/... certificates for NEGATIVE tests only: intruder (same CA, unknown identity),
|
||||
rogue-lb (identity "lb" from a foreign CA), lb-expired (expired "lb")
|
||||
|
||||
Commands:
|
||||
init create CA if missing, issue missing/expiring leaves, test certs, CRL (idempotent)
|
||||
issue <id> [--days N] re-issue one leaf IN PLACE (rotation; files are overwritten, not replaced,
|
||||
so bind mounts see the new content). Prints old and new serial.
|
||||
revoke <id> --serial S add serial S of <id> to the CRL (S from "show" or from "issue" output)
|
||||
revoke-previous <id> revoke the certificate that "issue" replaced (rotation: issue -> reload -> revoke)
|
||||
crl re-sign the CRL (it is valid CRL_DAYS days; run before it expires)
|
||||
show identities, serials, expiry dates
|
||||
|
||||
Plain ASCII on purpose (scripts/check_contract.py reads scripts/*.py with the default encoding).
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
|
||||
ROOT = Path(os.environ.get("PKI_ROOT", Path(__file__).resolve().parents[1] / "secrets"))
|
||||
CA_DIR, PUB_DIR, TLS_DIR, TEST_DIR = ROOT / "ca", ROOT / "tls" / "ca", ROOT / "tls", ROOT / "tls-test"
|
||||
CA_DAYS, LEAF_DAYS, CRL_DAYS, RENEW_BEFORE_DAYS = 365, 30, 30, 7
|
||||
LOCAL = ["localhost", "127.0.0.1"]
|
||||
|
||||
# identity -> extra SAN names. CN = first SAN = identity. All leaves: serverAuth + clientAuth.
|
||||
IDENTITIES = {
|
||||
"lb": LOCAL, # external TLS ingress (8443) and mTLS CLIENT of every API replica
|
||||
"api": [], # mTLS server for LB; client of postgres/s3/valkey/broker/collector/openbao
|
||||
"worker": [], # client of postgres/s3/broker/collector/openbao
|
||||
"postgres": LOCAL,
|
||||
"valkey": LOCAL,
|
||||
"s3": LOCAL,
|
||||
"broker": LOCAL,
|
||||
"otel-collector": LOCAL, # OTLP gateway server; client of prometheus/loki/tempo
|
||||
"prometheus": LOCAL,
|
||||
"loki": LOCAL,
|
||||
"tempo": LOCAL,
|
||||
"grafana": LOCAL, # HTTPS for the operator; client of prometheus/loki/tempo
|
||||
"openbao": LOCAL,
|
||||
}
|
||||
|
||||
|
||||
def now():
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def write(path: Path, data: bytes, mode: int = 0o644):
|
||||
"""Overwrite in place (same inode): containers that bind-mount the directory see the change."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "r+b" if path.exists() else "wb") as fh:
|
||||
fh.seek(0)
|
||||
fh.write(data)
|
||||
fh.truncate()
|
||||
os.chmod(path, mode)
|
||||
|
||||
|
||||
def pem_key(key) -> bytes:
|
||||
return key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption())
|
||||
|
||||
|
||||
def pem_cert(cert) -> bytes:
|
||||
return cert.public_bytes(serialization.Encoding.PEM)
|
||||
|
||||
|
||||
def load_state() -> dict:
|
||||
path = CA_DIR / "state.json"
|
||||
return json.loads(path.read_text()) if path.exists() else {"issued": {}, "revoked": []}
|
||||
|
||||
|
||||
def save_state(state: dict):
|
||||
write(CA_DIR / "state.json", json.dumps(state, indent=2, sort_keys=True).encode(), 0o600)
|
||||
|
||||
|
||||
def new_ca(common_name: str):
|
||||
key = ec.generate_private_key(ec.SECP256R1())
|
||||
name = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI lab7"),
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, common_name)])
|
||||
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now() - dt.timedelta(minutes=5)).not_valid_after(now() + dt.timedelta(days=CA_DAYS))
|
||||
.add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True)
|
||||
.add_extension(x509.KeyUsage(digital_signature=False, content_commitment=False, key_encipherment=False,
|
||||
data_encipherment=False, key_agreement=False, key_cert_sign=True,
|
||||
crl_sign=True, encipher_only=False, decipher_only=False), critical=True)
|
||||
.add_extension(x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False)
|
||||
.sign(key, hashes.SHA256()))
|
||||
return key, cert
|
||||
|
||||
|
||||
def load_ca():
|
||||
key = serialization.load_pem_private_key((CA_DIR / "ca.key").read_bytes(), None)
|
||||
cert = x509.load_pem_x509_certificate((PUB_DIR / "ca.crt").read_bytes())
|
||||
return key, cert
|
||||
|
||||
|
||||
def leaf(ca_key, ca_cert, identity: str, extra: list, not_before=None, not_after=None):
|
||||
key = ec.generate_private_key(ec.SECP256R1())
|
||||
names = []
|
||||
for value in [identity] + list(extra):
|
||||
try:
|
||||
names.append(x509.IPAddress(ipaddress.ip_address(value)))
|
||||
except ValueError:
|
||||
names.append(x509.DNSName(value))
|
||||
not_before = not_before or now() - dt.timedelta(minutes=5)
|
||||
not_after = not_after or now() + dt.timedelta(days=LEAF_DAYS)
|
||||
cert = (x509.CertificateBuilder()
|
||||
.subject_name(x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI lab7"),
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, identity)]))
|
||||
.issuer_name(ca_cert.subject).public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(not_before).not_valid_after(not_after)
|
||||
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
|
||||
.add_extension(x509.KeyUsage(digital_signature=True, content_commitment=False, key_encipherment=False,
|
||||
data_encipherment=False, key_agreement=True, key_cert_sign=False,
|
||||
crl_sign=False, encipher_only=False, decipher_only=False), critical=True)
|
||||
.add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH,
|
||||
ExtendedKeyUsageOID.CLIENT_AUTH]), critical=False)
|
||||
.add_extension(x509.SubjectAlternativeName(names), critical=False)
|
||||
.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), critical=False)
|
||||
.sign(ca_key, hashes.SHA256()))
|
||||
return key, cert
|
||||
|
||||
|
||||
def store_leaf(directory: Path, key, cert):
|
||||
write(directory / "tls.key", pem_key(key))
|
||||
write(directory / "tls.crt", pem_cert(cert))
|
||||
write(directory / "tls.pem", pem_cert(cert) + pem_key(key))
|
||||
|
||||
|
||||
def serial_hex(cert) -> str:
|
||||
return format(cert.serial_number, "X")
|
||||
|
||||
|
||||
def write_crl(ca_key, ca_cert, state):
|
||||
builder = (x509.CertificateRevocationListBuilder().issuer_name(ca_cert.subject)
|
||||
.last_update(now() - dt.timedelta(minutes=5)).next_update(now() + dt.timedelta(days=CRL_DAYS)))
|
||||
for item in state["revoked"]:
|
||||
builder = builder.add_revoked_certificate(
|
||||
x509.RevokedCertificateBuilder().serial_number(int(item["serial"], 16))
|
||||
.revocation_date(dt.datetime.fromisoformat(item["at"])).build())
|
||||
crl = builder.sign(ca_key, hashes.SHA256())
|
||||
write(PUB_DIR / "crl.pem", crl.public_bytes(serialization.Encoding.PEM))
|
||||
return len(state["revoked"])
|
||||
|
||||
|
||||
def issue(identity: str, days: int | None, state: dict, quiet=False) -> str:
|
||||
ca_key, ca_cert = load_ca()
|
||||
old = state["issued"].get(identity, {}).get("serial")
|
||||
not_after = now() + dt.timedelta(days=days) if days else None
|
||||
key, cert = leaf(ca_key, ca_cert, identity, IDENTITIES[identity], not_after=not_after)
|
||||
store_leaf(TLS_DIR / identity, key, cert)
|
||||
state["issued"][identity] = {"serial": serial_hex(cert), "not_after": cert.not_valid_after_utc.isoformat(),
|
||||
"previous": old}
|
||||
if not quiet:
|
||||
print(f"{identity}: issued serial {serial_hex(cert)} (valid until {cert.not_valid_after_utc:%Y-%m-%d %H:%M} UTC)"
|
||||
+ (f", previous serial {old}" if old else ""))
|
||||
return serial_hex(cert)
|
||||
|
||||
|
||||
def cmd_init(_args):
|
||||
os.umask(0o022)
|
||||
CA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(CA_DIR, 0o700)
|
||||
state = load_state()
|
||||
if not (CA_DIR / "ca.key").exists():
|
||||
key, cert = new_ca("MTUSI lab7 local CA")
|
||||
write(CA_DIR / "ca.key", pem_key(key), 0o600)
|
||||
write(PUB_DIR / "ca.crt", pem_cert(cert))
|
||||
state = {"issued": {}, "revoked": []}
|
||||
print("CA created: secrets/tls/ca/ca.crt (public), secrets/ca/ca.key (private, operator only)")
|
||||
ca_key, ca_cert = load_ca()
|
||||
for identity in IDENTITIES:
|
||||
path = TLS_DIR / identity / "tls.crt"
|
||||
if path.exists():
|
||||
cert = x509.load_pem_x509_certificate(path.read_bytes())
|
||||
fresh = cert.not_valid_after_utc - now() > dt.timedelta(days=RENEW_BEFORE_DAYS)
|
||||
if fresh and cert.issuer == ca_cert.subject:
|
||||
continue
|
||||
issue(identity, None, state)
|
||||
# Negative-test material: never mounted into services.
|
||||
if not (TEST_DIR / "intruder" / "tls.crt").exists():
|
||||
store_leaf(TEST_DIR / "intruder", *leaf(ca_key, ca_cert, "intruder", []))
|
||||
rogue_key, rogue_cert = new_ca("Rogue CA (negative tests)")
|
||||
write(TEST_DIR / "rogue-ca" / "ca.crt", pem_cert(rogue_cert))
|
||||
store_leaf(TEST_DIR / "rogue-lb", *leaf(rogue_key, rogue_cert, "lb", LOCAL))
|
||||
store_leaf(TEST_DIR / "lb-expired", *leaf(ca_key, ca_cert, "lb", LOCAL,
|
||||
not_before=now() - dt.timedelta(days=2),
|
||||
not_after=now() - dt.timedelta(days=1)))
|
||||
print("test certificates: secrets/tls-test/{intruder,rogue-lb,lb-expired,rogue-ca}")
|
||||
revoked = write_crl(ca_key, ca_cert, state)
|
||||
save_state(state)
|
||||
print(f"CRL signed ({revoked} revoked serials, valid {CRL_DAYS} days). Private keys were not printed.")
|
||||
|
||||
|
||||
def cmd_issue(args):
|
||||
if args.identity not in IDENTITIES:
|
||||
sys.exit(f"unknown identity {args.identity}; known: {', '.join(IDENTITIES)}")
|
||||
state = load_state()
|
||||
issue(args.identity, args.days, state)
|
||||
save_state(state)
|
||||
|
||||
|
||||
def cmd_revoke(args):
|
||||
state = load_state()
|
||||
serial = args.serial.upper().lstrip("0") or "0"
|
||||
if state["issued"].get(args.identity, {}).get("serial") == serial:
|
||||
sys.exit("refusing to revoke the CURRENT certificate of this identity: issue a new one first")
|
||||
if not any(r["serial"] == serial for r in state["revoked"]):
|
||||
state["revoked"].append({"identity": args.identity, "serial": serial, "at": now().isoformat()})
|
||||
ca_key, ca_cert = load_ca()
|
||||
count = write_crl(ca_key, ca_cert, state)
|
||||
save_state(state)
|
||||
print(f"revoked {args.identity} serial {serial}; CRL now lists {count} serials")
|
||||
|
||||
|
||||
def cmd_revoke_previous(args):
|
||||
"""Отозвать ПРЕДЫДУЩИЙ сертификат identity (после того как сервис перешёл на новый)."""
|
||||
state = load_state()
|
||||
previous = state["issued"].get(args.identity, {}).get("previous")
|
||||
if not previous:
|
||||
sys.exit(f"{args.identity}: no previous certificate recorded (run issue first)")
|
||||
args.serial = previous
|
||||
cmd_revoke(args)
|
||||
|
||||
|
||||
def cmd_crl(_args):
|
||||
state = load_state()
|
||||
ca_key, ca_cert = load_ca()
|
||||
print(f"CRL re-signed: {write_crl(ca_key, ca_cert, state)} revoked serials")
|
||||
|
||||
|
||||
def cmd_show(_args):
|
||||
state = load_state()
|
||||
for identity, item in sorted(state["issued"].items()):
|
||||
print(f"{identity:15} serial {item['serial']:>40} until {item['not_after'][:16]}")
|
||||
for item in state["revoked"]:
|
||||
print(f"revoked: {item['identity']} {item['serial']} at {item['at'][:19]}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("init").set_defaults(fn=cmd_init)
|
||||
p = sub.add_parser("issue")
|
||||
p.add_argument("identity")
|
||||
p.add_argument("--days", type=int)
|
||||
p.set_defaults(fn=cmd_issue)
|
||||
p = sub.add_parser("revoke")
|
||||
p.add_argument("identity")
|
||||
p.add_argument("--serial", required=True)
|
||||
p.set_defaults(fn=cmd_revoke)
|
||||
p = sub.add_parser("revoke-previous")
|
||||
p.add_argument("identity")
|
||||
p.set_defaults(fn=cmd_revoke_previous)
|
||||
sub.add_parser("crl").set_defaults(fn=cmd_crl)
|
||||
sub.add_parser("show").set_defaults(fn=cmd_show)
|
||||
args = parser.parse_args()
|
||||
args.fn(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
@echo off
|
||||
rem Windows restart hook for make test-persistence: same steps as scripts/restart.sh.
|
||||
rem Python on Windows cannot run .sh directly (WinError 193), so this .cmd is used instead.
|
||||
rem Restarts ALL api replicas, the worker and ALL stores (PostgreSQL, Valkey, S3 SeaweedFS, RabbitMQ),
|
||||
rem then the balancer. Volumes are kept (no down -v, no queue purge). Same containers, new start time.
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop api worker || exit /b 1
|
||||
docker compose restart postgres valkey s3 broker || exit /b 1
|
||||
docker compose start worker api || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Restart hook для make test-persistence (Linux/macOS).
|
||||
# Перезапускает ВСЕ реплики API, worker и ВСЕ используемые хранилища (PostgreSQL, Valkey, S3, RabbitMQ),
|
||||
# а затем балансировщик. Volumes сохраняются: нет down -v, flush, purge очереди или повторного seed.
|
||||
# Контейнеры те же (restart/stop/start, не recreate): ID не меняется, меняется время старта.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose stop api worker # API: readiness 503 -> уход из балансировки; worker доделывает задания
|
||||
docker compose restart postgres valkey s3 broker # корректная остановка (SIGINT/SIGTERM) и запуск тех же контейнеров
|
||||
docker compose start worker api # ждут PostgreSQL и брокер сами (повторные подключения)
|
||||
docker compose restart lb # балансировщик заново находит реплики по DNS
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Failover hook for make test-failover on Windows (same as scripts/stop-one.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Stops ONE api replica from TARGET_INSTANCE_ID; the logic is in scripts\stop_one.py.
|
||||
python "%~dp0stop_one.py"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Failover hook для make test-failover (Linux/macOS): останавливает ОДНУ реплику API —
|
||||
# ту, чей X-Instance-Id тест передал в TARGET_INSTANCE_ID, и оставляет её остановленной.
|
||||
# Логика (проверка значения, поиск контейнера, docker stop) — в scripts/stop_one.py,
|
||||
# общем для этого скрипта и для scripts/stop-one.cmd (Windows).
|
||||
# Восстановить реплику после теста: make restore (docker compose up -d --wait).
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
exec "${PYTHON:-python3}" "$(dirname "$0")/stop_one.py"
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Failover hook logic for tests/resilience.py --mode failover (lab3, updated for lab7).
|
||||
|
||||
Called by scripts/stop-one.sh (Linux/macOS) and scripts/stop-one.cmd (Windows).
|
||||
The test puts X-Instance-Id of the replica that served its first message into
|
||||
TARGET_INSTANCE_ID. Since lab7 that header is a random per-process alias, not the
|
||||
container hostname (contracts/README.md: "use an opaque replica alias, no hostname/IP").
|
||||
Each API process writes its alias into /tmp/instance-id inside its own container
|
||||
(tmpfs), so the operator -- who already has Docker access -- can map alias to
|
||||
container, while an API client cannot. This script reads that file in every running
|
||||
container of the "api" service of this Compose project and stops the matching one.
|
||||
Postgres, Valkey, S3, the broker, OpenBao, the balancer and volumes are not touched.
|
||||
The replica stays stopped; restore it with `make restore` or `docker start <id>`.
|
||||
|
||||
docker stop sends SIGTERM: the replica first answers 503 on /health/ready for
|
||||
SHUTDOWN_DRAIN_SECONDS (HAProxy takes it out of rotation), then finishes started
|
||||
requests and exits. --time 8: SIGKILL after 8 s at most, so the hook always returns
|
||||
within the 10 s the test allows.
|
||||
|
||||
This file is plain ASCII on purpose: scripts/check_contract.py reads every
|
||||
scripts/*.py with the default encoding, which is cp1251 on a Russian Windows.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STOP_TIMEOUT_SECONDS = 8
|
||||
ALIAS_FILE = os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id")
|
||||
|
||||
|
||||
def alias_of(container: str) -> str:
|
||||
result = subprocess.run(["docker", "exec", container, "cat", ALIAS_FILE],
|
||||
capture_output=True, text=True)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
|
||||
# The value comes from an HTTP header: accept only our own alias format.
|
||||
if not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", target):
|
||||
print("TARGET_INSTANCE_ID must be a replica alias (X-Instance-Id)", file=sys.stderr)
|
||||
return 2
|
||||
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout.split()
|
||||
matches = [container for container in running if alias_of(container) == target]
|
||||
if len(matches) != 1:
|
||||
print("no running api container with alias " + target, file=sys.stderr)
|
||||
return 1
|
||||
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
print("stopped api container " + matches[0][:12])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user