This commit is contained in:
2026-11-08 00:33:25 +03:00
parent 20afc2c790
commit 3d5c6a6689
148 changed files with 21441 additions and 79 deletions
+252
View File
@@ -0,0 +1,252 @@
"""OpenBao operator tool for the lab7 stand (Python standard library only).
Runs in the short-lived "ops" container (make bao-init / bao-unseal / bao-bootstrap / ...).
It is the ONLY place that reads the operator material in secrets/operator/:
unseal-key-1..3 Shamir shares (threshold 2). In real life: three different people/places.
root-token initial root token (used only by these operator commands)
The files are created with mode 600, never mounted into services, never printed.
Commands:
status initialized / sealed / version (no token needed)
init operator init (3 shares, threshold 2); refuses to run twice
unseal submit shares until unsealed (needed after every OpenBao restart)
seal seal OpenBao (incident drill: sealed OpenBao)
bootstrap idempotent: kv-v2 at secret/, cert auth, audit device, policies,
cert roles for api/worker, application secrets from secrets/*
rotate-db new password for DB role messenger_api: ALTER ROLE + new KV version
revoke-tokens revoke every token issued by the cert auth method (forces re-login)
token-ttl ROLE TTL change token TTL of a cert role (e.g. expiry drill: api 30s)
Plain ASCII on purpose (scripts/check_contract.py reads scripts/*.py with the default encoding).
"""
import json
import os
import secrets as rnd
import ssl
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = Path(os.environ.get("SECRETS_DIR", ROOT / "secrets"))
OPERATOR = SECRETS / "operator"
POLICIES = Path(os.environ.get("BAO_POLICY_DIR", ROOT / "infra" / "openbao" / "policies"))
ADDR = os.environ.get("BAO_ADDR", "https://openbao:8200").rstrip("/")
CACERT = os.environ.get("BAO_CACERT", str(SECRETS / "tls" / "ca" / "ca.crt"))
SHARES, THRESHOLD = 3, 2
# role -> (certificate CN, policy, secret path). One policy per service, nothing shared.
ROLES = {"messenger-api": ("api", "messenger-api", "messenger/api"),
"messenger-worker": ("worker", "messenger-worker", "messenger/worker")}
TOKEN_TTL, TOKEN_MAX_TTL = os.environ.get("BAO_TOKEN_TTL", "15m"), os.environ.get("BAO_TOKEN_MAX_TTL", "1h")
CTX = ssl.create_default_context(cafile=CACERT) # server certificate and hostname are verified
class BaoError(Exception):
def __init__(self, status, errors):
super().__init__(f"HTTP {status}: {'; '.join(errors) or 'no details'}")
self.status = status
def call(method, path, body=None, token=None, ok=(200, 204)):
data = None if body is None else json.dumps(body).encode()
request = urllib.request.Request(ADDR + "/v1/" + path.lstrip("/"), data=data, method=method)
request.add_header("Content-Type", "application/json")
if token:
request.add_header("X-Vault-Token", token)
try:
with urllib.request.urlopen(request, context=CTX, timeout=10) as response:
raw = response.read()
status = response.status
except urllib.error.HTTPError as exc:
raw, status = exc.read(), exc.code
payload = json.loads(raw) if raw.strip() else {}
if status not in ok:
# Error bodies of OpenBao contain no secret values; tokens are never echoed.
raise BaoError(status, payload.get("errors", []))
return payload
def secret_file(name: str) -> str:
return (SECRETS / name).read_text(encoding="utf-8").strip()
def write_private(path: Path, value: str):
path.parent.mkdir(parents=True, exist_ok=True)
os.chmod(path.parent, 0o700)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w") as fh:
fh.write(value)
def root_token() -> str:
path = OPERATOR / "root-token"
if not path.exists():
sys.exit("secrets/operator/root-token not found: run make bao-init first")
return path.read_text().strip()
def cmd_status(_args):
health = call("GET", "sys/seal-status")
print(f"initialized={health['initialized']} sealed={health['sealed']} "
f"progress={health.get('progress', 0)}/{health.get('t', 0)} version={health.get('version')} "
f"storage={health.get('storage_type')}")
return 0 if not health["sealed"] else 2
def cmd_init(_args):
if call("GET", "sys/init")["initialized"]:
print("OpenBao already initialized: nothing to do (shares and root token stay with the operator)")
return 0
result = call("PUT", "sys/init", {"secret_shares": SHARES, "secret_threshold": THRESHOLD})
for number, share in enumerate(result["keys_base64"], start=1):
write_private(OPERATOR / f"unseal-key-{number}", share)
write_private(OPERATOR / "root-token", result["root_token"])
print(f"initialized: {SHARES} unseal shares (threshold {THRESHOLD}) and root token saved to secrets/operator/ "
"(mode 600, not printed). Keep the shares apart: in production every share goes to a different person.")
return 0
def cmd_unseal(_args):
status = call("GET", "sys/seal-status")
if not status["initialized"]:
sys.exit("not initialized: run make bao-init")
if not status["sealed"]:
print("already unsealed")
return 0
keys = sorted(OPERATOR.glob("unseal-key-*"))
if len(keys) < THRESHOLD:
sys.exit(f"need {THRESHOLD} shares in secrets/operator/unseal-key-N")
for path in keys[:THRESHOLD]:
status = call("PUT", "sys/unseal", {"key": path.read_text().strip()})
print(f"sealed={status['sealed']} (used {THRESHOLD} of {len(keys)} shares)")
return 0 if not status["sealed"] else 1
def cmd_seal(_args):
call("PUT", "sys/seal", token=root_token())
print("sealed: services keep their cached secrets, new logins and reads fail until make bao-unseal")
return 0
def cmd_bootstrap(_args):
token = root_token()
if "secret/" not in call("GET", "sys/mounts", token=token)["data"]:
call("POST", "sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}, token=token)
print("enabled kv-v2 at secret/")
if "cert/" not in call("GET", "sys/auth", token=token)["data"]:
call("POST", "sys/auth/cert", {"type": "cert"}, token=token)
print("enabled cert auth at auth/cert/")
# Audit device is declared in infra/openbao/bao.hcl: OpenBao 2.4 refuses to enable it via the API.
ca_pem = Path(CACERT).read_text()
for role, (common_name, policy, secret_path) in ROLES.items():
call("PUT", f"sys/policies/acl/{policy}",
{"policy": (POLICIES / f"{policy}.hcl").read_text(encoding="utf-8")}, token=token)
call("POST", f"auth/cert/certs/{role}", {
"display_name": role,
"certificate": ca_pem, # trust anchor: our CA only
"allowed_common_names": [common_name], # and only this identity
"token_policies": [policy],
"token_ttl": TOKEN_TTL, "token_max_ttl": TOKEN_MAX_TTL,
"token_no_default_policy": False, # default policy = lookup-self / renew-self only
}, token=token)
put_kv(token, "messenger/api", {
"db_password": secret_file("db_api_password"),
"s3_access_key_id": secret_file("s3_api_access_key_id"),
"s3_secret_access_key": secret_file("s3_api_secret_access_key"),
"cache_password": secret_file("valkey_api_password"),
})
put_kv(token, "messenger/worker", {
"db_password": secret_file("db_worker_password"),
"s3_access_key_id": secret_file("s3_worker_access_key_id"),
"s3_secret_access_key": secret_file("s3_worker_secret_access_key"),
})
print(f"policies messenger-api, messenger-worker; cert roles (CN api / worker, token ttl {TOKEN_TTL}, "
f"max {TOKEN_MAX_TTL}); secrets secret/messenger/api and secret/messenger/worker written (values not printed)")
return 0
def put_kv(token, path, data):
current = None
try:
current = call("GET", f"secret/data/{path}", token=token)["data"]["data"]
except BaoError as exc:
if exc.status != 404:
raise
if current != data:
call("POST", f"secret/data/{path}", {"data": data}, token=token)
def cmd_rotate_db(_args):
"""Rotate the password of DB role messenger_api without rebuilding images or restarting the API.
Order: 1) ALTER ROLE in PostgreSQL, 2) new KV version in OpenBao, 3) the operator copy.
Between 1) and the next API refresh (BAO_REFRESH_SECONDS) NEW DB connections of the API fail
and are retried; connections that are already open keep working (PostgreSQL checks the password
only at login). The API drains its pool when it sees the new value, so every connection after
that is a new login with the new password.
"""
import psycopg # available in the application image the ops container runs from
from psycopg import sql
token = root_token()
new = rnd.token_hex(24)
owner = secret_file("postgres_password")
with psycopg.connect(host=os.environ.get("DB_HOST", "postgres"), port=int(os.environ.get("DB_PORT", "5432")),
dbname="messenger", user="messenger", password=owner,
sslmode="verify-full", sslrootcert=CACERT, connect_timeout=5, autocommit=True) as conn:
conn.execute(sql.SQL("ALTER ROLE messenger_api PASSWORD {}").format(sql.Literal(new)))
data = call("GET", "secret/data/messenger/api", token=token)["data"]["data"]
data["db_password"] = new
result = call("POST", "secret/data/messenger/api", {"data": data}, token=token)
(SECRETS / "db_api_password").write_text(new)
print(f"rotated DB password of messenger_api: PostgreSQL updated, KV secret/messenger/api now version "
f"{result['data']['version']} (value not printed)")
return 0
def cmd_revoke_tokens(_args):
call("PUT", "sys/leases/revoke-prefix/auth/cert/login", token=root_token())
print("revoked all tokens issued by auth/cert: services must log in again with their certificates")
return 0
def cmd_token_ttl(args):
if len(args) != 2 or args[0] not in ("api", "worker"):
sys.exit("usage: token-ttl api|worker TTL (e.g. 30s, 15m)")
role = f"messenger-{args[0]}"
token = root_token()
current = call("GET", f"auth/cert/certs/{role}", token=token)["data"]
max_ttl = args[1] if args[1].endswith("s") else TOKEN_MAX_TTL
call("POST", f"auth/cert/certs/{role}", {
"display_name": role, "certificate": current["certificate"],
"allowed_common_names": current["allowed_common_names"], "token_policies": current["token_policies"],
"token_ttl": args[1], "token_max_ttl": max_ttl}, token=token)
print(f"{role}: token_ttl={args[1]} token_max_ttl={max_ttl} (applies to NEW logins)")
return 0
COMMANDS = {"status": cmd_status, "init": cmd_init, "unseal": cmd_unseal, "seal": cmd_seal,
"bootstrap": cmd_bootstrap, "rotate-db": cmd_rotate_db, "revoke-tokens": cmd_revoke_tokens,
"token-ttl": cmd_token_ttl}
def main():
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
sys.exit("usage: bao.py " + "|".join(COMMANDS))
deadline = time.monotonic() + 60
while True: # OpenBao may still be starting after docker compose up
try:
return COMMANDS[sys.argv[1]](sys.argv[2:])
except (urllib.error.URLError, ConnectionError, TimeoutError) as exc:
if time.monotonic() > deadline:
sys.exit(f"OpenBao at {ADDR} is not reachable: {getattr(exc, 'reason', exc)}")
time.sleep(2)
except BaoError as exc:
sys.exit(f"OpenBao refused: {exc}")
if __name__ == "__main__":
sys.exit(main())
+68
View File
@@ -0,0 +1,68 @@
#!/bin/sh
# Создаёт локальные секреты стенда в каталоге secrets/ (он в .gitignore и в Git не попадает).
# Запуск один раз: sh scripts/init-secrets.sh (затем сертификаты: make pki)
# Повторный запуск безопасен: существующие файлы не меняются, недостающие досоздаются.
#
# Кто что получает (lab7):
# оператор (этот каталог) — всё; root token и unseal shares OpenBao появятся в secrets/operator/ (make bao-init)
# postgres — пароль владельца схемы и пароли ролей messenger_api/messenger_worker (только для
# первой инициализации volume, infra/postgres/initdb/10-roles.sh)
# migrate — пароль владельца схемы messenger (одноразовый сервис, не приложение)
# valkey — ACL-файл (пароли пользователей)
# s3 (SeaweedFS) — s3_config.json (пользователи и права); s3-init — ключи администратора
# grafana — пароль администратора
# api / worker — НИЧЕГО из этого каталога: свои пароли и ключи они читают из OpenBao
# (make bao-bootstrap переносит их туда), а входят в OpenBao сертификатом.
set -eu
cd "$(dirname "$0")/.."
umask 077 # каталог secrets/ доступен только владельцу
mkdir -p secrets
# 24 случайных байта -> 48 hex-символов. /dev/urandom есть и в Linux/macOS, и в Git Bash.
random_hex() { od -An -tx1 -N24 /dev/urandom | tr -d ' \n'; }
# Файл создаётся, только если его ещё нет (printf без перевода строки: в файле только значение).
make_secret() { [ -e "secrets/$1" ] || printf '%s%s' "$2" "$(random_hex | cut -c1-"$3")" > "secrets/$1"; }
# PostgreSQL: владелец схемы (migrate, оператор) и две роли приложения (с lab7 — раздельные).
make_secret postgres_password "" 48
make_secret db_api_password "" 48
make_secret db_worker_password "" 48
# Valkey: пользователь messenger (API). health — без пароля, только ping/info/dbsize.
make_secret valkey_api_password "" 48
printf '%s\n' \
'user default off' \
'user health on nopass -@all +ping +info +dbsize' \
"user messenger on >$(cat secrets/valkey_api_password) ~sess:* ~rl:* resetchannels -@all +get +mget +set +del +ping +incrby +ttl +multi +exec" \
> secrets/valkey_users.acl
# S3 (SeaweedFS): администратор (только s3-init) и РАЗНЫЕ пользователи для API и worker.
make_secret s3_root_user admin 12
make_secret s3_root_password "" 32
make_secret s3_api_access_key_id api 16
make_secret s3_api_secret_access_key "" 32
make_secret s3_worker_access_key_id wrk 16
make_secret s3_worker_secret_access_key "" 32
# Права по префиксам (с lab7):
# API — пишет только staging/ (принятый файл), читает только originals/ и variants/ (скачивание);
# не может подделать оригинал или вариант и не видит список объектов;
# worker — читает staging/ и originals/, пишет и удаляет в bucket (оригиналы, варианты, уборка),
# видит список (уборщик); не читает variants/ и не управляет bucket.
BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env 2>/dev/null | tr -d '\r')
BUCKET=${BUCKET:-messenger-attachments}
printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","List","Tagging","Write"]},{"name":"messenger-api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Write:%s/staging/*","Read:%s/originals/*","Read:%s/variants/*"]},{"name":"messenger-worker","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s/staging/*","Read:%s/originals/*","Write:%s","List:%s"]}]}\n' \
"$(cat secrets/s3_root_user)" "$(cat secrets/s3_root_password)" \
"$(cat secrets/s3_api_access_key_id)" "$(cat secrets/s3_api_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" \
"$(cat secrets/s3_worker_access_key_id)" "$(cat secrets/s3_worker_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" "$BUCKET" \
> secrets/s3_config.json
# Grafana: пароль администратора (https://127.0.0.1:3000, пользователь admin).
make_secret grafana_admin_password "" 24
# С lab7 у брокера паролей нет: вход по клиентским сертификатам (make pki).
# Файлы читают процессы в контейнерах под своими пользователями (postgres, valkey, seaweedfs, grafana),
# поэтому сами файлы доступны на чтение; посторонних на хосте не пускает каталог (700).
chmod 644 secrets/*.json secrets/*.acl secrets/*_password secrets/*_key_id secrets/*_access_key secrets/s3_root_* 2>/dev/null || true
echo "Готово: secrets/ (PostgreSQL x3, Valkey, S3 admin/API/worker, Grafana; значения не выводятся). Дальше: make pki"
+284
View File
@@ -0,0 +1,284 @@
"""Local PKI of the lab7 stand: one CA, one leaf certificate per service identity, CRL.
Operator tool. Runs in the "pki" container (docker compose --profile tools run --rm pki ...),
writes ONLY into ./secrets (in .gitignore), never prints private keys.
Layout:
secrets/ca/ca.key CA private key: only this tool reads it (never mounted into services)
secrets/ca/state.json issued serials and revoked serials (no secrets inside)
secrets/tls/ca/ca.crt CA certificate: public, mounted into every service as /run/ca/ca.crt
secrets/tls/ca/crl.pem CRL signed by the CA: revoked leaf serials (API checks it for LB identity)
secrets/tls/<id>/tls.crt leaf certificate of identity <id> (CN = SAN DNS = <id>)
secrets/tls/<id>/tls.key its private key (mounted only into the container of that service)
secrets/tls/<id>/tls.pem cert + key in one file (HAProxy format)
secrets/tls-test/... certificates for NEGATIVE tests only: intruder (same CA, unknown identity),
rogue-lb (identity "lb" from a foreign CA), lb-expired (expired "lb")
Commands:
init create CA if missing, issue missing/expiring leaves, test certs, CRL (idempotent)
issue <id> [--days N] re-issue one leaf IN PLACE (rotation; files are overwritten, not replaced,
so bind mounts see the new content). Prints old and new serial.
revoke <id> --serial S add serial S of <id> to the CRL (S from "show" or from "issue" output)
revoke-previous <id> revoke the certificate that "issue" replaced (rotation: issue -> reload -> revoke)
crl re-sign the CRL (it is valid CRL_DAYS days; run before it expires)
show identities, serials, expiry dates
Plain ASCII on purpose (scripts/check_contract.py reads scripts/*.py with the default encoding).
"""
import argparse
import datetime as dt
import ipaddress
import json
import os
import sys
from pathlib import Path
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
ROOT = Path(os.environ.get("PKI_ROOT", Path(__file__).resolve().parents[1] / "secrets"))
CA_DIR, PUB_DIR, TLS_DIR, TEST_DIR = ROOT / "ca", ROOT / "tls" / "ca", ROOT / "tls", ROOT / "tls-test"
CA_DAYS, LEAF_DAYS, CRL_DAYS, RENEW_BEFORE_DAYS = 365, 30, 30, 7
LOCAL = ["localhost", "127.0.0.1"]
# identity -> extra SAN names. CN = first SAN = identity. All leaves: serverAuth + clientAuth.
IDENTITIES = {
"lb": LOCAL, # external TLS ingress (8443) and mTLS CLIENT of every API replica
"api": [], # mTLS server for LB; client of postgres/s3/valkey/broker/collector/openbao
"worker": [], # client of postgres/s3/broker/collector/openbao
"postgres": LOCAL,
"valkey": LOCAL,
"s3": LOCAL,
"broker": LOCAL,
"otel-collector": LOCAL, # OTLP gateway server; client of prometheus/loki/tempo
"prometheus": LOCAL,
"loki": LOCAL,
"tempo": LOCAL,
"grafana": LOCAL, # HTTPS for the operator; client of prometheus/loki/tempo
"openbao": LOCAL,
}
def now():
return dt.datetime.now(dt.timezone.utc)
def write(path: Path, data: bytes, mode: int = 0o644):
"""Overwrite in place (same inode): containers that bind-mount the directory see the change."""
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "r+b" if path.exists() else "wb") as fh:
fh.seek(0)
fh.write(data)
fh.truncate()
os.chmod(path, mode)
def pem_key(key) -> bytes:
return key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
serialization.NoEncryption())
def pem_cert(cert) -> bytes:
return cert.public_bytes(serialization.Encoding.PEM)
def load_state() -> dict:
path = CA_DIR / "state.json"
return json.loads(path.read_text()) if path.exists() else {"issued": {}, "revoked": []}
def save_state(state: dict):
write(CA_DIR / "state.json", json.dumps(state, indent=2, sort_keys=True).encode(), 0o600)
def new_ca(common_name: str):
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI lab7"),
x509.NameAttribute(NameOID.COMMON_NAME, common_name)])
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now() - dt.timedelta(minutes=5)).not_valid_after(now() + dt.timedelta(days=CA_DAYS))
.add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True)
.add_extension(x509.KeyUsage(digital_signature=False, content_commitment=False, key_encipherment=False,
data_encipherment=False, key_agreement=False, key_cert_sign=True,
crl_sign=True, encipher_only=False, decipher_only=False), critical=True)
.add_extension(x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False)
.sign(key, hashes.SHA256()))
return key, cert
def load_ca():
key = serialization.load_pem_private_key((CA_DIR / "ca.key").read_bytes(), None)
cert = x509.load_pem_x509_certificate((PUB_DIR / "ca.crt").read_bytes())
return key, cert
def leaf(ca_key, ca_cert, identity: str, extra: list, not_before=None, not_after=None):
key = ec.generate_private_key(ec.SECP256R1())
names = []
for value in [identity] + list(extra):
try:
names.append(x509.IPAddress(ipaddress.ip_address(value)))
except ValueError:
names.append(x509.DNSName(value))
not_before = not_before or now() - dt.timedelta(minutes=5)
not_after = not_after or now() + dt.timedelta(days=LEAF_DAYS)
cert = (x509.CertificateBuilder()
.subject_name(x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI lab7"),
x509.NameAttribute(NameOID.COMMON_NAME, identity)]))
.issuer_name(ca_cert.subject).public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(not_before).not_valid_after(not_after)
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
.add_extension(x509.KeyUsage(digital_signature=True, content_commitment=False, key_encipherment=False,
data_encipherment=False, key_agreement=True, key_cert_sign=False,
crl_sign=False, encipher_only=False, decipher_only=False), critical=True)
.add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH,
ExtendedKeyUsageOID.CLIENT_AUTH]), critical=False)
.add_extension(x509.SubjectAlternativeName(names), critical=False)
.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), critical=False)
.sign(ca_key, hashes.SHA256()))
return key, cert
def store_leaf(directory: Path, key, cert):
write(directory / "tls.key", pem_key(key))
write(directory / "tls.crt", pem_cert(cert))
write(directory / "tls.pem", pem_cert(cert) + pem_key(key))
def serial_hex(cert) -> str:
return format(cert.serial_number, "X")
def write_crl(ca_key, ca_cert, state):
builder = (x509.CertificateRevocationListBuilder().issuer_name(ca_cert.subject)
.last_update(now() - dt.timedelta(minutes=5)).next_update(now() + dt.timedelta(days=CRL_DAYS)))
for item in state["revoked"]:
builder = builder.add_revoked_certificate(
x509.RevokedCertificateBuilder().serial_number(int(item["serial"], 16))
.revocation_date(dt.datetime.fromisoformat(item["at"])).build())
crl = builder.sign(ca_key, hashes.SHA256())
write(PUB_DIR / "crl.pem", crl.public_bytes(serialization.Encoding.PEM))
return len(state["revoked"])
def issue(identity: str, days: int | None, state: dict, quiet=False) -> str:
ca_key, ca_cert = load_ca()
old = state["issued"].get(identity, {}).get("serial")
not_after = now() + dt.timedelta(days=days) if days else None
key, cert = leaf(ca_key, ca_cert, identity, IDENTITIES[identity], not_after=not_after)
store_leaf(TLS_DIR / identity, key, cert)
state["issued"][identity] = {"serial": serial_hex(cert), "not_after": cert.not_valid_after_utc.isoformat(),
"previous": old}
if not quiet:
print(f"{identity}: issued serial {serial_hex(cert)} (valid until {cert.not_valid_after_utc:%Y-%m-%d %H:%M} UTC)"
+ (f", previous serial {old}" if old else ""))
return serial_hex(cert)
def cmd_init(_args):
os.umask(0o022)
CA_DIR.mkdir(parents=True, exist_ok=True)
os.chmod(CA_DIR, 0o700)
state = load_state()
if not (CA_DIR / "ca.key").exists():
key, cert = new_ca("MTUSI lab7 local CA")
write(CA_DIR / "ca.key", pem_key(key), 0o600)
write(PUB_DIR / "ca.crt", pem_cert(cert))
state = {"issued": {}, "revoked": []}
print("CA created: secrets/tls/ca/ca.crt (public), secrets/ca/ca.key (private, operator only)")
ca_key, ca_cert = load_ca()
for identity in IDENTITIES:
path = TLS_DIR / identity / "tls.crt"
if path.exists():
cert = x509.load_pem_x509_certificate(path.read_bytes())
fresh = cert.not_valid_after_utc - now() > dt.timedelta(days=RENEW_BEFORE_DAYS)
if fresh and cert.issuer == ca_cert.subject:
continue
issue(identity, None, state)
# Negative-test material: never mounted into services.
if not (TEST_DIR / "intruder" / "tls.crt").exists():
store_leaf(TEST_DIR / "intruder", *leaf(ca_key, ca_cert, "intruder", []))
rogue_key, rogue_cert = new_ca("Rogue CA (negative tests)")
write(TEST_DIR / "rogue-ca" / "ca.crt", pem_cert(rogue_cert))
store_leaf(TEST_DIR / "rogue-lb", *leaf(rogue_key, rogue_cert, "lb", LOCAL))
store_leaf(TEST_DIR / "lb-expired", *leaf(ca_key, ca_cert, "lb", LOCAL,
not_before=now() - dt.timedelta(days=2),
not_after=now() - dt.timedelta(days=1)))
print("test certificates: secrets/tls-test/{intruder,rogue-lb,lb-expired,rogue-ca}")
revoked = write_crl(ca_key, ca_cert, state)
save_state(state)
print(f"CRL signed ({revoked} revoked serials, valid {CRL_DAYS} days). Private keys were not printed.")
def cmd_issue(args):
if args.identity not in IDENTITIES:
sys.exit(f"unknown identity {args.identity}; known: {', '.join(IDENTITIES)}")
state = load_state()
issue(args.identity, args.days, state)
save_state(state)
def cmd_revoke(args):
state = load_state()
serial = args.serial.upper().lstrip("0") or "0"
if state["issued"].get(args.identity, {}).get("serial") == serial:
sys.exit("refusing to revoke the CURRENT certificate of this identity: issue a new one first")
if not any(r["serial"] == serial for r in state["revoked"]):
state["revoked"].append({"identity": args.identity, "serial": serial, "at": now().isoformat()})
ca_key, ca_cert = load_ca()
count = write_crl(ca_key, ca_cert, state)
save_state(state)
print(f"revoked {args.identity} serial {serial}; CRL now lists {count} serials")
def cmd_revoke_previous(args):
"""Отозвать ПРЕДЫДУЩИЙ сертификат identity (после того как сервис перешёл на новый)."""
state = load_state()
previous = state["issued"].get(args.identity, {}).get("previous")
if not previous:
sys.exit(f"{args.identity}: no previous certificate recorded (run issue first)")
args.serial = previous
cmd_revoke(args)
def cmd_crl(_args):
state = load_state()
ca_key, ca_cert = load_ca()
print(f"CRL re-signed: {write_crl(ca_key, ca_cert, state)} revoked serials")
def cmd_show(_args):
state = load_state()
for identity, item in sorted(state["issued"].items()):
print(f"{identity:15} serial {item['serial']:>40} until {item['not_after'][:16]}")
for item in state["revoked"]:
print(f"revoked: {item['identity']} {item['serial']} at {item['at'][:19]}")
def main():
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("init").set_defaults(fn=cmd_init)
p = sub.add_parser("issue")
p.add_argument("identity")
p.add_argument("--days", type=int)
p.set_defaults(fn=cmd_issue)
p = sub.add_parser("revoke")
p.add_argument("identity")
p.add_argument("--serial", required=True)
p.set_defaults(fn=cmd_revoke)
p = sub.add_parser("revoke-previous")
p.add_argument("identity")
p.set_defaults(fn=cmd_revoke_previous)
sub.add_parser("crl").set_defaults(fn=cmd_crl)
sub.add_parser("show").set_defaults(fn=cmd_show)
args = parser.parse_args()
args.fn(args)
if __name__ == "__main__":
main()
+11
View File
@@ -0,0 +1,11 @@
@echo off
rem Windows restart hook for make test-persistence: same steps as scripts/restart.sh.
rem Python on Windows cannot run .sh directly (WinError 193), so this .cmd is used instead.
rem Restarts ALL api replicas, the worker and ALL stores (PostgreSQL, Valkey, S3 SeaweedFS, RabbitMQ),
rem then the balancer. Volumes are kept (no down -v, no queue purge). Same containers, new start time.
cd /d "%~dp0.."
docker compose stop api worker || exit /b 1
docker compose restart postgres valkey s3 broker || exit /b 1
docker compose start worker api || exit /b 1
docker compose restart lb || exit /b 1
exit /b 0
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
# Restart hook для make test-persistence (Linux/macOS).
# Перезапускает ВСЕ реплики API, worker и ВСЕ используемые хранилища (PostgreSQL, Valkey, S3, RabbitMQ),
# а затем балансировщик. Volumes сохраняются: нет down -v, flush, purge очереди или повторного seed.
# Контейнеры те же (restart/stop/start, не recreate): ID не меняется, меняется время старта.
set -eu
cd "$(dirname "$0")/.."
docker compose stop api worker # API: readiness 503 -> уход из балансировки; worker доделывает задания
docker compose restart postgres valkey s3 broker # корректная остановка (SIGINT/SIGTERM) и запуск тех же контейнеров
docker compose start worker api # ждут PostgreSQL и брокер сами (повторные подключения)
docker compose restart lb # балансировщик заново находит реплики по DNS
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Failover hook for make test-failover on Windows (same as scripts/stop-one.sh).
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
rem Stops ONE api replica from TARGET_INSTANCE_ID; the logic is in scripts\stop_one.py.
python "%~dp0stop_one.py"
exit /b %ERRORLEVEL%
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Failover hook для make test-failover (Linux/macOS): останавливает ОДНУ реплику API —
# ту, чей X-Instance-Id тест передал в TARGET_INSTANCE_ID, и оставляет её остановленной.
# Логика (проверка значения, поиск контейнера, docker stop) — в scripts/stop_one.py,
# общем для этого скрипта и для scripts/stop-one.cmd (Windows).
# Восстановить реплику после теста: make restore (docker compose up -d --wait).
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
exec "${PYTHON:-python3}" "$(dirname "$0")/stop_one.py"
+58
View File
@@ -0,0 +1,58 @@
"""Failover hook logic for tests/resilience.py --mode failover (lab3, updated for lab7).
Called by scripts/stop-one.sh (Linux/macOS) and scripts/stop-one.cmd (Windows).
The test puts X-Instance-Id of the replica that served its first message into
TARGET_INSTANCE_ID. Since lab7 that header is a random per-process alias, not the
container hostname (contracts/README.md: "use an opaque replica alias, no hostname/IP").
Each API process writes its alias into /tmp/instance-id inside its own container
(tmpfs), so the operator -- who already has Docker access -- can map alias to
container, while an API client cannot. This script reads that file in every running
container of the "api" service of this Compose project and stops the matching one.
Postgres, Valkey, S3, the broker, OpenBao, the balancer and volumes are not touched.
The replica stays stopped; restore it with `make restore` or `docker start <id>`.
docker stop sends SIGTERM: the replica first answers 503 on /health/ready for
SHUTDOWN_DRAIN_SECONDS (HAProxy takes it out of rotation), then finishes started
requests and exits. --time 8: SIGKILL after 8 s at most, so the hook always returns
within the 10 s the test allows.
This file is plain ASCII on purpose: scripts/check_contract.py reads every
scripts/*.py with the default encoding, which is cp1251 on a Russian Windows.
"""
import os
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
STOP_TIMEOUT_SECONDS = 8
ALIAS_FILE = os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id")
def alias_of(container: str) -> str:
result = subprocess.run(["docker", "exec", container, "cat", ALIAS_FILE],
capture_output=True, text=True)
return result.stdout.strip() if result.returncode == 0 else ""
def main() -> int:
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
# The value comes from an HTTP header: accept only our own alias format.
if not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", target):
print("TARGET_INSTANCE_ID must be a replica alias (X-Instance-Id)", file=sys.stderr)
return 2
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
capture_output=True, text=True, check=True).stdout.split()
matches = [container for container in running if alias_of(container) == target]
if len(matches) != 1:
print("no running api container with alias " + target, file=sys.stderr)
return 1
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
print("stopped api container " + matches[0][:12])
return 0
if __name__ == "__main__":
sys.exit(main())