From f2ea9d65a85879debdcc5e0ca019943a2747e936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Fri, 20 Nov 2026 19:02:00 +0300 Subject: [PATCH] feat(security): add operator rotation and recovery commands --- .github/pull_request_template.md | 33 +- REPORT.md | 2 +- compose.test.yaml | 26 ++ compose.yaml | 52 ++- scripts/bootstrap_security.py | 22 +- scripts/credentials.py | 231 +++++++++++++ scripts/observe.py | 40 ++- scripts/operator_task.py | 57 ++++ scripts/pki.py | 359 ++++++++++++++++++++ scripts/prepare_dependencies.py | 15 +- scripts/research_slo.py | 3 + scripts/restart.sh | 3 +- scripts/restart_secure.py | 23 ++ scripts/stand.py | 39 ++- scripts/tls_client.py | 23 ++ scripts/up_security.py | 9 + scripts/verify_cleanup.py | 8 +- scripts/verify_observability_persistence.py | 10 +- scripts/verify_pipeline.py | 7 +- scripts/verify_telemetry.py | 15 +- src/security_probe.py | 8 +- student-unit/test_security.py | 44 +++ 22 files changed, 983 insertions(+), 46 deletions(-) create mode 100644 scripts/credentials.py create mode 100644 scripts/operator_task.py create mode 100644 scripts/pki.py create mode 100644 scripts/restart_secure.py create mode 100644 scripts/tls_client.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8aef0ff..d8018f3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,23 @@ ## Лабораторная 7 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Маслов, БВТ2301. +Целевая оценка: 5, трек 2 — жизненный цикл сертификатов и секретов. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +**Черновик. Работа пока не готова к сдаче.** -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Перенесено собственное приложение ЛР6: API, очередь задач, обработка изображений отдельным worker, наблюдаемость. Добавлены TLS/mTLS, раздельные удостоверения и права сервисов, конфигурация OpenBao, короткие сертификаты и токены, операторские команды ротации и восстановления, ограничения входа перед хешированием пароля, обновление старого хеша при успешном входе. Границы доверия и ограничения описаны в REPORT.md, contracts/trust-matrix.md и security/THREAT_MODEL.md. + +Пройдены 29 unit-тестов, проверки структуры/OpenAPI/собственного Python-кода и конфигурации Compose. Первый вариант образов собран. pip-audit проверил 40 закреплённых runtime-зависимостей Python без известных уязвимостей по своей базе на момент запуска. Это не проверка безопасности образов. + +Не завершены: запуск всего защищённого стенда, HTTP-регрессия, положительные и отрицательные проверки mTLS/прав, исследование ротации/отзыва/истечения сертификатов и замены CA под нагрузкой, проверка образов и воспроизведение из чистого клона. Пока нет результатов, подтверждающих выполнение требований на оценку 5. + +Запуск оператора: `uv sync --locked`, `make up PYTHON=.venv/bin/python`. Предполагаемый API: `https://localhost:8443`; доверенный CA создаётся локально в `trust/ca.pem`. Команды и подробности: [REPORT.md](https://gitea.privcloud.tech/wh1tet3a/mtusi2026lab7/src/branch/solution/bvt2301-maslov/REPORT.md). + +Источники: задание, собственная ЛР6, официальная документация OpenBao/HAProxy/OWASP. Использована помощь ИИ при проектировании, реализации, проверках и подготовке отчёта; чужие студенческие решения не использовались. + +- [ ] Весь обязательный минимум подтверждён запуском и регрессией. +- [x] Контракт и публичные тесты сохранены. +- [ ] make test и сценарии безопасности выполнены на защищённом стенде. +- [ ] Исследование выбранного трека выполнено и измерения приложены. +- [x] Секреты и приватные ключи находятся вне Git. +- [x] README задания сохранён, собственный REPORT.md заполнен. diff --git a/REPORT.md b/REPORT.md index bcab577..8467b59 100644 --- a/REPORT.md +++ b/REPORT.md @@ -2,7 +2,7 @@ Цель — оценка 5, трек 2: жизненный цикл PKI и секретов. Собственный код ЛР6 перенесён поверх независимого skeleton ЛР7. README/COURSE/публичные tests/OpenAPI/check/validate сохранены; trust matrix заполнена как требует задание. -Текущий статус: реализация готовится к первому запуску. Пройдены 28 unit tests, offline/OpenAPI/lint, проверка конфигурации Compose. Сканирование pip-audit2.10.1 проверило 40 закреплённых runtime зависимостей и не нашло известных уязвимостей в своей базе на момент запуска (`evidence/dependency-scan-first.json`). Это не результат проверки образов или полного стенда. HTTP/mTLS/OpenBao/ротация/исследование/чистый clone пока не подтверждены. +Текущий статус: реализация готовится к первому запуску. Пройдены 29 unit tests, offline/OpenAPI/lint, проверка конфигурации Compose. Сканирование pip-audit2.10.1 проверило 40 закреплённых runtime зависимостей и не нашло известных уязвимостей в своей базе на момент запуска (`evidence/dependency-scan-first.json`). Это не результат проверки образов или полного стенда. HTTP/mTLS/OpenBao/ротация/исследование/чистый clone пока не подтверждены. ## Запуск оператора diff --git a/compose.test.yaml b/compose.test.yaml index 9de164b..608dfca 100644 --- a/compose.test.yaml +++ b/compose.test.yaml @@ -8,3 +8,29 @@ services: worker: environment: PIPELINE_FAULTS: '1' + controller: + image: ${API_IMAGE:-mtusi-messenger:lab7} + pull_policy: never + entrypoint: + - python + user: 10001:10001 + environment: + IDENTITY_ENABLED: '1' + volumes: + - ./secrets/operator-task:/run/identity:ro + - ./trust:/trust:ro + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + cpus: 0.5 + mem_limit: 192m + pids_limit: 64 + healthcheck: + disable: true + command: + - -c + - import time;time.sleep(86400) diff --git a/compose.yaml b/compose.yaml index 88a67c1..8fa8e65 100644 --- a/compose.yaml +++ b/compose.yaml @@ -123,7 +123,7 @@ services: depends_on: identity-api: condition: service_started - cpus: ${API_CPUS:-0.5} + cpus: ${API_CPUS:-1} mem_limit: 768m memswap_limit: 768m pids_limit: 128 @@ -144,6 +144,16 @@ services: - -m - src.runtime - src + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).read() + interval: 3s + timeout: 3s + start_period: 45s + retries: 15 s3: image: ${S3_IMAGE:-mtusi-storage:lab7-minio-20251015} build: @@ -1450,13 +1460,22 @@ services: volumes: - ./trust:/trust:ro - ./secrets/probe:/probe-fixtures:ro + - identity_openbao:/identities/openbao:ro - identity_api:/identities/api:ro - identity_worker:/identities/worker:ro - identity_relay:/identities/relay:ro - - identity_lb:/identities/lb:ro - - identity_reconciler:/identities/reconciler:ro - identity_observer:/identities/observer:ro + - identity_reconciler:/identities/reconciler:ro + - identity_migrate:/identities/migrate:ro + - identity_lb:/identities/lb:ro + - identity_postgres:/identities/postgres:ro + - identity_s3:/identities/s3:ro + - identity_broker:/identities/broker:ro + - identity_cache:/identities/cache:ro - identity_otel:/identities/otel:ro + - identity_prometheus:/identities/prometheus:ro + - identity_loki:/identities/loki:ro + - identity_tempo:/identities/tempo:ro - identity_grafana:/identities/grafana:ro cap_drop: - ALL @@ -1472,6 +1491,33 @@ services: pids_limit: 64 healthcheck: disable: true + operator-task: + image: ${API_IMAGE:-mtusi-messenger:lab7} + pull_policy: never + profiles: + - operator + entrypoint: + - python + - -m + - src.operator + user: 10001:10001 + environment: + IDENTITY_ENABLED: '1' + volumes: + - ./secrets/operator-task:/run/identity:ro + - ./trust:/trust:ro + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + cpus: 0.5 + mem_limit: 192m + pids_limit: 64 + healthcheck: + disable: true secrets: postgres_password: file: ./secrets/postgres_password diff --git a/scripts/bootstrap_security.py b/scripts/bootstrap_security.py index f828c2a..df45cd1 100644 --- a/scripts/bootstrap_security.py +++ b/scripts/bootstrap_security.py @@ -270,7 +270,27 @@ def generate(): ).not_valid_after_utc < datetime.now(UTC) + timedelta(hours=1) ) - if not (directory / "key.pem").exists() or renew_operator: + renew_bootstrap = ( + role != "operator" + and (directory / "cert.pem").exists() + and x509.load_pem_x509_certificate( + (directory / "cert.pem").read_bytes() + ).not_valid_after_utc + < datetime.now(UTC) + timedelta(minutes=5) + ) + wrong_issuer = ( + directory / "cert.pem" + ).exists() and x509.load_pem_x509_certificate( + (directory / "cert.pem").read_bytes() + ).issuer != cert.subject + # Only this explicit operator command has the offline CA. Agents + # cannot refresh a bootstrap credential by themselves after expiry. + if ( + not (directory / "key.pem").exists() + or renew_operator + or renew_bootstrap + or wrong_issuer + ): leaf_key, leaf = sign_leaf( key, cert, diff --git a/scripts/credentials.py b/scripts/credentials.py new file mode 100644 index 0000000..089395f --- /dev/null +++ b/scripts/credentials.py @@ -0,0 +1,231 @@ +"""Operator helpers for explicit certificate repair and working-key rotation.""" + +import argparse +import io +import json +import os +import secrets +import ssl +import time +from uuid import uuid4 + +import urllib3 +from minio import Minio +from minio.credentials import StaticProvider +from minio.error import MinioAdminException, S3Error +from minio.minioadmin import MinioAdmin + +from scripts.bao_operator import Operator +from scripts.bootstrap_security import OPERATOR, ROLES, TRUST, write_private +from scripts.prepare_dependencies import attach_policy +from scripts.stand import compose + +INSTALL = """import json,os,pathlib,signal,sys,tempfile +data=json.load(sys.stdin) +base=pathlib.Path('/run/identity') +directory=pathlib.Path(tempfile.mkdtemp(prefix='generation-',dir=base)) +certificate=data['certificate']+'\\n'+'\\n'.join(data.get('ca_chain',[])) +for name,value in [('cert.pem',certificate),('key.pem',data['private_key']),('bundle.pem',certificate+'\\n'+data['private_key'])]: + path=directory/name;path.write_text(value);path.chmod(0o600) +link=base/'operator-next';link.unlink(missing_ok=True);link.symlink_to(directory.name);link.replace(base/'current') +if data.get('reload_lb'):os.kill(1,signal.SIGUSR2) +print('Operator installed a scoped short-lived leaf') +""" + + +def install(role, data): + payload = {**data, "reload_lb": role == "lb"} + args = ["--profile", "maintenance"] if role == "migrate" else [] + compose( + *args, + "run", + "--rm", + "--no-deps", + "-T", + "--entrypoint", + "python", + "identity-" + role, + "-c", + INSTALL, + input=json.dumps(payload), + ) + + +def issue(operator, role): + cfg = ROLES[role] + request = { + "common_name": role, + "alt_names": ",".join(cfg["dns"]), + "uri_sans": "spiffe://messenger/" + role, + "ttl": "120s", + } + if cfg.get("ip"): + request["ip_sans"] = ",".join(cfg["ip"]) + return operator.authorized("POST", "pki/issue/" + role, request)["data"] + + +def repair(operator, role): + result = issue(operator, role) + install(role, result) + return {key: result[key] for key in ["serial_number", "expiration"]} + + +def admin(): + raw = json.loads((OPERATOR / "service-secrets.json").read_text()) + context = ssl.create_default_context(cafile=TRUST / "ca.pem") + context.minimum_version = ssl.TLSVersion.TLSv1_3 + context.load_cert_chain( + OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem" + ) + pool = urllib3.PoolManager( + ssl_context=context, + timeout=urllib3.Timeout(connect=2, read=8), + retries=False, + ) + return ( + raw, + MinioAdmin( + "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), + credentials=StaticProvider( + raw["s3_root_user"], raw["s3_root_password"] + ), + secure=True, + http_client=pool, + ), + pool, + ) + + +def rotate_s3(operator): + raw, client, pool = admin() + try: + journal_path = OPERATOR / "s3-rotation.json" + resumed = journal_path.exists() and not json.loads( + journal_path.read_text() + ).get("complete") + journal = json.loads(journal_path.read_text()) if resumed else {} + values = journal.get( + "previous_values", raw["services"]["api"]["values"] + ) + previous = values["S3_ACCESS_KEY"] + old = Minio( + "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), + credentials=StaticProvider(previous, values["S3_SECRET_KEY"]), + secure=True, + http_client=pool, + region="us-east-1", + ) + probe_key = journal.get( + "probe_key", "objects/staging/rotation-" + uuid4().hex + ) + current = journal.get("next_values") or { + **values, + "S3_ACCESS_KEY": "api-" + secrets.token_hex(12), + "S3_SECRET_KEY": secrets.token_urlsafe(32), + } + if not resumed: + old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1) + journal = { + "previous_values": values, + "next_values": current, + "probe_key": probe_key, + "complete": False, + } + write_private(journal_path, json.dumps(journal) + "\n") + client.user_add(current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"]) + attach_policy(client, "messenger-api", current["S3_ACCESS_KEY"]) + operator.authorized("POST", "secret/data/api", {"data": current}) + # Compare a fingerprint; credential values are not placed in argv/output. + import hashlib + + expected = hashlib.sha256( + current["S3_ACCESS_KEY"].encode() + ).hexdigest() + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + observed = compose( + "exec", + "-T", + "identity-api", + "python", + "-c", + "import hashlib,json;print(hashlib.sha256(json.load(open('/run/identity/secrets.json'))['values']['S3_ACCESS_KEY'].encode()).hexdigest())", + ) + if observed == expected: + break + time.sleep(1) + else: + raise TimeoutError("New scoped S3 key not observed") + raw["services"]["api"]["values"] = current + write_private( + OPERATOR / "service-secrets.json", json.dumps(raw, indent=2) + "\n" + ) + try: + client.user_remove(previous) + except MinioAdminException as error: + if json.loads(error._body).get("Code") != "XMinioAdminNoSuchUser": + raise + pool.clear() + try: + old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1) + except S3Error as error: + assert error.code in {"AccessDenied", "InvalidAccessKeyId"}, ( + error.code + ) + else: + raise AssertionError( + "Removed working key accepted on a fresh connection" + ) + fresh = Minio( + "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), + credentials=StaticProvider( + current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"] + ), + secure=True, + http_client=pool, + region="us-east-1", + ) + fresh.put_object("messenger", probe_key, io.BytesIO(b"x"), 1) + root = Minio( + "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), + credentials=StaticProvider( + raw["s3_root_user"], raw["s3_root_password"] + ), + secure=True, + http_client=pool, + region="us-east-1", + ) + root.remove_object("messenger", probe_key) + write_private( + journal_path, json.dumps({**journal, "complete": True}) + "\n" + ) + return { + "new_key_observed": True, + "old_account_removed": True, + "old_key_fresh_connection_denied": True, + "new_key_put_succeeded": True, + "image_rebuilt": False, + } + finally: + pool.clear() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["repair-leaf", "rotate-s3"]) + parser.add_argument("--role", choices=ROLES, default="lb") + args = parser.parse_args() + operator = Operator() + try: + result = ( + repair(operator, args.role) + if args.action == "repair-leaf" + else rotate_s3(operator) + ) + print(json.dumps(result)) + finally: + operator.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/observe.py b/scripts/observe.py index bc334e5..2a77643 100644 --- a/scripts/observe.py +++ b/scripts/observe.py @@ -8,10 +8,10 @@ from urllib.parse import urlencode from scripts.stand import compose ENDPOINTS = { - "prometheus": "http://prometheus:9090", - "loki": "http://loki:3100", - "tempo": "http://tempo:3200", - "otel": "http://otel:13133", + "prometheus": "https://prometheus:9900", + "loki": "https://loki:3310", + "tempo": "https://tempo:3320", + "otel": "https://otel:13333", } @@ -20,11 +20,20 @@ def fetch(service, path, *, params=None, json_result=True): if params: url += "?" + urlencode(params, doseq=True) code = ( - "import urllib.request; " - f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); " + f"import urllib.request; from src.security_probe import context; service={service!r}; " + f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8,context=context('operator' if 'otel' == service else 'grafana')); " "print(r.read().decode())" ) - text = compose("exec", "-T", "observer", "python", "-c", code) + text = compose( + "--profile", + "operator", + "run", + "--rm", + "--no-deps", + "operator-probe", + "-c", + code, + ) return json.loads(text) if json_result else text @@ -50,14 +59,25 @@ def queries(expressions): for name, expression in expressions.items() } code = ( - "import json,urllib.request\n" + "import json,urllib.request\nfrom src.security_probe import context\n" f"urls={urls!r}\n" "result={}\n" "for name,url in urls.items():\n" - " with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n" + " with urllib.request.urlopen(url,timeout=8,context=context('grafana')) as r: result[name]=json.load(r)['data']['result']\n" "print(json.dumps(result))" ) - return json.loads(compose("exec", "-T", "observer", "python", "-c", code)) + return json.loads( + compose( + "--profile", + "operator", + "run", + "--rm", + "--no-deps", + "operator-probe", + "-c", + code, + ) + ) def logs(trace_id, *, start=None): diff --git a/scripts/operator_task.py b/scripts/operator_task.py new file mode 100644 index 0000000..baa69f8 --- /dev/null +++ b/scripts/operator_task.py @@ -0,0 +1,57 @@ +"""Prepare a 60-second operator-only snapshot and run the audited pipeline CLI.""" + +import json +import sys +import time +from urllib.parse import quote + +from scripts.bootstrap_security import OPERATOR, SECRET, write_private +from scripts.stand import compose + + +def prepare(): + raw = json.loads((OPERATOR / "service-secrets.json").read_text()) + directory = SECRET / "operator-task" + directory.mkdir(mode=0o755, exist_ok=True) + for name in ["cert.pem", "key.pem"]: + write_private( + directory / "current" / name, + (OPERATOR / "client" / name).read_bytes(), + mode=0o644, + ) + values = { + "DATABASE_URL": "postgresql://messenger:" + + quote(raw["postgres_password"], safe="") + + "@postgres:5443/messenger?sslmode=verify-full&gssencmode=disable&sslrootcert=/trust/ca.pem&sslcert=/run/identity/current/cert.pem&sslkey=/run/identity/current/key.pem", + "BROKER_URL": "rediss://admin:" + + raw["broker_admin"] + + "@broker:6380/0", + "S3_ACCESS_KEY": raw["s3_root_user"], + "S3_SECRET_KEY": raw["s3_root_password"], + } + write_private( + directory / "secrets.json", + json.dumps({"valid_until": time.time() + 60, "values": values}) + "\n", + mode=0o644, + ) + + +def run(*args): + prepare() + return compose( + "--profile", + "operator", + "run", + "--rm", + "--no-deps", + "operator-task", + *args, + ) + + +def main(): + print(run(*sys.argv[1:])) + + +if __name__ == "__main__": + main() diff --git a/scripts/pki.py b/scripts/pki.py new file mode 100644 index 0000000..e8aec19 --- /dev/null +++ b/scripts/pki.py @@ -0,0 +1,359 @@ +"""Explicit operator PKI lifecycle; trust is never disabled.""" + +import argparse +import json +import time +from uuid import uuid4 + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes + +from scripts.bao_operator import Operator +from scripts.bootstrap_security import ( + OPERATOR, + ROLES, + SECRET, + TRUST, + ca, + key_bytes, + make_root, + public, + sign_intermediate, + sign_leaf, + write_private, +) +from scripts.credentials import install, repair +from scripts.stand import compose + +REQUEST = ( + b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n" +) + + +def fixture_check(role, *, reject=False): + expression = f"exchange('api',8443,{role!r},{REQUEST!r})" + assertion = ( + "denied(lambda:" + expression + ")" + if reject + else "b' 200 ' in " + expression + ) + code = ( + "from src.security_probe import exchange,denied;assert " + + assertion + + ";print('Fresh TLS fixture check passed')" + ) + compile(code, "fixture-check", "exec") + return compose( + "--profile", + "operator", + "run", + "--rm", + "--no-deps", + "operator-probe", + "-c", + code, + ) + + +META = """import json,pathlib +from cryptography import x509 +result={} +for directory in pathlib.Path('/identities').iterdir(): + try: + cert=x509.load_pem_x509_certificate((directory/'current/cert.pem').read_bytes()) + result[directory.name]={'serial':format(cert.serial_number,'x'),'issuer':cert.issuer.rfc4514_string(),'expires_at':cert.not_valid_after_utc.timestamp()} + except FileNotFoundError:pass +print(json.dumps(result)) +""" + + +def metadata(): + return json.loads( + compose( + "--profile", + "operator", + "run", + "--rm", + "--no-deps", + "operator-probe", + "-c", + META, + ) + ) + + +def reload_lb(): + compose( + "exec", + "-T", + "identity-lb", + "python", + "-c", + "import os,signal;os.kill(1,signal.SIGUSR2)", + ) + + +def preserve_lb(): + data = json.loads( + compose( + "exec", + "-T", + "identity-lb", + "python", + "-c", + "import json,pathlib;p=pathlib.Path('/run/identity/current');print(json.dumps({name:(p/name).read_text() for name in ['cert.pem','key.pem']}))", + ) + ) + directory = SECRET / "probe/revoked-lb" + for name, value in data.items(): + write_private(directory / name, value, mode=0o644) + leaf = x509.load_pem_x509_certificate(data["cert.pem"].encode()) + return { + "serial": format(leaf.serial_number, "x"), + "expires_at": leaf.not_valid_after_utc.timestamp(), + } + + +def revoke(operator, leaf): + assert leaf["expires_at"] > time.time() + 5, ( + "Revocation must be tested before natural expiry" + ) + serial = leaf["serial"] + serial = ("0" + serial) if len(serial) % 2 else serial + colon = ":".join(serial[i : i + 2] for i in range(0, len(serial), 2)) + operator.authorized("POST", "pki/revoke", {"serial_number": colon}) + response = operator.http.get( + "pki/crl/pem", + headers={"X-Vault-Token": operator.token or operator.authenticate()}, + ) + response.raise_for_status() + operator.authorized( + "PUT", "auth/cert/crls/service-issuer", {"crl": response.text} + ) + revoked = json.loads((TRUST / "revoked.json").read_text()) + revoked[leaf["serial"]] = True + write_private( + TRUST / "revoked.json", json.dumps(revoked) + "\n", mode=0o644 + ) + fixture_check("revoked-lb", reject=True) + return { + "revoked_serial": leaf["serial"], + "still_unexpired_at_check": leaf["expires_at"] > time.time(), + "new_tls_connection_denied": True, + "bao_crl_updated": True, + } + + +def reload_bao(): + compose("restart", "openbao") + last = None + for _ in range(60): + operator = Operator() + try: + operator.unseal() + operator.authenticate() + return + except Exception as error: + last = type(error).__name__ + finally: + operator.close() + time.sleep(1) + raise TimeoutError("Verified Bao restart/unseal unavailable: " + str(last)) + + +def recover_bao(): + # Explicit operator recovery after the server leaf itself has expired. + key, root = ca() + leaf_key, leaf = sign_leaf( + key, + root, + "openbao", + ["openbao", "localhost"], + ["127.0.0.1"], + seconds=120, + ) + install( + "openbao", + { + "certificate": public(leaf).decode(), + "private_key": key_bytes(leaf_key).decode(), + "ca_chain": [public(root).decode()], + }, + ) + time.sleep(6) + operator = Operator() + try: + operator.unseal() + for role in ROLES: + if role != "migrate": + repair(operator, role) + finally: + operator.close() + + +def rollover(): + old_key, old_root = ca() + new_key, new_root = make_root() + directory = OPERATOR / ("rollover-" + uuid4().hex[:12]) + directory.mkdir(mode=0o700) + fixture_key, fixture = sign_leaf( + old_key, old_root, "lb", ["lb"], seconds=900 + ) + write_private( + SECRET / "probe/retired-ca-lb/cert.pem", + public(fixture) + public(old_root), + mode=0o644, + ) + write_private( + SECRET / "probe/retired-ca-lb/key.pem", + key_bytes(fixture_key), + mode=0o644, + ) + fixture_check("retired-ca-lb") + for name, value in [ + ("old-key.pem", key_bytes(old_key)), + ("old-ca.pem", public(old_root)), + ("new-key.pem", key_bytes(new_key)), + ("new-ca.pem", public(new_root)), + ]: + write_private(directory / name, value) + journal = { + "phase": "dual_trust", + "old_ca_sha256": old_root.fingerprint(hashes.SHA256()).hex(), + "new_ca_sha256": new_root.fingerprint(hashes.SHA256()).hex(), + } + write_private(directory / "state.json", json.dumps(journal) + "\n") + write_private( + TRUST / "ca.pem", public(old_root) + public(new_root), mode=0o644 + ) + reload_lb() # Load both roots before any API server leaf changes issuer. + reload_bao() # Native client CA requires an explicit listener restart. + operator = Operator() + try: + for role in [*ROLES, "operator"]: + operator.authorized( + "POST", + "auth/cert/certs/" + role + "-v2", + { + "display_name": role, + "certificate": public(new_root).decode(), + "allowed_common_names": [role], + "allowed_uri_sans": ["spiffe://messenger/" + role], + "token_policies": ["messenger-" + role], + "token_ttl": "900s" if role == "operator" else "120s", + "token_max_ttl": "3600s" if role == "operator" else "600s", + "token_no_default_policy": True, + }, + ) + csr = operator.authorized( + "POST", + "pki/intermediate/generate/internal", + { + "common_name": "Messenger rollover intermediate " + + uuid4().hex[:12], + "key_type": "ec", + "key_bits": 256, + }, + )["data"]["csr"] + intermediate = sign_intermediate( + new_key, new_root, x509.load_pem_x509_csr(csr.encode()) + ) + imported = operator.authorized( + "POST", + "pki/intermediate/set-signed", + { + "certificate": ( + public(intermediate) + public(new_root) + ).decode() + }, + )["data"]["imported_issuers"] + chosen = None + for issuer in imported: + details = operator.authorized("GET", "pki/issuer/" + issuer)[ + "data" + ] + if ( + x509.load_pem_x509_certificate( + details["certificate"].encode() + ).serial_number + == intermediate.serial_number + ): + chosen = issuer + assert chosen + operator.authorized( + "POST", + "pki/config/issuers", + {"default": chosen, "default_follows_latest_issuer": False}, + ) + journal["phase"] = "new_issuer_active" + write_private(directory / "state.json", json.dumps(journal) + "\n") + deadline = time.monotonic() + 120 + active = set(ROLES) - {"migrate"} + while time.monotonic() < deadline: + observed = metadata() + if active <= observed.keys() and all( + observed[r]["issuer"] == intermediate.subject.rfc4514_string() + for r in active + ): + break + time.sleep(3) + else: + raise TimeoutError( + "Not all active services renewed into new issuer" + ) + fixture_check("retired-ca-lb") + # Allow old L4 connections to expire, then remove old trust everywhere. + time.sleep(46) + for role in [*ROLES, "operator"]: + operator.authorized("DELETE", "auth/cert/certs/" + role + "-v1") + client_key, client_cert = sign_leaf( + new_key, new_root, "operator", ["operator"], seconds=86400 + ) + write_private(OPERATOR / "client/key.pem", key_bytes(client_key)) + write_private( + OPERATOR / "client/cert.pem", + public(client_cert) + public(new_root), + ) + write_private(OPERATOR / "ca-key.pem", key_bytes(new_key)) + write_private(OPERATOR / "ca.pem", public(new_root)) + write_private(TRUST / "ca.pem", public(new_root), mode=0o644) + journal["phase"] = "old_trust_retired" + write_private(directory / "state.json", json.dumps(journal) + "\n") + finally: + operator.close() + reload_lb() + reload_bao() + fixture_check("retired-ca-lb", reject=True) + fixture_check("lb") + journal["phase"] = "complete" + write_private(directory / "state.json", json.dumps(journal) + "\n") + return { + **journal, + "issuer_id": chosen, + "active_service_count": len(active), + "old_l4_connections_waited_seconds": 46, + "old_ca_fixture_valid_during_dual_trust": True, + "old_ca_new_connection_denied_after_retirement": True, + "services": metadata(), + "image_rebuilt": False, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "action", choices=["metadata", "rollover", "recover-bao"] + ) + args = parser.parse_args() + if args.action == "metadata": + result = metadata() + elif args.action == "rollover": + result = rollover() + else: + recover_bao() + result = {"recovered": True} + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_dependencies.py b/scripts/prepare_dependencies.py index f015e26..f87b60d 100644 --- a/scripts/prepare_dependencies.py +++ b/scripts/prepare_dependencies.py @@ -1,6 +1,7 @@ """Operator-only, idempotent SQL roles and scoped S3 accounts.""" import json +import os import ssl import time @@ -14,6 +15,12 @@ from scripts.bootstrap_security import OPERATOR, TRUST from scripts.stand import compose +def attach_policy(client, policy, user): + attached = json.loads(client.user_info(user)).get("policyName", "") or "" + if policy not in attached.split(","): + client.attach_policy([policy], user=user) + + def postgres(raw): # The bootstrap SQL channel is the server's local Unix socket. No password # or SQL literal is placed in argv; stdin and output are never recorded. @@ -190,13 +197,13 @@ def s3(raw): ) credentials = StaticProvider(raw["s3_root_user"], raw["s3_root_password"]) admin = MinioAdmin( - "localhost:9007", + "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), credentials=credentials, secure=True, http_client=pool, ) client = Minio( - "localhost:9007", + "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), credentials=credentials, secure=True, http_client=pool, @@ -248,9 +255,7 @@ def s3(raw): policy={"Version": "2012-10-17", "Statement": statements}, ) admin.user_add(values["S3_ACCESS_KEY"], values["S3_SECRET_KEY"]) - admin.attach_policy( - ["messenger-" + role], user=values["S3_ACCESS_KEY"] - ) + attach_policy(admin, "messenger-" + role, values["S3_ACCESS_KEY"]) pool.clear() diff --git a/scripts/research_slo.py b/scripts/research_slo.py index 483a26f..12b7bcc 100644 --- a/scripts/research_slo.py +++ b/scripts/research_slo.py @@ -16,6 +16,8 @@ from uuid import uuid4 import httpx from PIL import Image +from scripts.tls_client import FreshTrustTransport + os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") from scripts.observe import fetch, logs, queries, spans, wait from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql @@ -215,6 +217,7 @@ def main(): load = None trigger = "lab6_slo_delay_" + uuid4().hex[:8] with httpx.Client( + transport=FreshTrustTransport(), base_url=BASE, timeout=8, trust_env=False, diff --git a/scripts/restart.sh b/scripts/restart.sh index f123bd9..91dd041 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,4 +1,3 @@ #!/bin/sh set -eu -docker compose restart api postgres cache s3 broker relay worker lb -docker compose up -d --no-build --no-recreate --wait +exec python3 -m scripts.restart_secure diff --git a/scripts/restart_secure.py b/scripts/restart_secure.py new file mode 100644 index 0000000..2e216ec --- /dev/null +++ b/scripts/restart_secure.py @@ -0,0 +1,23 @@ +"""Restart persistent services and rejoin their protected local namespaces.""" + +from scripts.stand import compose, ready + + +def main(): + compose( + "restart", + "api", + "postgres", + "cache", + "s3", + "broker", + "relay", + "worker", + "lb", + ) + compose("up", "-d", "--no-build", "--no-recreate", "--wait") + ready(timeout=90) + + +if __name__ == "__main__": + main() diff --git a/scripts/stand.py b/scripts/stand.py index 315d830..b7fe5f0 100644 --- a/scripts/stand.py +++ b/scripts/stand.py @@ -26,7 +26,44 @@ def command(args, *, input=None): def compose(*args, input=None): - return command(["docker", "compose", *args], input=input) + if args[0] == "exec" and "controller" in args: + from scripts.operator_task import prepare + + prepare() + result = command(["docker", "compose", *args], input=input) + # A restarted native container can get a new network namespace. Rejoin + # its TLS gateway explicitly instead of keeping an obsolete loopback hop. + native = { + "postgres", + "cache", + "broker", + "s3", + "lb", + "otel", + "prometheus", + "loki", + "tempo", + "grafana", + } + if args[0] in {"restart", "up"}: + targets = sorted(native.intersection(args[1:])) + if targets: + command( + [ + "docker", + "compose", + "up", + "-d", + "--no-deps", + "--force-recreate", + *[role + "-tls" for role in targets], + ] + ) + if "lb" in targets: + command( + ["docker", "compose", "up", "-d", "--no-deps", "identity-lb"] + ) + return result def sql(query, *, database="messenger"): diff --git a/scripts/tls_client.py b/scripts/tls_client.py new file mode 100644 index 0000000..d32fed4 --- /dev/null +++ b/scripts/tls_client.py @@ -0,0 +1,23 @@ +"""Verified public HTTPS with trust reloaded on every fresh connection.""" + +import ssl + +import httpx + +from scripts.bootstrap_security import TRUST + + +class FreshTrustTransport(httpx.BaseTransport): + def handle_request(self, request): + context = ssl.create_default_context(cafile=TRUST / "ca.pem") + context.minimum_version = ssl.TLSVersion.TLSv1_3 + with httpx.HTTPTransport( + verify=context, trust_env=False, retries=0 + ) as transport: + response = transport.handle_request(request) + return httpx.Response( + response.status_code, + headers=response.headers, + content=response.read(), + request=request, + ) diff --git a/scripts/up_security.py b/scripts/up_security.py index c41964f..a036fae 100644 --- a/scripts/up_security.py +++ b/scripts/up_security.py @@ -8,6 +8,7 @@ import time from scripts.bao_operator import Operator from scripts.bootstrap_security import OPERATOR, generate +from scripts.credentials import repair from scripts.prepare_dependencies import main as dependencies from scripts.prepare_dependencies import postgres from scripts.stand import ROOT, command, compose, ready @@ -42,6 +43,9 @@ def main(): operator.init() operator.unseal() operator.configure() + # The maintenance identity is intentionally stopped between migrations. + # Its expired leaf is repaired explicitly by the operator before reuse. + repair(operator, "migrate") finally: operator.close() agents = [ @@ -107,6 +111,11 @@ def main(): "observer", "reconciler", ) + if "compose.test.yaml" in os.getenv("COMPOSE_FILE", ""): + from scripts.operator_task import prepare + + prepare() + compose("up", "-d", "--no-deps", "controller") ready(timeout=120) print("Secure stand ready at https://localhost:8443; values remain local") diff --git a/scripts/verify_cleanup.py b/scripts/verify_cleanup.py index e4e622c..e9bacaf 100644 --- a/scripts/verify_cleanup.py +++ b/scripts/verify_cleanup.py @@ -10,6 +10,7 @@ from uuid import uuid4 import httpx from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready +from scripts.tls_client import FreshTrustTransport from scripts.verify_pipeline import inside, operator, wait @@ -19,7 +20,12 @@ def seed(path): ready() compose("stop", "--timeout", "1", "reconciler") raw = (ROOT / "tests/fixtures/sample.png").read_bytes() - with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c: + with httpx.Client( + transport=FreshTrustTransport(), + base_url=BASE, + trust_env=False, + timeout=20, + ) as c: user, _, session = actor(c) headers = auth(session) result = c.post( diff --git a/scripts/verify_observability_persistence.py b/scripts/verify_observability_persistence.py index 13e2a59..174fe06 100644 --- a/scripts/verify_observability_persistence.py +++ b/scripts/verify_observability_persistence.py @@ -8,6 +8,8 @@ from datetime import UTC, datetime import httpx +from scripts.tls_client import FreshTrustTransport + os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") from scripts.observe import fetch, logs, query, spans, wait from scripts.stand import ROOT, command, compose @@ -57,9 +59,13 @@ def main(): "complete": False, } password = (ROOT / "secrets/grafana_password").read_text().strip() - base = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006") + base = "https://localhost:" + os.getenv("GRAFANA_PORT", "3447") with httpx.Client( - base_url=base, auth=("admin", password), timeout=10, trust_env=False + transport=FreshTrustTransport(), + base_url=base, + auth=("admin", password), + timeout=10, + trust_env=False, ) as grafana: preferences = grafana.get("/api/user/preferences") preferences.raise_for_status() diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index 28e9385..57d9f1d 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -13,6 +13,8 @@ from uuid import uuid4 import httpx from PIL import Image +from scripts.tls_client import FreshTrustTransport + os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1") os.environ.setdefault("no_proxy", "localhost,127.0.0.1") @@ -149,7 +151,10 @@ def main(): ) stop("reconciler") with httpx.Client( - base_url=BASE, trust_env=False, timeout=25 + transport=FreshTrustTransport(), + base_url=BASE, + trust_env=False, + timeout=25, ) as client: user, _, session = actor(client) h = auth(session) diff --git a/scripts/verify_telemetry.py b/scripts/verify_telemetry.py index d2a8765..cce0d17 100644 --- a/scripts/verify_telemetry.py +++ b/scripts/verify_telemetry.py @@ -11,6 +11,8 @@ from uuid import uuid4 import httpx from PIL import Image +from scripts.tls_client import FreshTrustTransport + os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") from scripts.observe import fetch, logs, query, spans, wait from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql @@ -78,7 +80,10 @@ def main(): raw = buffer.getvalue() sentinel = "PrivacyProbe-" + uuid4().hex with httpx.Client( - base_url=BASE, timeout=15, trust_env=False + transport=FreshTrustTransport(), + base_url=BASE, + timeout=15, + trust_env=False, ) as client: user, _, session = actor(client) headers = auth(session) @@ -266,10 +271,14 @@ def main(): assert all(sentinel not in json.dumps(s) for s in series) report["metric_series_count"] = len(series) report["metric_labels_low_cardinality"] = True - url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006") + url = "https://localhost:" + os.getenv("GRAFANA_PORT", "3447") password = (ROOT / "secrets/grafana_password").read_text().strip() with httpx.Client( - base_url=url, auth=("admin", password), trust_env=False, timeout=15 + transport=FreshTrustTransport(), + base_url=url, + auth=("admin", password), + trust_env=False, + timeout=15, ) as grafana: dashboard = grafana.get("/api/dashboards/uid/messenger-lab6") dashboard.raise_for_status() diff --git a/src/security_probe.py b/src/security_probe.py index 6bd6725..3fd214f 100644 --- a/src/security_probe.py +++ b/src/security_probe.py @@ -30,7 +30,11 @@ def context(role=None, *, foreign=False, wrong_ca=False): directory = ( Path("/probe-fixtures/foreign") if foreign - else Path("/identities") / role / "current" + else ( + Path("/probe-fixtures") / role + if role in {"revoked-lb", "retired-ca-lb"} + else Path("/identities") / role / "current" + ) ) ctx.load_cert_chain(directory / "cert.pem", directory / "key.pem") return ctx @@ -308,7 +312,7 @@ def main(): "worker_staging_write_denied": s3_denied( lambda: put("worker", staging) ), - "worker_other_bucket_denied": s3_denied( + "worker_list_all_buckets_denied": s3_denied( lambda: clients["worker"].list_buckets() ), "worker_delete_denied": s3_denied( diff --git a/student-unit/test_security.py b/student-unit/test_security.py index d01aeaf..2655ba7 100644 --- a/student-unit/test_security.py +++ b/student-unit/test_security.py @@ -1,4 +1,7 @@ +import io import json +import ssl +import sys import time import pytest @@ -58,3 +61,44 @@ def test_bootstrap_repairs_config_without_changing_credentials( assert config.read_bytes() == expected assert bootstrap.OPERATOR.stat().st_mode & 0o777 == 0o700 assert private.stat().st_mode & 0o777 == 0o600 + + +def test_operator_leaf_install_preserves_pem_and_matching_pair( + tmp_path, monkeypatch +): + from cryptography import x509 + + from scripts.bootstrap_security import ( + key_bytes, + make_root, + public, + sign_leaf, + ) + from scripts.credentials import INSTALL + + root_key, root_cert = make_root() + key, leaf = sign_leaf(root_key, root_cert, "worker", ["worker"]) + payload = { + "certificate": public(leaf).decode(), + "private_key": key_bytes(key).decode(), + "ca_chain": [public(root_cert).decode()], + } + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(payload))) + code = INSTALL.replace( + "pathlib.Path('/run/identity')", + "pathlib.Path(" + repr(str(tmp_path)) + ")", + ) + exec(compile(code, "operator-install-test", "exec"), {}) + current = tmp_path / "current" + assert current.is_symlink() + assert ( + len( + x509.load_pem_x509_certificates( + (current / "cert.pem").read_bytes() + ) + ) + == 2 + ) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(current / "cert.pem", current / "key.pem") + assert (current / "key.pem").stat().st_mode & 0o777 == 0o600