Close certificate auth before revoking worker tokens and rotating its secret

This commit is contained in:
2026-11-16 12:03:49 +03:00
parent d8455d9417
commit 040fa8d4da
6 changed files with 156 additions and 5 deletions
+33 -1
View File
@@ -78,9 +78,37 @@ def configure():
raise
def revoke_worker_tokens():
# Replacing the pinned certificate prevents NEW auth, but already issued
# tokens must also be revoked before publishing a new working secret.
accessors = call("auth/token/accessors", method="LIST").get("data", {}).get("keys", [])
count = 0
for accessor in accessors:
try:
record = call("auth/token/lookup-accessor", {"accessor": accessor})["data"]
except urllib.error.HTTPError as exc:
if exc.code == 400:
continue
raise
if "worker" in record.get("policies", []):
call("auth/token/revoke-accessor", {"accessor": accessor})
count += 1
return count
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["init", "unseal", "configure", "seal"])
parser.add_argument(
"action",
choices=[
"init",
"unseal",
"configure",
"seal",
"revoke-worker-tokens",
"disable-worker-auth",
],
)
args = parser.parse_args()
os.umask(0o077)
if args.action == "init":
@@ -106,6 +134,10 @@ def main():
unseal()
elif args.action == "configure":
configure()
elif args.action == "revoke-worker-tokens":
revoke_worker_tokens()
elif args.action == "disable-worker-auth":
call("auth/cert/certs/worker", method="DELETE")
else:
call("sys/seal", {}, method="PUT")
print("OpenBao operator action complete; credential values were not printed.")
+38 -1
View File
@@ -6,6 +6,9 @@ import json
import secrets
import ssl
import subprocess
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
@@ -22,8 +25,22 @@ def main():
for name in ("cert.pem", "key.pem"):
(snapshot / name).write_bytes((old / name).read_bytes())
(snapshot / name).chmod(0o600)
context = ssl.create_default_context(cafile=str(ROOT / "secrets/ca.pem"))
context.load_cert_chain(snapshot / "cert.pem", snapshot / "key.pem")
auth_request = urllib.request.Request(
"https://localhost:18207/v1/auth/cert/login",
data=b'{"name":"worker"}',
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(auth_request, context=context, timeout=5) as response:
old_auth = json.load(response)["auth"]
old_token = old_auth["client_token"]
old_token_expiry = time.time() + old_auth["lease_duration"]
seed_before = json.loads((ROOT / "secrets/bao_seed.json").read_text())
(snapshot / "material.json").write_text(json.dumps(seed_before["worker"]))
material = seed_before["worker"]
material["REVOKED_BAO_TOKEN"] = old_token
material["REVOKED_BAO_TOKEN_EXPIRES_AT"] = old_token_expiry
(snapshot / "material.json").write_text(json.dumps(material))
(snapshot / "material.json").chmod(0o644)
# The containing operator-private directory stays 0700 on the host. This
# disposable old-material mount is used only by the explicit negative probe.
@@ -37,6 +54,26 @@ def main():
subprocess.run(
["docker", "compose", "stop", "worker"], cwd=ROOT, check=True, capture_output=True
)
# Close OLD certificate authentication before enumerating/revoking tokens;
# otherwise an attacker could mint another token during the rotation gap.
subprocess.run(
["python3", "scripts/bao_operator.py", "disable-worker-auth"],
cwd=ROOT,
check=True,
capture_output=True,
)
try:
with urllib.request.urlopen(auth_request, context=context, timeout=5):
raise RuntimeError("Old worker certificate authentication remained enabled")
except urllib.error.HTTPError as exc:
if exc.code not in {400, 403}:
raise
subprocess.run(
["python3", "scripts/bao_operator.py", "revoke-worker-tokens"],
cwd=ROOT,
check=True,
capture_output=True,
)
bootstrap.certificate("worker", replace=True)
password = secrets.token_urlsafe(32)
# psql receives SQL through stdin, never a process argument/output containing
+13
View File
@@ -2,6 +2,7 @@
import json
import ssl
import time
import urllib.error
import urllib.request
from pathlib import Path
@@ -51,6 +52,18 @@ def main():
checks[name] = exc.code in {400, 403}
except (OSError, urllib.error.URLError):
checks[name] = True
try:
request = urllib.request.Request(
"https://openbao:8200/v1/secret/data/services/worker",
headers={"X-Vault-Token": old["REVOKED_BAO_TOKEN"]},
)
with urllib.request.urlopen(request, context=ctx, timeout=3):
checks["previously_issued_bao_token_denied"] = False
except urllib.error.HTTPError as exc:
checks["previously_issued_bao_token_denied"] = exc.code == 403
checks["old_bao_token_rejected_before_original_lease_expiry"] = (
time.time() < old["REVOKED_BAO_TOKEN_EXPIRES_AT"]
)
print(json.dumps({"checks": checks, "passed": all(checks.values())}))
if not all(checks.values()):
raise SystemExit(1)
+5 -3
View File
@@ -16,12 +16,14 @@ def honor_login_retry_after():
self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None
):
global _paced_logins
if method != "POST" or path != "/api/v1/auth/sessions" or expected != 201:
if method != "POST" or path != "/api/v1/auth/sessions" or expected not in (201, 401):
return original(self, method, path, expected, body, actor, headers, raw, schema)
deadline = time.monotonic() + 90
while True:
response = original(self, method, path, [201, 429], body, actor, headers, raw, schema)
if response["status"] == 201:
response = original(
self, method, path, [expected, 429], body, actor, headers, raw, schema
)
if response["status"] == expected:
return response
if time.monotonic() >= deadline:
pytest.fail("Shared login budget did not recover within test deadline")
+45
View File
@@ -0,0 +1,45 @@
"""Real exporter failure must remain bounded and account for lost sampled spans."""
import json
import os
import subprocess
import sys
from pathlib import Path
def test_span_queue_loss_conservation_when_collector_absent():
code = """
import json, time
from messenger.telemetry import setup
class Pool:
def get_stats(self): return {}
class Store:
pool=Pool()
t=setup("messenger-api")
t.start_background(Store())
for index in range(1200):
with t.tracer.start_as_current_span("synthetic-safe-span"):
pass
time.sleep(7)
counts=t.export_stats.snapshot()
pending=len(t.batch_spans._batch_processor._queue)
loss=counts["spans_recorded"]-counts["spans_exported"]-counts["spans_export_failed"]-counts["spans_in_flight"]-pending
assert counts["spans_recorded"]==1200
assert 0<=pending<=256
assert loss>0
assert counts["spans_export_failed"]>0
print(json.dumps({"recorded":1200,"pending":pending,"queue_lost":loss,"export_failed":counts["spans_export_failed"]}))
t.close()
"""
env = {
**os.environ,
"PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src"),
"OTEL_ENABLED": "1",
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",
}
result = subprocess.run(
[sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=15
)
assert result.returncode == 0, result.stderr[-1000:]
payload = json.loads(next(line for line in result.stdout.splitlines() if line.startswith("{")))
assert payload["queue_lost"] > 0
+22
View File
@@ -0,0 +1,22 @@
import pytest
from pydantic import ValidationError
from messenger.models import CreateChat, CreateMessage, CreateUser
@pytest.mark.parametrize("scalar", ["\ud800", "\udfff"])
def test_unpaired_surrogates_rejected(scalar):
for factory in (
lambda: CreateUser(username="valid_user", display_name=scalar, password="valid-pass-123"),
lambda: CreateUser(
username="valid_user", display_name="Valid", password="valid-pass-" + scalar
),
lambda: CreateChat(title=scalar, member_ids=["00000000-0000-4000-8000-000000000001"]),
lambda: CreateMessage(text=scalar),
):
with pytest.raises(ValidationError):
factory()
def test_valid_unicode_and_nul():
assert CreateMessage(text="😀\x00").text == "😀\x00"