Защитить мессенджер TLS и OpenBao и проверить отзыв прав worker
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator-only init, unseal, service policies and scoped secrets."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from bootstrap import ROLES, SECRETS
|
||||
|
||||
ADDRESS = f"https://localhost:{int(os.getenv('BAO_HOST_PORT', '8207'))}/v1/"
|
||||
CA = SECRETS / "tls/trust/ca.crt"
|
||||
RECEIPT = SECRETS / "operator/bao.json"
|
||||
|
||||
|
||||
def call(path, data=None, *, token=None, method=None):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
req = Request(
|
||||
ADDRESS + path,
|
||||
json.dumps(data).encode() if data is not None else None,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
with urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=30) as response:
|
||||
body = response.read()
|
||||
return json.loads(body) if body else {}
|
||||
|
||||
|
||||
def secret(name):
|
||||
return (SECRETS / name).read_text().strip()
|
||||
|
||||
|
||||
def config(role):
|
||||
ca = "/run/tls/trust/ca.crt"
|
||||
db = (
|
||||
f"postgresql://{role}:{secret(f'db_{role}_password')}@postgres:5432/messenger"
|
||||
f"?sslmode=verify-full&sslrootcert={ca}"
|
||||
)
|
||||
cache = (
|
||||
f"rediss://{role}:{secret(f'cache_{role}_password')}@valkey:6379/0"
|
||||
f"?ssl_ca_certs={ca}&ssl_check_hostname=True"
|
||||
if role == "api"
|
||||
else ""
|
||||
)
|
||||
broker = (
|
||||
f"rediss://{role}:{secret(f'broker_{role}_password')}@broker:6380/0"
|
||||
f"?ssl_ca_certs={ca}&ssl_check_hostname=True&ssl_certfile=/run/tls/client/service.crt"
|
||||
"&ssl_keyfile=/run/tls/client/service.key"
|
||||
if role in ("api", "worker", "relay")
|
||||
else ""
|
||||
)
|
||||
return {
|
||||
"database_url": db,
|
||||
"cache_url": cache,
|
||||
"broker_url": broker,
|
||||
"s3_access_key": secret(f"s3_{role}_access_key_id") if role != "relay" else "",
|
||||
"s3_secret_key": secret(f"s3_{role}_secret_access_key") if role != "relay" else "",
|
||||
}
|
||||
|
||||
|
||||
def setup():
|
||||
os.umask(0o077)
|
||||
for attempt in range(20):
|
||||
try:
|
||||
state = call("sys/init")
|
||||
break
|
||||
except URLError:
|
||||
if attempt == 19:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
if not state["initialized"]:
|
||||
result = call("sys/init", {"secret_shares": 1, "secret_threshold": 1}, method="PUT")
|
||||
RECEIPT.write_text(
|
||||
json.dumps({"root_token": result["root_token"], "unseal_key": result["keys_base64"][0]})
|
||||
)
|
||||
if not RECEIPT.exists():
|
||||
raise SystemExit("This OpenBao volume requires its original operator receipt")
|
||||
receipt = json.loads(RECEIPT.read_text())
|
||||
if call("sys/seal-status")["sealed"]:
|
||||
call("sys/unseal", {"key": receipt["unseal_key"]}, method="PUT")
|
||||
for attempt in range(30):
|
||||
try:
|
||||
call("sys/health")
|
||||
break
|
||||
except HTTPError as exc:
|
||||
if exc.code not in (429, 503) or attempt == 29:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
token = receipt["root_token"]
|
||||
if "cert/" not in call("sys/auth", token=token):
|
||||
call("sys/auth/cert", {"type": "cert"}, token=token)
|
||||
if "services/" not in call("sys/mounts", token=token):
|
||||
call("sys/mounts/services", {"type": "kv", "options": {"version": "2"}}, token=token)
|
||||
for role in ROLES:
|
||||
policy = (
|
||||
f'path "services/data/{role}" {{ capabilities = ["read"] }}\n'
|
||||
'path "auth/token/revoke-self" { capabilities = ["update"] }\n'
|
||||
)
|
||||
call("sys/policies/acl/" + role, {"policy": policy}, token=token, method="PUT")
|
||||
cert = secret(f"tls/{role}-client/service.crt")
|
||||
call(
|
||||
f"auth/cert/certs/{role}",
|
||||
{
|
||||
"certificate": cert,
|
||||
"allowed_uri_sans": f"urn:mtusi:lab7:{role}-client",
|
||||
"token_policies": [role],
|
||||
"token_ttl": "90s",
|
||||
"token_max_ttl": "120s",
|
||||
"token_no_default_policy": True,
|
||||
},
|
||||
token=token,
|
||||
)
|
||||
call(f"services/data/{role}", {"data": config(role)}, token=token)
|
||||
print("OpenBao initialized and unsealed; four scoped identities configured.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--revoke", choices=ROLES)
|
||||
parser.add_argument("--seal", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.revoke or args.seal:
|
||||
token = json.loads(RECEIPT.read_text())["root_token"]
|
||||
if args.revoke:
|
||||
call(f"auth/cert/certs/{args.revoke}", token=token, method="DELETE")
|
||||
print(f"Certificate role {args.revoke} revoked.")
|
||||
else:
|
||||
call("sys/seal", {}, token=token, method="PUT")
|
||||
print("OpenBao sealed.")
|
||||
else:
|
||||
setup()
|
||||
except HTTPError as exc:
|
||||
raise SystemExit(f"OpenBao operator action failed (HTTP {exc.code})") from None
|
||||
except (URLError, TimeoutError):
|
||||
raise SystemExit("OpenBao unavailable over verified TLS") from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local operator credentials and TLS files for this stand."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pki
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
ROLES = ("api", "worker", "relay", "reconciler")
|
||||
S3_ROLES = ("api", "worker", "reconciler", "test")
|
||||
|
||||
|
||||
def ensure(name, value):
|
||||
path = SECRETS / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
path.chmod(0o644)
|
||||
return path.read_text().strip()
|
||||
|
||||
|
||||
def initialize():
|
||||
os.umask(0o077)
|
||||
SECRETS.mkdir(mode=0o700, exist_ok=True)
|
||||
pki.init_ca()
|
||||
for name in (*pki.SERVERS, *pki.CLIENTS):
|
||||
pki.issue(name)
|
||||
env = ROOT / ".env"
|
||||
if not env.exists():
|
||||
shutil.copyfile(ROOT / ".env.example", env)
|
||||
for name in ("postgres_password", "grafana_password", "s3_root_password"):
|
||||
ensure(name, secrets.token_hex(24))
|
||||
ensure("s3_root_user", "operator-" + secrets.token_hex(8))
|
||||
for role in S3_ROLES:
|
||||
ensure(f"s3_{role}_access_key_id", "svc-" + secrets.token_hex(8))
|
||||
ensure(f"s3_{role}_secret_access_key", secrets.token_hex(24))
|
||||
for role in ROLES:
|
||||
for store in ("db", "cache", "broker"):
|
||||
ensure(f"{store}_{role}_password", secrets.token_hex(24))
|
||||
write_acls()
|
||||
provision_datasources()
|
||||
print("Local PKI, credentials and provisioning ready.")
|
||||
|
||||
|
||||
def write_acls():
|
||||
cache_ops = "reset on ~session-user:* +get +set +del +ping +hello +client|setinfo"
|
||||
broker_shared = "+ping +hello +client|setinfo"
|
||||
worker_ops = (
|
||||
"~attachment-jobs +xreadgroup +xautoclaim +xgroup|create "
|
||||
"+xack +xdel +eval +multi +exec (~invalid-jobs +xadd)"
|
||||
)
|
||||
for store, rules in {
|
||||
"cache": {"api": cache_ops, "operator": "on ~* +@all"},
|
||||
"broker": {
|
||||
"api": f"on {broker_shared}",
|
||||
"relay": f"on ~attachment-jobs +xadd {broker_shared}",
|
||||
"worker": f"on {worker_ops} {broker_shared}",
|
||||
"operator": "on ~* +@all",
|
||||
},
|
||||
}.items():
|
||||
lines = ["user default off"]
|
||||
for role, permissions in rules.items():
|
||||
password = ensure(f"{store}_{role}_password", secrets.token_hex(24))
|
||||
lines.append(f"user {role} {permissions} >{password}")
|
||||
target = SECRETS / f"{store}.acl"
|
||||
target.write_text("\n".join(lines) + "\n")
|
||||
target.chmod(0o644)
|
||||
|
||||
|
||||
def provision_datasources():
|
||||
trust = (pki.TLS / "trust" / "ca.crt").read_text()
|
||||
identity = pki.TLS / "grafana-client"
|
||||
secure = {
|
||||
"tlsCACert": trust,
|
||||
"tlsClientCert": (identity / "service.crt").read_text(),
|
||||
"tlsClientKey": (identity / "service.key").read_text(),
|
||||
}
|
||||
datasources = []
|
||||
for name, port in (("prometheus", 9090), ("loki", 3100), ("tempo", 3200)):
|
||||
config = {"tlsAuth": True, "tlsAuthWithCACert": True}
|
||||
if name == "loki":
|
||||
config["derivedFields"] = [
|
||||
{
|
||||
"name": "Trace",
|
||||
"matcherType": "label",
|
||||
"matcherRegex": "trace_id",
|
||||
"datasourceUid": "tempo",
|
||||
"url": "$${__value.raw}",
|
||||
}
|
||||
]
|
||||
if name == "tempo":
|
||||
config["tracesToLogsV2"] = {
|
||||
"datasourceUid": "loki",
|
||||
"spanStartTimeShift": "-2m",
|
||||
"spanEndTimeShift": "2m",
|
||||
"filterByTraceID": True,
|
||||
"tags": [{"key": "service.name", "value": "service_name"}],
|
||||
}
|
||||
datasources.append(
|
||||
{
|
||||
"name": name.title(),
|
||||
"uid": name,
|
||||
"type": name,
|
||||
"access": "proxy",
|
||||
"url": f"https://{name}:{port}",
|
||||
"editable": False,
|
||||
"isDefault": name == "prometheus",
|
||||
"jsonData": config,
|
||||
"secureJsonData": secure,
|
||||
}
|
||||
)
|
||||
directory = SECRETS / "grafana-datasources"
|
||||
directory.mkdir(exist_ok=True)
|
||||
directory.chmod(0o755)
|
||||
target = directory / "local.yaml"
|
||||
target.write_text(json.dumps({"apiVersion": 1, "datasources": datasources}, indent=2))
|
||||
target.chmod(0o644)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
initialize()
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
|
||||
|
||||
import json
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = "from pathlib import Path; print(Path('/tmp/api-instance-id').read_text())"
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="https://localhost:8443", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
tls = ssl.create_default_context(cafile=ROOT / "secrets/tls/trust/ca.crt")
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2, context=tls) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator PKI. Only service-specific files are mounted into containers."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] / "secrets"
|
||||
CA = ROOT / "operator" / "ca"
|
||||
TLS = ROOT / "tls"
|
||||
SERVERS = {
|
||||
"lb": "DNS:localhost,DNS:lb,IP:127.0.0.1",
|
||||
"api": "DNS:api",
|
||||
"postgres": "DNS:postgres",
|
||||
"valkey": "DNS:valkey",
|
||||
"broker": "DNS:broker",
|
||||
"s3": "DNS:s3",
|
||||
"openbao": "DNS:openbao,DNS:localhost,IP:127.0.0.1",
|
||||
"otel-collector": "DNS:otel-collector",
|
||||
"prometheus": "DNS:prometheus",
|
||||
"loki": "DNS:loki",
|
||||
"tempo": "DNS:tempo",
|
||||
"grafana": "DNS:grafana,DNS:localhost,IP:127.0.0.1",
|
||||
}
|
||||
CLIENTS = (
|
||||
"lb-client",
|
||||
"api-client",
|
||||
"worker-client",
|
||||
"relay-client",
|
||||
"reconciler-client",
|
||||
"collector-client",
|
||||
"prometheus-client",
|
||||
"grafana-client",
|
||||
"probe-client",
|
||||
)
|
||||
|
||||
|
||||
def openssl(*args):
|
||||
subprocess.run(
|
||||
["openssl", *map(str, args)],
|
||||
cwd=CA,
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def init_ca():
|
||||
CA.mkdir(parents=True, exist_ok=True)
|
||||
TLS.mkdir(parents=True, exist_ok=True)
|
||||
if (CA / "ca.key").exists():
|
||||
return
|
||||
(CA / "issued").mkdir(exist_ok=True)
|
||||
for name, value in {
|
||||
"index.txt": "",
|
||||
"index.txt.attr": "unique_subject = no\n",
|
||||
"serial": "1000\n",
|
||||
"crlnumber": "1000\n",
|
||||
}.items():
|
||||
(CA / name).write_text(value)
|
||||
openssl(
|
||||
"req",
|
||||
"-new",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-x509",
|
||||
"-days",
|
||||
90,
|
||||
"-subj",
|
||||
"/CN=MTUSI lab7 local root",
|
||||
"-addext",
|
||||
"basicConstraints=critical,CA:TRUE,pathlen:0",
|
||||
"-addext",
|
||||
"keyUsage=critical,keyCertSign,cRLSign",
|
||||
"-keyout",
|
||||
"ca.key",
|
||||
"-out",
|
||||
"ca.crt",
|
||||
)
|
||||
(CA / "ca.cnf").write_text("""[ca]
|
||||
default_ca = signer
|
||||
[signer]
|
||||
database = index.txt
|
||||
new_certs_dir = issued
|
||||
certificate = ca.crt
|
||||
private_key = ca.key
|
||||
serial = serial
|
||||
crlnumber = crlnumber
|
||||
default_md = sha256
|
||||
default_days = 7
|
||||
default_crl_days = 30
|
||||
policy = policy
|
||||
unique_subject = no
|
||||
[policy]
|
||||
commonName = supplied
|
||||
""")
|
||||
refresh_crl()
|
||||
|
||||
|
||||
def refresh_crl():
|
||||
openssl("ca", "-config", "ca.cnf", "-gencrl", "-out", "current.crl")
|
||||
trust = TLS / "trust"
|
||||
trust.mkdir(exist_ok=True)
|
||||
trust.chmod(0o755)
|
||||
for source, name in (("ca.crt", "ca.crt"), ("current.crl", "revoked.crl")):
|
||||
data = (CA / source).read_bytes()
|
||||
(trust / name).write_bytes(data)
|
||||
(trust / name).chmod(0o644)
|
||||
|
||||
|
||||
def issue(name, *, rotate=False):
|
||||
target = TLS / name
|
||||
target.mkdir(exist_ok=True)
|
||||
target.chmod(0o755)
|
||||
key, cert = target / "service.key", target / "service.crt"
|
||||
if cert.exists() and not rotate:
|
||||
return
|
||||
if rotate and cert.exists():
|
||||
openssl("ca", "-config", "ca.cnf", "-revoke", cert)
|
||||
purpose = "serverAuth" if name in SERVERS else "clientAuth"
|
||||
san = SERVERS.get(name, "URI:urn:mtusi:lab7:" + name)
|
||||
(CA / "leaf.cnf").write_text(
|
||||
"[leaf]\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n"
|
||||
f"extendedKeyUsage={purpose}\nsubjectAltName={san}\n"
|
||||
"subjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid,issuer\n"
|
||||
)
|
||||
openssl(
|
||||
"req",
|
||||
"-new",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-subj",
|
||||
"/CN=" + name,
|
||||
"-keyout",
|
||||
"leaf.key",
|
||||
"-out",
|
||||
"leaf.csr",
|
||||
)
|
||||
openssl(
|
||||
"ca",
|
||||
"-config",
|
||||
"ca.cnf",
|
||||
"-batch",
|
||||
"-notext",
|
||||
"-extfile",
|
||||
"leaf.cnf",
|
||||
"-extensions",
|
||||
"leaf",
|
||||
"-in",
|
||||
"leaf.csr",
|
||||
"-out",
|
||||
"leaf.crt",
|
||||
)
|
||||
# A directory mount sees atomic leaf replacement. Host parent remains 0700.
|
||||
for source, dest in (("leaf.key", key), ("leaf.crt", cert)):
|
||||
temp = dest.with_suffix(".new")
|
||||
shutil.move(CA / source, temp)
|
||||
temp.chmod(0o644)
|
||||
temp.replace(dest)
|
||||
(target / "service.pem").write_bytes(cert.read_bytes() + key.read_bytes())
|
||||
(target / "service.pem").chmod(0o644)
|
||||
if rotate:
|
||||
refresh_crl()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--rotate", choices=[*SERVERS, *CLIENTS])
|
||||
args = parser.parse_args()
|
||||
os.umask(0o077)
|
||||
ROOT.mkdir(exist_ok=True, mode=0o700)
|
||||
init_ca()
|
||||
if args.rotate:
|
||||
issue(args.rotate, rotate=True)
|
||||
else:
|
||||
for name in (*SERVERS, *CLIENTS):
|
||||
issue(name)
|
||||
print("Local PKI ready.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop api worker relay lb
|
||||
docker compose restart --timeout 7 postgres valkey s3 broker openbao
|
||||
python3 scripts/bao_operator.py
|
||||
docker compose restart broker-tls
|
||||
docker compose up -d --no-build --wait --wait-timeout 90 api worker relay lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8084"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,471 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible negative checks and worker revocation on the local lab7 stand."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCAL = ROOT / "secrets"
|
||||
CA = LOCAL / "tls/trust/ca.crt"
|
||||
INSIDE_CA = "/run/tls/trust/ca.crt"
|
||||
|
||||
|
||||
def checked(*command, **kwargs):
|
||||
try:
|
||||
return subprocess.run(command, cwd=ROOT, check=True, **kwargs)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
if exc.stderr:
|
||||
sys.stderr.write(exc.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return checked("docker", "compose", *args, **kwargs)
|
||||
|
||||
|
||||
def context(ca, certificate=None):
|
||||
ctx = ssl.create_default_context(cafile=ca)
|
||||
if certificate:
|
||||
ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key")
|
||||
return ctx
|
||||
|
||||
|
||||
def response(url, ctx, data=None, headers=None, method=None):
|
||||
request = Request(url, data, headers or {}, method=method)
|
||||
try:
|
||||
with urlopen(request, context=ctx, timeout=4) as r:
|
||||
return r.status, r.read(262144)
|
||||
except HTTPError as e:
|
||||
return e.code, b""
|
||||
|
||||
|
||||
def denied(action):
|
||||
try:
|
||||
status = action()
|
||||
assert status in (401, 403) or status == b"", "Unexpected success in negative probe"
|
||||
except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc:
|
||||
# Preserve DNS/connectivity errors as real test failures.
|
||||
cause = exc.reason if isinstance(exc, URLError) else exc
|
||||
if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)):
|
||||
raise
|
||||
|
||||
|
||||
def redis_ping(ctx):
|
||||
with socket.create_connection(("broker", 6380), timeout=3) as raw:
|
||||
with ctx.wrap_socket(raw, server_hostname="broker") as tls:
|
||||
tls.sendall(b"*1\r\n$4\r\nPING\r\n")
|
||||
return tls.recv(256)
|
||||
|
||||
|
||||
def audit_tls():
|
||||
results = {}
|
||||
endpoints = {
|
||||
"api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"),
|
||||
"collector": (
|
||||
lambda ctx: response(
|
||||
"https://otel-collector:4319/v1/traces",
|
||||
ctx,
|
||||
b"",
|
||||
{"Content-Type": "application/x-protobuf"},
|
||||
)[0],
|
||||
"worker-client",
|
||||
),
|
||||
"broker": (redis_ping, "worker-client"),
|
||||
}
|
||||
for name, (probe, identity) in endpoints.items():
|
||||
status = probe(context(INSIDE_CA, Path("/probe") / identity))
|
||||
assert status == 200 or status.startswith(b"-NOAUTH")
|
||||
for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")):
|
||||
denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert)))
|
||||
results[name] = [
|
||||
"allowed",
|
||||
"no_client_denied",
|
||||
"same_ca_wrong_role_denied",
|
||||
"rogue_ca_denied",
|
||||
]
|
||||
|
||||
for host, port in (
|
||||
("broker", 6379),
|
||||
("otel-collector", 4318),
|
||||
("prometheus", 19090),
|
||||
("loki", 13100),
|
||||
("tempo", 13200),
|
||||
("tempo", 14318),
|
||||
("tempo", 9095),
|
||||
("loki", 9095),
|
||||
):
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1):
|
||||
raise AssertionError("Reachable plaintext " + host)
|
||||
except (ConnectionRefusedError, TimeoutError):
|
||||
pass
|
||||
results["network_plaintext_bypass"] = "denied"
|
||||
results["grafana_backends"] = {}
|
||||
for host, port, path in (
|
||||
("prometheus", 9090, "/api/v1/status/buildinfo"),
|
||||
("loki", 3100, "/loki/api/v1/status/buildinfo"),
|
||||
("tempo", 3200, "/api/echo"),
|
||||
):
|
||||
endpoint = f"https://{host}:{port}{path}"
|
||||
assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200
|
||||
denied(
|
||||
lambda endpoint=endpoint: response(
|
||||
endpoint, context(INSIDE_CA, Path("/probe/probe-client"))
|
||||
)[0]
|
||||
)
|
||||
results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny"
|
||||
return results
|
||||
|
||||
|
||||
def worker_scope():
|
||||
import psycopg
|
||||
import redis
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
settings = Settings.from_env()
|
||||
with psycopg.connect(settings.database_url, autocommit=True) as conn:
|
||||
assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]
|
||||
conn.execute("SELECT count(*) FROM attachment_jobs").fetchone()
|
||||
for query in (
|
||||
"SELECT count(*) FROM users",
|
||||
"SELECT count(*) FROM messages",
|
||||
"SELECT count(*) FROM sessions",
|
||||
"CREATE TEMP TABLE forbidden_probe(id int)",
|
||||
"UPDATE attachments SET owner_id=owner_id WHERE false",
|
||||
"SET ROLE messenger",
|
||||
):
|
||||
try:
|
||||
conn.execute(query)
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
continue
|
||||
raise AssertionError("Unexpected database grant")
|
||||
|
||||
broker = redis.Redis.from_url(settings.broker_url)
|
||||
assert broker.ping()
|
||||
for op in (
|
||||
lambda: broker.config_get("appendonly"),
|
||||
lambda: broker.xrange("unrelated-key"),
|
||||
lambda: broker.get("session-user:synthetic"),
|
||||
):
|
||||
try:
|
||||
op()
|
||||
except redis.exceptions.NoPermissionError:
|
||||
continue
|
||||
raise AssertionError("Unexpected broker grant")
|
||||
|
||||
store = ObjectStore(settings)
|
||||
suffix = uuid.uuid4().hex
|
||||
key = "objects/audit/" + suffix
|
||||
store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic")
|
||||
assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9
|
||||
for op in (
|
||||
lambda: store.client.put_object(
|
||||
Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x"
|
||||
),
|
||||
lambda: store.client.delete_object(Bucket=store.bucket, Key=key),
|
||||
lambda: store.client.get_bucket_policy(Bucket=store.bucket),
|
||||
):
|
||||
try:
|
||||
op()
|
||||
except ClientError as exc:
|
||||
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
||||
continue
|
||||
raise AssertionError("Unexpected S3 grant")
|
||||
|
||||
cert = Path("/run/tls/client")
|
||||
ctx = context(INSIDE_CA, cert)
|
||||
base = "https://openbao:8200/v1/"
|
||||
ct = {"Content-Type": "application/json"}
|
||||
code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct)
|
||||
assert code == 200
|
||||
auth = json.loads(raw)["auth"]
|
||||
token = auth["client_token"]
|
||||
headers = {**ct, "X-Vault-Token": token}
|
||||
assert 0 < auth["lease_duration"] <= 120
|
||||
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200
|
||||
assert response(base + "services/data/api", ctx, headers=headers)[0] == 403
|
||||
assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403
|
||||
assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204
|
||||
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403
|
||||
return {
|
||||
"postgres": {"tls": True, "job_read": True, "denied_operations": 6},
|
||||
"broker": {"tls": True, "ping": True, "denied_operations": 3},
|
||||
"s3": {"put_objects": True, "denied_operations": 3},
|
||||
"openbao": {
|
||||
"own_read": True,
|
||||
"foreign_read_and_write_denied": True,
|
||||
"token_ttl_s": auth["lease_duration"],
|
||||
"revoked_token_denied": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def old_identity():
|
||||
ctx = context(INSIDE_CA, Path("/probe/old"))
|
||||
denied(lambda: redis_ping(ctx))
|
||||
denied(
|
||||
lambda: response(
|
||||
"https://otel-collector:4319/v1/traces",
|
||||
ctx,
|
||||
b"",
|
||||
{"Content-Type": "application/x-protobuf"},
|
||||
)[0]
|
||||
)
|
||||
code, _ = response(
|
||||
"https://openbao:8200/v1/auth/cert/login",
|
||||
ctx,
|
||||
b'{"name":"worker"}',
|
||||
{"Content-Type": "application/json"},
|
||||
)
|
||||
assert code in (400, 403)
|
||||
return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"}
|
||||
|
||||
|
||||
def worker_probe():
|
||||
raw = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"worker",
|
||||
"/test/audit.py",
|
||||
"--inside",
|
||||
"worker",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def isolated(mode, mounts):
|
||||
cmd = [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"mtusi-lab7_backend",
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges:true",
|
||||
"--memory",
|
||||
"256m",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
||||
"-v",
|
||||
f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro",
|
||||
]
|
||||
for name, source in mounts.items():
|
||||
cmd += ["-v", f"{source}:/probe/{name}:ro"]
|
||||
image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0]
|
||||
raw = checked(
|
||||
*cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def check_api(client):
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, attachment)
|
||||
path = "/api/v1/attachments/" + attachment["id"]
|
||||
client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]})
|
||||
client.call("GET", path + "/content", 404, actor=eve)
|
||||
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=bob)
|
||||
reg = alice["registration"]
|
||||
bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode()
|
||||
statuses, replicas, times = [], set(), []
|
||||
for _ in range(6):
|
||||
r = client.call(
|
||||
"POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad}
|
||||
)
|
||||
statuses.append(r["status"])
|
||||
times.append(r["elapsed"] * 1000)
|
||||
replicas.add(r["headers"]["X-Instance-Id"])
|
||||
if r["status"] == 429:
|
||||
assert int(r["headers"]["Retry-After"]) > 0
|
||||
assert 429 in statuses and 401 in statuses and len(replicas) >= 2
|
||||
too_big = client.call(
|
||||
"POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
assert too_big["status"] == 413
|
||||
return {
|
||||
"statuses": statuses,
|
||||
"replicas": len(replicas),
|
||||
"latency_ms": times,
|
||||
"outsider_denied": True,
|
||||
"logout_revoked": True,
|
||||
"oversize_json_denied": True,
|
||||
}
|
||||
|
||||
|
||||
def rotate(client, temp):
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
old = temp / "old"
|
||||
shutil.copytree(LOCAL / "tls/worker-client", old)
|
||||
compose("stop", "worker")
|
||||
accepted = []
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for _ in range(5):
|
||||
att, data = client.attachment(alice, chat)
|
||||
accepted.append((att, data))
|
||||
checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker")
|
||||
checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client")
|
||||
password = LOCAL / "broker_worker_password"
|
||||
password.write_text(secrets.token_hex(24) + "\n")
|
||||
checked(sys.executable, "scripts/bootstrap.py")
|
||||
# Load the changed ACL without restarting durable Valkey or erasing its stream.
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"sh",
|
||||
"-ec",
|
||||
"""p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)"
|
||||
export REDISCLI_AUTH="$p"
|
||||
valkey-cli --user operator ACL LOAD | grep -q '^OK$'
|
||||
valkey-cli --user operator CLIENT KILL USER worker >/dev/null""",
|
||||
)
|
||||
compose("restart", "broker-tls", "otel-collector-tls")
|
||||
result = isolated("old", {"old": old})
|
||||
finally:
|
||||
checked(sys.executable, "scripts/bao_operator.py")
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
||||
for attachment, original in accepted:
|
||||
item = client.wait_ready(alice, attachment)
|
||||
file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice)
|
||||
assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest()
|
||||
return {
|
||||
"old_identity": result,
|
||||
"accepted": len(accepted),
|
||||
"ready_with_original": len(accepted),
|
||||
"elapsed_seconds": round(time.monotonic() - start, 3),
|
||||
"broker_acl_reloaded": True,
|
||||
"images_rebuilt": False,
|
||||
"new_worker": worker_probe(),
|
||||
}
|
||||
|
||||
|
||||
def check_sealed(client):
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
start = time.monotonic()
|
||||
checked(sys.executable, "scripts/bao_operator.py", "--seal")
|
||||
try:
|
||||
deadline = start + 42
|
||||
status = 200
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
||||
if status == 503:
|
||||
break
|
||||
time.sleep(1)
|
||||
assert status == 503, "API did not stop serving after the credential grace period"
|
||||
unavailable = time.monotonic() - start
|
||||
finally:
|
||||
checked(sys.executable, "scripts/bao_operator.py")
|
||||
deadline = time.monotonic() + 22
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
||||
if status == 200:
|
||||
return {
|
||||
"unready_after_seconds": round(unavailable, 3),
|
||||
"recovered_after_seconds": round(time.monotonic() - start, 3),
|
||||
}
|
||||
time.sleep(1)
|
||||
raise AssertionError("API readiness did not recover after unseal")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--inside", choices=("tls", "worker", "old"))
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--rotate", action="store_true")
|
||||
parser.add_argument("--seal", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.inside:
|
||||
print(
|
||||
json.dumps(
|
||||
{"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]()
|
||||
)
|
||||
)
|
||||
return
|
||||
if not args.output or args.output.exists():
|
||||
parser.error("choose a new --output")
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(CA))
|
||||
with tempfile.TemporaryDirectory(dir=LOCAL) as folder:
|
||||
tmp = Path(folder)
|
||||
tmp.chmod(0o755)
|
||||
rogue = tmp / "rogue"
|
||||
rogue.mkdir(mode=0o755)
|
||||
checked(
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=worker-client",
|
||||
"-addext",
|
||||
"extendedKeyUsage=clientAuth",
|
||||
"-keyout",
|
||||
str(rogue / "service.key"),
|
||||
"-out",
|
||||
str(rogue / "service.crt"),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
(rogue / "service.key").chmod(0o644)
|
||||
identities = {
|
||||
name: LOCAL / "tls" / name
|
||||
for name in ("lb-client", "worker-client", "probe-client", "grafana-client")
|
||||
}
|
||||
result = {
|
||||
"tls": isolated("tls", {**identities, "rogue": rogue}),
|
||||
"worker_scope": worker_probe(),
|
||||
"api": check_api(client),
|
||||
}
|
||||
print("PASS: verified TLS and distinct service privileges", flush=True)
|
||||
if args.rotate:
|
||||
result["worker_compromise"] = rotate(client, tmp)
|
||||
print("PASS: old identity denied; all accepted uploads recovered", flush=True)
|
||||
if args.seal:
|
||||
result["sealed"] = check_sealed(client)
|
||||
print(
|
||||
"PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True
|
||||
)
|
||||
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = "from pathlib import Path; print(Path('/tmp/api-instance-id').read_text())"
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Use an operator connection only to seed a legacy hash and an expired test session."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def inside():
|
||||
import psycopg
|
||||
|
||||
from messenger.db_roles import admin_conninfo
|
||||
|
||||
fixture = json.load(sys.stdin)
|
||||
with psycopg.connect(admin_conninfo()) as db:
|
||||
if fixture["mode"] == "seed":
|
||||
db.execute(
|
||||
"UPDATE users SET password_hash=%s WHERE id=%s",
|
||||
(fixture["hash"], fixture["user"]),
|
||||
)
|
||||
elif fixture["mode"] == "check":
|
||||
row = db.execute(
|
||||
"SELECT password_hash FROM users WHERE id=%s", (fixture["user"],)
|
||||
).fetchone()
|
||||
assert row and row[0].startswith("$argon2id$")
|
||||
assert row[0].split("$")[3] == "m=19456,t=2,p=1"
|
||||
db.execute(
|
||||
"UPDATE sessions SET expires_at=created_at+interval '1 millisecond' "
|
||||
"WHERE token_hash=%s",
|
||||
(bytes.fromhex(fixture["digest"]),),
|
||||
)
|
||||
else:
|
||||
raise AssertionError("Unknown fixture mode")
|
||||
|
||||
|
||||
def db_fixture(**data):
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/verify_sessions.py:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"migrate",
|
||||
"/test/verify_sessions.py",
|
||||
"--inside",
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
input=json.dumps(data),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
if "--inside" in sys.argv:
|
||||
inside()
|
||||
return
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(ROOT / "secrets/tls/trust/ca.crt"))
|
||||
actor = client.user("legacy")
|
||||
reg = actor["registration"]
|
||||
legacy = PasswordHasher(memory_cost=65536, time_cost=3, parallelism=4).hash(reg["password"])
|
||||
db_fixture(mode="seed", user=actor["id"], hash=legacy)
|
||||
auth = base64.b64encode((reg["username"] + ":" + reg["password"]).encode()).decode()
|
||||
session = client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
schema="Session",
|
||||
headers={"Authorization": "Basic " + auth},
|
||||
)["json"]
|
||||
actor["token"] = session["token"]
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
db_fixture(
|
||||
mode="check", user=actor["id"], digest=hashlib.sha256(actor["token"].encode()).hexdigest()
|
||||
)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
print("PASS: successful login rehashed old Argon2id parameters; expired session rejected.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify the real Grafana data paths over authenticated HTTPS."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CA = ROOT / "secrets/tls/trust/ca.crt"
|
||||
|
||||
|
||||
def grafana(uid, path, **params):
|
||||
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
||||
basic = base64.b64encode(("admin:" + password).encode()).decode()
|
||||
url = f"https://localhost:3007/api/datasources/proxy/uid/{uid}{path}"
|
||||
if params:
|
||||
url += "?" + urlencode(params)
|
||||
request = Request(url, headers={"Authorization": "Basic " + basic})
|
||||
with urlopen(request, context=ssl.create_default_context(cafile=CA), timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def until(fn):
|
||||
end = time.monotonic() + 35
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
value = fn()
|
||||
except HTTPError as exc:
|
||||
if exc.code not in (404, 502, 503):
|
||||
raise
|
||||
else:
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("Timed out waiting for Grafana datasource results")
|
||||
|
||||
|
||||
def logs(service, field, value):
|
||||
# The query is restricted to a generated ID, which is validated by the probe.
|
||||
return grafana(
|
||||
"loki",
|
||||
"/loki/api/v1/query_range",
|
||||
query=f'{{service_name="messenger-{service}"}} | {field}="{value}"',
|
||||
start=str(time.time_ns() - 120_000_000_000),
|
||||
limit=10,
|
||||
)["data"]["result"]
|
||||
|
||||
|
||||
def main():
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(CA))
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
marker = "probe_" + uuid.uuid4().hex
|
||||
client.message(actor, chat, marker)
|
||||
upload = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename={marker}.png",
|
||||
202,
|
||||
actor=actor,
|
||||
headers={"Content-Type": "image/png"},
|
||||
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
schema="Attachment",
|
||||
)
|
||||
client.wait_ready(actor, upload["json"])
|
||||
request_id = str(uuid.UUID(upload["headers"]["X-Request-Id"]))
|
||||
api_logs = until(lambda: logs("api", "request_id", request_id))
|
||||
stream = api_logs[0]
|
||||
value = stream["values"][0]
|
||||
trace_id = (value[2] if len(value) > 2 else stream["stream"])["trace_id"]
|
||||
assert len(trace_id) == 32 and all(ch in "0123456789abcdef" for ch in trace_id)
|
||||
worker_logs = until(lambda: logs("worker", "trace_id", trace_id))
|
||||
trace = until(lambda: grafana("tempo", "/api/traces/" + trace_id))
|
||||
collected = json.dumps([api_logs, worker_logs, trace])
|
||||
assert "jobs.consume" in collected and "s3.put" in collected and "db.transaction" in collected
|
||||
for forbidden in (marker, actor["registration"]["password"], actor["token"]):
|
||||
assert forbidden not in collected
|
||||
until(
|
||||
lambda: grafana("prometheus", "/api/v1/query", query="sum(messenger_http_requests_total)")[
|
||||
"data"
|
||||
]["result"]
|
||||
)
|
||||
print("PASS: three Grafana datasources over mTLS; correlated payloads are clean.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user