feat: complete lab 7 grade 3 security controls

This commit is contained in:
Dmitrii Grishko
2026-11-14 20:08:27 +03:00
parent 8986ba6bef
commit c5e3ac3aff
49 changed files with 1670 additions and 149 deletions
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Initialize or unseal the local OpenBao; keep operator material on the host."""
import argparse
import json
import os
import ssl
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
SECRETS = Path(__file__).resolve().parents[1] / "secrets"
MATERIAL = SECRETS / "bao-operator.json"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("init", "unseal"))
action = parser.parse_args().action
context = ssl.create_default_context(cafile=str(SECRETS / "tls/lab7-ca.crt"))
def request(path, data):
req = Request(
"https://localhost:8200/v1/sys/" + path,
method="PUT",
data=json.dumps(data).encode(),
headers={"Content-Type": "application/json"},
)
with urlopen(req, context=context, timeout=15) as response:
return json.load(response)
if action == "init":
# Refuse to overwrite the operator's only copy of the material.
fd = os.open(MATERIAL, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w") as output:
try:
data = request("init", {"secret_shares": 3, "secret_threshold": 2})
except HTTPError:
MATERIAL.unlink()
raise
json.dump(data, output)
output.flush()
os.fsync(output.fileno())
print("Operator material saved in secrets/bao-operator.json")
return
data = json.loads(MATERIAL.read_text())
for share in data["keys_base64"]:
if not request("unseal", {"key": share})["sealed"]:
print("OpenBao unsealed")
return
raise SystemExit("OpenBao remains sealed")
if __name__ == "__main__":
main()
+27
View File
@@ -36,6 +36,33 @@ make_ca untrusted-ca
chmod 644 "$dir/lab7-ca.crt"
make_leaf ingress lab7-ca serverAuth DNS:localhost,IP:127.0.0.1
make_leaf api lab7-ca serverAuth DNS:api
for service in db redis s3 broker; do
san="DNS:$service"
if [ "$service" = s3 ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi
make_leaf "$service" lab7-ca serverAuth "$san"
done
for service in otel-collector loki tempo prometheus grafana; do
san="DNS:$service"
if [ "$service" = grafana ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi
make_leaf "$service" lab7-ca serverAuth "$san"
done
make_ca telemetry-read-ca
make_ca telemetry-export-ca
chmod 644 "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt"
make_leaf grafana-reader telemetry-read-ca clientAuth URI:urn:lab7:grafana
make_leaf collector-exporter telemetry-export-ca clientAuth URI:urn:lab7:collector
make_leaf prometheus-scraper telemetry-export-ca clientAuth URI:urn:lab7:prometheus
cat "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt" > "$dir/loki-client-ca.crt"
chmod 644 "$dir/loki-client-ca.crt"
make_ca telemetry-ingest-ca
chmod 644 "$dir/telemetry-ingest-ca.crt"
for role in api worker dispatcher; do
make_leaf "otel-$role" telemetry-ingest-ca clientAuth "URI:urn:lab7:otel:$role"
done
make_leaf openbao lab7-ca serverAuth DNS:openbao,DNS:localhost,IP:127.0.0.1
for role in api worker dispatcher cleanup; do
make_leaf "bao-$role" lab7-ca clientAuth "URI:urn:lab7:bao:$role"
done
make_leaf lb lab7-ca clientAuth URI:urn:lab7:load-balancer
make_leaf wrong-role lab7-ca clientAuth URI:urn:lab7:probe
make_leaf wrong-ca untrusted-ca clientAuth URI:urn:lab7:load-balancer
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Prove each runtime identity can read only its own OpenBao secret."""
import json
import ssl
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
ROLES = ("api", "worker", "dispatcher", "cleanup")
def request(context, path, *, body=None, token="", status=200):
req = Request(
f"https://localhost:8200/v1/{path}",
data=None if body is None else json.dumps(body).encode(),
headers={"X-Vault-Token": token, "Content-Type": "application/json"},
)
try:
with urlopen(req, context=context, timeout=5) as response:
assert response.status == status, (path, response.status, status)
return json.load(response)
except HTTPError as error:
assert error.code == status, (path, error.code, status)
def main():
for role in ROLES:
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
context.load_cert_chain(TLS / f"bao-{role}.crt", TLS / f"bao-{role}.key")
auth = request(context, "auth/cert/login", body={"name": role})["auth"]
assert set(auth["policies"]) == {role}
assert 0 < auth["lease_duration"] <= 900
token = auth["client_token"]
request(context, f"messenger/data/{role}", token=token)
request(context, f"messenger/data/{role}", token=token, body={"data": {}}, status=403)
request(context, f"sys/policies/acl/{role}", token=token, body={"policy": ""}, status=403)
for other in set(ROLES) - {role}:
request(context, f"messenger/data/{other}", token=token, status=403)
request(context, "auth/cert/login", body={"name": other}, status=400)
print(f"PASS {role}: scoped read, cross-role login/read and write denied, TTL <= 900 s")
if __name__ == "__main__":
main()
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose exec -T db sh -ec '
export PGPASSWORD=$(cat /run/secrets/database_worker_password)
export PGCHANNELBINDING=disable PGSSLMODE=verify-full PGSSLROOTCERT=/run/tls/ca.crt
psql -X -h db -U messenger_worker "$POSTGRES_DB" -Atc "
SELECT has_table_privilege(current_user, '\''attachments'\'', '\''UPDATE'\'')
AND NOT has_table_privilege(current_user, '\''users'\'', '\''SELECT'\'')
AND NOT has_schema_privilege(current_user, '\''public'\'', '\''CREATE'\'');
" | grep -qx t
'
printf '%s\n' 'PASS PostgreSQL: worker can update attachments; user table and schema creation denied'
docker compose exec -T broker rabbitmqctl -q list_permissions -p / |
awk -F '\t' '
$1=="messenger_dispatcher" && $2=="^attachments\\.v1$" && $3=="^amq\\.default$" && $4=="^$" { p=1 }
$1=="messenger_worker" && $2=="^attachments\\.v1$" && $3=="^$" && $4=="^attachments\\.v1$" { c=1 }
END {exit !(p && c)}
'
printf '%s\n' 'PASS RabbitMQ: producer can publish and consumer can read attachments.v1'
docker compose exec -T redis sh -ec '
export REDISCLI_AUTH=$(cat /run/secrets/redis_password)
redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger CONFIG GET dir | grep -q NOPERM
'
printf '%s\n' 'PASS Redis: application role cannot read server configuration'
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Negative HTTP cases beyond the supplied smoke."""
import ssl
import socket
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, run # noqa: E402
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, _, eve, chat = client.bootstrap()
upload = f"/api/v1/chats/{chat['id']}/attachments?filename=probe.png"
headers = {"Content-Type": "image/png"}
client.call("POST", upload, 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers=headers)
client.call("POST", upload, 415, actor=alice, raw=b"not an image", headers=headers)
attachment, _ = client.attachment(alice, chat)
path = "/api/v1/attachments/" + attachment["id"]
spoofed = {"X-User-Id": alice["id"]}
for suffix in ("", "/content"):
client.call("GET", path + suffix, 404, actor=eve, headers=spoofed)
other = client.call("POST", "/api/v1/chats", 201, actor=eve,
body={"title": "ACL probe", "member_ids": [eve["id"], alice["id"]]}, schema="Chat")["json"]
client.call("POST", f"/api/v1/chats/{other['id']}/messages", 404, actor=eve,
body={"text": "", "attachment_ids": [attachment["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex, **spoofed})
context = ssl.create_default_context(cafile=args.ca_file)
from urllib.parse import urlsplit
endpoint = urlsplit(client.base)
with socket.create_connection((endpoint.hostname, endpoint.port or 443), timeout=5) as connection:
try:
with context.wrap_socket(connection, server_hostname="wrong.lab.invalid"):
raise AssertionError("TLS accepted a wrong hostname")
except ssl.SSLCertVerificationError as error:
if error.verify_code != 62:
raise
print("PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname")
if __name__ == "__main__":
run(main)
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Inspect runtime constraints without printing inspect output or secrets."""
import json
import subprocess
APP_ROLES = {"api", "worker", "dispatcher", "upload-cleanup"}
ALLOWED_PORTS = {("haproxy", "8443/tcp"), ("grafana", "3000/tcp"),
("openbao", "8200/tcp"), ("s3", "9000/tcp")}
def main():
ids = subprocess.check_output(["docker", "compose", "ps", "-q"], text=True).split()
assert ids, "No running Compose services"
services = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
for service in services:
name = service["Config"]["Labels"]["com.docker.compose.service"]
config = service["HostConfig"]
assert not config["Privileged"] and config["NetworkMode"] != "host", name
mounts = [mount["Destination"] for mount in service["Mounts"]]
assert not any("docker.sock" in path for path in mounts), name
for port, bindings in (config.get("PortBindings") or {}).items():
assert (name, port) in ALLOWED_PORTS, (name, port)
assert all(binding["HostIp"] == "127.0.0.1" for binding in bindings), name
if name in APP_ROLES:
assert service["Config"]["User"] == "app:app", name
assert config["CapDrop"] == ["ALL"] and not config["CapAdd"], name
assert config["Memory"] > 0 and config["NanoCpus"] > 0, name
assert config["ReadonlyRootfs"] and "/run/runtime-secrets" in config["Tmpfs"], name
assert not any(path.startswith("/run/secrets") for path in mounts), name
assert not any("BAO_TOKEN=" in field for field in service["Config"]["Env"]), name
assert APP_ROLES <= {svc["Config"]["Labels"]["com.docker.compose.service"] for svc in services}
print("PASS: local-only ports, runtime secrets on tmpfs, non-root app roles with limits and dropped capabilities")
if __name__ == "__main__":
main()
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Confirm metrics, logs and traces reach their HTTPS stores."""
import json
import ssl
import subprocess
import time
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen
from urllib.error import HTTPError
import http.client
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
def query(service, port, path, params):
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
context.load_cert_chain(TLS / "grafana-reader.crt", TLS / "grafana-reader.key")
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
address = subprocess.check_output(["docker", "inspect", "-f",
'{{(index .NetworkSettings.Networks "highload-lab7_telemetry").IPAddress}}', container],
text=True).strip()
connection = http.client.HTTPSConnection(service, port, context=context, timeout=5)
create_connection = connection._create_connection
connection._create_connection = lambda _, *args: create_connection((address, port), *args)
try:
connection.request("GET", f"{path}?{urlencode(params)}")
response = connection.getresponse()
assert response.status == 200, (service, response.status)
return json.load(response)
finally:
connection.close()
def main():
try:
with urlopen("https://localhost:8443/api/v1/users/me", context=ssl.create_default_context(
cafile=str(TLS / "lab7-ca.crt")), timeout=5):
raise AssertionError("Anonymous request should be rejected")
except HTTPError as error:
assert error.code == 401
since = int(time.time()) - 10
for _ in range(30):
now = int(time.time())
metrics = query("prometheus", 9090, "/api/v1/query", {
"query": 'up{job="otel-collector"} == 1'})["data"]["result"]
logs = query("loki", 3100, "/loki/api/v1/query_range", {
"query": '{service_name="messenger-api"}', "start": since * 10**9,
"end": now * 10**9, "limit": 1})["data"]["result"]
traces = query("tempo", 3200, "/api/search", {
"q": '{ resource.service.name = "messenger-api" }',
"start": since, "end": now, "limit": 1}).get("traces")
if metrics and logs and traces:
print("PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS")
return
time.sleep(2)
raise SystemExit("Telemetry signal missing within 60 seconds")
if __name__ == "__main__":
main()
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""Verify trusted TLS on data-plane endpoints, from the Linux Docker host."""
import json
import socket
import ssl
import subprocess
from pathlib import Path
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
ENDPOINTS = (
("s3", 9000, None, False),
("broker", 5671, None, False),
("redis", 6379, None, False),
("db", 5432, None, True),
("otel-collector", 4318, "otel-api", False),
("otel-collector", 8889, "prometheus-scraper", False),
("prometheus", 9090, "grafana-reader", False),
("loki", 3100, "collector-exporter", False),
("loki", 3100, "grafana-reader", False),
("tempo", 4318, "collector-exporter", False),
("tempo", 3200, "grafana-reader", False),
("grafana", 3000, None, False),
("openbao", 8200, None, False),
)
def probe(address, service, port, identity, pg, ca):
context = ssl.create_default_context(cafile=str(TLS / f"{ca}.crt"))
if identity:
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
with socket.create_connection((address, port), timeout=5) as conn:
if pg:
conn.sendall(bytes.fromhex("0000000804d2162f"))
assert conn.recv(1) == b"S", "PostgreSQL did not accept TLS negotiation"
with context.wrap_socket(conn, server_hostname=service) as encrypted:
assert encrypted.version() in {"TLSv1.2", "TLSv1.3"}
if identity:
encrypted.sendall(b"GET / HTTP/1.0\r\n\r\n")
encrypted.recv(1)
def main():
for service, port, identity, pg in ENDPOINTS:
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
networks = json.loads(subprocess.check_output(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", container], text=True))
address = next(network["IPAddress"] for network in networks.values() if network["IPAddress"])
probe(address, service, port, identity, pg, "lab7-ca")
try:
probe(address, service, port, identity, pg, "untrusted-ca")
except ssl.SSLCertVerificationError:
pass
else:
raise AssertionError(f"{service}:{port} accepted a foreign CA")
if identity:
try:
probe(address, service, port, "wrong-role", pg, "lab7-ca")
except ssl.SSLError:
pass
else:
raise AssertionError(f"{service}:{port} accepted a foreign client identity")
print(f"PASS {service}:{port}: verified service SAN; foreign CA rejected"
+ ("; foreign client identity rejected" if identity else ""))
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replica=$(docker compose ps -q api | xargs docker inspect -f '{{.Name}} {{.Config.Hostname}}' |
awk '$1=="/highload-lab7-api-2" {print $2}')
test -n "$replica"
trap 'docker compose start api >/dev/null' EXIT
TARGET_INSTANCE_ID="$replica" ./scripts/stop-one.sh
./scripts/wait_ingress.sh
make test CA_FILE=secrets/tls/lab7-ca.crt
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""Use a stand started with SESSION_TTL_SECONDS between 3 and 15."""
import base64
import datetime as dt
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run, timestamp # noqa: E402
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
actor = client.user("expiry")
registration = actor["registration"]
credentials = base64.b64encode(
f"{registration['username']}:{registration['password']}".encode()
).decode()
session = client.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + credentials}, schema="Session")["json"]
actor["token"] = session["token"]
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
remaining = (timestamp(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds()
check(0 < remaining <= 15, "Start rehearsal API with a short SESSION_TTL_SECONDS")
time.sleep(remaining + 0.3)
client.call("GET", "/api/v1/users/me", 401, actor=actor)
print("PASS: valid session accepted before expiry and rejected after expiry")
if __name__ == "__main__":
run(main)
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Run level-3 checks on the local rehearsal stand; write sanitized Markdown."""
import argparse
import os
import shlex
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
CA = "secrets/tls/lab7-ca.crt"
OUTPUT = ROOT / "evidence"
VALIDATION = {"CA_FILE": str(ROOT / CA)}
CHECKS = {
"grade3-https": ("Бизнес-API, проверка CA, logout и вложения по HTTPS", [
["make", "check"], ["make", "test", "test-security", f"CA_FILE={CA}"],
]),
"grade3-mtls": ("Допуск только роли LB на каждой реплике", [
["python3", "scripts/check_ingress_mtls.py"],
]),
"grade3-dependencies": ("TLS хранилищ, брокера и телеметрии", [
["python3", "scripts/check_grade3_tls.py"],
["./scripts/check_dependency_roles.sh"],
]),
"grade3-signals": ("Доставка логов, метрик и трейсов по защищённым связям", [
["python3", "scripts/check_grade3_signals.py"],
]),
"grade3-openbao": ("Сервисная аутентификация и scoped policy OpenBao", [
["python3", "scripts/check_bao_roles.py"],
]),
"grade3-runtime": ("Изоляция, порты, лимиты и локальное размещение секретов", [
["python3", "scripts/check_grade3_runtime.py"],
]),
"grade3-http-security": ("Лимиты, MIME, чужие вложения и неверный TLS hostname", [
["python3", "scripts/check_grade3_http.py", "--ca-file", CA],
]),
"grade3-resilience": ("Сохранность после перезапуска хранилищ и отказ API", [
["sleep", "60"],
["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh", f"CA_FILE={CA}"],
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"],
["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"],
["sleep", "60"],
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"],
["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"],
]),
"grade3-other-replica": ("Работа HTTPS API при остановленной второй реплике", [
["sleep", "60"], ["./scripts/check_other_replica.sh"], ["./scripts/wait_ingress.sh"],
]),
"grade3-login": ("Общий лимит входа по адресу клиента", [
["sleep", "60"], ["python3", "scripts/check_login_rate.py"],
]),
"grade3-expiry": ("Истечение сессии на стенде с коротким TTL", [
["sleep", "60"],
["env", "SESSION_TTL_SECONDS=5", "docker", "compose", "up", "-d", "--no-deps", "api"],
["./scripts/wait_ingress.sh"],
["python3", "scripts/check_session_expiry.py", "--ca-file", CA],
["docker", "compose", "up", "-d", "--no-deps", "api"], ["./scripts/wait_ingress.sh"],
]),
"grade3-argon2": ("Стоимость проверки хеша Argon2id на лимитах API", [
["docker", "compose", "--profile", "test", "build", "backend-tests"],
["docker", "run", "--rm", "--network", "none", "--cpus", "0.5", "--memory", "512m",
"-e", "GOMAXPROCS=1", "highload-lab7-backend-tests", "go", "test", "./backend/services",
"-run", "^$", "-bench", "^BenchmarkPasswordVerification$", "-benchtime=3x", "-count=3"],
]),
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--only", nargs="+", choices=CHECKS)
selected = parser.parse_args().only or CHECKS
OUTPUT.mkdir(exist_ok=True)
if "grade3-resilience" in selected:
for name in ("restart", "failover"):
path = ROOT / ".memory" / f"{name}-containers.txt"
path.parent.mkdir(exist_ok=True)
path.write_text("")
VALIDATION[f"{name.upper()}_EVIDENCE_FILE"] = str(path)
passed = True
for slug in selected:
title, commands = CHECKS[slug]
report = [f"# {title}", "", "Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.", ""]
for command in commands:
result = subprocess.run(command, cwd=ROOT, env=os.environ | VALIDATION,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
passed &= result.returncode == 0
report.extend([f"`{shlex.join(command)}`", "", f"Код завершения: {result.returncode}.", "",
"```text", "\n".join(line.rstrip() for line in result.stdout.splitlines()).rstrip(), "```", ""])
if slug == "grade3-resilience":
for name in ("restart", "failover"):
rows = Path(VALIDATION[f"{name.upper()}_EVIDENCE_FILE"]).read_text().strip()
report.extend([f"Состояния контейнеров ({name}):", "", "```text", rows, "```", ""])
report.extend(["Итог: " + ("все команды завершились успешно." if all(
line == "Код завершения: 0." for line in report if line.startswith("Код завершения:")
) else "есть команды с ошибками."), ""])
(OUTPUT / f"{slug}.md").write_text("\n".join(report), encoding="utf-8")
print(f"Recorded {slug}", flush=True)
raise SystemExit(0 if passed else 1)
if __name__ == "__main__":
main()
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Configure an initialized, unsealed local OpenBao using an operator token."""
import argparse
import getpass
import json
import ssl
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
TLS = SECRETS / "tls"
def main():
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--operator-file", type=Path)
args = parser.parse_args()
token = (json.loads(args.operator_file.read_text())["root_token"]
if args.operator_file else getpass.getpass("OpenBao operator token: "))
def request(path, body=None):
data = None if body is None else json.dumps(body).encode()
req = Request(
"https://localhost:8200/v1/" + path,
data=data,
headers={"Content-Type": "application/json", "X-Vault-Token": token},
)
try:
with urlopen(req, context=context, timeout=5) as response:
return json.load(response) if response.status != 204 else {}
except HTTPError as error:
raise SystemExit(f"OpenBao {path}: HTTP {error.code}")
if "cert/" not in request("sys/auth"):
request("sys/auth/cert", {"type": "cert"})
if "messenger/" not in request("sys/mounts"):
request("sys/mounts/messenger", {"type": "kv", "options": {"version": "2"}})
for role in ("api", "worker", "dispatcher", "cleanup"):
request(f"sys/policies/acl/{role}", {
"policy": f'path "messenger/data/{role}" {{ capabilities = ["read"] }}',
})
request(f"auth/cert/certs/{role}", {
"certificate": (TLS / f"bao-{role}.crt").read_text(),
"token_policies": [role],
"token_no_default_policy": True,
"token_ttl": "15m",
"token_max_ttl": "15m",
})
names = {"database_url": f"database_{role}_url"}
if role == "api":
names["cache_url"] = "cache_tls_url"
if role in ("worker", "dispatcher"):
names["broker_url"] = f"broker_{role}_url"
if role != "dispatcher":
for key in ("access_key_id", "secret_access_key"):
names[f"s3_{key}"] = f"s3_{role}_{key}"
values = {key: (SECRETS / name).read_text().strip() for key, name in names.items()}
request(f"messenger/data/{role}", {"data": values})
print(f"Configured OpenBao role {role}")
if __name__ == "__main__":
main()
+36 -51
View File
@@ -26,9 +26,9 @@ def main():
POSTGRES_USER="messenger",
POSTGRES_PASSWORD=secrets.token_hex(24),
)
defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="http://s3:9000",
OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
S3_PUBLIC_ENDPOINT="http://127.0.0.1:9000",
defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="https://s3:9000",
OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
S3_PUBLIC_ENDPOINT="https://localhost:9000",
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
values = defaults | existing
for name, lower, upper in (
@@ -63,60 +63,45 @@ def main():
os.chmod(path, 0o600)
secret_dir = ROOT / "secrets"
secret_dir.mkdir(mode=0o700, exist_ok=True)
database_url = "postgres://{}:{}@db:5432/{}?sslmode=disable".format(
def create_secret(name, value):
target = secret_dir / name
if not target.exists():
target.write_text(value + "\n", encoding="utf-8")
target.chmod(0o644)
return target.read_text(encoding="utf-8").strip()
create_secret("grafana_admin_password", secrets.token_hex(24))
database_url = "postgres://{}:{}@db:5432/{}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt".format(
quote(values["POSTGRES_USER"], safe=""),
quote(values["POSTGRES_PASSWORD"], safe=""),
quote(values["POSTGRES_DB"], safe=""),
)
for filename,content in (("broker_user","messenger"),("broker_password",secrets.token_hex(32))):
target=secret_dir/filename
if not target.exists():
target.write_text(content+"\n",encoding="utf-8")
os.chmod(target,0o644)
broker_url="amqp://{}:{}@broker:5672/%2F".format(
quote((secret_dir/"broker_user").read_text().strip(),safe=""),
quote((secret_dir/"broker_password").read_text().strip(),safe=""))
# Migrate only the generated legacy Redis URL; preserve custom AMQP settings.
broker_file=secret_dir/"broker_url"
if broker_file.exists() and broker_file.read_text().strip()=="redis://broker:6379/0":
broker_file.write_text(broker_url+"\n",encoding="utf-8")
create_secret("broker_user", "messenger")
create_secret("broker_password", secrets.token_hex(32))
redis_password = create_secret("redis_password", secrets.token_hex(32))
cache_url = f"rediss://messenger:{redis_password}@redis:6379/0"
create_secret("redis_acl", "user default off\nuser messenger on >" + redis_password
+ " ~messenger:* +ping +get +set +del +incr +expire +ttl +eval +evalsha +hello +client|setinfo")
for name, content in (
("DATABASE_URL_FILE", database_url),
("CACHE_URL_FILE", "redis://redis:6379/0"),
("BROKER_URL_FILE", broker_url),
("S3_ACCESS_KEY_ID_FILE", secrets.token_hex(12)),
("S3_SECRET_ACCESS_KEY_FILE", secrets.token_hex(32)),
("database_tls_url", database_url),
("cache_tls_url", cache_url),
):
container_path = Path(values[name])
if container_path.parent != Path("/run/secrets") or not container_path.name:
raise SystemExit(f"{name} must name a file directly inside /run/secrets")
target = (
secret_dir
/ {
"DATABASE_URL_FILE": "database_url",
"CACHE_URL_FILE": "cache_url",
"BROKER_URL_FILE": "broker_url",
"S3_ACCESS_KEY_ID_FILE": "s3_access_key_id",
"S3_SECRET_ACCESS_KEY_FILE": "s3_secret_access_key",
}[name]
)
if not target.exists():
# Containers run as an unprivileged user; the private host directory
# protects the files, while read permission lets the API consume them.
with target.open("x", encoding="utf-8") as file:
file.write(content + "\n")
os.chmod(target, 0o644)
# Existing S3 files remain the root credentials; API gets a new, separate identity.
for filename, content in (
("s3_api_access_key_id", secrets.token_hex(12)),
("s3_api_secret_access_key", secrets.token_hex(32)),
):
target = secret_dir / filename
if not target.exists():
with target.open("x", encoding="utf-8") as file:
file.write(content + "\n")
os.chmod(target, 0o644)
print("Local .env and connection files ready; existing values and files preserved.")
create_secret(name, content)
create_secret("s3_access_key_id", secrets.token_hex(12))
create_secret("s3_secret_access_key", secrets.token_hex(32))
database = quote(values["POSTGRES_DB"], safe="")
for role in ("api", "worker", "dispatcher", "cleanup"):
password = create_secret(f"database_{role}_password", secrets.token_hex(24))
create_secret(f"database_{role}_url",
f"postgres://messenger_{role}:{password}@db:5432/{database}"
"?sslmode=verify-full&sslrootcert=/run/tls/ca.crt")
if role in ("worker", "dispatcher"):
password = create_secret(f"broker_{role}_password", secrets.token_hex(24))
create_secret(f"broker_{role}_url", f"amqps://messenger_{role}:{password}@broker:5671/%2F")
if role != "dispatcher":
create_secret(f"s3_{role}_access_key_id", secrets.token_hex(12))
create_secret(f"s3_{role}_secret_access_key", secrets.token_hex(32))
print("Local connection files ready; existing values and files preserved.")
if __name__ == "__main__":
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --wait db redis broker s3
docker compose run --rm migrate
docker compose exec -T db sh -ec '
for role in api worker dispatcher cleanup; do
export "${role}_password=$(cat "/run/secrets/database_${role}_password")"
done
psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -f -
' < infra/postgres-roles.sql
docker compose exec -T broker sh /bootstrap.sh
docker compose run --rm s3-bootstrap
+8
View File
@@ -1,7 +1,15 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then
docker compose ps -q db redis s3 broker api | xargs docker inspect \
-f '{{.Name}} {{.State.StartedAt}}' > "$RESTART_EVIDENCE_FILE"
fi
docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart db redis s3 broker >/dev/null
docker compose --project-name highload-lab7 --env-file .env -f compose.yml up -d --wait db redis s3 broker >/dev/null
docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart api upload-cleanup worker dispatcher >/dev/null
docker compose --project-name highload-lab7 --env-file .env -f compose.yml up -d --wait >/dev/null
if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then
docker compose ps -q db redis s3 broker api | xargs docker inspect \
-f '{{.Name}} {{.State.StartedAt}}' >> "$RESTART_EVIDENCE_FILE"
fi
+3
View File
@@ -8,6 +8,9 @@ for container in $(docker compose --project-name highload-lab7 --env-file .env -
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
# Only containers from this Compose project's api service are eligible.
docker stop --timeout 8 "$container" >/dev/null
if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then
docker inspect -f '{{.Name}} {{.State.Status}}' "$container" >> "$FAILOVER_EVIDENCE_FILE"
fi
printf 'Stopped API instance %s\n' "$hostname"
exit 0
fi