Files
mtusi2026lab7/scripts/init_env.py
T

189 lines
7.9 KiB
Python

"""Generate persistent local credentials and rendered configs once per project."""
import base64
import hashlib
import json
import secrets
import subprocess
from pathlib import Path
ROOT = Path.cwd()
PRIVATE = ROOT / "secrets"
BUCKET = "messenger-attachments"
def write(name, text):
path = PRIVATE / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8", newline="\n")
def encode(name, value):
write(name, json.dumps(value, indent=2) + "\n")
def get_or_create(name):
path = PRIVATE / name
if not path.exists():
write(name, secrets.token_hex(24))
return path.read_text(encoding="utf-8").strip()
def rabbit_user(name, password):
salt = secrets.token_bytes(4)
digest = salt + hashlib.sha256(salt + password.encode()).digest()
return {
"name": name, "password_hash": base64.b64encode(digest).decode(),
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": [],
}
def switch(service, port, routes):
header = """global
ssl-default-bind-options ssl-min-ver TLSv1.2
maxconn 64
defaults
mode http
timeout connect 2s
timeout client 10s
timeout server 10s
"""
userlist = ""
frontend = f"frontend secure\n bind :{port} ssl crt /run/tls/server.pem\n"
backends = ""
for role, username, password, method, paths, local_port in routes:
userlist += f"userlist {role}\n user {username} insecure-password {password}\n"
frontend += (
f" acl {role}_method method {method}\n"
f" acl {role}_path {paths}\n"
f" use_backend {role} if {role}_method {role}_path {{ http_auth({role}) }}\n"
)
backends += (
f"backend {role}\n http-request del-header Authorization\n"
f" server local 127.0.0.1:{local_port}\n"
)
backends += "backend rejected\n http-request deny deny_status 403\n"
write(f"generated/{service}.cfg", header + userlist + frontend + " default_backend rejected\n" + backends)
def telemetry(api, worker, collector, grafana):
switch("otel-collector", 4318, [
(role, role, value, "POST", "path /v1/metrics /v1/logs /v1/traces", 14318)
for role, value in (("api", api), ("worker", worker))
])
backends = [
("prometheus", 9090, "/api/v1/write", 19090, 19090,
"/api/v1/query /api/v1/label /api/v1/series /api/v1/metadata /-/ready"),
("loki", 3100, "/otlp/v1/logs", 13100, 13100,
"/loki/api/v1/query /loki/api/v1/label /loki/api/v1/series /loki/api/v1/index/stats /ready"),
("tempo", 3200, "/v1/traces", 14318, 13200,
"/api/traces /api/v2/traces /api/search /api/echo /api/metrics /ready"),
]
ca = (PRIVATE / "ca/server.crt").read_text(encoding="ascii")
sources = []
for service, port, ingest_path, ingest_port, query_port, query_paths in backends:
switch(service, port, [
("ingest", "collector", collector, "POST", f"path {ingest_path}", ingest_port),
("query", "grafana", grafana, "GET POST", f"path_beg {query_paths}", query_port),
])
source = {
"name": service.title(), "uid": f"lab6-{service}", "type": service, "access": "proxy",
"url": f"https://{service}:{port}", "isDefault": service == "prometheus", "editable": False,
"basicAuth": True, "basicAuthUser": "grafana",
"jsonData": {"tlsAuthWithCACert": True},
"secureJsonData": {"basicAuthPassword": grafana, "tlsCACert": ca},
}
if service == "prometheus":
source["jsonData"]["timeInterval"] = "5s"
if service == "loki":
source["jsonData"]["derivedFields"] = [{
"name": "Trace", "matcherType": "label", "matcherRegex": "trace_id",
"datasourceUid": "lab6-tempo", "url": "$${__value.raw}",
}]
sources.append(source)
encode("generated/datasources.json", {"apiVersion": 1, "datasources": sources})
basic = base64.b64encode(f"collector:{collector}".encode()).decode()
base = {"tls": {"ca_file": "/run/ca/server.crt"}, "headers": {"Authorization": f"Basic {basic}"}}
exporters = {
"prometheusremotewrite": {**base, "endpoint": "https://prometheus:9090/api/v1/write",
"resource_to_telemetry_conversion": {"enabled": True}},
"otlphttp/loki": {**base, "endpoint": "https://loki:3100/otlp"},
"otlphttp/tempo": {**base, "endpoint": "https://tempo:3200"},
}
encode("generated/collector.json", {
"receivers": {"otlp": {"protocols": {"http": {"endpoint": "127.0.0.1:14318"}}}},
"processors": {"memory_limiter": {"check_interval": "1s", "limit_mib": 128, "spike_limit_mib": 32},
"batch": {"timeout": "1s", "send_batch_size": 256, "send_batch_max_size": 512}},
"exporters": exporters,
"service": {"telemetry": {"metrics": {"level": "none"}}, "pipelines": {
signal: {"receivers": ["otlp"], "processors": ["memory_limiter", "batch"], "exporters": [exporter]}
for signal, exporter in zip(("metrics", "logs", "traces"), exporters)
}},
})
def policy(grants):
return {"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": [f"s3:{op}" for op in actions],
"Resource": [f"arn:aws:s3:::{BUCKET}/{prefix}" for prefix in prefixes]}
for actions, prefixes in grants
]}
def main():
if not (PRIVATE / "ca/public.crt").exists():
subprocess.run([
"docker", "run", "--rm", "--network", "none",
"-v", f"{PRIVATE}:/private", "-v", f"{ROOT / 'scripts/pki.sh'}:/pki.sh:ro",
"--entrypoint", "sh", "postgres:16.8-bookworm", "/pki.sh",
], check=True)
if not (ROOT / ".env").exists():
(ROOT / ".env").write_bytes((ROOT / ".env.example").read_bytes())
get_or_create("postgres/pg_admin")
get_or_create("grafana/password")
write("s3-admin/access", "lab7-bootstrap")
get_or_create("s3-admin/password")
api_otel, worker_otel, collector, grafana = (
get_or_create(f"telemetry/{role}") for role in ("api", "worker", "collector", "grafana")
)
telemetry(api_otel, worker_otel, collector, grafana)
passwords = {}
for role in ("api", "worker", "migrate"):
pg = get_or_create(f"postgres/pg_{role}")
dsn = f"postgresql://lab_{role}:{pg}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/ca/server.crt"
if role == "migrate":
write("postgres/pg_url", dsn)
continue
amqp = passwords[f"lab_{role}"] = get_or_create(f"broker/{role}")
s3_password = get_or_create(f"s3/{role}")
encode(f"bao-seed/{role}.json", {
"DATABASE_URL": dsn, "BROKER_URL": f"amqps://lab_{role}:{amqp}@broker:5671/jobs",
"S3_ACCESS_KEY_ID": f"lab7-{role}", "S3_SECRET_ACCESS_KEY": s3_password,
"OTEL_LOGIN": f"{role}:{api_otel if role == 'api' else worker_otel}",
})
encode("generated/definitions.json", {
"users": [rabbit_user(name, password) for name, password in passwords.items()],
"vhosts": [{"name": "jobs"}],
"permissions": [
{"user": name, "vhost": "jobs", "configure": "^$", "write": "^amq\\.default$",
"read": "^attachment\\.process\\.v1$" if name == "lab_worker" else "^$"}
for name in passwords
],
"queues": [
{"name": "attachment.process.v1", "vhost": "jobs", "durable": True,
"auto_delete": False, "arguments": {"x-queue-type": "classic"}},
],
})
encode("s3/api.json", policy([
(["PutObject"], ["staging/*"]),
(["GetObject"], ["health/ready", "originals/*", "variants/*"]),
]))
encode("s3/worker.json", policy([
(["GetObject", "DeleteObject"], ["staging/*"]),
(["PutObject"], ["originals/*", "variants/*"]),
]))
print("Local PKI and scoped service configuration ready (values withheld).")
if __name__ == "__main__":
main()