forked from mtusi2026/mtusi2026lab7
189 lines
7.9 KiB
Python
189 lines
7.9 KiB
Python
"""Generate persistent local credentials and rendered configs once per project."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import secrets
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
ROOT = Path.cwd()
|
|
PRIVATE = ROOT / "secrets"
|
|
BUCKET = "messenger-attachments"
|
|
|
|
|
|
def write(name, text):
|
|
path = PRIVATE / name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text, encoding="utf-8", newline="\n")
|
|
|
|
|
|
def encode(name, value):
|
|
write(name, json.dumps(value, indent=2) + "\n")
|
|
|
|
|
|
def get_or_create(name):
|
|
path = PRIVATE / name
|
|
if not path.exists():
|
|
write(name, secrets.token_hex(24))
|
|
return path.read_text(encoding="utf-8").strip()
|
|
|
|
|
|
def rabbit_user(name, password):
|
|
salt = secrets.token_bytes(4)
|
|
digest = salt + hashlib.sha256(salt + password.encode()).digest()
|
|
return {
|
|
"name": name, "password_hash": base64.b64encode(digest).decode(),
|
|
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": [],
|
|
}
|
|
|
|
|
|
def switch(service, port, routes):
|
|
header = """global
|
|
ssl-default-bind-options ssl-min-ver TLSv1.2
|
|
maxconn 64
|
|
defaults
|
|
mode http
|
|
timeout connect 2s
|
|
timeout client 10s
|
|
timeout server 10s
|
|
"""
|
|
userlist = ""
|
|
frontend = f"frontend secure\n bind :{port} ssl crt /run/tls/server.pem\n"
|
|
backends = ""
|
|
for role, username, password, method, paths, local_port in routes:
|
|
userlist += f"userlist {role}\n user {username} insecure-password {password}\n"
|
|
frontend += (
|
|
f" acl {role}_method method {method}\n"
|
|
f" acl {role}_path {paths}\n"
|
|
f" use_backend {role} if {role}_method {role}_path {{ http_auth({role}) }}\n"
|
|
)
|
|
backends += (
|
|
f"backend {role}\n http-request del-header Authorization\n"
|
|
f" server local 127.0.0.1:{local_port}\n"
|
|
)
|
|
backends += "backend rejected\n http-request deny deny_status 403\n"
|
|
write(f"generated/{service}.cfg", header + userlist + frontend + " default_backend rejected\n" + backends)
|
|
|
|
|
|
def telemetry(api, worker, collector, grafana):
|
|
switch("otel-collector", 4318, [
|
|
(role, role, value, "POST", "path /v1/metrics /v1/logs /v1/traces", 14318)
|
|
for role, value in (("api", api), ("worker", worker))
|
|
])
|
|
backends = [
|
|
("prometheus", 9090, "/api/v1/write", 19090, 19090,
|
|
"/api/v1/query /api/v1/label /api/v1/series /api/v1/metadata /-/ready"),
|
|
("loki", 3100, "/otlp/v1/logs", 13100, 13100,
|
|
"/loki/api/v1/query /loki/api/v1/label /loki/api/v1/series /loki/api/v1/index/stats /ready"),
|
|
("tempo", 3200, "/v1/traces", 14318, 13200,
|
|
"/api/traces /api/v2/traces /api/search /api/echo /api/metrics /ready"),
|
|
]
|
|
ca = (PRIVATE / "ca/server.crt").read_text(encoding="ascii")
|
|
sources = []
|
|
for service, port, ingest_path, ingest_port, query_port, query_paths in backends:
|
|
switch(service, port, [
|
|
("ingest", "collector", collector, "POST", f"path {ingest_path}", ingest_port),
|
|
("query", "grafana", grafana, "GET POST", f"path_beg {query_paths}", query_port),
|
|
])
|
|
source = {
|
|
"name": service.title(), "uid": f"lab6-{service}", "type": service, "access": "proxy",
|
|
"url": f"https://{service}:{port}", "isDefault": service == "prometheus", "editable": False,
|
|
"basicAuth": True, "basicAuthUser": "grafana",
|
|
"jsonData": {"tlsAuthWithCACert": True},
|
|
"secureJsonData": {"basicAuthPassword": grafana, "tlsCACert": ca},
|
|
}
|
|
if service == "prometheus":
|
|
source["jsonData"]["timeInterval"] = "5s"
|
|
if service == "loki":
|
|
source["jsonData"]["derivedFields"] = [{
|
|
"name": "Trace", "matcherType": "label", "matcherRegex": "trace_id",
|
|
"datasourceUid": "lab6-tempo", "url": "$${__value.raw}",
|
|
}]
|
|
sources.append(source)
|
|
encode("generated/datasources.json", {"apiVersion": 1, "datasources": sources})
|
|
basic = base64.b64encode(f"collector:{collector}".encode()).decode()
|
|
base = {"tls": {"ca_file": "/run/ca/server.crt"}, "headers": {"Authorization": f"Basic {basic}"}}
|
|
exporters = {
|
|
"prometheusremotewrite": {**base, "endpoint": "https://prometheus:9090/api/v1/write",
|
|
"resource_to_telemetry_conversion": {"enabled": True}},
|
|
"otlphttp/loki": {**base, "endpoint": "https://loki:3100/otlp"},
|
|
"otlphttp/tempo": {**base, "endpoint": "https://tempo:3200"},
|
|
}
|
|
encode("generated/collector.json", {
|
|
"receivers": {"otlp": {"protocols": {"http": {"endpoint": "127.0.0.1:14318"}}}},
|
|
"processors": {"memory_limiter": {"check_interval": "1s", "limit_mib": 128, "spike_limit_mib": 32},
|
|
"batch": {"timeout": "1s", "send_batch_size": 256, "send_batch_max_size": 512}},
|
|
"exporters": exporters,
|
|
"service": {"telemetry": {"metrics": {"level": "none"}}, "pipelines": {
|
|
signal: {"receivers": ["otlp"], "processors": ["memory_limiter", "batch"], "exporters": [exporter]}
|
|
for signal, exporter in zip(("metrics", "logs", "traces"), exporters)
|
|
}},
|
|
})
|
|
|
|
|
|
def policy(grants):
|
|
return {"Version": "2012-10-17", "Statement": [
|
|
{"Effect": "Allow", "Action": [f"s3:{op}" for op in actions],
|
|
"Resource": [f"arn:aws:s3:::{BUCKET}/{prefix}" for prefix in prefixes]}
|
|
for actions, prefixes in grants
|
|
]}
|
|
|
|
|
|
def main():
|
|
if not (PRIVATE / "ca/public.crt").exists():
|
|
subprocess.run([
|
|
"docker", "run", "--rm", "--network", "none",
|
|
"-v", f"{PRIVATE}:/private", "-v", f"{ROOT / 'scripts/pki.sh'}:/pki.sh:ro",
|
|
"--entrypoint", "sh", "postgres:16.8-bookworm", "/pki.sh",
|
|
], check=True)
|
|
if not (ROOT / ".env").exists():
|
|
(ROOT / ".env").write_bytes((ROOT / ".env.example").read_bytes())
|
|
get_or_create("postgres/pg_admin")
|
|
get_or_create("grafana/password")
|
|
write("s3-admin/access", "lab7-bootstrap")
|
|
get_or_create("s3-admin/password")
|
|
api_otel, worker_otel, collector, grafana = (
|
|
get_or_create(f"telemetry/{role}") for role in ("api", "worker", "collector", "grafana")
|
|
)
|
|
telemetry(api_otel, worker_otel, collector, grafana)
|
|
passwords = {}
|
|
for role in ("api", "worker", "migrate"):
|
|
pg = get_or_create(f"postgres/pg_{role}")
|
|
dsn = f"postgresql://lab_{role}:{pg}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/ca/server.crt"
|
|
if role == "migrate":
|
|
write("postgres/pg_url", dsn)
|
|
continue
|
|
amqp = passwords[f"lab_{role}"] = get_or_create(f"broker/{role}")
|
|
s3_password = get_or_create(f"s3/{role}")
|
|
encode(f"bao-seed/{role}.json", {
|
|
"DATABASE_URL": dsn, "BROKER_URL": f"amqps://lab_{role}:{amqp}@broker:5671/jobs",
|
|
"S3_ACCESS_KEY_ID": f"lab7-{role}", "S3_SECRET_ACCESS_KEY": s3_password,
|
|
"OTEL_LOGIN": f"{role}:{api_otel if role == 'api' else worker_otel}",
|
|
})
|
|
encode("generated/definitions.json", {
|
|
"users": [rabbit_user(name, password) for name, password in passwords.items()],
|
|
"vhosts": [{"name": "jobs"}],
|
|
"permissions": [
|
|
{"user": name, "vhost": "jobs", "configure": "^$", "write": "^amq\\.default$",
|
|
"read": "^attachment\\.process\\.v1$" if name == "lab_worker" else "^$"}
|
|
for name in passwords
|
|
],
|
|
"queues": [
|
|
{"name": "attachment.process.v1", "vhost": "jobs", "durable": True,
|
|
"auto_delete": False, "arguments": {"x-queue-type": "classic"}},
|
|
],
|
|
})
|
|
encode("s3/api.json", policy([
|
|
(["PutObject"], ["staging/*"]),
|
|
(["GetObject"], ["health/ready", "originals/*", "variants/*"]),
|
|
]))
|
|
encode("s3/worker.json", policy([
|
|
(["GetObject", "DeleteObject"], ["staging/*"]),
|
|
(["PutObject"], ["originals/*", "variants/*"]),
|
|
]))
|
|
print("Local PKI and scoped service configuration ready (values withheld).")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|