Files
mtusi2026lab7/scripts/topology.ts

458 lines
17 KiB
TypeScript

import { resolve } from "node:path";
import { mkdir, writeFile } from "node:fs/promises";
import { root, local } from "./runtime";
import { openobserveQueryPolicy } from "./openobserve-query-policy";
export const images = {
haproxy:
"haproxy:3.2-alpine@sha256:1c3568e520453cc8ba98c39501febcf450429c80e6cc27ecad779961d76e597c",
postgres:
"postgres:18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336",
broker:
"valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd",
s3: "rustfs/rustfs:1.0.0-rc.1@sha256:f53d700fc16809070326f6ab3fb565d48e722364a7bd5a12e47d0a5abe079ce6",
bao: "openbao/openbao:2.7.2@sha256:a78eb12a9fc7a3ebb4bec1724e2bdd6a28e33394820641d061166919a3f1866b",
collector:
"otel/opentelemetry-collector-k8s:0.163.2@sha256:a65df9129a5b3f99ee4434c5e53aee1965ff0129eda6a2da75ea917165b7fa6b",
openobserve:
"public.ecr.aws/zinclabs/openobserve:v1.1.2@sha256:26618f379c86b253c0d91624c487dd14b71293699845ecbc8457c94dca14819f",
grafana:
"grafana/grafana@sha256:1c88285f1dd391a62232c010e6c224846e6a73bdc583b3365be2eba614b7919e",
};
export const roles = [
"lb",
"api",
"worker",
"api1-gateway",
"api2-gateway",
"postgres",
"broker-gateway",
"s3-gateway",
"collector-gateway",
"collector",
"openobserve-gateway",
"grafana",
"openbao",
"operator",
"forbidden",
];
const base = {
restart: "unless-stopped",
read_only: true,
cap_drop: ["ALL"],
security_opt: ["no-new-privileges:true"],
pids_limit: 128,
cpu_shares: 128,
cpus: 0.5,
mem_limit: "128m",
tmpfs: ["/tmp:size=16777216,mode=1777"],
logging: {
driver: "json-file",
options: { "max-size": "2m", "max-file": "2" },
},
};
const runUser = "${LOCAL_UID:?run bootstrap}:${LOCAL_GID:?run bootstrap}";
const ro = (src: string, dst: string) => ({
type: "bind",
source: src,
target: dst,
read_only: true,
bind: { create_host_path: false },
});
const rw = (src: string, dst: string) => ({
type: "bind",
source: src,
target: dst,
bind: { create_host_path: false },
});
const pki = (role: string) => ro(`./.local/pki/${role}`, "/pki");
function gateway(role: string, configName = role) {
return {
...base,
image: images.haproxy,
pull_policy: "never",
user: runUser,
networks: ["private"],
volumes: [
pki(role),
ro(`./config/${configName}.cfg`, "/usr/local/etc/haproxy/haproxy.cfg"),
],
command: [
"haproxy",
"-W",
"-db",
"-f",
"/usr/local/etc/haproxy/haproxy.cfg",
],
};
}
const global = `global\n maxconn 256\n nbthread 1\n hard-stop-after 2s\n ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets\n ssl-default-server-options ssl-min-ver TLSv1.2\ndefaults\n mode tcp\n timeout connect 2s\n timeout client 30s\n timeout server 30s\nresolvers docker\n nameserver dns 127.0.0.11:53\n resolve_retries 3\n timeout resolve 1s\n timeout retry 1s\n hold valid 1s\n`;
function incoming(
name: string,
port: number,
upstream: number,
allowed: string[],
publicTLS = false,
) {
return `\nlisten ${name}\n bind :${port} ssl crt /pki/server.pem${publicTLS ? "" : " ca-file /pki/ca.crt crl-file /pki/ca.crl verify required"}\n${publicTLS ? "" : ` tcp-request content reject unless { ssl_c_s_dn(CN) -m str ${allowed.join(" ")} }\n`} server local 127.0.0.1:${upstream}\n`;
}
function outgoing(
name: string,
localPort: number,
host: string,
port: number,
alpn = "",
) {
return `\nlisten ${name}\n bind 127.0.0.1:${localPort}\n server remote ${host}:${port} ssl verify required ca-file /pki/ca.crt crt /pki/server.pem crl-file /pki/ca.crl verifyhost ${host} sni str(${host})${alpn ? " alpn " + alpn : ""} resolvers docker init-addr none\n`;
}
export function gatewayConfigs() {
const out: Record<string, string> = {};
for (const role of ["api1-gateway", "api2-gateway", "worker"])
out[role] =
global +
(role === "worker" ? "" : incoming("api", 8443, 8080, ["lb"])) +
outgoing("postgres", 15432, "postgres", 5432, "postgresql") +
outgoing("broker", 16379, "broker-gateway", 6379) +
outgoing("s3", 19000, "s3-gateway", 9000) +
outgoing("otel", 14318, "collector-gateway", 4318);
// API gateways use the api client identity outbound, while their inbound server certificate SAN remains replica-specific.
out["broker-gateway"] =
global + incoming("broker", 6379, 16379, ["api", "worker", "operator"]);
out["s3-gateway"] =
global + incoming("s3", 9000, 19000, ["api", "worker", "operator"]);
out["collector-gateway"] =
global +
incoming("otlp", 4318, 14318, ["api", "worker", "operator"]) +
outgoing("openobserve", 15080, "openobserve-gateway", 5080);
out["openobserve-gateway"] =
global.replace(" maxconn 256", " maxconn 16\n tune.bufsize 131072") +
incoming("ingest", 5080, 15080, ["collector", "operator"])
.replace(" bind :5080", " mode http\n bind :5080")
.replace(
" server local",
` acl collector ssl_c_s_dn(CN) -m str collector\n acl ingest path /api/default/v1/logs /api/default/v1/metrics /api/default/v1/traces\n acl post method POST\n http-request deny status 403 if collector !ingest\n http-request deny status 403 if collector !post\n server local`,
) +
incoming("query", 5082, 15080, ["grafana", "operator"])
.replace(" bind :5082", " mode http\n bind :5082")
.replace(" server local", openobserveQueryPolicy + " server local");
out.grafana =
global +
incoming("grafana", 3443, 3000, [], true) +
outgoing("openobserve", 15080, "openobserve-gateway", 5082);
out.lb =
global +
`\nfrontend public\n mode http\n bind :8443 ssl crt /pki/server.pem\n http-response set-header Strict-Transport-Security max-age=86400\n default_backend apis\nbackend apis\n mode http\n balance roundrobin\n option httpchk\n http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost\n http-check expect status 200\n default-server inter 500ms fall 2 rise 2\n server api1 api1-gateway:8443 check ssl verify required ca-file /pki/ca.crt crt /pki/server.pem crl-file /pki/ca.crl verifyhost api1-gateway sni str(api1-gateway) resolvers docker init-addr none\n server api2 api2-gateway:8443 check ssl verify required ca-file /pki/ca.crt crt /pki/server.pem crl-file /pki/ca.crl verifyhost api2-gateway sni str(api2-gateway) resolvers docker init-addr none\n`;
return out;
}
export function manifest() {
const services: Record<string, any> = {};
for (const name of [
"lb",
"api1-gateway",
"api2-gateway",
"broker-gateway",
"s3-gateway",
"collector-gateway",
"openobserve-gateway",
"grafana",
])
services[name === "grafana" ? "grafana-gateway" : name] = gateway(name);
services["worker-gateway"] = gateway("worker");
services.lb.ports = ["127.0.0.1:${HTTPS_PORT:-18443}:8443"];
services.lb.networks = ["private", "edge"];
services["grafana-gateway"].ports = ["127.0.0.1:${GRAFANA_PORT:-18447}:3443"];
services["grafana-gateway"].networks = ["private", "edge"];
// Replica gateways need server SAN and a separate scoped outbound client identity.
for (const role of ["api1-gateway", "api2-gateway"])
services[role].volumes.push(ro("./.local/pki/api", "/client-pki"));
services["collector-gateway"].volumes.push(
ro("./.local/pki/collector", "/client-pki"),
);
services.postgres = {
...base,
image: images.postgres,
pull_policy: "never",
read_only: true,
cap_drop: ["ALL"],
cap_add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"],
mem_limit: "512m",
cpus: 1,
networks: ["private"],
environment: {
POSTGRES_USER: "operator",
POSTGRES_DB: "messenger",
POSTGRES_PASSWORD_FILE: "/bootstrap/db-password",
},
volumes: [
rw("./.local/data/postgres", "/var/lib/postgresql"),
ro("./.local/pki/postgres", "/source-tls"),
ro("./.local/operator/db-password", "/bootstrap/db-password"),
ro("./config/pg_hba.conf", "/config/pg_hba.conf"),
ro("./scripts/postgres-entrypoint.sh", "/config/entrypoint.sh"),
],
entrypoint: ["/bin/sh", "/config/entrypoint.sh"],
tmpfs: [
"/run/tls:size=1048576,mode=0755",
"/var/run/postgresql:size=1048576,mode=0777",
"/tmp:size=16777216,mode=1777",
],
healthcheck: {
test: ["CMD-SHELL", "pg_isready -U operator -d messenger"],
interval: "2s",
timeout: "2s",
retries: 30,
},
};
services.broker = {
...base,
image: images.broker,
pull_policy: "never",
user: runUser,
network_mode: "service:broker-gateway",
volumes: [
rw("./.local/data/broker", "/data"),
ro("./.local/config/valkey.conf", "/config/valkey.conf"),
ro("./.local/config/users.acl", "/config/users.acl"),
],
command: ["valkey-server", "/config/valkey.conf"],
mem_limit: "256m",
};
services.s3 = {
...base,
image: images.s3,
pull_policy: "never",
user: runUser,
network_mode: "service:s3-gateway",
volumes: [
rw("./.local/data/s3", "/data"),
rw("./.local/data/s3-logs", "/logs"),
ro("./.local/config/s3.env", "/config/s3.env"),
ro("./scripts/s3-entrypoint.sh", "/config/entrypoint.sh"),
],
entrypoint: ["/bin/sh", "/config/entrypoint.sh"],
mem_limit: "512m",
cpus: 1,
};
services.openbao = {
...base,
image: images.bao,
pull_policy: "never",
user: runUser,
networks: ["private"],
ports: ["127.0.0.1:${BAO_PORT:-18445}:8200"],
volumes: [
pki("openbao"),
rw("./.local/data/bao", "/bao/data"),
ro("./infra/openbao/server.hcl", "/bao/config/server.hcl"),
],
environment: {
BAO_ADDR: "https://openbao:8200",
BAO_CACERT: "/pki/ca.crt",
SKIP_SETCAP: "true",
},
entrypoint: ["bao"],
command: ["server", "-config=/bao/config/server.hcl"],
mem_limit: "256m",
healthcheck: {
test: ["CMD", "bao", "status", "-format=json"],
interval: "2s",
timeout: "2s",
retries: 30,
},
};
for (const service of ["api1", "api2", "worker"]) {
const role = service === "worker" ? "worker" : "api",
gate = service === "worker" ? "worker-gateway" : service + "-gateway";
services[service + "-agent"] = {
...base,
image: images.bao,
pull_policy: "never",
user: runUser,
network_mode: `service:${gate}`,
volumes: [
pki(role),
ro(`./infra/openbao/${role}-agent.hcl`, "/config/agent.hcl"),
rw(`./.local/rendered/${service}`, "/run/app"),
],
environment: { SKIP_SETCAP: "true" },
entrypoint: ["bao"],
command: ["agent", "-config=/config/agent.hcl"],
depends_on: { openbao: { condition: "service_healthy" } },
healthcheck: {
test: [
"CMD-SHELL",
"test -s /run/app/database_url && test -s /run/app/cache_url && test -s /run/app/s3.json",
],
interval: "2s",
timeout: "1s",
retries: 30,
},
mem_limit: "96m",
};
services[service] = {
...base,
image: "mtuci-vp7-security-app:local",
pull_policy: "never",
build: { context: ".", dockerfile: "Dockerfile" },
user: runUser,
network_mode: `service:${gate}`,
volumes: [ro(`./.local/rendered/${service}`, "/run/app")],
environment: {
APP_HOST: "127.0.0.1",
APP_PORT: "8080",
APP_ENV: "lab",
DATABASE_URL_FILE: "/run/app/database_url",
CACHE_URL_FILE: "/run/app/cache_url",
S3_CONFIG_FILE: "/run/app/s3.json",
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}",
OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:14318",
OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf",
OTEL_SERVICE_NAME:
service === "worker" ? "messenger-worker" : "messenger-api",
OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=lab",
PG_MAX_CONNS: "4",
WORKER_MAX_ATTEMPTS: "5",
},
depends_on: { [service + "-agent"]: { condition: "service_healthy" } },
command: service === "worker" ? ["/worker"] : ["/app"],
mem_limit: service === "worker" ? "2048m" : "256m",
cpus: service === "worker" ? 1 : 0.5,
stop_grace_period: "15s",
};
}
for (const name of ["api1", "api2", "worker"]) {
services[name].environment.SECRET_FRESHNESS_FILE = "/run/app/freshness";
services[name].environment.SECRET_MAX_AGE_SECONDS = "120";
services[name].environment.GOMEMLIMIT = "1536MiB";
services[name].mem_limit = "2048m";
}
services.worker.environment.BROKER_PUBLISH_URL_FILE =
"/run/app/publisher_url";
for (const name of ["api1", "api2", "worker"])
services[name + "-agent"].healthcheck.test = [
"CMD-SHELL",
"test -s /run/app/database_url && test -s /run/app/cache_url && test -s /run/app/s3.json && test -s /run/app/freshness" +
(name === "worker" ? " && test -s /run/app/publisher_url" : ""),
];
services.collector = {
...base,
image: `${"${"}VP7_COLLECTOR_IMAGE:-${images.collector}}`,
pull_policy: "never",
user: runUser,
network_mode: "service:collector-gateway",
volumes: [
ro("./infra/otel/collector.yaml", "/etc/otelcol/config.yaml"),
rw("./.local/data/collector", "/data"),
ro(
"./.local/config/openobserve_ingest_auth",
"/run/secrets/openobserve_ingest_auth",
),
],
command: ["--config=/etc/otelcol/config.yaml"],
mem_limit: "256m",
};
services.openobserve = {
...base,
image: `${"${"}VP7_OPENOBSERVE_IMAGE:-${images.openobserve}}`,
pull_policy: "never",
user: runUser,
network_mode: "service:openobserve-gateway",
environment: {
ZO_LOCAL_MODE: "true",
ZO_LOCAL_MODE_STORAGE: "disk",
ZO_DATA_DIR: "/data/",
ZO_HTTP_ADDR: "127.0.0.1",
ZO_HTTP_PORT: "15080",
ZO_GRPC_ADDR: "127.0.0.1",
ZO_TELEMETRY: "false",
ZO_MEMORY_CACHE_ENABLED: "false",
ZO_MEM_TABLE_MAX_SIZE: "128",
ZO_MAX_FILE_SIZE_IN_MEMORY: "16",
ZO_MEMORY_CACHE_DATAFUSION_MAX_SIZE: "256",
ZO_QUERY_TIMEOUT: "8",
ZO_HTTP_WORKER_NUM: "2",
ZO_SERVICE_ACCOUNT_ENABLED: "true",
},
env_file: ["./.local/config/openobserve.env"],
volumes: [rw("./.local/data/openobserve", "/data")],
mem_limit: "1024m",
cpus: 1,
};
services.grafana = {
...base,
image: "${VP7_GRAFANA_IMAGE:-" + images.grafana + "}",
pull_policy: "never",
user: runUser,
network_mode: "service:grafana-gateway",
environment: {
GF_SERVER_HTTP_ADDR: "127.0.0.1",
GF_SERVER_HTTP_PORT: "3000",
GF_AUTH_ANONYMOUS_ENABLED: "false",
GF_USERS_ALLOW_SIGN_UP: "false",
GF_SECURITY_ADMIN_USER: "operator",
GF_SECURITY_ADMIN_PASSWORD__FILE: "/run/admin/password",
GF_SECURITY_SECRET_KEY__FILE: "/run/admin/secret-key",
GF_SECURITY_COOKIE_SECURE: "true",
GF_ANALYTICS_REPORTING_ENABLED: "false",
GF_ANALYTICS_CHECK_FOR_UPDATES: "false",
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false",
GF_LOG_MODE: "console",
GF_LOG_LEVEL: "warn",
GF_DATAPROXY_TIMEOUT: "10",
GF_DATAPROXY_RESPONSE_LIMIT: "8388608",
GF_DATAPROXY_MAX_CONNS_PER_HOST: "8",
GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "openobserve",
GF_PLUGINS_PREINSTALL_DISABLED: "true",
GF_PLUGINS_PUBLIC_KEY_RETRIEVAL_DISABLED: "true",
},
env_file: ["./.local/config/grafana-query.env"],
volumes: [
rw("./.local/data/grafana", "/var/lib/grafana"),
ro("./.local/operator/grafana-password", "/run/admin/password"),
ro("./.local/operator/grafana-secret-key", "/run/admin/secret-key"),
ro("./infra/grafana/provisioning", "/etc/grafana/provisioning"),
ro("./infra/grafana/dashboards", "/var/lib/grafana/dashboards"),
ro("./.local/plugins", "/var/lib/grafana/plugins"),
],
mem_limit: "256m",
};
services.migrate = {
...base,
image: "mtuci-vp7-security-app:local",
pull_policy: "never",
user: runUser,
profiles: ["tools"],
networks: ["private"],
volumes: [
pki("operator"),
ro("./.local/operator/migration-url", "/run/operator/database_url"),
],
environment: { DATABASE_URL_FILE: "/run/operator/database_url" },
command: ["/app", "-migrate"],
restart: "no",
mem_limit: "256m",
};
services.openbao.networks = ["private", "edge"];
return {
name: "mtuci-vp7-security",
services,
networks: { private: { internal: true }, edge: { internal: false } },
};
}
export async function writeTopology() {
await mkdir(resolve(root, "config"), { recursive: true });
for (const [name, config] of Object.entries(gatewayConfigs())) {
let value = config;
if (["api1-gateway", "api2-gateway", "collector-gateway"].includes(name))
value = value.replaceAll(
"crt /pki/server.pem crl-file",
"crt /client-pki/server.pem crl-file",
);
await writeFile(resolve(root, "config", name + ".cfg"), value);
}
await writeFile(
resolve(root, "compose.yaml"),
JSON.stringify(manifest(), null, 2) + "\n",
);
}
if (import.meta.main) await writeTopology();