forked from mtusi2026/mtusi2026lab7
458 lines
17 KiB
TypeScript
458 lines
17 KiB
TypeScript
import { resolve } from "node:path";
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { root, local } from "./runtime";
|
|
import { openobserveQueryPolicy } from "./openobserve-query-policy";
|
|
export const images = {
|
|
haproxy:
|
|
"haproxy:3.2-alpine@sha256:1c3568e520453cc8ba98c39501febcf450429c80e6cc27ecad779961d76e597c",
|
|
postgres:
|
|
"postgres:18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336",
|
|
broker:
|
|
"valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd",
|
|
s3: "rustfs/rustfs:1.0.0-rc.1@sha256:f53d700fc16809070326f6ab3fb565d48e722364a7bd5a12e47d0a5abe079ce6",
|
|
bao: "openbao/openbao:2.7.2@sha256:a78eb12a9fc7a3ebb4bec1724e2bdd6a28e33394820641d061166919a3f1866b",
|
|
collector:
|
|
"otel/opentelemetry-collector-k8s:0.163.2@sha256:a65df9129a5b3f99ee4434c5e53aee1965ff0129eda6a2da75ea917165b7fa6b",
|
|
openobserve:
|
|
"public.ecr.aws/zinclabs/openobserve:v1.1.2@sha256:26618f379c86b253c0d91624c487dd14b71293699845ecbc8457c94dca14819f",
|
|
grafana:
|
|
"grafana/grafana@sha256:1c88285f1dd391a62232c010e6c224846e6a73bdc583b3365be2eba614b7919e",
|
|
};
|
|
export const roles = [
|
|
"lb",
|
|
"api",
|
|
"worker",
|
|
"api1-gateway",
|
|
"api2-gateway",
|
|
"postgres",
|
|
"broker-gateway",
|
|
"s3-gateway",
|
|
"collector-gateway",
|
|
"collector",
|
|
"openobserve-gateway",
|
|
"grafana",
|
|
"openbao",
|
|
"operator",
|
|
"forbidden",
|
|
];
|
|
const base = {
|
|
restart: "unless-stopped",
|
|
read_only: true,
|
|
cap_drop: ["ALL"],
|
|
security_opt: ["no-new-privileges:true"],
|
|
pids_limit: 128,
|
|
cpu_shares: 128,
|
|
cpus: 0.5,
|
|
mem_limit: "128m",
|
|
tmpfs: ["/tmp:size=16777216,mode=1777"],
|
|
logging: {
|
|
driver: "json-file",
|
|
options: { "max-size": "2m", "max-file": "2" },
|
|
},
|
|
};
|
|
const runUser = "${LOCAL_UID:?run bootstrap}:${LOCAL_GID:?run bootstrap}";
|
|
const ro = (src: string, dst: string) => ({
|
|
type: "bind",
|
|
source: src,
|
|
target: dst,
|
|
read_only: true,
|
|
bind: { create_host_path: false },
|
|
});
|
|
const rw = (src: string, dst: string) => ({
|
|
type: "bind",
|
|
source: src,
|
|
target: dst,
|
|
bind: { create_host_path: false },
|
|
});
|
|
const pki = (role: string) => ro(`./.local/pki/${role}`, "/pki");
|
|
function gateway(role: string, configName = role) {
|
|
return {
|
|
...base,
|
|
image: images.haproxy,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
networks: ["private"],
|
|
volumes: [
|
|
pki(role),
|
|
ro(`./config/${configName}.cfg`, "/usr/local/etc/haproxy/haproxy.cfg"),
|
|
],
|
|
command: [
|
|
"haproxy",
|
|
"-W",
|
|
"-db",
|
|
"-f",
|
|
"/usr/local/etc/haproxy/haproxy.cfg",
|
|
],
|
|
};
|
|
}
|
|
const global = `global\n maxconn 256\n nbthread 1\n hard-stop-after 2s\n ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets\n ssl-default-server-options ssl-min-ver TLSv1.2\ndefaults\n mode tcp\n timeout connect 2s\n timeout client 30s\n timeout server 30s\nresolvers docker\n nameserver dns 127.0.0.11:53\n resolve_retries 3\n timeout resolve 1s\n timeout retry 1s\n hold valid 1s\n`;
|
|
function incoming(
|
|
name: string,
|
|
port: number,
|
|
upstream: number,
|
|
allowed: string[],
|
|
publicTLS = false,
|
|
) {
|
|
return `\nlisten ${name}\n bind :${port} ssl crt /pki/server.pem${publicTLS ? "" : " ca-file /pki/ca.crt crl-file /pki/ca.crl verify required"}\n${publicTLS ? "" : ` tcp-request content reject unless { ssl_c_s_dn(CN) -m str ${allowed.join(" ")} }\n`} server local 127.0.0.1:${upstream}\n`;
|
|
}
|
|
function outgoing(
|
|
name: string,
|
|
localPort: number,
|
|
host: string,
|
|
port: number,
|
|
alpn = "",
|
|
) {
|
|
return `\nlisten ${name}\n bind 127.0.0.1:${localPort}\n server remote ${host}:${port} ssl verify required ca-file /pki/ca.crt crt /pki/server.pem crl-file /pki/ca.crl verifyhost ${host} sni str(${host})${alpn ? " alpn " + alpn : ""} resolvers docker init-addr none\n`;
|
|
}
|
|
export function gatewayConfigs() {
|
|
const out: Record<string, string> = {};
|
|
for (const role of ["api1-gateway", "api2-gateway", "worker"])
|
|
out[role] =
|
|
global +
|
|
(role === "worker" ? "" : incoming("api", 8443, 8080, ["lb"])) +
|
|
outgoing("postgres", 15432, "postgres", 5432, "postgresql") +
|
|
outgoing("broker", 16379, "broker-gateway", 6379) +
|
|
outgoing("s3", 19000, "s3-gateway", 9000) +
|
|
outgoing("otel", 14318, "collector-gateway", 4318);
|
|
// API gateways use the api client identity outbound, while their inbound server certificate SAN remains replica-specific.
|
|
out["broker-gateway"] =
|
|
global + incoming("broker", 6379, 16379, ["api", "worker", "operator"]);
|
|
out["s3-gateway"] =
|
|
global + incoming("s3", 9000, 19000, ["api", "worker", "operator"]);
|
|
out["collector-gateway"] =
|
|
global +
|
|
incoming("otlp", 4318, 14318, ["api", "worker", "operator"]) +
|
|
outgoing("openobserve", 15080, "openobserve-gateway", 5080);
|
|
out["openobserve-gateway"] =
|
|
global.replace(" maxconn 256", " maxconn 16\n tune.bufsize 131072") +
|
|
incoming("ingest", 5080, 15080, ["collector", "operator"])
|
|
.replace(" bind :5080", " mode http\n bind :5080")
|
|
.replace(
|
|
" server local",
|
|
` acl collector ssl_c_s_dn(CN) -m str collector\n acl ingest path /api/default/v1/logs /api/default/v1/metrics /api/default/v1/traces\n acl post method POST\n http-request deny status 403 if collector !ingest\n http-request deny status 403 if collector !post\n server local`,
|
|
) +
|
|
incoming("query", 5082, 15080, ["grafana", "operator"])
|
|
.replace(" bind :5082", " mode http\n bind :5082")
|
|
.replace(" server local", openobserveQueryPolicy + " server local");
|
|
out.grafana =
|
|
global +
|
|
incoming("grafana", 3443, 3000, [], true) +
|
|
outgoing("openobserve", 15080, "openobserve-gateway", 5082);
|
|
out.lb =
|
|
global +
|
|
`\nfrontend public\n mode http\n bind :8443 ssl crt /pki/server.pem\n http-response set-header Strict-Transport-Security max-age=86400\n default_backend apis\nbackend apis\n mode http\n balance roundrobin\n option httpchk\n http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost\n http-check expect status 200\n default-server inter 500ms fall 2 rise 2\n server api1 api1-gateway:8443 check ssl verify required ca-file /pki/ca.crt crt /pki/server.pem crl-file /pki/ca.crl verifyhost api1-gateway sni str(api1-gateway) resolvers docker init-addr none\n server api2 api2-gateway:8443 check ssl verify required ca-file /pki/ca.crt crt /pki/server.pem crl-file /pki/ca.crl verifyhost api2-gateway sni str(api2-gateway) resolvers docker init-addr none\n`;
|
|
return out;
|
|
}
|
|
export function manifest() {
|
|
const services: Record<string, any> = {};
|
|
for (const name of [
|
|
"lb",
|
|
"api1-gateway",
|
|
"api2-gateway",
|
|
"broker-gateway",
|
|
"s3-gateway",
|
|
"collector-gateway",
|
|
"openobserve-gateway",
|
|
"grafana",
|
|
])
|
|
services[name === "grafana" ? "grafana-gateway" : name] = gateway(name);
|
|
services["worker-gateway"] = gateway("worker");
|
|
services.lb.ports = ["127.0.0.1:${HTTPS_PORT:-18443}:8443"];
|
|
services.lb.networks = ["private", "edge"];
|
|
services["grafana-gateway"].ports = ["127.0.0.1:${GRAFANA_PORT:-18447}:3443"];
|
|
services["grafana-gateway"].networks = ["private", "edge"];
|
|
// Replica gateways need server SAN and a separate scoped outbound client identity.
|
|
for (const role of ["api1-gateway", "api2-gateway"])
|
|
services[role].volumes.push(ro("./.local/pki/api", "/client-pki"));
|
|
services["collector-gateway"].volumes.push(
|
|
ro("./.local/pki/collector", "/client-pki"),
|
|
);
|
|
services.postgres = {
|
|
...base,
|
|
image: images.postgres,
|
|
pull_policy: "never",
|
|
read_only: true,
|
|
cap_drop: ["ALL"],
|
|
cap_add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"],
|
|
mem_limit: "512m",
|
|
cpus: 1,
|
|
networks: ["private"],
|
|
environment: {
|
|
POSTGRES_USER: "operator",
|
|
POSTGRES_DB: "messenger",
|
|
POSTGRES_PASSWORD_FILE: "/bootstrap/db-password",
|
|
},
|
|
volumes: [
|
|
rw("./.local/data/postgres", "/var/lib/postgresql"),
|
|
ro("./.local/pki/postgres", "/source-tls"),
|
|
ro("./.local/operator/db-password", "/bootstrap/db-password"),
|
|
ro("./config/pg_hba.conf", "/config/pg_hba.conf"),
|
|
ro("./scripts/postgres-entrypoint.sh", "/config/entrypoint.sh"),
|
|
],
|
|
entrypoint: ["/bin/sh", "/config/entrypoint.sh"],
|
|
tmpfs: [
|
|
"/run/tls:size=1048576,mode=0755",
|
|
"/var/run/postgresql:size=1048576,mode=0777",
|
|
"/tmp:size=16777216,mode=1777",
|
|
],
|
|
healthcheck: {
|
|
test: ["CMD-SHELL", "pg_isready -U operator -d messenger"],
|
|
interval: "2s",
|
|
timeout: "2s",
|
|
retries: 30,
|
|
},
|
|
};
|
|
services.broker = {
|
|
...base,
|
|
image: images.broker,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
network_mode: "service:broker-gateway",
|
|
volumes: [
|
|
rw("./.local/data/broker", "/data"),
|
|
ro("./.local/config/valkey.conf", "/config/valkey.conf"),
|
|
ro("./.local/config/users.acl", "/config/users.acl"),
|
|
],
|
|
command: ["valkey-server", "/config/valkey.conf"],
|
|
mem_limit: "256m",
|
|
};
|
|
services.s3 = {
|
|
...base,
|
|
image: images.s3,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
network_mode: "service:s3-gateway",
|
|
volumes: [
|
|
rw("./.local/data/s3", "/data"),
|
|
rw("./.local/data/s3-logs", "/logs"),
|
|
ro("./.local/config/s3.env", "/config/s3.env"),
|
|
ro("./scripts/s3-entrypoint.sh", "/config/entrypoint.sh"),
|
|
],
|
|
entrypoint: ["/bin/sh", "/config/entrypoint.sh"],
|
|
mem_limit: "512m",
|
|
cpus: 1,
|
|
};
|
|
services.openbao = {
|
|
...base,
|
|
image: images.bao,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
networks: ["private"],
|
|
ports: ["127.0.0.1:${BAO_PORT:-18445}:8200"],
|
|
volumes: [
|
|
pki("openbao"),
|
|
rw("./.local/data/bao", "/bao/data"),
|
|
ro("./infra/openbao/server.hcl", "/bao/config/server.hcl"),
|
|
],
|
|
environment: {
|
|
BAO_ADDR: "https://openbao:8200",
|
|
BAO_CACERT: "/pki/ca.crt",
|
|
SKIP_SETCAP: "true",
|
|
},
|
|
entrypoint: ["bao"],
|
|
command: ["server", "-config=/bao/config/server.hcl"],
|
|
mem_limit: "256m",
|
|
healthcheck: {
|
|
test: ["CMD", "bao", "status", "-format=json"],
|
|
interval: "2s",
|
|
timeout: "2s",
|
|
retries: 30,
|
|
},
|
|
};
|
|
for (const service of ["api1", "api2", "worker"]) {
|
|
const role = service === "worker" ? "worker" : "api",
|
|
gate = service === "worker" ? "worker-gateway" : service + "-gateway";
|
|
services[service + "-agent"] = {
|
|
...base,
|
|
image: images.bao,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
network_mode: `service:${gate}`,
|
|
volumes: [
|
|
pki(role),
|
|
ro(`./infra/openbao/${role}-agent.hcl`, "/config/agent.hcl"),
|
|
rw(`./.local/rendered/${service}`, "/run/app"),
|
|
],
|
|
environment: { SKIP_SETCAP: "true" },
|
|
entrypoint: ["bao"],
|
|
command: ["agent", "-config=/config/agent.hcl"],
|
|
depends_on: { openbao: { condition: "service_healthy" } },
|
|
healthcheck: {
|
|
test: [
|
|
"CMD-SHELL",
|
|
"test -s /run/app/database_url && test -s /run/app/cache_url && test -s /run/app/s3.json",
|
|
],
|
|
interval: "2s",
|
|
timeout: "1s",
|
|
retries: 30,
|
|
},
|
|
mem_limit: "96m",
|
|
};
|
|
services[service] = {
|
|
...base,
|
|
image: "mtuci-vp7-security-app:local",
|
|
pull_policy: "never",
|
|
build: { context: ".", dockerfile: "Dockerfile" },
|
|
user: runUser,
|
|
network_mode: `service:${gate}`,
|
|
volumes: [ro(`./.local/rendered/${service}`, "/run/app")],
|
|
environment: {
|
|
APP_HOST: "127.0.0.1",
|
|
APP_PORT: "8080",
|
|
APP_ENV: "lab",
|
|
DATABASE_URL_FILE: "/run/app/database_url",
|
|
CACHE_URL_FILE: "/run/app/cache_url",
|
|
S3_CONFIG_FILE: "/run/app/s3.json",
|
|
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}",
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:14318",
|
|
OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf",
|
|
OTEL_SERVICE_NAME:
|
|
service === "worker" ? "messenger-worker" : "messenger-api",
|
|
OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=lab",
|
|
PG_MAX_CONNS: "4",
|
|
WORKER_MAX_ATTEMPTS: "5",
|
|
},
|
|
depends_on: { [service + "-agent"]: { condition: "service_healthy" } },
|
|
command: service === "worker" ? ["/worker"] : ["/app"],
|
|
mem_limit: service === "worker" ? "2048m" : "256m",
|
|
cpus: service === "worker" ? 1 : 0.5,
|
|
stop_grace_period: "15s",
|
|
};
|
|
}
|
|
for (const name of ["api1", "api2", "worker"]) {
|
|
services[name].environment.SECRET_FRESHNESS_FILE = "/run/app/freshness";
|
|
services[name].environment.SECRET_MAX_AGE_SECONDS = "120";
|
|
services[name].environment.GOMEMLIMIT = "1536MiB";
|
|
services[name].mem_limit = "2048m";
|
|
}
|
|
services.worker.environment.BROKER_PUBLISH_URL_FILE =
|
|
"/run/app/publisher_url";
|
|
for (const name of ["api1", "api2", "worker"])
|
|
services[name + "-agent"].healthcheck.test = [
|
|
"CMD-SHELL",
|
|
"test -s /run/app/database_url && test -s /run/app/cache_url && test -s /run/app/s3.json && test -s /run/app/freshness" +
|
|
(name === "worker" ? " && test -s /run/app/publisher_url" : ""),
|
|
];
|
|
services.collector = {
|
|
...base,
|
|
image: `${"${"}VP7_COLLECTOR_IMAGE:-${images.collector}}`,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
network_mode: "service:collector-gateway",
|
|
volumes: [
|
|
ro("./infra/otel/collector.yaml", "/etc/otelcol/config.yaml"),
|
|
rw("./.local/data/collector", "/data"),
|
|
ro(
|
|
"./.local/config/openobserve_ingest_auth",
|
|
"/run/secrets/openobserve_ingest_auth",
|
|
),
|
|
],
|
|
command: ["--config=/etc/otelcol/config.yaml"],
|
|
mem_limit: "256m",
|
|
};
|
|
services.openobserve = {
|
|
...base,
|
|
image: `${"${"}VP7_OPENOBSERVE_IMAGE:-${images.openobserve}}`,
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
network_mode: "service:openobserve-gateway",
|
|
environment: {
|
|
ZO_LOCAL_MODE: "true",
|
|
ZO_LOCAL_MODE_STORAGE: "disk",
|
|
ZO_DATA_DIR: "/data/",
|
|
ZO_HTTP_ADDR: "127.0.0.1",
|
|
ZO_HTTP_PORT: "15080",
|
|
ZO_GRPC_ADDR: "127.0.0.1",
|
|
ZO_TELEMETRY: "false",
|
|
ZO_MEMORY_CACHE_ENABLED: "false",
|
|
ZO_MEM_TABLE_MAX_SIZE: "128",
|
|
ZO_MAX_FILE_SIZE_IN_MEMORY: "16",
|
|
ZO_MEMORY_CACHE_DATAFUSION_MAX_SIZE: "256",
|
|
ZO_QUERY_TIMEOUT: "8",
|
|
ZO_HTTP_WORKER_NUM: "2",
|
|
ZO_SERVICE_ACCOUNT_ENABLED: "true",
|
|
},
|
|
env_file: ["./.local/config/openobserve.env"],
|
|
volumes: [rw("./.local/data/openobserve", "/data")],
|
|
mem_limit: "1024m",
|
|
cpus: 1,
|
|
};
|
|
services.grafana = {
|
|
...base,
|
|
image: "${VP7_GRAFANA_IMAGE:-" + images.grafana + "}",
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
network_mode: "service:grafana-gateway",
|
|
environment: {
|
|
GF_SERVER_HTTP_ADDR: "127.0.0.1",
|
|
GF_SERVER_HTTP_PORT: "3000",
|
|
GF_AUTH_ANONYMOUS_ENABLED: "false",
|
|
GF_USERS_ALLOW_SIGN_UP: "false",
|
|
GF_SECURITY_ADMIN_USER: "operator",
|
|
GF_SECURITY_ADMIN_PASSWORD__FILE: "/run/admin/password",
|
|
GF_SECURITY_SECRET_KEY__FILE: "/run/admin/secret-key",
|
|
GF_SECURITY_COOKIE_SECURE: "true",
|
|
GF_ANALYTICS_REPORTING_ENABLED: "false",
|
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false",
|
|
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false",
|
|
GF_LOG_MODE: "console",
|
|
GF_LOG_LEVEL: "warn",
|
|
GF_DATAPROXY_TIMEOUT: "10",
|
|
GF_DATAPROXY_RESPONSE_LIMIT: "8388608",
|
|
GF_DATAPROXY_MAX_CONNS_PER_HOST: "8",
|
|
GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "openobserve",
|
|
GF_PLUGINS_PREINSTALL_DISABLED: "true",
|
|
GF_PLUGINS_PUBLIC_KEY_RETRIEVAL_DISABLED: "true",
|
|
},
|
|
env_file: ["./.local/config/grafana-query.env"],
|
|
volumes: [
|
|
rw("./.local/data/grafana", "/var/lib/grafana"),
|
|
ro("./.local/operator/grafana-password", "/run/admin/password"),
|
|
ro("./.local/operator/grafana-secret-key", "/run/admin/secret-key"),
|
|
ro("./infra/grafana/provisioning", "/etc/grafana/provisioning"),
|
|
ro("./infra/grafana/dashboards", "/var/lib/grafana/dashboards"),
|
|
ro("./.local/plugins", "/var/lib/grafana/plugins"),
|
|
],
|
|
mem_limit: "256m",
|
|
};
|
|
services.migrate = {
|
|
...base,
|
|
image: "mtuci-vp7-security-app:local",
|
|
pull_policy: "never",
|
|
user: runUser,
|
|
profiles: ["tools"],
|
|
networks: ["private"],
|
|
volumes: [
|
|
pki("operator"),
|
|
ro("./.local/operator/migration-url", "/run/operator/database_url"),
|
|
],
|
|
environment: { DATABASE_URL_FILE: "/run/operator/database_url" },
|
|
command: ["/app", "-migrate"],
|
|
restart: "no",
|
|
mem_limit: "256m",
|
|
};
|
|
services.openbao.networks = ["private", "edge"];
|
|
return {
|
|
name: "mtuci-vp7-security",
|
|
services,
|
|
networks: { private: { internal: true }, edge: { internal: false } },
|
|
};
|
|
}
|
|
export async function writeTopology() {
|
|
await mkdir(resolve(root, "config"), { recursive: true });
|
|
for (const [name, config] of Object.entries(gatewayConfigs())) {
|
|
let value = config;
|
|
if (["api1-gateway", "api2-gateway", "collector-gateway"].includes(name))
|
|
value = value.replaceAll(
|
|
"crt /pki/server.pem crl-file",
|
|
"crt /client-pki/server.pem crl-file",
|
|
);
|
|
await writeFile(resolve(root, "config", name + ".cfg"), value);
|
|
}
|
|
await writeFile(
|
|
resolve(root, "compose.yaml"),
|
|
JSON.stringify(manifest(), null, 2) + "\n",
|
|
);
|
|
}
|
|
if (import.meta.main) await writeTopology();
|