forked from mtusi2026/mtusi2026lab7
150 lines
11 KiB
Python
150 lines
11 KiB
Python
"""lab3–lab7, уровни 3–4: устройство стенда по реальным контейнерам и docker compose config.
|
||
|
||
make check-stack (= python student-tests/check_stack.py)
|
||
|
||
С lab7 добавлено (hardening, README уровни 3–4):
|
||
- ни один контейнер не privileged, без host network/pid и без Docker socket;
|
||
- у всех: no-new-privileges и cap_drop ALL; cap_add только у объяснённых исключений
|
||
(entrypoint образа стартует от root и переключается на своего пользователя);
|
||
- read_only корень + tmpfs у всех сервисов, кроме объяснённых stateful-исключений;
|
||
- api и worker работают не от root (uid процесса в контейнере);
|
||
- у каждого работающего контейнера лимиты CPU и RAM;
|
||
- наружу опубликованы только lb (8443, 8444, 8404) и grafana (3000), и только на 127.0.0.1;
|
||
- loki и tempo без своей сети: только loopback шлюза tsdb-gw (allowlist identity, infra/tsdb/gateway.cfg);
|
||
Prometheus с lab7 наружу НЕ опубликован (у него mTLS с allowlist identity);
|
||
- все сети, кроме public и ops, internal; api/worker видят только нужные сети;
|
||
- файлы-секреты (Docker secrets) получает только тот, кому они нужны; у api и worker их нет совсем
|
||
(секреты — из OpenBao), TLS-ключ каждой identity смонтирован только в свой сервис.
|
||
Сохранены проверки lab3–lab6: >= 2 реплик api без портов, все реплики под проверкой готовности,
|
||
нет container_name, закреплённые версии образов, volumes хранилищ.
|
||
"""
|
||
import json
|
||
|
||
from helpers import check, run
|
||
from stand import api_containers, compose_config, compose_output, docker, inspect, lb_rows
|
||
|
||
# Сервисы, которым нужны capabilities: entrypoint образа делает chown volume и переключается
|
||
# на своего пользователя (gosu / su-exec), либо одноразово меняет владельца volume.
|
||
CAP_EXCEPTIONS = {"postgres": {"CHOWN", "DAC_OVERRIDE", "FOWNER", "SETUID", "SETGID"},
|
||
"broker": {"CHOWN", "DAC_OVERRIDE", "FOWNER", "SETUID", "SETGID"},
|
||
"openbao-init": {"CHOWN", "DAC_OVERRIDE", "FOWNER"},
|
||
"tempo-init": {"CHOWN"}}
|
||
# Корень не read-only: stateful-сервисы, чей entrypoint пишет вне volume (/var/run/postgresql,
|
||
# конфиг RabbitMQ, ~/.erlang.cookie), и одноразовые init-контейнеры без сети.
|
||
RW_EXCEPTIONS = {"postgres", "broker", "openbao-init", "tempo-init"}
|
||
# Какие сервисы смонтированы с каким каталогом secrets/tls/<identity> (/run/tls).
|
||
TLS_IDENTITY = {"lb": "lb", "api": "api", "worker": "worker", "postgres": "postgres", "valkey": "valkey",
|
||
"s3": "s3", "broker": "broker", "otlp-gw": "otel-collector", "otel-collector": "otel-collector",
|
||
"prometheus": "prometheus", "grafana": "grafana",
|
||
"openbao": "openbao"}
|
||
SECRETS = {"api": [], "worker": [], "lb": [], "migrate": ["postgres_password"],
|
||
"postgres": ["db_api_password", "db_worker_password", "postgres_password"],
|
||
"valkey": ["valkey_acl"], "s3": ["s3_config"], "s3-init": ["s3_root_password", "s3_root_user"],
|
||
"broker": [], "openbao": [], "grafana": ["grafana_admin_password"], "otel-collector": [],
|
||
"otlp-gw": [], "tsdb-gw": [], "prometheus": [], "loki": [], "tempo": []}
|
||
NETWORKS = {"lb": ["edge", "public", "telemetry"], "api": ["backend", "edge", "otlp", "vault"],
|
||
"worker": ["backend", "otlp", "vault"], "postgres": ["backend"], "valkey": ["backend"],
|
||
"s3": ["backend"], "broker": ["backend"], "openbao": ["vault"], "otlp-gw": ["otlp", "telemetry"],
|
||
"prometheus": ["telemetry"], "tsdb-gw": ["telemetry"],
|
||
"grafana": ["ops", "telemetry"]}
|
||
PUBLISHED = {"lb": {"8443", "8444", "8404"}, "grafana": {"3000"}}
|
||
|
||
|
||
def main():
|
||
config = compose_config()
|
||
services = config["services"]
|
||
networks = config.get("networks", {})
|
||
print("docker: " + docker("version", "--format", "{{.Server.Version}}"))
|
||
print("compose: " + compose_output("version", "--short").strip())
|
||
|
||
check(not [n for n, svc in services.items() if "container_name" in svc], "container_name is set")
|
||
replicas = api_containers()
|
||
check(len(replicas) >= 2, f"need >= 2 running api replicas, got {len(replicas)}")
|
||
print(f"api replicas running: {len(replicas)}; container_name not set anywhere")
|
||
|
||
print(f"\n{'service':<15} {'cpus':>5} {'mem':>6} {'user':<12} {'ro':<3} {'caps added':<28} published")
|
||
published = {}
|
||
for container in sorted(compose_output("ps", "-q").split()):
|
||
service = inspect(container, '{{index .Config.Labels "com.docker.compose.service"}}')
|
||
host = json.loads(inspect(container, "{{json .HostConfig}}"))
|
||
ports = json.loads(inspect(container, "{{json .NetworkSettings.Ports}}") or "{}") or {}
|
||
bindings = [(b["HostIp"], b["HostPort"]) for items in ports.values() for b in (items or [])]
|
||
published.setdefault(service, []).extend(bindings)
|
||
caps = {c.removeprefix("CAP_") for c in host.get("CapAdd") or []}
|
||
user = docker("exec", container, "id", "-u", check_exit=False).strip() or "?"
|
||
print(f"{service:<15} {host['NanoCpus'] / 1e9:>5.2f} {host['Memory'] // 2**20:>4}Mi {user:<12} "
|
||
f"{'yes' if host.get('ReadonlyRootfs') else 'no':<3} {','.join(sorted(caps)) or '-':<28} "
|
||
f"{', '.join(f'{ip}:{port}' for ip, port in bindings) or '-'}")
|
||
check(host["NanoCpus"] > 0 and host["Memory"] > 0, f"{service}: no CPU/RAM limit")
|
||
check(not host.get("Privileged"), f"{service}: privileged")
|
||
check(host.get("NetworkMode") != "host" and host.get("PidMode") != "host", f"{service}: host namespace")
|
||
check("no-new-privileges:true" in (host.get("SecurityOpt") or []), f"{service}: no no-new-privileges")
|
||
check("ALL" in (host.get("CapDrop") or []), f"{service}: capabilities are not dropped")
|
||
check(caps <= CAP_EXCEPTIONS.get(service, set()), f"{service}: unexpected cap_add {caps}")
|
||
check(host.get("ReadonlyRootfs") or service in RW_EXCEPTIONS, f"{service}: root filesystem is writable")
|
||
mounts = json.loads(inspect(container, "{{json .Mounts}}"))
|
||
check(not any("docker.sock" in (m.get("Source") or "") for m in mounts), f"{service}: Docker socket mounted")
|
||
tls = [m["Source"].replace("\\", "/").rstrip("/").rsplit("/", 1)[-1] for m in mounts
|
||
if (m.get("Destination") == "/run/tls")]
|
||
if service in TLS_IDENTITY:
|
||
check(tls == [TLS_IDENTITY[service]], f"{service}: /run/tls must be secrets/tls/{TLS_IDENTITY[service]}, got {tls}")
|
||
if service in ("loki", "tempo"):
|
||
# Слушают только loopback шлюза tsdb-gw: своего сертификата и своей сети у них нет.
|
||
check(not tls and host.get("NetworkMode", "").startswith("container:"),
|
||
f"{service} must share the tsdb-gw namespace and have no TLS key of its own")
|
||
if service == "tsdb-gw":
|
||
keys = sorted(m["Destination"] for m in mounts if m.get("Destination", "").startswith("/run/tls-"))
|
||
check(keys == ["/run/tls-loki", "/run/tls-tempo"], f"tsdb-gw must hold the loki and tempo keys, got {keys}")
|
||
if service in ("api", "worker"):
|
||
check(user not in ("0", "?"), f"{service} runs as root")
|
||
print()
|
||
|
||
for service, bindings in published.items():
|
||
if not bindings:
|
||
continue
|
||
check(service in PUBLISHED, f"{service} must not publish ports: {bindings}")
|
||
check(all(ip == "127.0.0.1" for ip, _ in bindings), f"{service} must be published on 127.0.0.1 only")
|
||
check({port for _, port in bindings} <= PUBLISHED[service], f"{service}: unexpected ports {bindings}")
|
||
print("published: lb 127.0.0.1:8443 (HTTPS API), :8444 (short TTL), :8404 (stats, HTTPS); "
|
||
"grafana 127.0.0.1:3000 (HTTPS). api, OpenBao, Prometheus, Loki, Tempo, Collector, stores: none")
|
||
print("hardening: no privileged/host namespaces/Docker socket; no-new-privileges and cap_drop ALL everywhere; "
|
||
f"cap_add only for {', '.join(sorted(CAP_EXCEPTIONS))}; read-only root except {', '.join(sorted(RW_EXCEPTIONS))}; "
|
||
"api/worker are not root; CPU/RAM limits everywhere")
|
||
|
||
for name, wanted in NETWORKS.items():
|
||
got = sorted(services[name].get("networks", {}) or [])
|
||
check(got == wanted, f"{name} networks {got} != {wanted}")
|
||
for name, item in networks.items():
|
||
check(item.get("internal") is True or name in ("public", "ops"), f"network {name} must be internal")
|
||
print("networks: only public (lb) and ops (grafana) are not internal; " +
|
||
"; ".join(f"{n}: {'+'.join(w)}" for n, w in NETWORKS.items()))
|
||
|
||
servers = lb_rows("api")
|
||
check(not [r for r in servers if r.get("bck") == "1"], "backup servers in the balancer")
|
||
check(not [r for r in servers if not r.get("status", "").startswith("MAINT") and not r.get("check_status")],
|
||
"servers without a readiness check in the balancer")
|
||
print("balancer: every api server under the /health/ready check over mTLS, no backup servers")
|
||
|
||
for name, wanted in SECRETS.items():
|
||
got = sorted(item["source"] for item in services[name].get("secrets", []))
|
||
check(got == wanted, f"{name} secrets {got} != {wanted}")
|
||
for name in ("api", "worker"):
|
||
env = services[name].get("environment", {})
|
||
check(not any("TOKEN" in key or "SECRET_ID" in key or "PASSWORD" in key for key in env),
|
||
f"{name} has a token/password variable in its environment")
|
||
print("secrets: api and worker get NO Docker secrets and no token variables (only their TLS identity "
|
||
"-> OpenBao cert auth); " + "; ".join(f"{n} -> {', '.join(w)}" for n, w in SECRETS.items() if w))
|
||
|
||
images = {name: services[name]["image"] for name in services if "image" in services[name]}
|
||
check(all(":" in image and not image.endswith(":latest") for image in images.values()), "unpinned image")
|
||
volumes = {"postgres": "pgdata", "s3": "s3data", "broker": "rabbitmqdata", "openbao": "baodata",
|
||
"prometheus": "promdata", "loki": "lokidata", "tempo": "tempodata", "grafana": "grafanadata"}
|
||
for name, volume in volumes.items():
|
||
check(any(v.get("source") == volume for v in services[name].get("volumes", [])), f"{name} needs {volume}")
|
||
print("images: " + "; ".join(f"{n} {i}" for n, i in sorted(images.items()) if not i.startswith("mtusi")))
|
||
print("PASS: hardening, ports, networks, per-service secrets and TLS identities match the trust matrix")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main)
|