forked from mtusi2026/mtusi2026lab6
161 lines
11 KiB
Python
161 lines
11 KiB
Python
"""lab3–lab6, уровень 4: проверка устройства стенда по реальным контейнерам и docker compose config.
|
||
|
||
make check-stack (= python student-tests/check_stack.py)
|
||
|
||
- не меньше 2 работающих реплик api, у них нет опубликованных портов (обойти балансировщик нельзя);
|
||
- в балансировщике каждая реплика под проверкой готовности, резервных серверов без проверки нет
|
||
(неготовая реплика не получает запросы ни при каких условиях);
|
||
- ни у одного сервиса нет container_name (иначе нельзя запустить несколько реплик одного сервиса);
|
||
- наружу опубликованы только балансировщик и (с lab6) Grafana и Prometheus, и только на 127.0.0.1;
|
||
- у каждого работающего контейнера есть лимиты CPU и RAM;
|
||
- сети: postgres, valkey, s3, broker и worker только во внутренней сети backend, lb только в edge, api в обеих;
|
||
- (lab5) брокер — отдельный сервис с volume и постоянным hostname, без портов и без веб-консоли;
|
||
worker — отдельный сервис без портов;
|
||
- (lab6) otel-collector, prometheus, loki, tempo, grafana — отдельные сервисы с закреплёнными версиями;
|
||
хранилища телеметрии с volumes; OTLP, Loki и Tempo только во внутренней сети telemetry;
|
||
- секреты получает только тот, кому они нужны;
|
||
- версии Docker, Compose и образов (для отчёта).
|
||
"""
|
||
import json
|
||
|
||
from helpers import check, run
|
||
from stand import api_containers, compose_config, compose_output, docker, inspect, lb_rows
|
||
|
||
|
||
def main():
|
||
config = compose_config()
|
||
services = config["services"]
|
||
networks = config.get("networks", {})
|
||
|
||
print("docker: " + docker("version", "--format", "{{.Server.Version}}"))
|
||
print("compose: " + compose_output("version", "--short").strip())
|
||
|
||
check(not [name for name, svc in services.items() if "container_name" in svc], "container_name is set")
|
||
print("container_name: not set for any service (replicas of one service are possible)")
|
||
|
||
replicas = api_containers()
|
||
check(len(replicas) >= 2, f"need >= 2 running api replicas, got {len(replicas)}")
|
||
print(f"api replicas running: {len(replicas)} -> {[c[:12] for c in replicas]} "
|
||
f"(deploy.replicas = {services['api'].get('deploy', {}).get('replicas')})")
|
||
|
||
print()
|
||
print(f"{'service':<14} {'container':<13} {'cpus':>5} {'memory':>8} {'published ports':<34} networks")
|
||
published = {}
|
||
for container in sorted(compose_output("ps", "-q").split()):
|
||
service = inspect(container, '{{index .Config.Labels "com.docker.compose.service"}}')
|
||
nano_cpus = int(inspect(container, "{{.HostConfig.NanoCpus}}") or 0)
|
||
memory = int(inspect(container, "{{.HostConfig.Memory}}") or 0)
|
||
ports = json.loads(inspect(container, "{{json .NetworkSettings.Ports}}") or "{}") or {}
|
||
bindings = [f"{b['HostIp']}:{b['HostPort']}->{port}" for port, items in ports.items() for b in (items or [])]
|
||
nets = sorted(json.loads(inspect(container, "{{json .NetworkSettings.Networks}}") or "{}"))
|
||
nets = [name.split("_", 1)[-1] for name in nets]
|
||
published[service] = published.get(service, []) + bindings
|
||
print(f"{service:<14} {container[:12]:<13} {nano_cpus / 1e9:>5.2f} {memory // 2**20:>6}Mi "
|
||
f"{', '.join(bindings) or '-':<34} {', '.join(nets)}")
|
||
check(nano_cpus > 0 and memory > 0, f"{service} {container[:12]} has no CPU/RAM limit")
|
||
print()
|
||
|
||
check(not published.get("api"), "api publishes ports and can be reached bypassing the balancer")
|
||
allowed = ("lb", "grafana", "prometheus")
|
||
others = {service: ports for service, ports in published.items() if ports and service not in allowed}
|
||
check(not others, f"only lb, grafana and prometheus may publish ports, got {others}")
|
||
for service in allowed:
|
||
check(published.get(service) and all(b.startswith("127.0.0.1:") for b in published[service]),
|
||
f"{service} must be published on 127.0.0.1 only")
|
||
print("published: only lb, grafana (3000) and prometheus (9090), only on 127.0.0.1 (api replicas, OTLP, "
|
||
"Loki, Tempo have no ports)")
|
||
|
||
servers = lb_rows("api")
|
||
backups = [row["svname"] for row in servers if row.get("bck") == "1"]
|
||
unchecked = [row["svname"] for row in servers
|
||
if not row.get("status", "").startswith("MAINT") and not row.get("check_status")]
|
||
check(not backups, f"backup servers in the balancer: {backups}")
|
||
check(not unchecked, f"servers without a readiness check in the balancer: {unchecked}")
|
||
in_use = [f"{row['svname']} {row['status']} ({row['check_status']})" for row in servers
|
||
if not row.get("status", "").startswith("MAINT")]
|
||
print(f"balancer: every api server is under the /health/ready check, no backup servers -> {', '.join(in_use)}")
|
||
print("limits: every running container has CPU and RAM limits")
|
||
|
||
def nets_of(name):
|
||
return sorted(services[name].get("networks", {}))
|
||
|
||
check(all(nets_of(name) == ["backend"] for name in ("postgres", "valkey", "s3", "s3-init", "broker")),
|
||
"stores and broker must be on backend only")
|
||
check(nets_of("worker") == ["backend", "telemetry"], "worker must be on backend + telemetry")
|
||
check(nets_of("lb") == ["edge", "telemetry"], "lb must be on edge (+ telemetry for its metrics)")
|
||
check(nets_of("api") == ["backend", "edge", "telemetry"], "api must be on edge, backend and telemetry")
|
||
check(networks.get("backend", {}).get("internal") is True, "backend network must be internal")
|
||
print("networks: lb -> edge (+telemetry); api -> edge + backend + telemetry; postgres, valkey, s3, broker -> "
|
||
"backend (internal: no way out)")
|
||
|
||
# lab5: брокер и worker — отдельные сервисы общего стенда.
|
||
broker, worker = services["broker"], services["worker"]
|
||
check(not broker.get("ports") and not worker.get("ports"), "broker and worker must not publish ports")
|
||
check(":" in broker["image"] and not broker["image"].endswith(":latest") and "management" not in broker["image"],
|
||
"broker image must be pinned and without the management console")
|
||
check(any(v.get("source") == "rabbitmqdata" for v in broker.get("volumes", [])), "broker needs a volume")
|
||
check(broker.get("hostname") == "broker", "broker needs a fixed hostname (node name = data directory)")
|
||
check(worker.get("command", [])[-1:] == ["messenger.worker"], "worker must be a separate process")
|
||
workers = sorted(compose_output("ps", "-q", "worker").split())
|
||
check(len(workers) >= 1, "worker is not running")
|
||
print(f"broker: {broker['image']}, volume rabbitmqdata, hostname broker, no ports, no web console; "
|
||
f"worker: separate service ({len(workers)} running), no ports")
|
||
|
||
# lab4: S3 без портов наружу и без веб-консоли; версии образов закреплены.
|
||
s3 = services["s3"]
|
||
check(not s3.get("ports"), "s3 must not publish ports")
|
||
command = s3.get("command", [])
|
||
check("-ip.bind=127.0.0.1" in command and "-s3.iam=false" in command,
|
||
"SeaweedFS master/filer UIs must listen on 127.0.0.1 only and IAM API must be off")
|
||
check(not s3["image"].endswith(":latest") and ":" in s3["image"], "s3 image must be pinned")
|
||
check(any(v.get("source") == "s3data" for v in s3.get("volumes", [])), "s3 must use the persistent volume s3data")
|
||
print("s3: no published ports; master/filer web UIs only on 127.0.0.1 inside the container; IAM API off; "
|
||
"pinned image; volume s3data")
|
||
|
||
# lab6: наблюдаемость — отдельные сервисы общего стенда.
|
||
check(networks.get("telemetry", {}).get("internal") is True, "telemetry network must be internal")
|
||
for name in ("otel-collector", "loki", "tempo"):
|
||
check(nets_of(name) == ["telemetry"], f"{name} must be on the internal telemetry network only")
|
||
for name in ("prometheus", "grafana"):
|
||
check(nets_of(name) == ["ops", "telemetry"], f"{name}: telemetry + ops (for the 127.0.0.1 port only)")
|
||
for name in ("otel-collector", "prometheus", "loki", "tempo", "grafana"):
|
||
image = services[name]["image"]
|
||
check(":" in image and not image.endswith(":latest"), f"{name} image must be pinned")
|
||
running = compose_output("ps", "-q", name).split()
|
||
check(running, f"{name} is not running")
|
||
volumes = {"prometheus": "promdata", "loki": "lokidata", "tempo": "tempodata", "grafana": "grafanadata"}
|
||
for name, volume in volumes.items():
|
||
check(any(v.get("source") == volume for v in services[name].get("volumes", [])), f"{name} needs volume {volume}")
|
||
check(services["worker"].get("environment", {}).get("OTEL_SERVICE_NAME") == "messenger-worker",
|
||
"worker must have its own service.name")
|
||
print("telemetry: " + "; ".join(f"{n} {services[n]['image']}" for n in
|
||
("otel-collector", "prometheus", "loki", "tempo", "grafana")))
|
||
print(" volumes promdata, lokidata, tempodata, grafanadata; OTLP/Loki/Tempo only on internal "
|
||
"network telemetry; worker service.name=messenger-worker")
|
||
|
||
def secrets_of(name):
|
||
return sorted(item["source"] for item in services[name].get("secrets", []))
|
||
|
||
expected = {"api": ["broker_url", "cache_url", "database_url", "s3_access_key_id", "s3_secret_access_key"],
|
||
"worker": ["broker_url", "database_url", "s3_access_key_id", "s3_secret_access_key"],
|
||
"broker": ["broker_credentials"],
|
||
"migrate": ["database_url"], "lb": [],
|
||
"postgres": ["postgres_password"], "valkey": ["valkey_acl"],
|
||
"s3": ["s3_config"],
|
||
"s3-init": ["s3_root_password", "s3_root_user"],
|
||
"grafana": ["grafana_admin_password"], "otel-collector": [], "prometheus": [], "loki": [],
|
||
"tempo": []}
|
||
for name, wanted in expected.items():
|
||
check(secrets_of(name) == wanted, f"{name} secrets {secrets_of(name)} != {wanted}")
|
||
print("secrets: " + "; ".join(f"{name} -> {', '.join(wanted) or 'none'}" for name, wanted in expected.items()))
|
||
print("images: " + "; ".join(f"{name} {services[name]['image']}"
|
||
for name in ("lb", "api", "worker", "postgres", "valkey", "s3", "broker")))
|
||
print("PASS: one entry point (lb on 127.0.0.1), >= 2 api replicas without ports and container_name, "
|
||
"only ready replicas get traffic, CPU/RAM limits everywhere, stores (incl. S3) only on the internal network, "
|
||
"S3 admin credentials only in s3-init; broker and worker are separate services; collector and three "
|
||
"telemetry stores + Grafana pinned, with volumes, OTLP only on the internal network")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main)
|