ops: add PKI and HTTPS ingress

This commit is contained in:
2026-11-04 23:05:11 +03:00
parent 951e554e06
commit 58c9bb027b
26 changed files with 572 additions and 56 deletions
+1
View File
@@ -3,6 +3,7 @@
.env
.env.*
!.env.example
.runtime
secrets
certs
volumes
+2
View File
@@ -2,6 +2,7 @@
.env
.env.*
!.env.example
.runtime/
secrets/
certs/private/
*.key
@@ -32,3 +33,4 @@ coverage/
# Inspect git diff --cached: ignore rules are not a secret scanner.
evidence/runtime/
.runtime/
+12 -2
View File
@@ -1,9 +1,19 @@
FROM alpine:3.24.2 AS security-test
RUN apk add --no-cache curl openssl
WORKDIR /src
COPY scripts/test-tls-matrix.sh /src/scripts/test-tls-matrix.sh
CMD ["/bin/sh", "/src/scripts/test-tls-matrix.sh"]
FROM golang:1.26.5-alpine3.24 AS build
ARG GOPROXY=https://proxy.golang.org
WORKDIR /src
ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB
RUN apk add --no-cache git
COPY go.mod go.sum ./
RUN go mod download
RUN GOPROXY="$GOPROXY" go mod download
COPY . .
RUN go build -p=1 -o /out/messenger ./cmd/messenger
RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment
@@ -41,5 +51,5 @@ COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment
USER app
WORKDIR /app
HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \
CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1
CMD wget -q -O /dev/null "http://127.0.0.1:8082/health/ready" || exit 1
CMD ["/app/messenger"]
+20 -1
View File
@@ -2,9 +2,10 @@ PYTHON ?= python3
BASE_URL ?= https://localhost:8443
CA_FILE ?=
ACTION_SCRIPT ?=
ENV_FILE ?= $(if $(wildcard .env),.env,lab7.env.example)
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover test-security
.PHONY: help check validate test load test-persistence test-failover test-security test-tls-matrix test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -31,3 +32,21 @@ test-failover:
test-security:
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-tls-matrix:
docker compose --env-file "$(ENV_FILE)" --profile security run --rm security-tests
test-lab5:
./scripts/test-lab5.sh
test-lab5-acceptance:
./scripts/test-lab5-acceptance.sh
lab6-load:
$(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" $(TLS_ARGS) --mode mixed --duration 60 --concurrency 4 --wait-ready
test-lab6-smoke:
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh smoke
test-lab6:
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh full
+84 -24
View File
@@ -10,6 +10,20 @@ x-telemetry-environment: &telemetry-environment
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
services:
security-bootstrap:
image: alpine:3.24.2
entrypoint: ["/bin/sh", "/install.sh"]
volumes:
- ./.runtime/security:/source:ro
- ./infra/pki/install.sh:/install.sh:ro
- api_security:/target/api
- nginx_security:/target/nginx
- prometheus_security:/target/prometheus
- client_security:/target/client
healthcheck:
disable: true
restart: "no"
postgres:
image: postgres:17.6-alpine3.22
environment:
@@ -123,7 +137,7 @@ services:
networks: [data]
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:lab7}
command: ["/app/messenger", "migrate"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -142,13 +156,14 @@ services:
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:lab7}
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-api
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
APP_HEALTH_ADDRESS: ${APP_HEALTH_ADDRESS}
DATABASE_URL: ${DATABASE_URL}
CACHE_URL: ${CACHE_URL}
SWAGGER_PORT: ${SWAGGER_PORT}
@@ -159,9 +174,13 @@ services:
S3_SECRET_KEY: ${S3_SECRET_KEY}
volumes:
- *messenger-config-volume
- api_security:/run/security:ro
expose:
- "8080"
- "8443"
- "9091"
depends_on:
security-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
valkey:
@@ -169,7 +188,7 @@ services:
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8082/health/ready || exit 1"]
interval: 2s
timeout: 1s
start_period: 5s
@@ -180,10 +199,10 @@ services:
mem_limit: ${API_MEMORY}
stop_grace_period: 8s
restart: unless-stopped
networks: [application, data]
networks: [application, data, monitoring]
dispatcher:
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:lab7}
command: ["/app/messenger", "dispatch"]
environment:
<<: *telemetry-environment
@@ -211,7 +230,7 @@ services:
networks: [data, monitoring]
worker:
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:lab7}
command: ["/app/messenger", "worker"]
environment:
<<: *telemetry-environment
@@ -250,10 +269,14 @@ services:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
image: ${NGINX_IMAGE:-mtusi-nginx:lab7}
ports:
- "${API_BIND_ADDRESS}:8080"
- "${API_BIND_ADDRESS}:8443"
volumes:
- nginx_security:/run/security:ro
depends_on:
security-bootstrap:
condition: service_completed_successfully
api:
condition: service_healthy
healthcheck:
@@ -266,11 +289,29 @@ services:
restart: unless-stopped
networks: [edge, application, monitoring]
security-tests:
profiles: ["security"]
build:
context: .
target: security-test
environment:
PUBLIC_TLS_HOST: nginx
PUBLIC_TLS_PORT: "8443"
API_TLS_ADDRESS: api:8443
METRICS_TLS_ADDRESS: api:9091
SECURITY_DIR: /src/.runtime/security
volumes:
- ./.runtime/security:/src/.runtime/security:ro
depends_on:
nginx:
condition: service_healthy
networks: [edge, application]
swagger:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab7}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
@@ -300,7 +341,7 @@ services:
lab5-experiment:
profiles: ["experiments"]
image: ${API_IMAGE:-mtusi-messenger:lab6}
image: ${API_IMAGE:-mtusi-messenger:lab7}
command: ["/app/lab5experiment"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -313,6 +354,7 @@ services:
BROKER_URL: ${BROKER_URL}
volumes:
- *messenger-config-volume
- client_security:/run/security:ro
depends_on:
migrate:
condition: service_completed_successfully
@@ -386,7 +428,7 @@ services:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab7}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
@@ -394,6 +436,7 @@ services:
- --web.enable-remote-write-receiver
volumes:
- prometheus_data:/prometheus
- prometheus_security:/run/security:ro
ports:
- "${PROMETHEUS_BIND_ADDRESS}:9090"
healthcheck:
@@ -402,17 +445,30 @@ services:
timeout: 3s
retries: 20
depends_on:
- api
- nginx-exporter
- postgres-exporter
- valkey-exporter
- redis-broker-exporter
- dispatcher
- worker
- s3
- otel-collector
- loki
- tempo
security-bootstrap:
condition: service_completed_successfully
api:
condition: service_healthy
nginx-exporter:
condition: service_started
postgres-exporter:
condition: service_started
valkey-exporter:
condition: service_started
redis-broker-exporter:
condition: service_started
dispatcher:
condition: service_healthy
worker:
condition: service_healthy
s3:
condition: service_healthy
otel-collector:
condition: service_healthy
loki:
condition: service_healthy
tempo:
condition: service_healthy
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
@@ -422,7 +478,7 @@ services:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab7}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
@@ -507,6 +563,10 @@ services:
networks: [application, data, monitoring]
volumes:
api_security:
nginx_security:
prometheus_security:
client_security:
postgres_data:
valkey_data:
redis_broker_data:
+8
View File
@@ -1,6 +1,14 @@
version: 1
server:
health_address: 127.0.0.1:8082
metrics_address: 0.0.0.0:9091
tls:
ca_path: /run/security/internal-ca/ca.crt
certificate_path: /run/security/api/tls.crt
private_key_path: /run/security/api/tls.key
allowed_client_uri: spiffe://mtusi/lab7/load-balancer
allowed_metrics_client_uri: spiffe://mtusi/lab7/prometheus
session_store: valkey
session_ttl: 1h
idempotency_ttl: 24h
+7
View File
@@ -58,6 +58,13 @@ service:
telemetry:
logs:
level: info
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
+107
View File
@@ -0,0 +1,107 @@
#!/bin/sh
set -eu
umask 077
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/../.." && pwd)
security_dir="$repo_dir/.runtime/security"
staging_dir="$security_dir/.bootstrap.$$"
cleanup() {
rm -rf "$staging_dir"
}
trap cleanup EXIT HUP INT TERM
if [ "${1:-}" = "--force" ]; then
force=1
elif [ "$#" -eq 0 ]; then
force=0
else
echo "usage: $0 [--force]" >&2
exit 2
fi
if [ -d "$security_dir/public-ca" ] || [ -d "$security_dir/internal-ca" ]; then
if [ "$force" -ne 1 ]; then
echo "security material already exists; use --force to replace it" >&2
exit 1
fi
fi
mkdir -p "$security_dir" "$staging_dir"
chmod 0700 "$security_dir" "$staging_dir"
create_ca() {
name=$1
common_name=$2
directory="$staging_dir/$name"
mkdir -p "$directory"
openssl genpkey -quiet -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$directory/ca.key"
openssl req -new -x509 -sha256 -days 3650 \
-key "$directory/ca.key" \
-out "$directory/ca.crt" \
-subj "/CN=$common_name" \
-config "$script_dir/templates/ca.cnf" \
-extensions v3_ca
}
create_leaf() {
name=$1
common_name=$2
issuer=$3
template=$4
directory="$staging_dir/$name"
issuer_directory="$staging_dir/$issuer"
mkdir -p "$directory"
openssl genpkey -quiet -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$directory/tls.key"
openssl req -new -sha256 \
-key "$directory/tls.key" \
-out "$directory/tls.csr" \
-subj "/CN=$common_name"
openssl x509 -req -sha256 -days 30 \
-in "$directory/tls.csr" \
-CA "$issuer_directory/ca.crt" \
-CAkey "$issuer_directory/ca.key" \
-CAcreateserial \
-out "$directory/tls.crt" \
-extfile "$script_dir/templates/$template"
rm "$directory/tls.csr"
}
create_ca public-ca "MTUSI Lab 7 Public CA"
create_ca internal-ca "MTUSI Lab 7 Internal CA"
create_ca foreign-ca "MTUSI Lab 7 Foreign CA"
create_leaf ingress localhost public-ca ingress.cnf
create_leaf api api internal-ca api.cnf
create_leaf load-balancer load-balancer internal-ca load-balancer.cnf
create_leaf prometheus prometheus internal-ca prometheus.cnf
create_leaf forbidden-client worker internal-ca forbidden-client.cnf
create_leaf foreign-client load-balancer foreign-ca load-balancer.cnf
openssl verify -purpose sslserver -verify_hostname localhost \
-CAfile "$staging_dir/public-ca/ca.crt" "$staging_dir/ingress/tls.crt" >/dev/null
openssl verify -purpose sslserver -verify_hostname api \
-CAfile "$staging_dir/internal-ca/ca.crt" "$staging_dir/api/tls.crt" >/dev/null
openssl verify -purpose sslclient \
-CAfile "$staging_dir/internal-ca/ca.crt" \
"$staging_dir/load-balancer/tls.crt" \
"$staging_dir/prometheus/tls.crt" \
"$staging_dir/forbidden-client/tls.crt" >/dev/null
openssl verify -purpose sslclient \
-CAfile "$staging_dir/foreign-ca/ca.crt" "$staging_dir/foreign-client/tls.crt" >/dev/null
find "$staging_dir" -type d -exec chmod 0700 {} \;
find "$staging_dir" -type f -exec chmod 0600 {} \;
for directory in public-ca internal-ca foreign-ca ingress api load-balancer prometheus forbidden-client foreign-client; do
rm -rf "$security_dir/$directory"
mv "$staging_dir/$directory" "$security_dir/$directory"
done
echo "local PKI created in .runtime/security"
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
umask 077
rm -rf /target/api/internal-ca /target/api/api
mkdir -p /target/api/internal-ca /target/api/api
cp /source/internal-ca/ca.crt /target/api/internal-ca/ca.crt
cp /source/api/tls.crt /target/api/api/tls.crt
cp /source/api/tls.key /target/api/api/tls.key
chown -R 10001:10001 /target/api
find /target/api -type d -exec chmod 0700 {} \;
find /target/api -type f -exec chmod 0600 {} \;
rm -rf /target/nginx/ingress /target/nginx/public-ca /target/nginx/internal-ca /target/nginx/load-balancer
mkdir -p /target/nginx/ingress /target/nginx/public-ca /target/nginx/internal-ca /target/nginx/load-balancer
cp /source/ingress/tls.crt /target/nginx/ingress/tls.crt
cp /source/ingress/tls.key /target/nginx/ingress/tls.key
cp /source/public-ca/ca.crt /target/nginx/public-ca/ca.crt
cp /source/internal-ca/ca.crt /target/nginx/internal-ca/ca.crt
cp /source/load-balancer/tls.crt /target/nginx/load-balancer/tls.crt
cp /source/load-balancer/tls.key /target/nginx/load-balancer/tls.key
chown -R 0:0 /target/nginx
find /target/nginx -type d -exec chmod 0700 {} \;
find /target/nginx -type f -exec chmod 0600 {} \;
rm -rf /target/prometheus/internal-ca /target/prometheus/prometheus
mkdir -p /target/prometheus/internal-ca /target/prometheus/prometheus
cp /source/internal-ca/ca.crt /target/prometheus/internal-ca/ca.crt
cp /source/prometheus/tls.crt /target/prometheus/prometheus/tls.crt
cp /source/prometheus/tls.key /target/prometheus/prometheus/tls.key
chown -R 65534:65534 /target/prometheus
find /target/prometheus -type d -exec chmod 0700 {} \;
find /target/prometheus -type f -exec chmod 0600 {} \;
rm -rf /target/client/public-ca
mkdir -p /target/client/public-ca
cp /source/public-ca/ca.crt /target/client/public-ca/ca.crt
chown -R 10001:10001 /target/client
find /target/client -type d -exec chmod 0700 {} \;
find /target/client -type f -exec chmod 0600 {} \;
+10
View File
@@ -0,0 +1,10 @@
basicConstraints = critical,CA:FALSE
keyUsage = critical,digitalSignature,keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @subject_alt_names
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
[subject_alt_names]
DNS.1 = api
URI.1 = spiffe://mtusi/lab7/api
+12
View File
@@ -0,0 +1,12 @@
[req]
distinguished_name = distinguished_name
prompt = no
[distinguished_name]
CN = local-ca
[v3_ca]
basicConstraints = critical,CA:TRUE,pathlen:0
keyUsage = critical,keyCertSign,cRLSign
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always
+9
View File
@@ -0,0 +1,9 @@
basicConstraints = critical,CA:FALSE
keyUsage = critical,digitalSignature
extendedKeyUsage = clientAuth
subjectAltName = @subject_alt_names
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
[subject_alt_names]
URI.1 = spiffe://mtusi/lab7/worker
+11
View File
@@ -0,0 +1,11 @@
basicConstraints = critical,CA:FALSE
keyUsage = critical,digitalSignature,keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @subject_alt_names
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
[subject_alt_names]
DNS.1 = localhost
DNS.2 = nginx
IP.1 = 127.0.0.1
+9
View File
@@ -0,0 +1,9 @@
basicConstraints = critical,CA:FALSE
keyUsage = critical,digitalSignature
extendedKeyUsage = clientAuth
subjectAltName = @subject_alt_names
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
[subject_alt_names]
URI.1 = spiffe://mtusi/lab7/load-balancer
+4
View File
@@ -0,0 +1,4 @@
basicConstraints = critical,CA:FALSE
keyUsage = critical,digitalSignature
extendedKeyUsage = clientAuth
subjectAltName = URI:spiffe://mtusi/lab7/prometheus
+6 -5
View File
@@ -1,13 +1,14 @@
# Изолированное имя Compose-проекта и его volumes/networks.
COMPOSE_PROJECT_NAME=mtusi2026lab6
COMPOSE_PROJECT_NAME=mtusi2026lab7
# Адрес и внутренний порт HTTP-сервера.
# Адреса HTTPS API и локального health-only listener.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_PORT=8443
APP_HEALTH_ADDRESS=127.0.0.1:8082
MESSENGER_CONFIG_PATH=config/messenger.yaml
# Адреса публикации сервисов на локальной машине.
API_BIND_ADDRESS=127.0.0.1:8080
API_BIND_ADDRESS=127.0.0.1:8443
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
@@ -16,7 +17,7 @@ TEMPO_BIND_ADDRESS=127.0.0.1:3200
# Ресурсные атрибуты и внутренний OTLP endpoint приложений.
DEPLOYMENT_ENVIRONMENT=lab
SERVICE_VERSION=lab6
SERVICE_VERSION=lab7
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+8 -1
View File
@@ -12,8 +12,15 @@ scrape_configs:
dns_sd_configs:
- names: ["api"]
type: A
port: 8080
port: 9091
refresh_interval: 2s
scheme: https
tls_config:
ca_file: /run/security/internal-ca/ca.crt
cert_file: /run/security/prometheus/tls.crt
key_file: /run/security/prometheus/tls.key
server_name: api
min_version: TLS13
relabel_configs:
- target_label: service_name
replacement: messenger-api
+17 -3
View File
@@ -25,18 +25,32 @@ http {
}
server {
listen 8080;
listen 8443 ssl;
client_max_body_size 11m;
ssl_certificate /run/security/ingress/tls.crt;
ssl_certificate_key /run/security/ingress/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_tickets off;
location / {
proxy_pass http://messenger;
proxy_pass https://messenger;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Proto https;
proxy_request_buffering off;
proxy_ssl_server_name on;
proxy_ssl_name api;
proxy_ssl_trusted_certificate /run/security/internal-ca/ca.crt;
proxy_ssl_verify on;
proxy_ssl_verify_depth 1;
proxy_ssl_certificate /run/security/load-balancer/tls.crt;
proxy_ssl_certificate_key /run/security/load-balancer/tls.key;
proxy_ssl_session_reuse off;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
+1 -1
View File
@@ -1 +1 @@
server api:8080 resolve max_fails=1 fail_timeout=3s;
server api:8443 resolve max_fails=1 fail_timeout=3s;
+4 -2
View File
@@ -2,8 +2,9 @@
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
@@ -33,6 +34,7 @@ trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode messages \
--duration "$((outage_seconds + 20))" \
--concurrency 4 \
+5 -3
View File
@@ -2,8 +2,9 @@
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
@@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop tempo loki >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode mixed \
--duration "$outage_seconds" \
--concurrency 4 \
@@ -41,7 +43,7 @@ load_pid=$!
sleep "$outage_seconds"
wait "$load_pid"
load_pid=
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready")
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
+4 -2
View File
@@ -3,8 +3,9 @@
set -eu
mode=${1:-short}
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
@@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop worker >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode attachments \
--duration "$load_seconds" \
--requests "$requests" \
+9 -2
View File
@@ -7,6 +7,7 @@ import json
import math
from pathlib import Path
import secrets
import ssl
import struct
import threading
import time
@@ -16,6 +17,7 @@ import urllib.request
import zlib
MAX_RESPONSE_BYTES = 1 << 20
TLS_CONTEXT = None
def png_fixture(width=96, height=64):
@@ -37,7 +39,7 @@ def png_fixture(width=96, height=64):
def request(method, url, body=None, headers=None, timeout=10):
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
try:
with urllib.request.urlopen(req, timeout=timeout) as response:
with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response:
payload = response.read(MAX_RESPONSE_BYTES)
return response.status, payload
except urllib.error.HTTPError as error:
@@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout):
def main():
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--base-url", default="https://localhost:8443")
parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt")
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
parser.add_argument("--duration", type=float, default=30)
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
@@ -179,6 +182,10 @@ def main():
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
args = parser.parse_args()
global TLS_CONTEXT
if args.base_url.startswith("https://"):
TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file)
if args.duration <= 0 or args.duration > 1800:
parser.error("duration must be in (0, 1800]")
if args.requests < 0:
+9 -4
View File
@@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
fi
manifest=$1
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"}
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
current_manifest=$deploy_dir/current-images.env
active_containers=$deploy_dir/active-api-containers
previous_containers=$deploy_dir/previous-api-containers
@@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf
upstream_backup=$deploy_dir/upstream.previous.conf
candidate_file=$deploy_dir/candidate-api-containers
replicas=${API_REPLICAS:-2}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
switched=false
nginx_created=false
upstream_changed=false
@@ -52,7 +54,7 @@ write_upstream() {
: > "$output"
for container in $containers; do
name=$(container_name "$container")
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
done
}
@@ -81,7 +83,8 @@ wait_for_containers() {
}
smoke_check() {
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
curl --fail --silent --show-error --noproxy '*' \
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
}
remove_containers() {
@@ -123,6 +126,7 @@ trap 'exit 143' TERM
load_manifest "$manifest"
new_api_image=$API_IMAGE
test -r "$public_ca_file"
if [ -f "$previous_containers" ]; then
remove_containers "$(cat "$previous_containers")"
@@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup"
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
compose run --rm security-bootstrap
compose run --rm --no-deps s3-bootstrap
compose run --rm --no-deps migrate
+8 -6
View File
@@ -11,8 +11,9 @@ case "$profile" in
;;
esac
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
@@ -35,6 +36,7 @@ compose config --quiet
compose up -d --build --wait --wait-timeout 180
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode mixed \
--duration "${BASELINE_LOAD_SECONDS:-15}" \
--requests "${BASELINE_REQUESTS:-120}" \
@@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \
--require >/dev/null
if [ "$profile" = "full" ]; then
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
fi
arguments=
+163
View File
@@ -0,0 +1,163 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security}
public_port=${PUBLIC_TLS_PORT:-8443}
public_host=${PUBLIC_TLS_HOST:-localhost}
api_address=${API_TLS_ADDRESS:-api:8443}
metrics_address=${METRICS_TLS_ADDRESS:-api:9091}
empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$"
cleanup() {
rm -f "$empty_ca"
}
trap cleanup EXIT HUP INT TERM
require_file() {
if [ ! -r "$1" ]; then
echo "missing security material: $1" >&2
exit 1
fi
}
expect_failure() {
name=$1
shift
if "$@" >/dev/null 2>&1; then
echo "FAIL: $name unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: $name"
}
expect_tls_success() {
name=$1
certificate=$2
key=$3
if curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$certificate" --key "$key" \
"https://$api_address/health/ready" >/dev/null 2>&1; then
echo "PASS: $name"
return
fi
echo "FAIL: $name" >&2
exit 1
}
expect_tls_failure() {
name=$1
shift
if curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
"$@" "https://$api_address/health/ready" >/dev/null 2>&1; then
echo "FAIL: $name unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: $name"
}
expect_public_hostname_failure() {
if openssl s_client \
-connect "$public_host:$public_port" \
-servername wrong.local \
-verify_hostname wrong.local \
-verify_return_error \
-CAfile "$security_dir/public-ca/ca.crt" </dev/null >/dev/null 2>&1; then
echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: public TLS with wrong hostname"
}
for file in \
"$security_dir/public-ca/ca.crt" \
"$security_dir/internal-ca/ca.crt" \
"$security_dir/foreign-ca/ca.crt" \
"$security_dir/load-balancer/tls.crt" \
"$security_dir/load-balancer/tls.key" \
"$security_dir/prometheus/tls.crt" \
"$security_dir/prometheus/tls.key" \
"$security_dir/forbidden-client/tls.crt" \
"$security_dir/forbidden-client/tls.key" \
"$security_dir/foreign-client/tls.crt" \
"$security_dir/foreign-client/tls.key"; do
require_file "$file"
done
: >"$empty_ca"
chmod 0600 "$empty_ca"
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/public-ca/ca.crt" \
"https://$public_host:$public_port/health/ready" >/dev/null
echo "PASS: public TLS with trusted CA and hostname"
expect_failure "public TLS with empty CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$empty_ca" \
"https://$public_host:$public_port/health/ready"
expect_failure "public TLS with foreign CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/foreign-ca/ca.crt" \
"https://$public_host:$public_port/health/ready"
expect_public_hostname_failure
expect_tls_failure "API mTLS without client certificate"
expect_tls_failure "API mTLS with forbidden trusted identity" \
--cert "$security_dir/forbidden-client/tls.crt" \
--key "$security_dir/forbidden-client/tls.key"
expect_tls_failure "API mTLS with client certificate from foreign CA" \
--cert "$security_dir/foreign-client/tls.crt" \
--key "$security_dir/foreign-client/tls.key"
expect_tls_success "API mTLS with load-balancer identity" \
"$security_dir/load-balancer/tls.crt" \
"$security_dir/load-balancer/tls.key"
expect_tls_failure "business API with Prometheus identity" \
--cert "$security_dir/prometheus/tls.crt" \
--key "$security_dir/prometheus/tls.key"
expect_failure "metrics mTLS without client certificate" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
"https://$metrics_address/metrics"
expect_failure "metrics mTLS with load-balancer identity" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/load-balancer/tls.crt" \
--key "$security_dir/load-balancer/tls.key" \
"https://$metrics_address/metrics"
expect_failure "metrics mTLS with client certificate from foreign CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/foreign-client/tls.crt" \
--key "$security_dir/foreign-client/tls.key" \
"https://$metrics_address/metrics"
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/prometheus/tls.crt" \
--key "$security_dir/prometheus/tls.key" \
"https://$metrics_address/metrics" >/dev/null
echo "PASS: metrics mTLS with Prometheus identity"