forked from mtusi2026/mtusi2026lab7
ops: add PKI and HTTPS ingress
This commit is contained in:
@@ -2,8 +2,9 @@
|
||||
|
||||
set -eu
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
|
||||
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
|
||||
@@ -33,6 +34,7 @@ trap restore EXIT INT TERM
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode messages \
|
||||
--duration "$((outage_seconds + 20))" \
|
||||
--concurrency 4 \
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
set -eu
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
|
||||
@@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop tempo loki >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode mixed \
|
||||
--duration "$outage_seconds" \
|
||||
--concurrency 4 \
|
||||
@@ -41,7 +43,7 @@ load_pid=$!
|
||||
sleep "$outage_seconds"
|
||||
wait "$load_pid"
|
||||
load_pid=
|
||||
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
|
||||
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready")
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
|
||||
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
set -eu
|
||||
|
||||
mode=${1:-short}
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
|
||||
@@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop worker >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode attachments \
|
||||
--duration "$load_seconds" \
|
||||
--requests "$requests" \
|
||||
|
||||
@@ -7,6 +7,7 @@ import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import ssl
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
@@ -16,6 +17,7 @@ import urllib.request
|
||||
import zlib
|
||||
|
||||
MAX_RESPONSE_BYTES = 1 << 20
|
||||
TLS_CONTEXT = None
|
||||
|
||||
|
||||
def png_fixture(width=96, height=64):
|
||||
@@ -37,7 +39,7 @@ def png_fixture(width=96, height=64):
|
||||
def request(method, url, body=None, headers=None, timeout=10):
|
||||
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response:
|
||||
payload = response.read(MAX_RESPONSE_BYTES)
|
||||
return response.status, payload
|
||||
except urllib.error.HTTPError as error:
|
||||
@@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout):
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--base-url", default="https://localhost:8443")
|
||||
parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt")
|
||||
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
|
||||
parser.add_argument("--duration", type=float, default=30)
|
||||
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
|
||||
@@ -179,6 +182,10 @@ def main():
|
||||
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
|
||||
args = parser.parse_args()
|
||||
|
||||
global TLS_CONTEXT
|
||||
if args.base_url.startswith("https://"):
|
||||
TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file)
|
||||
|
||||
if args.duration <= 0 or args.duration > 1800:
|
||||
parser.error("duration must be in (0, 1800]")
|
||||
if args.requests < 0:
|
||||
|
||||
@@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
|
||||
fi
|
||||
|
||||
manifest=$1
|
||||
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"}
|
||||
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
|
||||
current_manifest=$deploy_dir/current-images.env
|
||||
active_containers=$deploy_dir/active-api-containers
|
||||
previous_containers=$deploy_dir/previous-api-containers
|
||||
@@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf
|
||||
upstream_backup=$deploy_dir/upstream.previous.conf
|
||||
candidate_file=$deploy_dir/candidate-api-containers
|
||||
replicas=${API_REPLICAS:-2}
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5}
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
|
||||
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
|
||||
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
switched=false
|
||||
nginx_created=false
|
||||
upstream_changed=false
|
||||
@@ -52,7 +54,7 @@ write_upstream() {
|
||||
: > "$output"
|
||||
for container in $containers; do
|
||||
name=$(container_name "$container")
|
||||
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -81,7 +83,8 @@ wait_for_containers() {
|
||||
}
|
||||
|
||||
smoke_check() {
|
||||
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
|
||||
}
|
||||
|
||||
remove_containers() {
|
||||
@@ -123,6 +126,7 @@ trap 'exit 143' TERM
|
||||
|
||||
load_manifest "$manifest"
|
||||
new_api_image=$API_IMAGE
|
||||
test -r "$public_ca_file"
|
||||
|
||||
if [ -f "$previous_containers" ]; then
|
||||
remove_containers "$(cat "$previous_containers")"
|
||||
@@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup"
|
||||
|
||||
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
|
||||
compose run --rm security-bootstrap
|
||||
compose run --rm --no-deps s3-bootstrap
|
||||
compose run --rm --no-deps migrate
|
||||
|
||||
|
||||
@@ -11,8 +11,9 @@ case "$profile" in
|
||||
;;
|
||||
esac
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
|
||||
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
|
||||
@@ -35,6 +36,7 @@ compose config --quiet
|
||||
compose up -d --build --wait --wait-timeout 180
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode mixed \
|
||||
--duration "${BASELINE_LOAD_SECONDS:-15}" \
|
||||
--requests "${BASELINE_REQUESTS:-120}" \
|
||||
@@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \
|
||||
--require >/dev/null
|
||||
|
||||
if [ "$profile" = "full" ]; then
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
|
||||
fi
|
||||
|
||||
arguments=
|
||||
|
||||
Executable
+163
@@ -0,0 +1,163 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security}
|
||||
public_port=${PUBLIC_TLS_PORT:-8443}
|
||||
public_host=${PUBLIC_TLS_HOST:-localhost}
|
||||
api_address=${API_TLS_ADDRESS:-api:8443}
|
||||
metrics_address=${METRICS_TLS_ADDRESS:-api:9091}
|
||||
empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$"
|
||||
|
||||
cleanup() {
|
||||
rm -f "$empty_ca"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
require_file() {
|
||||
if [ ! -r "$1" ]; then
|
||||
echo "missing security material: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
name=$1
|
||||
shift
|
||||
|
||||
if "$@" >/dev/null 2>&1; then
|
||||
echo "FAIL: $name unexpectedly succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: $name"
|
||||
}
|
||||
|
||||
expect_tls_success() {
|
||||
name=$1
|
||||
certificate=$2
|
||||
key=$3
|
||||
|
||||
if curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$certificate" --key "$key" \
|
||||
"https://$api_address/health/ready" >/dev/null 2>&1; then
|
||||
echo "PASS: $name"
|
||||
return
|
||||
fi
|
||||
echo "FAIL: $name" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
expect_tls_failure() {
|
||||
name=$1
|
||||
shift
|
||||
|
||||
if curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
"$@" "https://$api_address/health/ready" >/dev/null 2>&1; then
|
||||
echo "FAIL: $name unexpectedly succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: $name"
|
||||
}
|
||||
|
||||
expect_public_hostname_failure() {
|
||||
if openssl s_client \
|
||||
-connect "$public_host:$public_port" \
|
||||
-servername wrong.local \
|
||||
-verify_hostname wrong.local \
|
||||
-verify_return_error \
|
||||
-CAfile "$security_dir/public-ca/ca.crt" </dev/null >/dev/null 2>&1; then
|
||||
echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: public TLS with wrong hostname"
|
||||
}
|
||||
|
||||
for file in \
|
||||
"$security_dir/public-ca/ca.crt" \
|
||||
"$security_dir/internal-ca/ca.crt" \
|
||||
"$security_dir/foreign-ca/ca.crt" \
|
||||
"$security_dir/load-balancer/tls.crt" \
|
||||
"$security_dir/load-balancer/tls.key" \
|
||||
"$security_dir/prometheus/tls.crt" \
|
||||
"$security_dir/prometheus/tls.key" \
|
||||
"$security_dir/forbidden-client/tls.crt" \
|
||||
"$security_dir/forbidden-client/tls.key" \
|
||||
"$security_dir/foreign-client/tls.crt" \
|
||||
"$security_dir/foreign-client/tls.key"; do
|
||||
require_file "$file"
|
||||
done
|
||||
|
||||
: >"$empty_ca"
|
||||
chmod 0600 "$empty_ca"
|
||||
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/public-ca/ca.crt" \
|
||||
"https://$public_host:$public_port/health/ready" >/dev/null
|
||||
echo "PASS: public TLS with trusted CA and hostname"
|
||||
|
||||
expect_failure "public TLS with empty CA" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$empty_ca" \
|
||||
"https://$public_host:$public_port/health/ready"
|
||||
|
||||
expect_failure "public TLS with foreign CA" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/foreign-ca/ca.crt" \
|
||||
"https://$public_host:$public_port/health/ready"
|
||||
|
||||
expect_public_hostname_failure
|
||||
|
||||
expect_tls_failure "API mTLS without client certificate"
|
||||
expect_tls_failure "API mTLS with forbidden trusted identity" \
|
||||
--cert "$security_dir/forbidden-client/tls.crt" \
|
||||
--key "$security_dir/forbidden-client/tls.key"
|
||||
expect_tls_failure "API mTLS with client certificate from foreign CA" \
|
||||
--cert "$security_dir/foreign-client/tls.crt" \
|
||||
--key "$security_dir/foreign-client/tls.key"
|
||||
expect_tls_success "API mTLS with load-balancer identity" \
|
||||
"$security_dir/load-balancer/tls.crt" \
|
||||
"$security_dir/load-balancer/tls.key"
|
||||
|
||||
expect_tls_failure "business API with Prometheus identity" \
|
||||
--cert "$security_dir/prometheus/tls.crt" \
|
||||
--key "$security_dir/prometheus/tls.key"
|
||||
|
||||
expect_failure "metrics mTLS without client certificate" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
"https://$metrics_address/metrics"
|
||||
|
||||
expect_failure "metrics mTLS with load-balancer identity" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$security_dir/load-balancer/tls.crt" \
|
||||
--key "$security_dir/load-balancer/tls.key" \
|
||||
"https://$metrics_address/metrics"
|
||||
|
||||
expect_failure "metrics mTLS with client certificate from foreign CA" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$security_dir/foreign-client/tls.crt" \
|
||||
--key "$security_dir/foreign-client/tls.key" \
|
||||
"https://$metrics_address/metrics"
|
||||
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$security_dir/prometheus/tls.crt" \
|
||||
--key "$security_dir/prometheus/tls.key" \
|
||||
"https://$metrics_address/metrics" >/dev/null
|
||||
echo "PASS: metrics mTLS with Prometheus identity"
|
||||
Reference in New Issue
Block a user