ops: add PKI and HTTPS ingress

This commit is contained in:
2026-11-04 23:05:11 +03:00
parent 951e554e06
commit 58c9bb027b
26 changed files with 572 additions and 56 deletions
+4 -2
View File
@@ -2,8 +2,9 @@
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
@@ -33,6 +34,7 @@ trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode messages \
--duration "$((outage_seconds + 20))" \
--concurrency 4 \
+5 -3
View File
@@ -2,8 +2,9 @@
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
@@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop tempo loki >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode mixed \
--duration "$outage_seconds" \
--concurrency 4 \
@@ -41,7 +43,7 @@ load_pid=$!
sleep "$outage_seconds"
wait "$load_pid"
load_pid=
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready")
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
+4 -2
View File
@@ -3,8 +3,9 @@
set -eu
mode=${1:-short}
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
@@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop worker >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode attachments \
--duration "$load_seconds" \
--requests "$requests" \
+9 -2
View File
@@ -7,6 +7,7 @@ import json
import math
from pathlib import Path
import secrets
import ssl
import struct
import threading
import time
@@ -16,6 +17,7 @@ import urllib.request
import zlib
MAX_RESPONSE_BYTES = 1 << 20
TLS_CONTEXT = None
def png_fixture(width=96, height=64):
@@ -37,7 +39,7 @@ def png_fixture(width=96, height=64):
def request(method, url, body=None, headers=None, timeout=10):
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
try:
with urllib.request.urlopen(req, timeout=timeout) as response:
with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response:
payload = response.read(MAX_RESPONSE_BYTES)
return response.status, payload
except urllib.error.HTTPError as error:
@@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout):
def main():
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--base-url", default="https://localhost:8443")
parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt")
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
parser.add_argument("--duration", type=float, default=30)
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
@@ -179,6 +182,10 @@ def main():
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
args = parser.parse_args()
global TLS_CONTEXT
if args.base_url.startswith("https://"):
TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file)
if args.duration <= 0 or args.duration > 1800:
parser.error("duration must be in (0, 1800]")
if args.requests < 0:
+9 -4
View File
@@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
fi
manifest=$1
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"}
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
current_manifest=$deploy_dir/current-images.env
active_containers=$deploy_dir/active-api-containers
previous_containers=$deploy_dir/previous-api-containers
@@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf
upstream_backup=$deploy_dir/upstream.previous.conf
candidate_file=$deploy_dir/candidate-api-containers
replicas=${API_REPLICAS:-2}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
switched=false
nginx_created=false
upstream_changed=false
@@ -52,7 +54,7 @@ write_upstream() {
: > "$output"
for container in $containers; do
name=$(container_name "$container")
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
done
}
@@ -81,7 +83,8 @@ wait_for_containers() {
}
smoke_check() {
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
curl --fail --silent --show-error --noproxy '*' \
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
}
remove_containers() {
@@ -123,6 +126,7 @@ trap 'exit 143' TERM
load_manifest "$manifest"
new_api_image=$API_IMAGE
test -r "$public_ca_file"
if [ -f "$previous_containers" ]; then
remove_containers "$(cat "$previous_containers")"
@@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup"
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
compose run --rm security-bootstrap
compose run --rm --no-deps s3-bootstrap
compose run --rm --no-deps migrate
+8 -6
View File
@@ -11,8 +11,9 @@ case "$profile" in
;;
esac
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
@@ -35,6 +36,7 @@ compose config --quiet
compose up -d --build --wait --wait-timeout 180
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode mixed \
--duration "${BASELINE_LOAD_SECONDS:-15}" \
--requests "${BASELINE_REQUESTS:-120}" \
@@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \
--require >/dev/null
if [ "$profile" = "full" ]; then
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
fi
arguments=
+163
View File
@@ -0,0 +1,163 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security}
public_port=${PUBLIC_TLS_PORT:-8443}
public_host=${PUBLIC_TLS_HOST:-localhost}
api_address=${API_TLS_ADDRESS:-api:8443}
metrics_address=${METRICS_TLS_ADDRESS:-api:9091}
empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$"
cleanup() {
rm -f "$empty_ca"
}
trap cleanup EXIT HUP INT TERM
require_file() {
if [ ! -r "$1" ]; then
echo "missing security material: $1" >&2
exit 1
fi
}
expect_failure() {
name=$1
shift
if "$@" >/dev/null 2>&1; then
echo "FAIL: $name unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: $name"
}
expect_tls_success() {
name=$1
certificate=$2
key=$3
if curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$certificate" --key "$key" \
"https://$api_address/health/ready" >/dev/null 2>&1; then
echo "PASS: $name"
return
fi
echo "FAIL: $name" >&2
exit 1
}
expect_tls_failure() {
name=$1
shift
if curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
"$@" "https://$api_address/health/ready" >/dev/null 2>&1; then
echo "FAIL: $name unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: $name"
}
expect_public_hostname_failure() {
if openssl s_client \
-connect "$public_host:$public_port" \
-servername wrong.local \
-verify_hostname wrong.local \
-verify_return_error \
-CAfile "$security_dir/public-ca/ca.crt" </dev/null >/dev/null 2>&1; then
echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: public TLS with wrong hostname"
}
for file in \
"$security_dir/public-ca/ca.crt" \
"$security_dir/internal-ca/ca.crt" \
"$security_dir/foreign-ca/ca.crt" \
"$security_dir/load-balancer/tls.crt" \
"$security_dir/load-balancer/tls.key" \
"$security_dir/prometheus/tls.crt" \
"$security_dir/prometheus/tls.key" \
"$security_dir/forbidden-client/tls.crt" \
"$security_dir/forbidden-client/tls.key" \
"$security_dir/foreign-client/tls.crt" \
"$security_dir/foreign-client/tls.key"; do
require_file "$file"
done
: >"$empty_ca"
chmod 0600 "$empty_ca"
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/public-ca/ca.crt" \
"https://$public_host:$public_port/health/ready" >/dev/null
echo "PASS: public TLS with trusted CA and hostname"
expect_failure "public TLS with empty CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$empty_ca" \
"https://$public_host:$public_port/health/ready"
expect_failure "public TLS with foreign CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/foreign-ca/ca.crt" \
"https://$public_host:$public_port/health/ready"
expect_public_hostname_failure
expect_tls_failure "API mTLS without client certificate"
expect_tls_failure "API mTLS with forbidden trusted identity" \
--cert "$security_dir/forbidden-client/tls.crt" \
--key "$security_dir/forbidden-client/tls.key"
expect_tls_failure "API mTLS with client certificate from foreign CA" \
--cert "$security_dir/foreign-client/tls.crt" \
--key "$security_dir/foreign-client/tls.key"
expect_tls_success "API mTLS with load-balancer identity" \
"$security_dir/load-balancer/tls.crt" \
"$security_dir/load-balancer/tls.key"
expect_tls_failure "business API with Prometheus identity" \
--cert "$security_dir/prometheus/tls.crt" \
--key "$security_dir/prometheus/tls.key"
expect_failure "metrics mTLS without client certificate" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
"https://$metrics_address/metrics"
expect_failure "metrics mTLS with load-balancer identity" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/load-balancer/tls.crt" \
--key "$security_dir/load-balancer/tls.key" \
"https://$metrics_address/metrics"
expect_failure "metrics mTLS with client certificate from foreign CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/foreign-client/tls.crt" \
--key "$security_dir/foreign-client/tls.key" \
"https://$metrics_address/metrics"
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/prometheus/tls.crt" \
--key "$security_dir/prometheus/tls.key" \
"https://$metrics_address/metrics" >/dev/null
echo "PASS: metrics mTLS with Prometheus identity"