#!/bin/sh set -eu mode=${1:-short} env_file=${ENV_FILE:-lab7.env.example} base_url=${BASE_URL:-https://localhost:8443} ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} evidence_root=${EVIDENCE_DIR:-evidence/runtime} require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} case "$mode" in short) incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75} load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60} requests=${SHORT_WORKER_REQUESTS:-300} default_alert=AttachmentWorkerUnavailable ;; long) incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180} load_seconds=${LONG_WORKER_LOAD_SECONDS:-120} requests=${LONG_WORKER_REQUESTS:-1000} default_alert=AttachmentQueueBacklogGrowing ;; *) echo "usage: $0 [short|long]" >&2 exit 2 ;; esac worker_alert_name=${WORKER_ALERT_NAME:-$default_alert} run_id=$(date -u +%Y%m%dT%H%M%SZ) output_dir="$evidence_root/worker-$mode-$run_id" mkdir -p "$output_dir" load_pid= compose() { docker compose --env-file "$env_file" "$@" } restore() { if [ -n "$load_pid" ]; then kill "$load_pid" >/dev/null 2>&1 || true fi compose up -d worker >/dev/null 2>&1 || true } trap restore EXIT INT TERM python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose stop worker >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ --ca-file "$ca_file" \ --mode attachments \ --duration "$load_seconds" \ --requests "$requests" \ --concurrency 4 \ --rate "${WORKER_LOAD_RATE:-5}" \ --wait-ready \ --ready-timeout 480 \ --output "$output_dir/traffic.json" >/dev/null & load_pid=$! sleep "$incident_seconds" if [ "$require_alerts" = "1" ]; then python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null fi python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose up -d --wait --wait-timeout 90 worker >/dev/null wait "$load_pid" load_pid= if [ "$require_alerts" = "1" ]; then python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null if [ "$mode" = "long" ]; then python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null fi fi sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY' import datetime import json import pathlib import sys output, mode, started, recovery_started, recovered, duration = sys.argv[1:] parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) report = { "schema_version": 1, "incident": "worker_unavailable", "profile": mode, "started_at": started, "recovery_started_at": recovery_started, "recovered_at": recovered, "planned_injection_seconds": int(duration), "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"], } if mode == "long": report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"]) pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") PY trap - EXIT INT TERM printf '%s\n' "$output_dir"