121 lines
5.8 KiB
PowerShell
121 lines
5.8 KiB
PowerShell
param(
|
|
[int]$DurationSeconds = 165,
|
|
[string]$BaseUrl = "http://localhost:8080",
|
|
[string]$EnvFile = ".env"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$project = Split-Path -Parent $PSScriptRoot
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$evidenceDir = Join-Path $project "evidence\lab6-$stamp-database-stop"
|
|
New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null
|
|
|
|
Push-Location $project
|
|
try {
|
|
docker compose --env-file $EnvFile stop postgres
|
|
$samples = @()
|
|
$deadline = (Get-Date).AddSeconds($DurationSeconds)
|
|
while ((Get-Date) -lt $deadline) {
|
|
$started = Get-Date
|
|
try {
|
|
$response = Invoke-WebRequest -Uri "$BaseUrl/health/ready" `
|
|
-SkipHttpErrorCheck -TimeoutSec 5
|
|
$readyStatus = [int]$response.StatusCode
|
|
}
|
|
catch {
|
|
$readyStatus = 0
|
|
}
|
|
try {
|
|
$response = Invoke-WebRequest -Uri "$BaseUrl/api/v1/users/me" `
|
|
-Headers @{ Authorization = "Bearer incident-synthetic-token" } `
|
|
-SkipHttpErrorCheck -TimeoutSec 5
|
|
$businessStatus = [int]$response.StatusCode
|
|
}
|
|
catch {
|
|
$businessStatus = 0
|
|
}
|
|
$samples += [ordered]@{
|
|
at = $started.ToUniversalTime().ToString("o")
|
|
ready_status = $readyStatus
|
|
business_status = $businessStatus
|
|
duration_ms = [math]::Round(((Get-Date) - $started).TotalMilliseconds, 1)
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
$samples | ConvertTo-Json -Depth 5 | Set-Content -Encoding utf8 `
|
|
(Join-Path $evidenceDir "http-samples.json")
|
|
|
|
$alertsSnapshot = Invoke-RestMethod "http://localhost:9090/api/v1/alerts"
|
|
$highErrorAlert = @($alertsSnapshot.data.alerts | Where-Object {
|
|
$_.labels.alertname -eq "MessengerApiHighErrorRate" -and $_.state -eq "firing"
|
|
})
|
|
$observations = [ordered]@{
|
|
captured_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
api_5xx = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=sum(rate(http_server_request_duration_seconds_count%7Bhttp_response_status_code%3D~%225..%22%7D%5B2m%5D))"
|
|
alerts = $alertsSnapshot
|
|
alert_firing = ($highErrorAlert.Count -gt 0)
|
|
jdbc_spans_hint = "Grafana Explore -> Tempo: { resource.service.name = `"messenger-api`" && span.db.system.name = `"postgresql`" }"
|
|
correlated_logs_hint = "Grafana Explore -> Loki: {service_name=`"messenger-api`"} | json | level=`"ERROR`""
|
|
}
|
|
$observations | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
|
(Join-Path $evidenceDir "failure-observation.json")
|
|
if ($highErrorAlert.Count -eq 0) {
|
|
throw "MessengerApiHighErrorRate did not reach firing state"
|
|
}
|
|
|
|
$end = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() * 1000000
|
|
$start = [DateTimeOffset]::UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds() * 1000000
|
|
$logQuery = [uri]::EscapeDataString('{service_name="messenger-api"}')
|
|
Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$logQuery&start=$start&end=$end&limit=100&direction=backward" |
|
|
ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "loki-api-logs.json")
|
|
$dbTraceQuery = [uri]::EscapeDataString('{ span.db.system = "postgresql" && status = error }')
|
|
Invoke-RestMethod "http://localhost:3200/api/search?q=$dbTraceQuery&limit=100" |
|
|
ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "tempo-postgres-errors.json")
|
|
$httpTraceQuery = [uri]::EscapeDataString('{ name = "GET /health/ready" && status = error }')
|
|
Invoke-RestMethod "http://localhost:3200/api/search?q=$httpTraceQuery&limit=100" |
|
|
ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "tempo-http-errors.json")
|
|
}
|
|
finally {
|
|
docker compose --env-file $EnvFile start postgres
|
|
try {
|
|
$readyDeadline = (Get-Date).AddMinutes(2)
|
|
do {
|
|
Start-Sleep -Seconds 5
|
|
try {
|
|
$readyResponse = Invoke-WebRequest -Uri "$BaseUrl/health/ready" `
|
|
-SkipHttpErrorCheck -TimeoutSec 10
|
|
$ready = ([int]$readyResponse.StatusCode -eq 200)
|
|
}
|
|
catch {
|
|
$ready = $false
|
|
}
|
|
} while (-not $ready -and (Get-Date) -lt $readyDeadline)
|
|
|
|
$alertDeadline = (Get-Date).AddMinutes(6)
|
|
do {
|
|
Start-Sleep -Seconds 10
|
|
$recoveryAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts"
|
|
$activeHighError = @($recoveryAlerts.data.alerts | Where-Object {
|
|
$_.labels.alertname -eq "MessengerApiHighErrorRate"
|
|
})
|
|
} while ($activeHighError.Count -gt 0 -and (Get-Date) -lt $alertDeadline)
|
|
|
|
[ordered]@{
|
|
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
|
readiness_recovered = $ready
|
|
alerts = $recoveryAlerts
|
|
alert_resolved = ($activeHighError.Count -eq 0)
|
|
api_5xx_ratio = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=%28sum%28rate%28http_server_request_duration_seconds_count%7Bservice_name%3D%22messenger-api%22%2Chttp_response_status_code%3D~%225..%22%2Chttp_route%21~%22%2Fhealth%2F.%2A%22%7D%5B5m%5D%29%29%20or%20vector%280%29%29%20%2F%20clamp_min%28sum%28rate%28http_server_request_duration_seconds_count%7Bservice_name%3D%22messenger-api%22%2Chttp_route%21~%22%2Fhealth%2F.%2A%22%7D%5B5m%5D%29%29%2C%200.001%29"
|
|
} | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
|
(Join-Path $evidenceDir "recovery-observation.json")
|
|
if (-not $ready -or $activeHighError.Count -gt 0) {
|
|
throw "database or MessengerApiHighErrorRate did not recover before the deadline"
|
|
}
|
|
}
|
|
finally {
|
|
Pop-Location
|
|
}
|
|
}
|
|
|
|
Write-Host "Evidence saved to $evidenceDir"
|