исправлены тесты и сценарии отказов

This commit is contained in:
2026-11-17 15:51:21 +03:00
parent baee48072c
commit bdcf40aad5
14 changed files with 280 additions and 22 deletions
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/env python3
"""Lab6 failure scenarios. Prints statuses, ids and metric values. Never tokens."""
import base64
import json
import os
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PROM = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
def compose(*args: str) -> None:
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
if os.environ.get("COMPOSE_ALTPORTS") == "1":
command.extend(["-f", "compose.altports.yaml"])
subprocess.run([*command, *args], cwd=ROOT, check=True)
class Api:
def __init__(self) -> None:
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None, timeout=15):
hdrs = dict(headers or {})
data = raw
if body is not None:
data = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if content_type:
hdrs["Content-Type"] = content_type
if token:
hdrs["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
try:
response = self.opener.open(request, timeout=timeout)
except urllib.error.HTTPError as error:
response = error
with response:
payload = response.read()
return response.status, payload
def expect(status, got, label):
if got != status:
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
print("PASS", label, got)
def prom_query(expr: str):
url = PROM + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
with urllib.request.urlopen(url, timeout=10) as response:
body = json.loads(response.read().decode())
if body.get("status") != "success":
raise SystemExit("FAIL prometheus query")
result = body["data"]["result"]
if not result:
return None
return float(result[0]["value"][1])
def alerts():
with urllib.request.urlopen(PROM + "/api/v1/alerts", timeout=10) as response:
body = json.loads(response.read().decode())
names = []
for item in body["data"]["alerts"]:
if item.get("state") == "firing":
names.append(item["labels"].get("alertname", ""))
return sorted(names)
def wait_ready_http():
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
status, _ = Api().call("GET", "/health/ready", timeout=3)
if status == 200:
return
except Exception:
pass
time.sleep(1)
raise SystemExit("FAIL API did not become ready")
def user(api: Api):
username = "l6_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
status, _ = api.call(
"POST",
"/api/v1/users",
{"username": username, "display_name": "Lab6", "password": password},
)
expect(201, status, "register")
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, payload = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
expect(201, status, "login")
return json.loads(payload)["token"]
def main() -> None:
api = Api()
token = user(api)
status, chat_raw = api.call(
"POST",
"/api/v1/chats",
{"title": "lab6-incidents", "member_ids": []},
token=token,
)
# member_ids must include someone? empty may 400. Create with only self via API rules.
if status != 201:
me_status, me_raw = api.call("GET", "/api/v1/users/me", token=token)
expect(200, me_status, "me")
me = json.loads(me_raw)["id"]
status, chat_raw = api.call(
"POST",
"/api/v1/chats",
{"title": "lab6-incidents", "member_ids": [me]},
token=token,
)
expect(201, status, "chat")
chat_id = json.loads(chat_raw)["id"]
print("chat", chat_id)
print("-- worker stopped: queue age grows, content stays 409 --")
compose("stop", "worker")
content = FIXTURE.read_bytes()
status, attachment_raw = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
token=token,
raw=content,
content_type="image/png",
)
expect(202, status, "upload while worker stopped")
attachment_id = json.loads(attachment_raw)["id"]
print("attachment", attachment_id)
status, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment_id, token=token)
expect(409, status, "content before ready")
deadline = time.monotonic() + 80
age = None
while time.monotonic() < deadline:
age = prom_query("max(messenger_queue_oldest_age_seconds)")
print("queue_oldest_age_seconds", age, "firing", ",".join(alerts()) or "-")
if age is not None and age >= 15:
break
time.sleep(5)
if age is None or age < 5:
raise SystemExit("FAIL queue age did not grow")
compose("start", "worker")
ready_deadline = time.monotonic() + 90
while time.monotonic() < ready_deadline:
status, body = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll")
parsed = json.loads(body)
if parsed["status"] == "ready":
print("PASS attachment ready after worker start", attachment_id)
break
if parsed["status"] == "failed":
raise SystemExit("FAIL attachment failed")
time.sleep(0.5)
else:
raise SystemExit("FAIL attachment did not become ready")
time.sleep(12)
print("queue_oldest_age_seconds_after", prom_query("max(messenger_queue_oldest_age_seconds)"))
print("-- postgres stopped: live stays 200, ready becomes 503 --")
compose("stop", "postgres")
time.sleep(2)
status, _ = api.call("GET", "/health/live")
expect(200, status, "live while postgres stopped")
status, _ = api.call("GET", "/health/ready")
expect(503, status, "ready while postgres stopped")
burst_503 = 0
for _ in range(8):
status, _ = api.call(
"POST",
"/api/v1/users",
{
"username": "l6d_" + uuid.uuid4().hex[:8],
"display_name": "DbDown",
"password": "T3st-" + uuid.uuid4().hex,
},
)
if status == 503:
burst_503 += 1
else:
print("unexpected status during postgres stop", status)
if burst_503 < 8:
raise SystemExit("FAIL expected every business write to be 503")
print("PASS business 503 while postgres stopped", burst_503)
compose("start", "postgres")
wait_ready_http()
print("PASS ready after postgres start")
print("-- collector stopped: API still accepts a registration --")
compose("stop", "otel-collector")
time.sleep(1)
status, _ = api.call(
"POST",
"/api/v1/users",
{
"username": "l6c_" + uuid.uuid4().hex[:10],
"display_name": "CollectorDown",
"password": "T3st-" + uuid.uuid4().hex,
},
)
expect(201, status, "register while collector is down")
compose("start", "otel-collector")
print("PASS collector restarted")
print("firing_at_end", ",".join(alerts()) or "-")
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0\.."
docker compose --env-file .env restart postgres redis s3 rabbitmq api worker
exit /b %ERRORLEVEL%
+3
View File
@@ -0,0 +1,3 @@
@echo off
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1"
exit /b %ERRORLEVEL%
+23
View File
@@ -0,0 +1,23 @@
$ErrorActionPreference = "Stop"
if (-not $env:TARGET_INSTANCE_ID) {
Write-Error "TARGET_INSTANCE_ID is required"
exit 1
}
Set-Location (Join-Path $PSScriptRoot "..")
$ids = docker compose --env-file .env ps -q api
$found = $false
foreach ($id in $ids) {
if (-not $id) { continue }
$iid = ""
try { $iid = (docker exec $id cat /tmp/instance-id).Trim() } catch { $iid = "" }
$hostName = (docker inspect -f "{{.Config.Hostname}}" $id).Trim()
if ($iid -eq $env:TARGET_INSTANCE_ID -or $hostName -eq $env:TARGET_INSTANCE_ID) {
docker stop -t 5 $id | Out-Null
$found = $true
break
}
}
if (-not $found) {
Write-Error "no running API replica matched TARGET_INSTANCE_ID"
exit 1
}