forked from mtusi2026/mtusi2026lab6
исправлены тесты и сценарии отказов
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab6 failure scenarios. Prints statuses, ids and metric values. Never tokens."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
PROM = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
|
||||
|
||||
def compose(*args: str) -> None:
|
||||
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
|
||||
if os.environ.get("COMPOSE_ALTPORTS") == "1":
|
||||
command.extend(["-f", "compose.altports.yaml"])
|
||||
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None, timeout=15):
|
||||
hdrs = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
hdrs["Content-Type"] = content_type
|
||||
if token:
|
||||
hdrs["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
return response.status, payload
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
def prom_query(expr: str):
|
||||
url = PROM + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
body = json.loads(response.read().decode())
|
||||
if body.get("status") != "success":
|
||||
raise SystemExit("FAIL prometheus query")
|
||||
result = body["data"]["result"]
|
||||
if not result:
|
||||
return None
|
||||
return float(result[0]["value"][1])
|
||||
|
||||
|
||||
def alerts():
|
||||
with urllib.request.urlopen(PROM + "/api/v1/alerts", timeout=10) as response:
|
||||
body = json.loads(response.read().decode())
|
||||
names = []
|
||||
for item in body["data"]["alerts"]:
|
||||
if item.get("state") == "firing":
|
||||
names.append(item["labels"].get("alertname", ""))
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def wait_ready_http():
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, _ = Api().call("GET", "/health/ready", timeout=3)
|
||||
if status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("FAIL API did not become ready")
|
||||
|
||||
|
||||
def user(api: Api):
|
||||
username = "l6_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Lab6", "password": password},
|
||||
)
|
||||
expect(201, status, "register")
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, payload = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
expect(201, status, "login")
|
||||
return json.loads(payload)["token"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api = Api()
|
||||
token = user(api)
|
||||
status, chat_raw = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "lab6-incidents", "member_ids": []},
|
||||
token=token,
|
||||
)
|
||||
# member_ids must include someone? empty may 400. Create with only self via API rules.
|
||||
if status != 201:
|
||||
me_status, me_raw = api.call("GET", "/api/v1/users/me", token=token)
|
||||
expect(200, me_status, "me")
|
||||
me = json.loads(me_raw)["id"]
|
||||
status, chat_raw = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "lab6-incidents", "member_ids": [me]},
|
||||
token=token,
|
||||
)
|
||||
expect(201, status, "chat")
|
||||
chat_id = json.loads(chat_raw)["id"]
|
||||
print("chat", chat_id)
|
||||
|
||||
print("-- worker stopped: queue age grows, content stays 409 --")
|
||||
compose("stop", "worker")
|
||||
content = FIXTURE.read_bytes()
|
||||
status, attachment_raw = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
|
||||
token=token,
|
||||
raw=content,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "upload while worker stopped")
|
||||
attachment_id = json.loads(attachment_raw)["id"]
|
||||
print("attachment", attachment_id)
|
||||
status, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment_id, token=token)
|
||||
expect(409, status, "content before ready")
|
||||
deadline = time.monotonic() + 80
|
||||
age = None
|
||||
while time.monotonic() < deadline:
|
||||
age = prom_query("max(messenger_queue_oldest_age_seconds)")
|
||||
print("queue_oldest_age_seconds", age, "firing", ",".join(alerts()) or "-")
|
||||
if age is not None and age >= 15:
|
||||
break
|
||||
time.sleep(5)
|
||||
if age is None or age < 5:
|
||||
raise SystemExit("FAIL queue age did not grow")
|
||||
compose("start", "worker")
|
||||
ready_deadline = time.monotonic() + 90
|
||||
while time.monotonic() < ready_deadline:
|
||||
status, body = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll")
|
||||
parsed = json.loads(body)
|
||||
if parsed["status"] == "ready":
|
||||
print("PASS attachment ready after worker start", attachment_id)
|
||||
break
|
||||
if parsed["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed")
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise SystemExit("FAIL attachment did not become ready")
|
||||
time.sleep(12)
|
||||
print("queue_oldest_age_seconds_after", prom_query("max(messenger_queue_oldest_age_seconds)"))
|
||||
|
||||
print("-- postgres stopped: live stays 200, ready becomes 503 --")
|
||||
compose("stop", "postgres")
|
||||
time.sleep(2)
|
||||
status, _ = api.call("GET", "/health/live")
|
||||
expect(200, status, "live while postgres stopped")
|
||||
status, _ = api.call("GET", "/health/ready")
|
||||
expect(503, status, "ready while postgres stopped")
|
||||
burst_503 = 0
|
||||
for _ in range(8):
|
||||
status, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{
|
||||
"username": "l6d_" + uuid.uuid4().hex[:8],
|
||||
"display_name": "DbDown",
|
||||
"password": "T3st-" + uuid.uuid4().hex,
|
||||
},
|
||||
)
|
||||
if status == 503:
|
||||
burst_503 += 1
|
||||
else:
|
||||
print("unexpected status during postgres stop", status)
|
||||
if burst_503 < 8:
|
||||
raise SystemExit("FAIL expected every business write to be 503")
|
||||
print("PASS business 503 while postgres stopped", burst_503)
|
||||
compose("start", "postgres")
|
||||
wait_ready_http()
|
||||
print("PASS ready after postgres start")
|
||||
|
||||
print("-- collector stopped: API still accepts a registration --")
|
||||
compose("stop", "otel-collector")
|
||||
time.sleep(1)
|
||||
status, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{
|
||||
"username": "l6c_" + uuid.uuid4().hex[:10],
|
||||
"display_name": "CollectorDown",
|
||||
"password": "T3st-" + uuid.uuid4().hex,
|
||||
},
|
||||
)
|
||||
expect(201, status, "register while collector is down")
|
||||
compose("start", "otel-collector")
|
||||
print("PASS collector restarted")
|
||||
print("firing_at_end", ",".join(alerts()) or "-")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0\.."
|
||||
docker compose --env-file .env restart postgres redis s3 rabbitmq api worker
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,23 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
if (-not $env:TARGET_INSTANCE_ID) {
|
||||
Write-Error "TARGET_INSTANCE_ID is required"
|
||||
exit 1
|
||||
}
|
||||
Set-Location (Join-Path $PSScriptRoot "..")
|
||||
$ids = docker compose --env-file .env ps -q api
|
||||
$found = $false
|
||||
foreach ($id in $ids) {
|
||||
if (-not $id) { continue }
|
||||
$iid = ""
|
||||
try { $iid = (docker exec $id cat /tmp/instance-id).Trim() } catch { $iid = "" }
|
||||
$hostName = (docker inspect -f "{{.Config.Hostname}}" $id).Trim()
|
||||
if ($iid -eq $env:TARGET_INSTANCE_ID -or $hostName -eq $env:TARGET_INSTANCE_ID) {
|
||||
docker stop -t 5 $id | Out-Null
|
||||
$found = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $found) {
|
||||
Write-Error "no running API replica matched TARGET_INSTANCE_ID"
|
||||
exit 1
|
||||
}
|
||||
Reference in New Issue
Block a user