172 lines
8.2 KiB
Python
172 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url)
|
|
alice, _, _, chat = client.bootstrap()
|
|
|
|
def trace_for(request_id, matches):
|
|
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
|
|
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
|
|
deadline = time.monotonic() + 35
|
|
while time.monotonic() < deadline:
|
|
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
|
|
logs = json.load(response)["data"]["result"]
|
|
if logs:
|
|
trace_id = logs[0]["stream"]["trace_id"]
|
|
try:
|
|
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
|
|
trace = json.load(response)
|
|
except urllib.error.HTTPError as error:
|
|
if error.code != 404:
|
|
raise
|
|
else:
|
|
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
|
if matches(spans):
|
|
return trace_id
|
|
time.sleep(2)
|
|
raise Failure("expected trace was not found")
|
|
|
|
def alerts(name):
|
|
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
|
|
with urllib.request.urlopen(endpoint, timeout=5) as response:
|
|
active = json.load(response)["data"]["alerts"]
|
|
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
|
|
|
|
def wait_alert(name, firing, exercise=lambda: None):
|
|
deadline = time.monotonic() + 100
|
|
while time.monotonic() < deadline:
|
|
exercise()
|
|
if alerts(name) == firing:
|
|
print(name, "firing" if firing else "resolved", flush=True)
|
|
return
|
|
time.sleep(2)
|
|
raise Failure(name + ": expected alert state not observed")
|
|
|
|
if args.incident == "worker":
|
|
compose("stop", "worker")
|
|
try:
|
|
attachment, _ = client.attachment(alice, chat)
|
|
wait_alert("MessengerQueueLag", True)
|
|
finally:
|
|
compose("start", "worker")
|
|
client.wait_ready(alice, attachment)
|
|
wait_alert("MessengerQueueLag", False)
|
|
elif args.incident == "database":
|
|
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
|
|
marker = "lab6-fault-" + uuid.uuid4().hex
|
|
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
|
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
|
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
|
|
"sh", marker]
|
|
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
|
|
try:
|
|
time.sleep(2)
|
|
|
|
request_ids = []
|
|
|
|
def exercise():
|
|
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
|
|
request_ids.append(response["headers"]["X-Request-Id"])
|
|
|
|
wait_alert("MessengerHighErrorRate", True, exercise)
|
|
finally:
|
|
# Terminate only the synthetic lock session so the transaction rolls back immediately.
|
|
compose("exec", "-T", "db", "sh", "-c",
|
|
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
|
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
|
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
|
|
lock.wait(timeout=10)
|
|
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
|
|
trace_id = trace_for(request_ids[-1], lambda spans: any(
|
|
span["name"] == "postgresql query"
|
|
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
|
|
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
|
|
wait_alert("MessengerHighErrorRate", False)
|
|
elif args.incident == "retry":
|
|
compose("stop", "worker")
|
|
try:
|
|
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
|
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
|
headers={"Content-Type": "image/png"}, schema="Attachment")
|
|
attachment = upload["json"]
|
|
compose("stop", "s3")
|
|
try:
|
|
compose("up", "-d", "--no-deps", "worker")
|
|
deadline = time.monotonic() + 25
|
|
while time.monotonic() < deadline:
|
|
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
|
|
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
|
|
output = subprocess.check_output(
|
|
["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
|
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
|
|
cwd=ROOT, text=True, timeout=10)
|
|
if int(output.strip()) > 0:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise Failure("durable retry was not observed")
|
|
finally:
|
|
compose("up", "-d", "--wait", "s3")
|
|
client.wait_ready(alice, attachment)
|
|
def retried(spans):
|
|
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
|
|
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
|
|
return len(set(attempts)) >= 2
|
|
|
|
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
|
|
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
|
|
finally:
|
|
compose("start", "worker")
|
|
else:
|
|
since = str(int(time.time()))
|
|
compose("stop", "tempo")
|
|
try:
|
|
for _ in range(8):
|
|
client.message(alice, chat, "telemetry-backend-off")
|
|
time.sleep(5)
|
|
attachment, _ = client.attachment(alice, chat)
|
|
client.wait_ready(alice, attachment)
|
|
compose("stats", "--no-stream", "otel-collector")
|
|
print("API и worker работали при остановленном Tempo.", flush=True)
|
|
finally:
|
|
compose("start", "tempo")
|
|
output = subprocess.check_output(
|
|
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
|
|
cwd=ROOT, text=True, timeout=15)
|
|
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
|
|
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
|
|
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
|
|
print("FAIL:", error, file=sys.stderr)
|
|
sys.exit(1)
|