Files
mtusi2026lab6/scripts/check_observability.py
T
2026-11-22 19:55:50 +03:00

729 lines
26 KiB
Python

"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
В evidence сохраняются только агрегаты и выбранные технические поля.
"""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import math
import subprocess
import threading
from datetime import datetime, timezone
import json
from pathlib import Path
import time
import urllib.parse
import urllib.request
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, check
ROOT = Path(__file__).resolve().parents[1]
def eventually(operation, message, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
value = operation()
if value:
return value
except (OSError, ValueError, KeyError, AssertionError):
pass
time.sleep(1)
raise AssertionError(message)
class Grafana:
def __init__(self, base):
self.base = base.rstrip("/")
secret = (
(ROOT / "secrets/grafana_admin_password")
.read_text(encoding="utf-8")
.strip()
)
self.authorization = (
"Basic " + base64.b64encode(("admin:" + secret).encode()).decode()
)
def get(self, path, **params):
url = self.base + path
if params:
url += "?" + urllib.parse.urlencode(params)
request = urllib.request.Request(
url,
headers={"Authorization": self.authorization, "Accept": "application/json"},
)
with urllib.request.urlopen(request, timeout=10) as response:
return json.load(response)
def proxy(self, source, path, **params):
return self.get("/api/datasources/proxy/uid/" + source + path, **params)
def query(self, expression):
value = self.proxy("prometheus", "/api/v1/query", query=expression)
check(value["status"] == "success", "Prometheus query failed")
return value["data"]["result"]
def logs(self, selector, start):
value = self.proxy(
"loki",
"/loki/api/v1/query_range",
query=selector,
start=start,
end=time.time_ns(),
limit=1000,
direction="forward",
)
check(value["status"] == "success", "Loki query failed")
return [
json.loads(line)
for stream in value["data"]["result"]
for _, line, *_ in stream["values"]
]
def trace(self, trace_id):
return self.proxy("tempo", "/api/traces/" + trace_id)
def identifier_hex(value, byte_length):
if len(value) == byte_length * 2 and all(
c in "0123456789abcdefABCDEF" for c in value
):
return value.lower()
return base64.b64decode(value, validate=True).hex()
def spans_in(value):
found = []
if isinstance(value, dict):
for key, item in value.items():
if key == "spans":
found.extend(item)
else:
found.extend(spans_in(item))
elif isinstance(value, list):
for item in value:
found.extend(spans_in(item))
return found
def correlation(
grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None
):
selector = '{service_name="' + service + '"} | json'
if service == "messenger-api":
selector += ' | event="http.request"'
if trace_id:
selector += ' | trace_id="' + trace_id + '"'
if attachment_id:
selector += ' | attachment_id="' + attachment_id + '"'
if request_id:
selector += ' | request_id="' + request_id + '"'
records = eventually(
lambda: grafana.logs(selector, start), "No correlated log for " + service
)
record = records[0]
trace_id = record["trace_id"]
def matching_trace():
candidate = grafana.trace(trace_id)
return (
candidate
if any(
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
for span in spans_in(candidate)
)
else None
)
trace = eventually(matching_trace, "Matching span is missing in Tempo")
spans = [
span
for span in spans_in(trace)
if identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
]
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
check(
any(
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
for span in spans
),
"Log span_id not found in real trace",
)
check(
all(
int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"])
for span in spans
),
"Trace has no real duration",
)
return {
"trace_id": trace_id,
"span_id": record["span_id"],
"log": record,
"spans": [
{
key: span.get(key)
for key in (
"name",
"kind",
"spanId",
"startTimeUnixNano",
"endTimeUnixNano",
"status",
)
}
for span in spans
],
}, trace
def run_load(args):
check(
1 <= args.duration <= 300
and 1 <= args.rate <= 40
and 1 <= args.concurrency <= 16,
"Load profile outside bounds",
)
client = Client(args.base_url)
actor, _, _, chat = client.bootstrap()
project = ComposeProject()
container_ids = project.run("ps", "--quiet").splitlines()
samples = []
stop = threading.Event()
def sample():
while not stop.is_set():
result = subprocess.run(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{json .}}",
*container_ids,
],
capture_output=True,
encoding="utf-8",
timeout=15,
)
if result.returncode == 0:
samples.append(
{
"elapsed_seconds": round(time.monotonic() - started, 3),
"containers": [
{
k: row[k]
for k in ("Name", "CPUPerc", "MemUsage", "MemPerc")
}
for row in (
json.loads(line) for line in result.stdout.splitlines()
)
],
}
)
stop.wait(3)
def write(index):
local = Client(args.base_url)
before = time.monotonic()
try:
local.message(actor, chat, "synthetic-observability-" + str(index))
return time.monotonic() - before
except Failure:
return None
started = time.monotonic()
sampler = threading.Thread(target=sample)
sampler.start()
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = []
for index in range(args.duration * args.rate):
time.sleep(max(0, started + index / args.rate - time.monotonic()))
futures.append(pool.submit(write, index))
results = [future.result() for future in futures]
elapsed = time.monotonic() - started
finally:
stop.set()
sampler.join()
latencies = sorted(value for value in results if value is not None)
failed = len(results) - len(latencies)
def percentile(q):
return (
round(latencies[max(0, math.ceil(q * len(latencies)) - 1)] * 1000, 3)
if latencies
else None
)
report = {
"date_utc": datetime.now(timezone.utc).isoformat(),
"workload": "synthetic POST messages; fixed arrival rate; no retries",
"duration_seconds": args.duration,
"elapsed_seconds": round(elapsed, 3),
"arrival_rate": args.rate,
"concurrency_limit": args.concurrency,
"successful_requests": len(latencies),
"failed_requests": failed,
"error_ratio": failed / len(results),
"successful_rps": round(len(latencies) / elapsed, 3),
"successful_latency_ms": {
name: percentile(q)
for name, q in [("p50", 0.5), ("p95", 0.95), ("p99", 0.99)]
},
"dataset": {"users": 3, "chats": 1, "messages": len(latencies)},
"resource_samples": samples,
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.",
}
check(bool(latencies) and failed == 0, "Controlled load contains errors")
return report
def metric(grafana, expression):
rows = grafana.query(expression)
return float(rows[0]["value"][1]) if rows else None
def alert_firing(grafana, name):
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
def trace_chain(grafana, trace_id, *, retry=False):
def inspect():
spans = spans_in(grafana.trace(trace_id))
by_id = {identifier_hex(span["spanId"], 8): span for span in spans}
workers = [
span
for span in spans
if span["name"] == "attachment.process.v1"
and span.get("kind") in ("SPAN_KIND_CONSUMER", 5)
]
for worker in workers:
publisher = by_id.get(identifier_hex(worker.get("parentSpanId", ""), 8), {})
if publisher.get("kind") not in ("SPAN_KIND_PRODUCER", 4):
return None
upload = by_id.get(identifier_hex(publisher.get("parentSpanId", ""), 8), {})
if upload.get("name") != "POST /api/v1/chats/{chat_id}/attachments":
return None
names = {span["name"] for span in spans}
if (
not workers
or not {"s3.read_staging", "s3.write_result", "postgres.commit_ready"}
<= names
):
return None
if retry:
outcomes = set()
for span in workers:
attrs = {row["key"]: row["value"] for row in span.get("attributes", [])}
outcomes.add(
(
int(attrs["job.attempt"]["intValue"]),
attrs.get("outcome", {}).get("stringValue"),
)
)
if not {(1, "error"), (2, "error")} <= outcomes or not any(
n > 2 and outcome == "success" for n, outcome in outcomes
):
return None
return [
{
"name": span["name"],
"kind": span.get("kind"),
"span_id": identifier_hex(span["spanId"], 8),
"parent_span_id": identifier_hex(span.get("parentSpanId", ""), 8),
"job": {
entry["key"]: entry["value"]
for entry in span.get("attributes", [])
if entry["key"] in {"job.attempt", "outcome"}
},
}
for span in spans
]
return eventually(inspect, "Missing upload -> producer -> consumer -> S3/DB chain")
def telemetry_outage(project, client, grafana, actor, chat):
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
baseline = [metric(grafana, expr) for expr in (failed, refused)]
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
samples = []
successful = 0
try:
project.run("stop", "loki")
start = time.monotonic()
for second in range(51):
for index in range(4):
client.message(actor, chat, f"outage-{second}-{index}")
successful += 1
if second % 5 == 0:
samples.append(
{
"seconds": round(time.monotonic() - start, 2),
"rss_bytes": metric(grafana, rss),
"queued_batches": metric(
grafana,
'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})',
),
"capacity_batches": metric(
grafana,
'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})',
),
}
)
time.sleep(max(0, start + second + 1 - time.monotonic()))
losses = [
metric(grafana, expr) - count
for expr, count in zip((failed, refused), baseline)
]
check(sum(losses) > 0, "No measured Collector log loss during prolonged outage")
check(
all(
0 <= row["queued_batches"] <= row["capacity_batches"] == 256
for row in samples
),
"Unexpected queue bound",
)
check(
max(row["rss_bytes"] for row in samples) < 256 * 1024**2,
"Collector crossed container memory limit",
)
finally:
project.run("start", "loki")
eventually(
lambda: grafana.proxy("loki", "/loki/api/v1/labels").get("status") == "success",
"Loki did not recover",
)
restored = client.message(actor, chat, "telemetry-backend-restored")
correlation(
grafana,
"messenger-api",
time.time_ns() - 30 * 10**9,
trace_id=restored["headers"]["X-Trace-Id"],
)
return {
"scenario": "loki_unavailable",
"successful_messages": successful,
"dropped_after_export_failure": losses[0],
"dropped_at_enqueue": losses[1],
"samples": samples,
"log_export_recovered": True,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--grafana-url", default="http://localhost:13000")
parser.add_argument("--incidents", action="store_true")
parser.add_argument("--restart-telemetry", action="store_true")
parser.add_argument("--output", default="evidence/observability.json")
parser.add_argument("--duration", type=int, default=10)
parser.add_argument("--rate", type=int, default=10)
parser.add_argument("--concurrency", type=int, default=2)
args = parser.parse_args()
load = run_load(args)
start = time.time_ns()
project = ComposeProject()
client = Client(args.base_url, timeout=40)
aliases = {r.instance_id for r in project.replicas() if r.running}
check(len(aliases) >= 2, "Need two real API containers")
check(
client.call("GET", "/health/live")["headers"].get("X-Instance-Id") in aliases,
"Wrong Compose project for BASE_URL",
)
grafana = Grafana(args.grafana_url)
eventually(
lambda: grafana.get("/api/health").get("database") == "ok",
"Grafana is not ready",
)
sources = grafana.get("/api/datasources")
check(
{"prometheus", "loki", "tempo"} <= {source["uid"] for source in sources},
"Missing provisioned datasource",
)
dashboard = grafana.get("/api/dashboards/uid/messenger-overview")["dashboard"]
check(len(dashboard["panels"]) >= 8, "Dashboard is missing panels")
actor, bob, eve, chat = client.bootstrap()
sentinel = "private-content-" + uuid4().hex
response = client.message(actor, chat, sentinel)
controlled_trace = response["headers"].get("X-Trace-Id")
check(bool(controlled_trace), "No API trace identifier")
attachment, _ = client.attachment(actor, chat)
client.wait_ready(actor, attachment)
api, api_trace = correlation(
grafana,
"messenger-api",
start,
trace_id=controlled_trace,
request_id=response["headers"]["X-Request-Id"],
)
worker, worker_trace = correlation(
grafana, "messenger-worker", start, attachment_id=attachment["id"]
)
accepted_chain = trace_chain(grafana, worker["trace_id"])
check(
api["log"]["request_id"] == response["headers"]["X-Request-Id"],
"Request id correlation lost",
)
check(
any(span["kind"] in ("SPAN_KIND_SERVER", 2) for span in api["spans"]),
"No server span",
)
check(
any(span["kind"] in ("SPAN_KIND_CONSUMER", 5) for span in worker["spans"]),
"No consumer span",
)
processed = eventually(
lambda: grafana.logs(
'{service_name="messenger-worker"} | json | event="job.processed" | trace_id="'
+ worker["trace_id"]
+ '"',
start,
),
"No worker result log",
)
check(processed[0]["outcome"] == "success", "Controlled worker did not succeed")
metrics = {}
expressions = {
"http_requests": "sum(messenger_http_requests_total)",
"http_duration": "sum(messenger_http_duration_seconds_count)",
"job_success": 'sum(messenger_job_attempts_total{outcome="success"})',
"job_duration": "sum(messenger_job_duration_seconds_count)",
}
for name, expression in expressions.items():
metrics[name] = eventually(
lambda expression=expression: grafana.query(expression),
"Missing metric: " + expression,
)
check(float(metrics[name][0]["value"][1]) > 0, "Empty metric: " + name)
# Все PromQL из dashboard должны давать численные данные после traffic.
for panel in dashboard["panels"]:
if panel["datasource"]["uid"] == "prometheus":
eventually(
lambda panel=panel: all(
grafana.query(target["expr"]) for target in panel["targets"]
),
"Empty dashboard panel: " + panel["title"],
)
forbidden_labels = {
"user_id",
"chat_id",
"message_id",
"attachment_id",
"request_id",
"filename",
"url",
"http_target",
"http_url",
}
series = grafana.query('{__name__=~"messenger_.*"}')
check(
not any(forbidden_labels & set(row["metric"]) for row in series),
"High cardinality metric label",
)
report = {
"date_utc": datetime.now(timezone.utc).isoformat(),
"project": project.name,
"load": load,
"api_replicas": len(aliases),
"dashboard_uid": dashboard["uid"],
"api_correlation": api,
"worker_correlation": worker,
"metrics": metrics,
"upload_trace": {"trace_id": worker["trace_id"], "chain": accepted_chain},
"incidents": [],
}
if args.incidents:
def fail_request():
return client.call(
"POST",
"/api/v1/users",
expected=503,
body={
"username": "fault_" + uuid4().hex[:12],
"display_name": "Synthetic",
"password": "not-exported-" + uuid4().hex,
},
)
before = time.monotonic()
try:
project.run("stop", "postgres")
failed = fail_request()
error_trace = failed["headers"].get("X-Trace-Id")
error, db_trace = correlation(
grafana,
"messenger-api",
start,
trace_id=error_trace,
request_id=failed["headers"]["X-Request-Id"],
)
diagnostics = eventually(
lambda: grafana.logs(
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="'
+ error_trace
+ '"',
start,
),
"No dependency diagnostic log",
)
checkout = next(
span
for span in spans_in(db_trace)
if span["name"] == "postgres.checkout"
and span.get("status", {}).get("code") in ("STATUS_CODE_ERROR", 2)
)
def db_alert():
for _ in range(4):
fail_request()
return alert_firing(grafana, "MessengerHighErrorRate")
eventually(db_alert, "Error rate alert did not fire", timeout=75)
report["incidents"].append(
{
"scenario": "postgres_unavailable",
"http_status": 503,
"correlation": error,
"diagnostic": diagnostics[0],
"db_span": checkout["name"],
"alert": "MessengerHighErrorRate",
"firing_after_seconds": round(time.monotonic() - before, 3),
}
)
finally:
project.run("start", "postgres")
eventually(lambda: _ready(client), "API did not recover after PostgreSQL")
eventually(
lambda: not alert_firing(grafana, "MessengerHighErrorRate"),
"Error rate alert not resolved",
)
report["incidents"][-1]["resolved_after_seconds"] = round(
time.monotonic() - before, 3
)
from scripts.check_jobs import retry_replay
queue = {}
def observe_queue(_):
observed = time.monotonic()
eventually(
lambda: alert_firing(grafana, "MessengerQueueLag"),
"Queue lag alert did not fire",
)
queue.update(
depth=metric(grafana, "max(messenger_queue_depth_ratio)"),
oldest_seconds=metric(grafana, "max(messenger_queue_oldest_seconds)"),
firing_after_seconds=round(time.monotonic() - observed, 3),
)
incident = retry_replay(
client, actor, chat, permanent=False, on_queued=observe_queue
)
retried_worker, retry_trace = correlation(
grafana, "messenger-worker", start, attachment_id=incident["attachment_id"]
)
chain = trace_chain(grafana, retried_worker["trace_id"], retry=True)
eventually(
lambda: not alert_firing(grafana, "MessengerQueueLag")
and metric(grafana, "max(messenger_queue_depth_ratio)") == 0,
"Backlog did not drain",
)
queue.update(resolved=True, final_depth=0)
report["incidents"].append(
{
"scenario": "worker_stopped_then_transient_retry",
"alert": "MessengerQueueLag",
"queue": queue,
"recovered_status": incident["after"]["status"],
"trace_id": retried_worker["trace_id"],
"chain": chain,
}
)
report["incidents"].append(
telemetry_outage(project, client, grafana, actor, chat)
)
else:
retry_trace = {}
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
private = [
sentinel,
*(person["token"] for person in (actor, bob, eve)),
*(person["registration"]["password"] for person in (actor, bob, eve)),
]
private.extend(
path.read_text(encoding="utf-8").strip()
for path in (ROOT / "secrets").iterdir()
if path.is_file()
)
check(
not any(secret and secret in exported for secret in private),
"Private value found in telemetry",
)
check(
all(
record.get("trace_id")
and record.get("span_id")
and record.get("timestamp")
and record.get("severity")
for record in logs
),
"Incomplete structured logs",
)
report["privacy_checked_logs"] = len(logs)
report["checks"] = {
"three_backends": True,
"dashboard_queries": True,
"server_consumer_traces": True,
"log_trace_correlation": True,
"low_cardinality": True,
"privacy": True,
}
if args.restart_telemetry:
historical_time = time.time()
expression = "sum(messenger_http_requests_total)"
before = grafana.proxy(
"prometheus", "/api/v1/query", query=expression, time=historical_time
)["data"]["result"]
project.run("restart", "prometheus", "loki", "tempo", "grafana")
after = eventually(
lambda: grafana.proxy(
"prometheus", "/api/v1/query", query=expression, time=historical_time
)["data"]["result"],
"Prometheus history lost",
)
check(before == after, "Historical metric changed after storage restart")
correlation(grafana, "messenger-api", start, trace_id=api["trace_id"])
correlation(grafana, "messenger-worker", start, trace_id=worker["trace_id"])
report["checks"]["history_survives_restart"] = True
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(
"PASS: telemetry, dashboard, privacy, correlations"
+ ("; alerts and outage recovery verified" if args.incidents else "")
)
def _ready(client):
return client.call("GET", "/health/ready", expected=[200, 503])["status"] == 200
if __name__ == "__main__":
main()