729 lines
26 KiB
Python
729 lines
26 KiB
Python
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
|
|
|
|
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
|
|
В evidence сохраняются только агрегаты и выбранные технические поля.
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import math
|
|
import subprocess
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from uuid import uuid4
|
|
|
|
from scripts.deployment import ComposeProject
|
|
from tests.client import Client, Failure, check
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def eventually(operation, message, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = operation()
|
|
if value:
|
|
return value
|
|
except (OSError, ValueError, KeyError, AssertionError):
|
|
pass
|
|
time.sleep(1)
|
|
raise AssertionError(message)
|
|
|
|
|
|
class Grafana:
|
|
def __init__(self, base):
|
|
self.base = base.rstrip("/")
|
|
secret = (
|
|
(ROOT / "secrets/grafana_admin_password")
|
|
.read_text(encoding="utf-8")
|
|
.strip()
|
|
)
|
|
self.authorization = (
|
|
"Basic " + base64.b64encode(("admin:" + secret).encode()).decode()
|
|
)
|
|
|
|
def get(self, path, **params):
|
|
url = self.base + path
|
|
if params:
|
|
url += "?" + urllib.parse.urlencode(params)
|
|
request = urllib.request.Request(
|
|
url,
|
|
headers={"Authorization": self.authorization, "Accept": "application/json"},
|
|
)
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
return json.load(response)
|
|
|
|
def proxy(self, source, path, **params):
|
|
return self.get("/api/datasources/proxy/uid/" + source + path, **params)
|
|
|
|
def query(self, expression):
|
|
value = self.proxy("prometheus", "/api/v1/query", query=expression)
|
|
check(value["status"] == "success", "Prometheus query failed")
|
|
return value["data"]["result"]
|
|
|
|
def logs(self, selector, start):
|
|
value = self.proxy(
|
|
"loki",
|
|
"/loki/api/v1/query_range",
|
|
query=selector,
|
|
start=start,
|
|
end=time.time_ns(),
|
|
limit=1000,
|
|
direction="forward",
|
|
)
|
|
check(value["status"] == "success", "Loki query failed")
|
|
return [
|
|
json.loads(line)
|
|
for stream in value["data"]["result"]
|
|
for _, line, *_ in stream["values"]
|
|
]
|
|
|
|
def trace(self, trace_id):
|
|
return self.proxy("tempo", "/api/traces/" + trace_id)
|
|
|
|
|
|
def identifier_hex(value, byte_length):
|
|
if len(value) == byte_length * 2 and all(
|
|
c in "0123456789abcdefABCDEF" for c in value
|
|
):
|
|
return value.lower()
|
|
return base64.b64decode(value, validate=True).hex()
|
|
|
|
|
|
def spans_in(value):
|
|
found = []
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
if key == "spans":
|
|
found.extend(item)
|
|
else:
|
|
found.extend(spans_in(item))
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
found.extend(spans_in(item))
|
|
return found
|
|
|
|
|
|
def correlation(
|
|
grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None
|
|
):
|
|
selector = '{service_name="' + service + '"} | json'
|
|
if service == "messenger-api":
|
|
selector += ' | event="http.request"'
|
|
if trace_id:
|
|
selector += ' | trace_id="' + trace_id + '"'
|
|
if attachment_id:
|
|
selector += ' | attachment_id="' + attachment_id + '"'
|
|
if request_id:
|
|
selector += ' | request_id="' + request_id + '"'
|
|
records = eventually(
|
|
lambda: grafana.logs(selector, start), "No correlated log for " + service
|
|
)
|
|
record = records[0]
|
|
trace_id = record["trace_id"]
|
|
|
|
def matching_trace():
|
|
candidate = grafana.trace(trace_id)
|
|
return (
|
|
candidate
|
|
if any(
|
|
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
|
|
for span in spans_in(candidate)
|
|
)
|
|
else None
|
|
)
|
|
|
|
trace = eventually(matching_trace, "Matching span is missing in Tempo")
|
|
spans = [
|
|
span
|
|
for span in spans_in(trace)
|
|
if identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
|
|
]
|
|
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
|
|
check(
|
|
any(
|
|
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
|
|
for span in spans
|
|
),
|
|
"Log span_id not found in real trace",
|
|
)
|
|
check(
|
|
all(
|
|
int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"])
|
|
for span in spans
|
|
),
|
|
"Trace has no real duration",
|
|
)
|
|
return {
|
|
"trace_id": trace_id,
|
|
"span_id": record["span_id"],
|
|
"log": record,
|
|
"spans": [
|
|
{
|
|
key: span.get(key)
|
|
for key in (
|
|
"name",
|
|
"kind",
|
|
"spanId",
|
|
"startTimeUnixNano",
|
|
"endTimeUnixNano",
|
|
"status",
|
|
)
|
|
}
|
|
for span in spans
|
|
],
|
|
}, trace
|
|
|
|
|
|
def run_load(args):
|
|
check(
|
|
1 <= args.duration <= 300
|
|
and 1 <= args.rate <= 40
|
|
and 1 <= args.concurrency <= 16,
|
|
"Load profile outside bounds",
|
|
)
|
|
client = Client(args.base_url)
|
|
actor, _, _, chat = client.bootstrap()
|
|
project = ComposeProject()
|
|
container_ids = project.run("ps", "--quiet").splitlines()
|
|
samples = []
|
|
stop = threading.Event()
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{json .}}",
|
|
*container_ids,
|
|
],
|
|
capture_output=True,
|
|
encoding="utf-8",
|
|
timeout=15,
|
|
)
|
|
if result.returncode == 0:
|
|
samples.append(
|
|
{
|
|
"elapsed_seconds": round(time.monotonic() - started, 3),
|
|
"containers": [
|
|
{
|
|
k: row[k]
|
|
for k in ("Name", "CPUPerc", "MemUsage", "MemPerc")
|
|
}
|
|
for row in (
|
|
json.loads(line) for line in result.stdout.splitlines()
|
|
)
|
|
],
|
|
}
|
|
)
|
|
stop.wait(3)
|
|
|
|
def write(index):
|
|
local = Client(args.base_url)
|
|
before = time.monotonic()
|
|
try:
|
|
local.message(actor, chat, "synthetic-observability-" + str(index))
|
|
return time.monotonic() - before
|
|
except Failure:
|
|
return None
|
|
|
|
started = time.monotonic()
|
|
sampler = threading.Thread(target=sample)
|
|
sampler.start()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futures = []
|
|
for index in range(args.duration * args.rate):
|
|
time.sleep(max(0, started + index / args.rate - time.monotonic()))
|
|
futures.append(pool.submit(write, index))
|
|
results = [future.result() for future in futures]
|
|
elapsed = time.monotonic() - started
|
|
finally:
|
|
stop.set()
|
|
sampler.join()
|
|
latencies = sorted(value for value in results if value is not None)
|
|
failed = len(results) - len(latencies)
|
|
|
|
def percentile(q):
|
|
return (
|
|
round(latencies[max(0, math.ceil(q * len(latencies)) - 1)] * 1000, 3)
|
|
if latencies
|
|
else None
|
|
)
|
|
|
|
report = {
|
|
"date_utc": datetime.now(timezone.utc).isoformat(),
|
|
"workload": "synthetic POST messages; fixed arrival rate; no retries",
|
|
"duration_seconds": args.duration,
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"arrival_rate": args.rate,
|
|
"concurrency_limit": args.concurrency,
|
|
"successful_requests": len(latencies),
|
|
"failed_requests": failed,
|
|
"error_ratio": failed / len(results),
|
|
"successful_rps": round(len(latencies) / elapsed, 3),
|
|
"successful_latency_ms": {
|
|
name: percentile(q)
|
|
for name, q in [("p50", 0.5), ("p95", 0.95), ("p99", 0.99)]
|
|
},
|
|
"dataset": {"users": 3, "chats": 1, "messages": len(latencies)},
|
|
"resource_samples": samples,
|
|
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.",
|
|
}
|
|
check(bool(latencies) and failed == 0, "Controlled load contains errors")
|
|
|
|
return report
|
|
|
|
|
|
def metric(grafana, expression):
|
|
rows = grafana.query(expression)
|
|
return float(rows[0]["value"][1]) if rows else None
|
|
|
|
|
|
def alert_firing(grafana, name):
|
|
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
|
|
|
|
|
|
def trace_chain(grafana, trace_id, *, retry=False):
|
|
def inspect():
|
|
spans = spans_in(grafana.trace(trace_id))
|
|
by_id = {identifier_hex(span["spanId"], 8): span for span in spans}
|
|
workers = [
|
|
span
|
|
for span in spans
|
|
if span["name"] == "attachment.process.v1"
|
|
and span.get("kind") in ("SPAN_KIND_CONSUMER", 5)
|
|
]
|
|
for worker in workers:
|
|
publisher = by_id.get(identifier_hex(worker.get("parentSpanId", ""), 8), {})
|
|
if publisher.get("kind") not in ("SPAN_KIND_PRODUCER", 4):
|
|
return None
|
|
upload = by_id.get(identifier_hex(publisher.get("parentSpanId", ""), 8), {})
|
|
if upload.get("name") != "POST /api/v1/chats/{chat_id}/attachments":
|
|
return None
|
|
names = {span["name"] for span in spans}
|
|
if (
|
|
not workers
|
|
or not {"s3.read_staging", "s3.write_result", "postgres.commit_ready"}
|
|
<= names
|
|
):
|
|
return None
|
|
if retry:
|
|
outcomes = set()
|
|
for span in workers:
|
|
attrs = {row["key"]: row["value"] for row in span.get("attributes", [])}
|
|
outcomes.add(
|
|
(
|
|
int(attrs["job.attempt"]["intValue"]),
|
|
attrs.get("outcome", {}).get("stringValue"),
|
|
)
|
|
)
|
|
if not {(1, "error"), (2, "error")} <= outcomes or not any(
|
|
n > 2 and outcome == "success" for n, outcome in outcomes
|
|
):
|
|
return None
|
|
return [
|
|
{
|
|
"name": span["name"],
|
|
"kind": span.get("kind"),
|
|
"span_id": identifier_hex(span["spanId"], 8),
|
|
"parent_span_id": identifier_hex(span.get("parentSpanId", ""), 8),
|
|
"job": {
|
|
entry["key"]: entry["value"]
|
|
for entry in span.get("attributes", [])
|
|
if entry["key"] in {"job.attempt", "outcome"}
|
|
},
|
|
}
|
|
for span in spans
|
|
]
|
|
|
|
return eventually(inspect, "Missing upload -> producer -> consumer -> S3/DB chain")
|
|
|
|
|
|
def telemetry_outage(project, client, grafana, actor, chat):
|
|
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
|
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
|
baseline = [metric(grafana, expr) for expr in (failed, refused)]
|
|
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
|
samples = []
|
|
successful = 0
|
|
try:
|
|
project.run("stop", "loki")
|
|
start = time.monotonic()
|
|
for second in range(51):
|
|
for index in range(4):
|
|
client.message(actor, chat, f"outage-{second}-{index}")
|
|
successful += 1
|
|
if second % 5 == 0:
|
|
samples.append(
|
|
{
|
|
"seconds": round(time.monotonic() - start, 2),
|
|
"rss_bytes": metric(grafana, rss),
|
|
"queued_batches": metric(
|
|
grafana,
|
|
'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})',
|
|
),
|
|
"capacity_batches": metric(
|
|
grafana,
|
|
'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})',
|
|
),
|
|
}
|
|
)
|
|
time.sleep(max(0, start + second + 1 - time.monotonic()))
|
|
losses = [
|
|
metric(grafana, expr) - count
|
|
for expr, count in zip((failed, refused), baseline)
|
|
]
|
|
check(sum(losses) > 0, "No measured Collector log loss during prolonged outage")
|
|
check(
|
|
all(
|
|
0 <= row["queued_batches"] <= row["capacity_batches"] == 256
|
|
for row in samples
|
|
),
|
|
"Unexpected queue bound",
|
|
)
|
|
check(
|
|
max(row["rss_bytes"] for row in samples) < 256 * 1024**2,
|
|
"Collector crossed container memory limit",
|
|
)
|
|
finally:
|
|
project.run("start", "loki")
|
|
eventually(
|
|
lambda: grafana.proxy("loki", "/loki/api/v1/labels").get("status") == "success",
|
|
"Loki did not recover",
|
|
)
|
|
restored = client.message(actor, chat, "telemetry-backend-restored")
|
|
correlation(
|
|
grafana,
|
|
"messenger-api",
|
|
time.time_ns() - 30 * 10**9,
|
|
trace_id=restored["headers"]["X-Trace-Id"],
|
|
)
|
|
return {
|
|
"scenario": "loki_unavailable",
|
|
"successful_messages": successful,
|
|
"dropped_after_export_failure": losses[0],
|
|
"dropped_at_enqueue": losses[1],
|
|
"samples": samples,
|
|
"log_export_recovered": True,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", required=True)
|
|
parser.add_argument("--grafana-url", default="http://localhost:13000")
|
|
parser.add_argument("--incidents", action="store_true")
|
|
parser.add_argument("--restart-telemetry", action="store_true")
|
|
parser.add_argument("--output", default="evidence/observability.json")
|
|
parser.add_argument("--duration", type=int, default=10)
|
|
parser.add_argument("--rate", type=int, default=10)
|
|
parser.add_argument("--concurrency", type=int, default=2)
|
|
args = parser.parse_args()
|
|
load = run_load(args)
|
|
start = time.time_ns()
|
|
project = ComposeProject()
|
|
client = Client(args.base_url, timeout=40)
|
|
aliases = {r.instance_id for r in project.replicas() if r.running}
|
|
check(len(aliases) >= 2, "Need two real API containers")
|
|
check(
|
|
client.call("GET", "/health/live")["headers"].get("X-Instance-Id") in aliases,
|
|
"Wrong Compose project for BASE_URL",
|
|
)
|
|
grafana = Grafana(args.grafana_url)
|
|
eventually(
|
|
lambda: grafana.get("/api/health").get("database") == "ok",
|
|
"Grafana is not ready",
|
|
)
|
|
sources = grafana.get("/api/datasources")
|
|
check(
|
|
{"prometheus", "loki", "tempo"} <= {source["uid"] for source in sources},
|
|
"Missing provisioned datasource",
|
|
)
|
|
dashboard = grafana.get("/api/dashboards/uid/messenger-overview")["dashboard"]
|
|
check(len(dashboard["panels"]) >= 8, "Dashboard is missing panels")
|
|
actor, bob, eve, chat = client.bootstrap()
|
|
sentinel = "private-content-" + uuid4().hex
|
|
response = client.message(actor, chat, sentinel)
|
|
controlled_trace = response["headers"].get("X-Trace-Id")
|
|
check(bool(controlled_trace), "No API trace identifier")
|
|
attachment, _ = client.attachment(actor, chat)
|
|
client.wait_ready(actor, attachment)
|
|
api, api_trace = correlation(
|
|
grafana,
|
|
"messenger-api",
|
|
start,
|
|
trace_id=controlled_trace,
|
|
request_id=response["headers"]["X-Request-Id"],
|
|
)
|
|
worker, worker_trace = correlation(
|
|
grafana, "messenger-worker", start, attachment_id=attachment["id"]
|
|
)
|
|
accepted_chain = trace_chain(grafana, worker["trace_id"])
|
|
check(
|
|
api["log"]["request_id"] == response["headers"]["X-Request-Id"],
|
|
"Request id correlation lost",
|
|
)
|
|
check(
|
|
any(span["kind"] in ("SPAN_KIND_SERVER", 2) for span in api["spans"]),
|
|
"No server span",
|
|
)
|
|
check(
|
|
any(span["kind"] in ("SPAN_KIND_CONSUMER", 5) for span in worker["spans"]),
|
|
"No consumer span",
|
|
)
|
|
processed = eventually(
|
|
lambda: grafana.logs(
|
|
'{service_name="messenger-worker"} | json | event="job.processed" | trace_id="'
|
|
+ worker["trace_id"]
|
|
+ '"',
|
|
start,
|
|
),
|
|
"No worker result log",
|
|
)
|
|
check(processed[0]["outcome"] == "success", "Controlled worker did not succeed")
|
|
metrics = {}
|
|
expressions = {
|
|
"http_requests": "sum(messenger_http_requests_total)",
|
|
"http_duration": "sum(messenger_http_duration_seconds_count)",
|
|
"job_success": 'sum(messenger_job_attempts_total{outcome="success"})',
|
|
"job_duration": "sum(messenger_job_duration_seconds_count)",
|
|
}
|
|
for name, expression in expressions.items():
|
|
metrics[name] = eventually(
|
|
lambda expression=expression: grafana.query(expression),
|
|
"Missing metric: " + expression,
|
|
)
|
|
check(float(metrics[name][0]["value"][1]) > 0, "Empty metric: " + name)
|
|
# Все PromQL из dashboard должны давать численные данные после traffic.
|
|
for panel in dashboard["panels"]:
|
|
if panel["datasource"]["uid"] == "prometheus":
|
|
eventually(
|
|
lambda panel=panel: all(
|
|
grafana.query(target["expr"]) for target in panel["targets"]
|
|
),
|
|
"Empty dashboard panel: " + panel["title"],
|
|
)
|
|
forbidden_labels = {
|
|
"user_id",
|
|
"chat_id",
|
|
"message_id",
|
|
"attachment_id",
|
|
"request_id",
|
|
"filename",
|
|
"url",
|
|
"http_target",
|
|
"http_url",
|
|
}
|
|
series = grafana.query('{__name__=~"messenger_.*"}')
|
|
check(
|
|
not any(forbidden_labels & set(row["metric"]) for row in series),
|
|
"High cardinality metric label",
|
|
)
|
|
report = {
|
|
"date_utc": datetime.now(timezone.utc).isoformat(),
|
|
"project": project.name,
|
|
"load": load,
|
|
"api_replicas": len(aliases),
|
|
"dashboard_uid": dashboard["uid"],
|
|
"api_correlation": api,
|
|
"worker_correlation": worker,
|
|
"metrics": metrics,
|
|
"upload_trace": {"trace_id": worker["trace_id"], "chain": accepted_chain},
|
|
"incidents": [],
|
|
}
|
|
if args.incidents:
|
|
|
|
def fail_request():
|
|
return client.call(
|
|
"POST",
|
|
"/api/v1/users",
|
|
expected=503,
|
|
body={
|
|
"username": "fault_" + uuid4().hex[:12],
|
|
"display_name": "Synthetic",
|
|
"password": "not-exported-" + uuid4().hex,
|
|
},
|
|
)
|
|
|
|
before = time.monotonic()
|
|
try:
|
|
project.run("stop", "postgres")
|
|
failed = fail_request()
|
|
error_trace = failed["headers"].get("X-Trace-Id")
|
|
error, db_trace = correlation(
|
|
grafana,
|
|
"messenger-api",
|
|
start,
|
|
trace_id=error_trace,
|
|
request_id=failed["headers"]["X-Request-Id"],
|
|
)
|
|
diagnostics = eventually(
|
|
lambda: grafana.logs(
|
|
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="'
|
|
+ error_trace
|
|
+ '"',
|
|
start,
|
|
),
|
|
"No dependency diagnostic log",
|
|
)
|
|
checkout = next(
|
|
span
|
|
for span in spans_in(db_trace)
|
|
if span["name"] == "postgres.checkout"
|
|
and span.get("status", {}).get("code") in ("STATUS_CODE_ERROR", 2)
|
|
)
|
|
|
|
def db_alert():
|
|
for _ in range(4):
|
|
fail_request()
|
|
return alert_firing(grafana, "MessengerHighErrorRate")
|
|
|
|
eventually(db_alert, "Error rate alert did not fire", timeout=75)
|
|
report["incidents"].append(
|
|
{
|
|
"scenario": "postgres_unavailable",
|
|
"http_status": 503,
|
|
"correlation": error,
|
|
"diagnostic": diagnostics[0],
|
|
"db_span": checkout["name"],
|
|
"alert": "MessengerHighErrorRate",
|
|
"firing_after_seconds": round(time.monotonic() - before, 3),
|
|
}
|
|
)
|
|
finally:
|
|
project.run("start", "postgres")
|
|
eventually(lambda: _ready(client), "API did not recover after PostgreSQL")
|
|
eventually(
|
|
lambda: not alert_firing(grafana, "MessengerHighErrorRate"),
|
|
"Error rate alert not resolved",
|
|
)
|
|
report["incidents"][-1]["resolved_after_seconds"] = round(
|
|
time.monotonic() - before, 3
|
|
)
|
|
|
|
from scripts.check_jobs import retry_replay
|
|
|
|
queue = {}
|
|
|
|
def observe_queue(_):
|
|
observed = time.monotonic()
|
|
eventually(
|
|
lambda: alert_firing(grafana, "MessengerQueueLag"),
|
|
"Queue lag alert did not fire",
|
|
)
|
|
queue.update(
|
|
depth=metric(grafana, "max(messenger_queue_depth_ratio)"),
|
|
oldest_seconds=metric(grafana, "max(messenger_queue_oldest_seconds)"),
|
|
firing_after_seconds=round(time.monotonic() - observed, 3),
|
|
)
|
|
|
|
incident = retry_replay(
|
|
client, actor, chat, permanent=False, on_queued=observe_queue
|
|
)
|
|
retried_worker, retry_trace = correlation(
|
|
grafana, "messenger-worker", start, attachment_id=incident["attachment_id"]
|
|
)
|
|
chain = trace_chain(grafana, retried_worker["trace_id"], retry=True)
|
|
eventually(
|
|
lambda: not alert_firing(grafana, "MessengerQueueLag")
|
|
and metric(grafana, "max(messenger_queue_depth_ratio)") == 0,
|
|
"Backlog did not drain",
|
|
)
|
|
queue.update(resolved=True, final_depth=0)
|
|
report["incidents"].append(
|
|
{
|
|
"scenario": "worker_stopped_then_transient_retry",
|
|
"alert": "MessengerQueueLag",
|
|
"queue": queue,
|
|
"recovered_status": incident["after"]["status"],
|
|
"trace_id": retried_worker["trace_id"],
|
|
"chain": chain,
|
|
}
|
|
)
|
|
report["incidents"].append(
|
|
telemetry_outage(project, client, grafana, actor, chat)
|
|
)
|
|
else:
|
|
retry_trace = {}
|
|
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
|
|
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
|
|
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
|
|
private = [
|
|
sentinel,
|
|
*(person["token"] for person in (actor, bob, eve)),
|
|
*(person["registration"]["password"] for person in (actor, bob, eve)),
|
|
]
|
|
private.extend(
|
|
path.read_text(encoding="utf-8").strip()
|
|
for path in (ROOT / "secrets").iterdir()
|
|
if path.is_file()
|
|
)
|
|
check(
|
|
not any(secret and secret in exported for secret in private),
|
|
"Private value found in telemetry",
|
|
)
|
|
check(
|
|
all(
|
|
record.get("trace_id")
|
|
and record.get("span_id")
|
|
and record.get("timestamp")
|
|
and record.get("severity")
|
|
for record in logs
|
|
),
|
|
"Incomplete structured logs",
|
|
)
|
|
report["privacy_checked_logs"] = len(logs)
|
|
report["checks"] = {
|
|
"three_backends": True,
|
|
"dashboard_queries": True,
|
|
"server_consumer_traces": True,
|
|
"log_trace_correlation": True,
|
|
"low_cardinality": True,
|
|
"privacy": True,
|
|
}
|
|
if args.restart_telemetry:
|
|
historical_time = time.time()
|
|
expression = "sum(messenger_http_requests_total)"
|
|
before = grafana.proxy(
|
|
"prometheus", "/api/v1/query", query=expression, time=historical_time
|
|
)["data"]["result"]
|
|
project.run("restart", "prometheus", "loki", "tempo", "grafana")
|
|
after = eventually(
|
|
lambda: grafana.proxy(
|
|
"prometheus", "/api/v1/query", query=expression, time=historical_time
|
|
)["data"]["result"],
|
|
"Prometheus history lost",
|
|
)
|
|
check(before == after, "Historical metric changed after storage restart")
|
|
correlation(grafana, "messenger-api", start, trace_id=api["trace_id"])
|
|
correlation(grafana, "messenger-worker", start, trace_id=worker["trace_id"])
|
|
report["checks"]["history_survives_restart"] = True
|
|
destination = ROOT / args.output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
print(
|
|
"PASS: telemetry, dashboard, privacy, correlations"
|
|
+ ("; alerts and outage recovery verified" if args.incidents else "")
|
|
)
|
|
|
|
|
|
def _ready(client):
|
|
return client.call("GET", "/health/ready", expected=[200, 503])["status"] == 200
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|