90 lines
3.7 KiB
Python
90 lines
3.7 KiB
Python
"""Query all three TLS backends from the operator probe namespace."""
|
|
import json
|
|
import sys
|
|
import time
|
|
from urllib.parse import urlencode
|
|
|
|
from trust import check, https
|
|
|
|
|
|
def query(host, port, path):
|
|
status, raw, _ = https(host, port, path, "operator")
|
|
return status, raw
|
|
|
|
|
|
def promql(expression):
|
|
status, raw = query("prometheus", 9090, "/api/v1/query?" + urlencode({"query": expression}))
|
|
if status != 200:
|
|
raise AssertionError("Prometheus TLS query failed")
|
|
return json.loads(raw)["data"]["result"]
|
|
|
|
|
|
def event_trace(trace):
|
|
return query("tempo", 3200, "/api/traces/" + trace)
|
|
|
|
|
|
def verify_signals(payload):
|
|
trace_id = payload["trace_id"]
|
|
loki_path = "/loki/api/v1/query_range?" + urlencode({
|
|
"query": '{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"',
|
|
"start": str(payload["start"]),
|
|
"end": str(time.time_ns() + 60_000_000_000),
|
|
"limit": "100",
|
|
})
|
|
spans = (b"POST /api/v1/chats/{chat_id}/attachments", b"attachments publish",
|
|
b"attachment.process", b"s3.", b"db.execute")
|
|
end = time.monotonic() + 40
|
|
while time.monotonic() < end:
|
|
loki_code, logs = query("loki", 3100, loki_path)
|
|
tempo_code, trace = event_trace(trace_id)
|
|
if loki_code == tempo_code == 200 and b"attachment_ready" in logs and all(fragment in trace for fragment in spans):
|
|
break
|
|
time.sleep(1.5)
|
|
check("upload HTTP request ID and ready worker event share one Loki trace ID",
|
|
payload["request_id"].encode() in logs and b"attachment_ready" in logs)
|
|
check("Tempo trace joins HTTP, producer, consumer, S3 and DB",
|
|
tempo_code == 200 and all(fragment in trace for fragment in spans))
|
|
window_path = "/loki/api/v1/query_range?" + urlencode({
|
|
"query": '{service_name=~"messenger-api|messenger-worker"}',
|
|
"start": str(payload["start"]), "end": str(time.time_ns()), "limit": "2000",
|
|
})
|
|
status, window_logs = query("loki", 3100, window_path)
|
|
message_code, message_trace = event_trace(payload["message_trace_id"])
|
|
check("Loki and Tempo contain none of the synthetic sensitive values",
|
|
status == message_code == 200 and not any(
|
|
item.encode() in window_logs or item.encode() in trace or item.encode() in message_trace
|
|
for item in payload["canaries"]))
|
|
|
|
for expression in (
|
|
"app_http_requests_total", "app_http_duration_seconds_bucket",
|
|
"jobs_processed_total", "attachment_ready_seconds_bucket", "queue_depth",
|
|
"queue_oldest_age_seconds", "worker_capacity", "db_pool_used",
|
|
"process_memory_bytes", "otelcol_receiver_accepted_spans_total",
|
|
):
|
|
for _ in range(18):
|
|
if promql(expression):
|
|
check("metric " + expression, True)
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise AssertionError("metric missing: " + expression)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1:2] == ["--trace-error"]:
|
|
trace_id = sys.argv[2]
|
|
for _ in range(15):
|
|
status, raw = event_trace(trace_id)
|
|
if status == 200 and all(x in raw for x in (b"LockNotAvailable", b"db.execute", b"POST /api/v1/chats/{chat_id}/messages")):
|
|
print("PASS Tempo HTTP 503 trace includes DB span with LockNotAvailable")
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise SystemExit("expected DB error trace missing")
|
|
elif sys.argv[1:]:
|
|
for expression in sys.argv[1:]:
|
|
result = promql(expression)
|
|
print(json.dumps({"expression": expression, "values": [row["value"][1] for row in result]}))
|
|
else:
|
|
verify_signals(json.load(sys.stdin))
|