64 lines
2.6 KiB
Python
64 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Confirm metrics, logs and traces reach their HTTPS stores."""
|
|
import json
|
|
import ssl
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode
|
|
from urllib.request import urlopen
|
|
from urllib.error import HTTPError
|
|
import http.client
|
|
|
|
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
|
|
|
|
|
|
def query(service, port, path, params, *, allow_missing=False):
|
|
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
|
|
context.load_cert_chain(TLS / "grafana-reader.crt", TLS / "grafana-reader.key")
|
|
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
|
|
address = subprocess.check_output(["docker", "inspect", "-f",
|
|
'{{(index .NetworkSettings.Networks "highload-lab7_telemetry").IPAddress}}', container],
|
|
text=True).strip()
|
|
connection = http.client.HTTPSConnection(service, port, context=context, timeout=5)
|
|
create_connection = connection._create_connection
|
|
connection._create_connection = lambda _, *args: create_connection((address, port), *args)
|
|
try:
|
|
connection.request("GET", f"{path}?{urlencode(params)}")
|
|
response = connection.getresponse()
|
|
if allow_missing and response.status == 404:
|
|
return None
|
|
assert response.status == 200, (service, response.status)
|
|
return json.load(response)
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def main():
|
|
try:
|
|
with urlopen("https://localhost:8443/api/v1/users/me", context=ssl.create_default_context(
|
|
cafile=str(TLS / "lab7-ingress-ca.crt")), timeout=5):
|
|
raise AssertionError("Anonymous request should be rejected")
|
|
except HTTPError as error:
|
|
assert error.code == 401
|
|
since = int(time.time()) - 10
|
|
for _ in range(30):
|
|
now = int(time.time())
|
|
metrics = query("prometheus", 9090, "/api/v1/query", {
|
|
"query": 'up{job="otel-collector"} == 1'})["data"]["result"]
|
|
logs = query("loki", 3100, "/loki/api/v1/query_range", {
|
|
"query": '{service_name="messenger-api"}', "start": since * 10**9,
|
|
"end": now * 10**9, "limit": 1})["data"]["result"]
|
|
traces = query("tempo", 3200, "/api/search", {
|
|
"q": '{ resource.service.name = "messenger-api" }',
|
|
"start": since, "end": now, "limit": 1}).get("traces")
|
|
if metrics and logs and traces:
|
|
print("PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS")
|
|
return
|
|
time.sleep(2)
|
|
raise SystemExit("Telemetry signal missing within 60 seconds")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|