Files
mtusi2026lab6-fork/scripts/check_observability.py
T
2026-11-14 05:00:23 +03:00

292 lines
17 KiB
Python

"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
--incidents явно разрешает два ограниченных отказа на выбранном Compose-проекте.
В evidence сохраняются только агрегаты и выбранные технические поля.
"""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import math
import subprocess
import threading
from datetime import datetime, timezone
import json
from pathlib import Path
import time
import urllib.parse
import urllib.request
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, check
ROOT = Path(__file__).resolve().parents[1]
def eventually(operation, message, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
value = operation()
if value:
return value
except (OSError, ValueError, KeyError, AssertionError):
pass
time.sleep(1)
raise AssertionError(message)
class Grafana:
def __init__(self, base):
self.base = base.rstrip('/')
secret = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
self.authorization = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode()
def get(self, path, **params):
url = self.base + path
if params:
url += '?' + urllib.parse.urlencode(params)
request = urllib.request.Request(url, headers={'Authorization': self.authorization, 'Accept': 'application/json'})
with urllib.request.urlopen(request, timeout=10) as response:
return json.load(response)
def proxy(self, source, path, **params):
return self.get('/api/datasources/proxy/uid/' + source + path, **params)
def query(self, expression):
value = self.proxy('prometheus', '/api/v1/query', query=expression)
check(value['status'] == 'success', 'Prometheus query failed')
return value['data']['result']
def logs(self, selector, start):
value = self.proxy('loki', '/loki/api/v1/query_range', query=selector, start=start, end=time.time_ns(), limit=1000, direction='forward')
check(value['status'] == 'success', 'Loki query failed')
return [json.loads(line) for stream in value['data']['result'] for _, line, *_ in stream['values']]
def trace(self, trace_id):
return self.proxy('tempo', '/api/traces/' + trace_id)
def identifier_hex(value, byte_length):
if len(value) == byte_length * 2 and all(c in '0123456789abcdefABCDEF' for c in value):
return value.lower()
return base64.b64decode(value, validate=True).hex()
def spans_in(value):
found = []
if isinstance(value, dict):
for key, item in value.items():
if key == 'spans':
found.extend(item)
else:
found.extend(spans_in(item))
elif isinstance(value, list):
for item in value:
found.extend(spans_in(item))
return found
def correlation(grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None):
selector = '{service_name="' + service + '"} | json'
if service == 'messenger-api':
selector += ' | event="http.request"'
if trace_id:
selector += ' | trace_id="' + trace_id + '"'
if attachment_id:
selector += ' | attachment_id="' + attachment_id + '"'
if request_id:
selector += ' | request_id="' + request_id + '"'
records = eventually(lambda: grafana.logs(selector, start), 'No correlated log for ' + service)
record = records[0]
trace_id = record['trace_id']
def matching_trace():
candidate = grafana.trace(trace_id)
return candidate if any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans_in(candidate)) else None
trace = eventually(matching_trace, 'Matching span is missing in Tempo')
spans = [span for span in spans_in(trace) if identifier_hex(span.get('spanId', ''), 8) == record['span_id']]
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
check(any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans), 'Log span_id not found in real trace')
check(all(int(span['endTimeUnixNano']) > int(span['startTimeUnixNano']) for span in spans), 'Trace has no real duration')
return {'trace_id': trace_id, 'span_id': record['span_id'], 'log': record,
'spans': [{key: span.get(key) for key in ('name', 'kind', 'spanId', 'startTimeUnixNano', 'endTimeUnixNano', 'status')} for span in spans]}, trace
def run_load(args):
check(1 <= args.duration <= 300 and 1 <= args.rate <= 40 and 1 <= args.concurrency <= 16, 'Load profile outside bounds')
client = Client(args.base_url)
actor, _, _, chat = client.bootstrap()
project = ComposeProject()
container_ids = project.run('ps', '--quiet').splitlines()
samples = []
stop = threading.Event()
def sample():
while not stop.is_set():
result = subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*container_ids],capture_output=True,text=True,timeout=15)
if result.returncode == 0:
samples.append({'elapsed_seconds':round(time.monotonic()-started,3), 'containers':[
{k:row[k] for k in ('Name','CPUPerc','MemUsage','MemPerc')} for row in (json.loads(line) for line in result.stdout.splitlines())]})
stop.wait(3)
def write(index):
local = Client(args.base_url)
before = time.monotonic()
try:
local.message(actor,chat,'synthetic-observability-' + str(index))
return time.monotonic()-before
except Failure:
return None
started = time.monotonic()
sampler = threading.Thread(target=sample)
sampler.start()
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = []
for index in range(args.duration * args.rate):
time.sleep(max(0, started + index/args.rate - time.monotonic()))
futures.append(pool.submit(write,index))
results = [future.result() for future in futures]
elapsed = time.monotonic()-started
finally:
stop.set()
sampler.join()
latencies = sorted(value for value in results if value is not None)
failed = len(results)-len(latencies)
def percentile(q):
return round(latencies[max(0, math.ceil(q*len(latencies))-1)]*1000,3) if latencies else None
report = {'date_utc':datetime.now(timezone.utc).isoformat(), 'workload':'synthetic POST messages; fixed arrival rate; no retries',
'duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'arrival_rate':args.rate,'concurrency_limit':args.concurrency,
'successful_requests':len(latencies),'failed_requests':failed,'error_ratio':failed/len(results),
'successful_rps':round(len(latencies)/elapsed,3), 'successful_latency_ms':{name:percentile(q) for name,q in [('p50',.5),('p95',.95),('p99',.99)]},
'dataset':{'users':3,'chats':1,'messages':len(latencies)},'resource_samples':samples,
'limits':'Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.'}
check(bool(latencies) and failed == 0, 'Controlled load contains errors')
return report
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', required=True)
parser.add_argument('--grafana-url', default='http://localhost:13000')
parser.add_argument('--incidents', action='store_true')
parser.add_argument('--restart-telemetry', action='store_true')
parser.add_argument('--output', default='evidence/observability.json')
parser.add_argument('--duration', type=int, default=10)
parser.add_argument('--rate', type=int, default=10)
parser.add_argument('--concurrency', type=int, default=2)
args = parser.parse_args()
load = run_load(args)
start = time.time_ns()
project = ComposeProject()
client = Client(args.base_url, timeout=40)
aliases = {r.instance_id for r in project.replicas() if r.running}
check(len(aliases) >= 2, 'Need two real API containers')
check(client.call('GET', '/health/live')['headers'].get('X-Instance-Id') in aliases, 'Wrong Compose project for BASE_URL')
grafana = Grafana(args.grafana_url)
eventually(lambda: grafana.get('/api/health').get('database') == 'ok', 'Grafana is not ready')
sources = grafana.get('/api/datasources')
check({'prometheus', 'loki', 'tempo'} <= {source['uid'] for source in sources}, 'Missing provisioned datasource')
dashboard = grafana.get('/api/dashboards/uid/messenger-overview')['dashboard']
check(len(dashboard['panels']) >= 8, 'Dashboard is missing panels')
actor, bob, eve, chat = client.bootstrap()
sentinel = 'private-content-' + uuid4().hex
response = client.message(actor, chat, sentinel)
controlled_trace = response['headers'].get('X-Trace-Id')
check(bool(controlled_trace), 'No API trace identifier')
attachment, _ = client.attachment(actor, chat)
client.wait_ready(actor, attachment)
api, api_trace = correlation(grafana, 'messenger-api', start, trace_id=controlled_trace, request_id=response['headers']['X-Request-Id'])
worker, worker_trace = correlation(grafana, 'messenger-worker', start, attachment_id=attachment['id'])
check(api['log']['request_id'] == response['headers']['X-Request-Id'], 'Request id correlation lost')
check(any(span['kind'] in ('SPAN_KIND_SERVER', 2) for span in api['spans']), 'No server span')
check(any(span['kind'] in ('SPAN_KIND_CONSUMER', 5) for span in worker['spans']), 'No consumer span')
processed = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | trace_id="' + worker['trace_id'] + '"', start), 'No worker result log')
check(processed[0]['outcome'] == 'success', 'Controlled worker did not succeed')
metrics = {}
expressions = {
'http_requests': 'sum(messenger_http_requests_total)',
'http_duration': 'sum(messenger_http_duration_seconds_count)',
'job_success': 'sum(messenger_job_attempts_total{outcome="success"})',
'job_duration': 'sum(messenger_job_duration_seconds_count)',
}
for name, expression in expressions.items():
metrics[name] = eventually(lambda expression=expression: grafana.query(expression), 'Missing metric: ' + expression)
check(float(metrics[name][0]['value'][1]) > 0, 'Empty metric: ' + name)
# Все PromQL из dashboard должны давать численные данные после traffic.
for panel in dashboard['panels']:
if panel['datasource']['uid'] == 'prometheus':
eventually(lambda panel=panel: all(grafana.query(target['expr']) for target in panel['targets']), 'Empty dashboard panel: ' + panel['title'])
forbidden_labels = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target','http_url'}
series = grafana.query('{__name__=~"messenger_.*"}')
check(not any(forbidden_labels & set(row['metric']) for row in series), 'High cardinality metric label')
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'project': project.name,
'load': load, 'api_replicas': len(aliases), 'dashboard_uid': dashboard['uid'],
'api_correlation': api, 'worker_correlation': worker, 'metrics': metrics, 'incidents': []}
if args.incidents:
# Восстановление выполняется даже при ошибке проверки.
try:
before = time.monotonic()
project.run('stop', 'postgres')
failed = client.call('POST', '/api/v1/users', expected=503,
body={'username':'fault_' + uuid4().hex[:12], 'display_name':'Synthetic', 'password':'not-exported-' + uuid4().hex})
error_trace = failed['headers'].get('X-Trace-Id')
error, _ = correlation(grafana, 'messenger-api', start, trace_id=error_trace, request_id=failed['headers']['X-Request-Id'])
diagnostics = eventually(lambda: grafana.logs('{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="' + error_trace + '"', start), 'No dependency diagnostic log')
check(error['log'].get('status_code') == 503 or any(s['status'].get('code') in ('STATUS_CODE_ERROR', 2) for s in error['spans']), 'DB failure not reflected in trace')
report['incidents'].append({'scenario':'postgres_unavailable','http_status':503,'correlation':error,'diagnostic':diagnostics[0], 'observed_after_seconds':round(time.monotonic()-before,3)})
finally:
project.run('start', 'postgres')
eventually(lambda: _ready(client), 'API did not recover after PostgreSQL')
report['incidents'][-1]['recovery_seconds'] = round(time.monotonic() - before, 3)
# Явный fault hook обработки заданий: один permanent processing failure, затем replay.
from scripts.check_jobs import retry_replay
failed_instance = []
def wait_failed_metric(sample, container_name):
instance = project.docker('inspect', '--format', '{{.Config.Hostname}}', container_name)
failed_instance.append(instance)
expression = 'sum(messenger_job_attempts_total{outcome="error",service_instance_id="' + instance + '"})'
eventually(lambda: grafana.query(expression), 'Failed job metric not exported before recovery')
incident = retry_replay(client, actor, chat, permanent=True, on_failed=wait_failed_metric)
error_logs = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | outcome="error" | error_code="injected_permanent"', start), 'No failed job telemetry')
error_trace = eventually(lambda: grafana.trace(error_logs[-1]['trace_id']), 'No failed job trace')
check(any(span.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2) for span in spans_in(error_trace)), 'Worker error trace status lost')
expression = 'sum(messenger_job_attempts_total{outcome="error",service_instance_id="' + failed_instance[0] + '"})'
errors = eventually(lambda: grafana.query(expression), 'No failed jobs counter')
recovery = (datetime.fromisoformat(incident['after']['updated_at']) - datetime.fromisoformat(incident['samples'][-1]['updated_at'])).total_seconds()
report['incidents'].append({'scenario':'permanent_processing_failure','recovered_status':incident['after']['status'], 'log':error_logs[-1], 'failed_jobs_metric':errors, 'metric_query':expression, 'failure_to_ready_seconds':round(recovery,3)})
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
exported = json.dumps([logs, api_trace, worker_trace])
private = [sentinel, *(person['token'] for person in (actor,bob,eve)), *(person['registration']['password'] for person in (actor,bob,eve))]
private.extend(path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file())
check(not any(secret and secret in exported for secret in private), 'Private value found in telemetry')
check(all(record.get('trace_id') and record.get('span_id') and record.get('timestamp') and record.get('severity') for record in logs), 'Incomplete structured logs')
report['privacy_checked_logs'] = len(logs)
report['checks'] = {'three_backends':True, 'dashboard_queries':True,'server_consumer_traces':True,'log_trace_correlation':True,'low_cardinality':True,'privacy':True}
if args.restart_telemetry:
historical_time = time.time()
expression = 'sum(messenger_http_requests_total)'
before = grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result']
project.run('restart', 'prometheus', 'loki', 'tempo', 'grafana')
after = eventually(lambda: grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result'], 'Prometheus history lost')
check(before == after, 'Historical metric changed after storage restart')
correlation(grafana, 'messenger-api', start, trace_id=api['trace_id'])
correlation(grafana, 'messenger-worker', start, trace_id=worker['trace_id'])
report['checks']['history_survives_restart'] = True
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
print('PASS: telemetry, dashboard, privacy, correlations' + ('; two incidents recovered' if args.incidents else ''))
def _ready(client):
return client.call('GET', '/health/ready', expected=[200,503])['status'] == 200
if __name__ == '__main__':
main()