forked from mtusi2026/mtusi2026lab6
292 lines
17 KiB
Python
292 lines
17 KiB
Python
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
|
|
|
|
--incidents явно разрешает два ограниченных отказа на выбранном Compose-проекте.
|
|
В evidence сохраняются только агрегаты и выбранные технические поля.
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import math
|
|
import subprocess
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from uuid import uuid4
|
|
|
|
from scripts.deployment import ComposeProject
|
|
from tests.client import Client, Failure, check
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def eventually(operation, message, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = operation()
|
|
if value:
|
|
return value
|
|
except (OSError, ValueError, KeyError, AssertionError):
|
|
pass
|
|
time.sleep(1)
|
|
raise AssertionError(message)
|
|
|
|
|
|
class Grafana:
|
|
def __init__(self, base):
|
|
self.base = base.rstrip('/')
|
|
secret = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
|
self.authorization = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode()
|
|
|
|
def get(self, path, **params):
|
|
url = self.base + path
|
|
if params:
|
|
url += '?' + urllib.parse.urlencode(params)
|
|
request = urllib.request.Request(url, headers={'Authorization': self.authorization, 'Accept': 'application/json'})
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
return json.load(response)
|
|
|
|
def proxy(self, source, path, **params):
|
|
return self.get('/api/datasources/proxy/uid/' + source + path, **params)
|
|
|
|
def query(self, expression):
|
|
value = self.proxy('prometheus', '/api/v1/query', query=expression)
|
|
check(value['status'] == 'success', 'Prometheus query failed')
|
|
return value['data']['result']
|
|
|
|
def logs(self, selector, start):
|
|
value = self.proxy('loki', '/loki/api/v1/query_range', query=selector, start=start, end=time.time_ns(), limit=1000, direction='forward')
|
|
check(value['status'] == 'success', 'Loki query failed')
|
|
return [json.loads(line) for stream in value['data']['result'] for _, line, *_ in stream['values']]
|
|
|
|
def trace(self, trace_id):
|
|
return self.proxy('tempo', '/api/traces/' + trace_id)
|
|
|
|
|
|
def identifier_hex(value, byte_length):
|
|
if len(value) == byte_length * 2 and all(c in '0123456789abcdefABCDEF' for c in value):
|
|
return value.lower()
|
|
return base64.b64decode(value, validate=True).hex()
|
|
|
|
|
|
def spans_in(value):
|
|
found = []
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
if key == 'spans':
|
|
found.extend(item)
|
|
else:
|
|
found.extend(spans_in(item))
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
found.extend(spans_in(item))
|
|
return found
|
|
|
|
|
|
def correlation(grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None):
|
|
selector = '{service_name="' + service + '"} | json'
|
|
if service == 'messenger-api':
|
|
selector += ' | event="http.request"'
|
|
if trace_id:
|
|
selector += ' | trace_id="' + trace_id + '"'
|
|
if attachment_id:
|
|
selector += ' | attachment_id="' + attachment_id + '"'
|
|
if request_id:
|
|
selector += ' | request_id="' + request_id + '"'
|
|
records = eventually(lambda: grafana.logs(selector, start), 'No correlated log for ' + service)
|
|
record = records[0]
|
|
trace_id = record['trace_id']
|
|
def matching_trace():
|
|
candidate = grafana.trace(trace_id)
|
|
return candidate if any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans_in(candidate)) else None
|
|
trace = eventually(matching_trace, 'Matching span is missing in Tempo')
|
|
spans = [span for span in spans_in(trace) if identifier_hex(span.get('spanId', ''), 8) == record['span_id']]
|
|
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
|
|
check(any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans), 'Log span_id not found in real trace')
|
|
check(all(int(span['endTimeUnixNano']) > int(span['startTimeUnixNano']) for span in spans), 'Trace has no real duration')
|
|
return {'trace_id': trace_id, 'span_id': record['span_id'], 'log': record,
|
|
'spans': [{key: span.get(key) for key in ('name', 'kind', 'spanId', 'startTimeUnixNano', 'endTimeUnixNano', 'status')} for span in spans]}, trace
|
|
|
|
|
|
def run_load(args):
|
|
check(1 <= args.duration <= 300 and 1 <= args.rate <= 40 and 1 <= args.concurrency <= 16, 'Load profile outside bounds')
|
|
client = Client(args.base_url)
|
|
actor, _, _, chat = client.bootstrap()
|
|
project = ComposeProject()
|
|
container_ids = project.run('ps', '--quiet').splitlines()
|
|
samples = []
|
|
stop = threading.Event()
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
result = subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*container_ids],capture_output=True,text=True,timeout=15)
|
|
if result.returncode == 0:
|
|
samples.append({'elapsed_seconds':round(time.monotonic()-started,3), 'containers':[
|
|
{k:row[k] for k in ('Name','CPUPerc','MemUsage','MemPerc')} for row in (json.loads(line) for line in result.stdout.splitlines())]})
|
|
stop.wait(3)
|
|
|
|
def write(index):
|
|
local = Client(args.base_url)
|
|
before = time.monotonic()
|
|
try:
|
|
local.message(actor,chat,'synthetic-observability-' + str(index))
|
|
return time.monotonic()-before
|
|
except Failure:
|
|
return None
|
|
|
|
started = time.monotonic()
|
|
sampler = threading.Thread(target=sample)
|
|
sampler.start()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futures = []
|
|
for index in range(args.duration * args.rate):
|
|
time.sleep(max(0, started + index/args.rate - time.monotonic()))
|
|
futures.append(pool.submit(write,index))
|
|
results = [future.result() for future in futures]
|
|
elapsed = time.monotonic()-started
|
|
finally:
|
|
stop.set()
|
|
sampler.join()
|
|
latencies = sorted(value for value in results if value is not None)
|
|
failed = len(results)-len(latencies)
|
|
def percentile(q):
|
|
return round(latencies[max(0, math.ceil(q*len(latencies))-1)]*1000,3) if latencies else None
|
|
report = {'date_utc':datetime.now(timezone.utc).isoformat(), 'workload':'synthetic POST messages; fixed arrival rate; no retries',
|
|
'duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'arrival_rate':args.rate,'concurrency_limit':args.concurrency,
|
|
'successful_requests':len(latencies),'failed_requests':failed,'error_ratio':failed/len(results),
|
|
'successful_rps':round(len(latencies)/elapsed,3), 'successful_latency_ms':{name:percentile(q) for name,q in [('p50',.5),('p95',.95),('p99',.99)]},
|
|
'dataset':{'users':3,'chats':1,'messages':len(latencies)},'resource_samples':samples,
|
|
'limits':'Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.'}
|
|
check(bool(latencies) and failed == 0, 'Controlled load contains errors')
|
|
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', required=True)
|
|
parser.add_argument('--grafana-url', default='http://localhost:13000')
|
|
parser.add_argument('--incidents', action='store_true')
|
|
parser.add_argument('--restart-telemetry', action='store_true')
|
|
parser.add_argument('--output', default='evidence/observability.json')
|
|
parser.add_argument('--duration', type=int, default=10)
|
|
parser.add_argument('--rate', type=int, default=10)
|
|
parser.add_argument('--concurrency', type=int, default=2)
|
|
args = parser.parse_args()
|
|
load = run_load(args)
|
|
start = time.time_ns()
|
|
project = ComposeProject()
|
|
client = Client(args.base_url, timeout=40)
|
|
aliases = {r.instance_id for r in project.replicas() if r.running}
|
|
check(len(aliases) >= 2, 'Need two real API containers')
|
|
check(client.call('GET', '/health/live')['headers'].get('X-Instance-Id') in aliases, 'Wrong Compose project for BASE_URL')
|
|
grafana = Grafana(args.grafana_url)
|
|
eventually(lambda: grafana.get('/api/health').get('database') == 'ok', 'Grafana is not ready')
|
|
sources = grafana.get('/api/datasources')
|
|
check({'prometheus', 'loki', 'tempo'} <= {source['uid'] for source in sources}, 'Missing provisioned datasource')
|
|
dashboard = grafana.get('/api/dashboards/uid/messenger-overview')['dashboard']
|
|
check(len(dashboard['panels']) >= 8, 'Dashboard is missing panels')
|
|
actor, bob, eve, chat = client.bootstrap()
|
|
sentinel = 'private-content-' + uuid4().hex
|
|
response = client.message(actor, chat, sentinel)
|
|
controlled_trace = response['headers'].get('X-Trace-Id')
|
|
check(bool(controlled_trace), 'No API trace identifier')
|
|
attachment, _ = client.attachment(actor, chat)
|
|
client.wait_ready(actor, attachment)
|
|
api, api_trace = correlation(grafana, 'messenger-api', start, trace_id=controlled_trace, request_id=response['headers']['X-Request-Id'])
|
|
worker, worker_trace = correlation(grafana, 'messenger-worker', start, attachment_id=attachment['id'])
|
|
check(api['log']['request_id'] == response['headers']['X-Request-Id'], 'Request id correlation lost')
|
|
check(any(span['kind'] in ('SPAN_KIND_SERVER', 2) for span in api['spans']), 'No server span')
|
|
check(any(span['kind'] in ('SPAN_KIND_CONSUMER', 5) for span in worker['spans']), 'No consumer span')
|
|
processed = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | trace_id="' + worker['trace_id'] + '"', start), 'No worker result log')
|
|
check(processed[0]['outcome'] == 'success', 'Controlled worker did not succeed')
|
|
metrics = {}
|
|
expressions = {
|
|
'http_requests': 'sum(messenger_http_requests_total)',
|
|
'http_duration': 'sum(messenger_http_duration_seconds_count)',
|
|
'job_success': 'sum(messenger_job_attempts_total{outcome="success"})',
|
|
'job_duration': 'sum(messenger_job_duration_seconds_count)',
|
|
}
|
|
for name, expression in expressions.items():
|
|
metrics[name] = eventually(lambda expression=expression: grafana.query(expression), 'Missing metric: ' + expression)
|
|
check(float(metrics[name][0]['value'][1]) > 0, 'Empty metric: ' + name)
|
|
# Все PromQL из dashboard должны давать численные данные после traffic.
|
|
for panel in dashboard['panels']:
|
|
if panel['datasource']['uid'] == 'prometheus':
|
|
eventually(lambda panel=panel: all(grafana.query(target['expr']) for target in panel['targets']), 'Empty dashboard panel: ' + panel['title'])
|
|
forbidden_labels = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target','http_url'}
|
|
series = grafana.query('{__name__=~"messenger_.*"}')
|
|
check(not any(forbidden_labels & set(row['metric']) for row in series), 'High cardinality metric label')
|
|
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'project': project.name,
|
|
'load': load, 'api_replicas': len(aliases), 'dashboard_uid': dashboard['uid'],
|
|
'api_correlation': api, 'worker_correlation': worker, 'metrics': metrics, 'incidents': []}
|
|
if args.incidents:
|
|
# Восстановление выполняется даже при ошибке проверки.
|
|
try:
|
|
before = time.monotonic()
|
|
project.run('stop', 'postgres')
|
|
failed = client.call('POST', '/api/v1/users', expected=503,
|
|
body={'username':'fault_' + uuid4().hex[:12], 'display_name':'Synthetic', 'password':'not-exported-' + uuid4().hex})
|
|
error_trace = failed['headers'].get('X-Trace-Id')
|
|
error, _ = correlation(grafana, 'messenger-api', start, trace_id=error_trace, request_id=failed['headers']['X-Request-Id'])
|
|
diagnostics = eventually(lambda: grafana.logs('{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="' + error_trace + '"', start), 'No dependency diagnostic log')
|
|
check(error['log'].get('status_code') == 503 or any(s['status'].get('code') in ('STATUS_CODE_ERROR', 2) for s in error['spans']), 'DB failure not reflected in trace')
|
|
report['incidents'].append({'scenario':'postgres_unavailable','http_status':503,'correlation':error,'diagnostic':diagnostics[0], 'observed_after_seconds':round(time.monotonic()-before,3)})
|
|
finally:
|
|
project.run('start', 'postgres')
|
|
eventually(lambda: _ready(client), 'API did not recover after PostgreSQL')
|
|
report['incidents'][-1]['recovery_seconds'] = round(time.monotonic() - before, 3)
|
|
# Явный fault hook обработки заданий: один permanent processing failure, затем replay.
|
|
from scripts.check_jobs import retry_replay
|
|
failed_instance = []
|
|
def wait_failed_metric(sample, container_name):
|
|
instance = project.docker('inspect', '--format', '{{.Config.Hostname}}', container_name)
|
|
failed_instance.append(instance)
|
|
expression = 'sum(messenger_job_attempts_total{outcome="error",service_instance_id="' + instance + '"})'
|
|
eventually(lambda: grafana.query(expression), 'Failed job metric not exported before recovery')
|
|
incident = retry_replay(client, actor, chat, permanent=True, on_failed=wait_failed_metric)
|
|
error_logs = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | outcome="error" | error_code="injected_permanent"', start), 'No failed job telemetry')
|
|
error_trace = eventually(lambda: grafana.trace(error_logs[-1]['trace_id']), 'No failed job trace')
|
|
check(any(span.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2) for span in spans_in(error_trace)), 'Worker error trace status lost')
|
|
expression = 'sum(messenger_job_attempts_total{outcome="error",service_instance_id="' + failed_instance[0] + '"})'
|
|
errors = eventually(lambda: grafana.query(expression), 'No failed jobs counter')
|
|
recovery = (datetime.fromisoformat(incident['after']['updated_at']) - datetime.fromisoformat(incident['samples'][-1]['updated_at'])).total_seconds()
|
|
report['incidents'].append({'scenario':'permanent_processing_failure','recovered_status':incident['after']['status'], 'log':error_logs[-1], 'failed_jobs_metric':errors, 'metric_query':expression, 'failure_to_ready_seconds':round(recovery,3)})
|
|
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
|
|
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
|
|
exported = json.dumps([logs, api_trace, worker_trace])
|
|
private = [sentinel, *(person['token'] for person in (actor,bob,eve)), *(person['registration']['password'] for person in (actor,bob,eve))]
|
|
private.extend(path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file())
|
|
check(not any(secret and secret in exported for secret in private), 'Private value found in telemetry')
|
|
check(all(record.get('trace_id') and record.get('span_id') and record.get('timestamp') and record.get('severity') for record in logs), 'Incomplete structured logs')
|
|
report['privacy_checked_logs'] = len(logs)
|
|
report['checks'] = {'three_backends':True, 'dashboard_queries':True,'server_consumer_traces':True,'log_trace_correlation':True,'low_cardinality':True,'privacy':True}
|
|
if args.restart_telemetry:
|
|
historical_time = time.time()
|
|
expression = 'sum(messenger_http_requests_total)'
|
|
before = grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result']
|
|
project.run('restart', 'prometheus', 'loki', 'tempo', 'grafana')
|
|
after = eventually(lambda: grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result'], 'Prometheus history lost')
|
|
check(before == after, 'Historical metric changed after storage restart')
|
|
correlation(grafana, 'messenger-api', start, trace_id=api['trace_id'])
|
|
correlation(grafana, 'messenger-worker', start, trace_id=worker['trace_id'])
|
|
report['checks']['history_survives_restart'] = True
|
|
destination = ROOT / args.output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS: telemetry, dashboard, privacy, correlations' + ('; two incidents recovered' if args.incidents else ''))
|
|
|
|
|
|
def _ready(client):
|
|
return client.call('GET', '/health/ready', expected=[200,503])['status'] == 200
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|