396 lines
22 KiB
Python
396 lines
22 KiB
Python
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
|
|
|
|
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
|
|
В evidence сохраняются только агрегаты и выбранные технические поля.
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import math
|
|
import subprocess
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from uuid import uuid4
|
|
|
|
from scripts.deployment import ComposeProject
|
|
from tests.client import Client, Failure, check
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def eventually(operation, message, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = operation()
|
|
if value:
|
|
return value
|
|
except (OSError, ValueError, KeyError, AssertionError):
|
|
pass
|
|
time.sleep(1)
|
|
raise AssertionError(message)
|
|
|
|
|
|
class Grafana:
|
|
def __init__(self, base):
|
|
self.base = base.rstrip('/')
|
|
secret = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
|
self.authorization = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode()
|
|
|
|
def get(self, path, **params):
|
|
url = self.base + path
|
|
if params:
|
|
url += '?' + urllib.parse.urlencode(params)
|
|
request = urllib.request.Request(url, headers={'Authorization': self.authorization, 'Accept': 'application/json'})
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
return json.load(response)
|
|
|
|
def proxy(self, source, path, **params):
|
|
return self.get('/api/datasources/proxy/uid/' + source + path, **params)
|
|
|
|
def query(self, expression):
|
|
value = self.proxy('prometheus', '/api/v1/query', query=expression)
|
|
check(value['status'] == 'success', 'Prometheus query failed')
|
|
return value['data']['result']
|
|
|
|
def logs(self, selector, start):
|
|
value = self.proxy('loki', '/loki/api/v1/query_range', query=selector, start=start, end=time.time_ns(), limit=1000, direction='forward')
|
|
check(value['status'] == 'success', 'Loki query failed')
|
|
return [json.loads(line) for stream in value['data']['result'] for _, line, *_ in stream['values']]
|
|
|
|
def trace(self, trace_id):
|
|
return self.proxy('tempo', '/api/traces/' + trace_id)
|
|
|
|
|
|
def identifier_hex(value, byte_length):
|
|
if len(value) == byte_length * 2 and all(c in '0123456789abcdefABCDEF' for c in value):
|
|
return value.lower()
|
|
return base64.b64decode(value, validate=True).hex()
|
|
|
|
|
|
def spans_in(value):
|
|
found = []
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
if key == 'spans':
|
|
found.extend(item)
|
|
else:
|
|
found.extend(spans_in(item))
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
found.extend(spans_in(item))
|
|
return found
|
|
|
|
|
|
def correlation(grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None):
|
|
selector = '{service_name="' + service + '"} | json'
|
|
if service == 'messenger-api':
|
|
selector += ' | event="http.request"'
|
|
if trace_id:
|
|
selector += ' | trace_id="' + trace_id + '"'
|
|
if attachment_id:
|
|
selector += ' | attachment_id="' + attachment_id + '"'
|
|
if request_id:
|
|
selector += ' | request_id="' + request_id + '"'
|
|
records = eventually(lambda: grafana.logs(selector, start), 'No correlated log for ' + service)
|
|
record = records[0]
|
|
trace_id = record['trace_id']
|
|
def matching_trace():
|
|
candidate = grafana.trace(trace_id)
|
|
return candidate if any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans_in(candidate)) else None
|
|
trace = eventually(matching_trace, 'Matching span is missing in Tempo')
|
|
spans = [span for span in spans_in(trace) if identifier_hex(span.get('spanId', ''), 8) == record['span_id']]
|
|
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
|
|
check(any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans), 'Log span_id not found in real trace')
|
|
check(all(int(span['endTimeUnixNano']) > int(span['startTimeUnixNano']) for span in spans), 'Trace has no real duration')
|
|
return {'trace_id': trace_id, 'span_id': record['span_id'], 'log': record,
|
|
'spans': [{key: span.get(key) for key in ('name', 'kind', 'spanId', 'startTimeUnixNano', 'endTimeUnixNano', 'status')} for span in spans]}, trace
|
|
|
|
|
|
def run_load(args):
|
|
check(1 <= args.duration <= 300 and 1 <= args.rate <= 40 and 1 <= args.concurrency <= 16, 'Load profile outside bounds')
|
|
client = Client(args.base_url)
|
|
actor, _, _, chat = client.bootstrap()
|
|
project = ComposeProject()
|
|
container_ids = project.run('ps', '--quiet').splitlines()
|
|
samples = []
|
|
stop = threading.Event()
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
result = subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*container_ids],capture_output=True,text=True,timeout=15)
|
|
if result.returncode == 0:
|
|
samples.append({'elapsed_seconds':round(time.monotonic()-started,3), 'containers':[
|
|
{k:row[k] for k in ('Name','CPUPerc','MemUsage','MemPerc')} for row in (json.loads(line) for line in result.stdout.splitlines())]})
|
|
stop.wait(3)
|
|
|
|
def write(index):
|
|
local = Client(args.base_url)
|
|
before = time.monotonic()
|
|
try:
|
|
local.message(actor,chat,'synthetic-observability-' + str(index))
|
|
return time.monotonic()-before
|
|
except Failure:
|
|
return None
|
|
|
|
started = time.monotonic()
|
|
sampler = threading.Thread(target=sample)
|
|
sampler.start()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futures = []
|
|
for index in range(args.duration * args.rate):
|
|
time.sleep(max(0, started + index/args.rate - time.monotonic()))
|
|
futures.append(pool.submit(write,index))
|
|
results = [future.result() for future in futures]
|
|
elapsed = time.monotonic()-started
|
|
finally:
|
|
stop.set()
|
|
sampler.join()
|
|
latencies = sorted(value for value in results if value is not None)
|
|
failed = len(results)-len(latencies)
|
|
def percentile(q):
|
|
return round(latencies[max(0, math.ceil(q*len(latencies))-1)]*1000,3) if latencies else None
|
|
report = {'date_utc':datetime.now(timezone.utc).isoformat(), 'workload':'synthetic POST messages; fixed arrival rate; no retries',
|
|
'duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'arrival_rate':args.rate,'concurrency_limit':args.concurrency,
|
|
'successful_requests':len(latencies),'failed_requests':failed,'error_ratio':failed/len(results),
|
|
'successful_rps':round(len(latencies)/elapsed,3), 'successful_latency_ms':{name:percentile(q) for name,q in [('p50',.5),('p95',.95),('p99',.99)]},
|
|
'dataset':{'users':3,'chats':1,'messages':len(latencies)},'resource_samples':samples,
|
|
'limits':'Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.'}
|
|
check(bool(latencies) and failed == 0, 'Controlled load contains errors')
|
|
|
|
return report
|
|
|
|
|
|
def metric(grafana, expression):
|
|
rows = grafana.query(expression)
|
|
return float(rows[0]['value'][1]) if rows else None
|
|
|
|
|
|
def alert_firing(grafana, name):
|
|
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
|
|
|
|
|
|
def trace_chain(grafana, trace_id, *, retry=False):
|
|
def inspect():
|
|
spans = spans_in(grafana.trace(trace_id))
|
|
by_id = {identifier_hex(span['spanId'], 8): span for span in spans}
|
|
workers = [span for span in spans if span['name'] == 'attachment.process.v1'
|
|
and span.get('kind') in ('SPAN_KIND_CONSUMER', 5)]
|
|
for worker in workers:
|
|
publisher = by_id.get(identifier_hex(worker.get('parentSpanId', ''), 8), {})
|
|
if publisher.get('kind') not in ('SPAN_KIND_PRODUCER', 4):
|
|
return None
|
|
upload = by_id.get(identifier_hex(publisher.get('parentSpanId', ''), 8), {})
|
|
if upload.get('name') != 'POST /api/v1/chats/{chat_id}/attachments':
|
|
return None
|
|
names = {span['name'] for span in spans}
|
|
if not workers or not {'s3.read_staging', 's3.write_result', 'postgres.commit_ready'} <= names:
|
|
return None
|
|
if retry:
|
|
outcomes = set()
|
|
for span in workers:
|
|
attrs = {row['key']: row['value'] for row in span.get('attributes', [])}
|
|
outcomes.add((int(attrs['job.attempt']['intValue']), attrs.get('outcome', {}).get('stringValue')))
|
|
if not {(1, 'error'), (2, 'error')} <= outcomes or not any(n > 2 and outcome == 'success' for n, outcome in outcomes):
|
|
return None
|
|
return [{'name': span['name'], 'kind': span.get('kind'),
|
|
'span_id': identifier_hex(span['spanId'], 8),
|
|
'parent_span_id': identifier_hex(span.get('parentSpanId', ''), 8),
|
|
'job': {entry['key']: entry['value'] for entry in span.get('attributes', [])
|
|
if entry['key'] in {'job.attempt', 'outcome'}}} for span in spans]
|
|
return eventually(inspect, 'Missing upload -> producer -> consumer -> S3/DB chain')
|
|
|
|
|
|
def telemetry_outage(project, client, grafana, actor, chat):
|
|
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
|
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
|
baseline = [metric(grafana, expr) for expr in (failed, refused)]
|
|
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
|
samples = []
|
|
successful = 0
|
|
try:
|
|
project.run('stop', 'loki')
|
|
start = time.monotonic()
|
|
for second in range(51):
|
|
for index in range(4):
|
|
client.message(actor, chat, f'outage-{second}-{index}')
|
|
successful += 1
|
|
if second % 5 == 0:
|
|
samples.append({
|
|
'seconds': round(time.monotonic() - start, 2),
|
|
'rss_bytes': metric(grafana, rss),
|
|
'queued_batches': metric(grafana, 'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})'),
|
|
'capacity_batches': metric(grafana, 'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})'),
|
|
})
|
|
time.sleep(max(0, start + second + 1 - time.monotonic()))
|
|
losses = [metric(grafana, expr) - count for expr, count in zip((failed, refused), baseline)]
|
|
check(sum(losses) > 0, 'No measured Collector log loss during prolonged outage')
|
|
check(all(0 <= row['queued_batches'] <= row['capacity_batches'] == 256 for row in samples), 'Unexpected queue bound')
|
|
check(max(row['rss_bytes'] for row in samples) < 256 * 1024**2, 'Collector crossed container memory limit')
|
|
finally:
|
|
project.run('start', 'loki')
|
|
eventually(lambda: grafana.proxy('loki', '/loki/api/v1/labels').get('status') == 'success',
|
|
'Loki did not recover')
|
|
restored = client.message(actor, chat, 'telemetry-backend-restored')
|
|
correlation(grafana, 'messenger-api', time.time_ns() - 30 * 10**9,
|
|
trace_id=restored['headers']['X-Trace-Id'])
|
|
return {'scenario':'loki_unavailable', 'successful_messages': successful,
|
|
'dropped_after_export_failure': losses[0], 'dropped_at_enqueue': losses[1],
|
|
'samples': samples, 'log_export_recovered': True}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', required=True)
|
|
parser.add_argument('--grafana-url', default='http://localhost:13000')
|
|
parser.add_argument('--incidents', action='store_true')
|
|
parser.add_argument('--restart-telemetry', action='store_true')
|
|
parser.add_argument('--output', default='evidence/observability.json')
|
|
parser.add_argument('--duration', type=int, default=10)
|
|
parser.add_argument('--rate', type=int, default=10)
|
|
parser.add_argument('--concurrency', type=int, default=2)
|
|
args = parser.parse_args()
|
|
load = run_load(args)
|
|
start = time.time_ns()
|
|
project = ComposeProject()
|
|
client = Client(args.base_url, timeout=40)
|
|
aliases = {r.instance_id for r in project.replicas() if r.running}
|
|
check(len(aliases) >= 2, 'Need two real API containers')
|
|
check(client.call('GET', '/health/live')['headers'].get('X-Instance-Id') in aliases, 'Wrong Compose project for BASE_URL')
|
|
grafana = Grafana(args.grafana_url)
|
|
eventually(lambda: grafana.get('/api/health').get('database') == 'ok', 'Grafana is not ready')
|
|
sources = grafana.get('/api/datasources')
|
|
check({'prometheus', 'loki', 'tempo'} <= {source['uid'] for source in sources}, 'Missing provisioned datasource')
|
|
dashboard = grafana.get('/api/dashboards/uid/messenger-overview')['dashboard']
|
|
check(len(dashboard['panels']) >= 8, 'Dashboard is missing panels')
|
|
actor, bob, eve, chat = client.bootstrap()
|
|
sentinel = 'private-content-' + uuid4().hex
|
|
response = client.message(actor, chat, sentinel)
|
|
controlled_trace = response['headers'].get('X-Trace-Id')
|
|
check(bool(controlled_trace), 'No API trace identifier')
|
|
attachment, _ = client.attachment(actor, chat)
|
|
client.wait_ready(actor, attachment)
|
|
api, api_trace = correlation(grafana, 'messenger-api', start, trace_id=controlled_trace, request_id=response['headers']['X-Request-Id'])
|
|
worker, worker_trace = correlation(grafana, 'messenger-worker', start, attachment_id=attachment['id'])
|
|
accepted_chain = trace_chain(grafana, worker['trace_id'])
|
|
check(api['log']['request_id'] == response['headers']['X-Request-Id'], 'Request id correlation lost')
|
|
check(any(span['kind'] in ('SPAN_KIND_SERVER', 2) for span in api['spans']), 'No server span')
|
|
check(any(span['kind'] in ('SPAN_KIND_CONSUMER', 5) for span in worker['spans']), 'No consumer span')
|
|
processed = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | trace_id="' + worker['trace_id'] + '"', start), 'No worker result log')
|
|
check(processed[0]['outcome'] == 'success', 'Controlled worker did not succeed')
|
|
metrics = {}
|
|
expressions = {
|
|
'http_requests': 'sum(messenger_http_requests_total)',
|
|
'http_duration': 'sum(messenger_http_duration_seconds_count)',
|
|
'job_success': 'sum(messenger_job_attempts_total{outcome="success"})',
|
|
'job_duration': 'sum(messenger_job_duration_seconds_count)',
|
|
}
|
|
for name, expression in expressions.items():
|
|
metrics[name] = eventually(lambda expression=expression: grafana.query(expression), 'Missing metric: ' + expression)
|
|
check(float(metrics[name][0]['value'][1]) > 0, 'Empty metric: ' + name)
|
|
# Все PromQL из dashboard должны давать численные данные после traffic.
|
|
for panel in dashboard['panels']:
|
|
if panel['datasource']['uid'] == 'prometheus':
|
|
eventually(lambda panel=panel: all(grafana.query(target['expr']) for target in panel['targets']), 'Empty dashboard panel: ' + panel['title'])
|
|
forbidden_labels = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target','http_url'}
|
|
series = grafana.query('{__name__=~"messenger_.*"}')
|
|
check(not any(forbidden_labels & set(row['metric']) for row in series), 'High cardinality metric label')
|
|
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'project': project.name,
|
|
'load': load, 'api_replicas': len(aliases), 'dashboard_uid': dashboard['uid'],
|
|
'api_correlation': api, 'worker_correlation': worker, 'metrics': metrics, 'upload_trace': {'trace_id':worker['trace_id'], 'chain':accepted_chain}, 'incidents': []}
|
|
if args.incidents:
|
|
def fail_request():
|
|
return client.call('POST', '/api/v1/users', expected=503,
|
|
body={'username':'fault_' + uuid4().hex[:12],
|
|
'display_name':'Synthetic', 'password':'not-exported-' + uuid4().hex})
|
|
|
|
before = time.monotonic()
|
|
try:
|
|
project.run('stop', 'postgres')
|
|
failed = fail_request()
|
|
error_trace = failed['headers'].get('X-Trace-Id')
|
|
error, db_trace = correlation(grafana, 'messenger-api', start, trace_id=error_trace,
|
|
request_id=failed['headers']['X-Request-Id'])
|
|
diagnostics = eventually(lambda: grafana.logs(
|
|
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="' + error_trace + '"', start),
|
|
'No dependency diagnostic log')
|
|
checkout = next(span for span in spans_in(db_trace) if span['name'] == 'postgres.checkout'
|
|
and span.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2))
|
|
|
|
def db_alert():
|
|
for _ in range(4):
|
|
fail_request()
|
|
return alert_firing(grafana, 'MessengerHighErrorRate')
|
|
|
|
eventually(db_alert, 'Error rate alert did not fire', timeout=75)
|
|
report['incidents'].append({'scenario':'postgres_unavailable','http_status':503,'correlation':error,
|
|
'diagnostic':diagnostics[0], 'db_span':checkout['name'],
|
|
'alert':'MessengerHighErrorRate', 'firing_after_seconds':round(time.monotonic()-before,3)})
|
|
finally:
|
|
project.run('start', 'postgres')
|
|
eventually(lambda: _ready(client), 'API did not recover after PostgreSQL')
|
|
eventually(lambda: not alert_firing(grafana, 'MessengerHighErrorRate'), 'Error rate alert not resolved')
|
|
report['incidents'][-1]['resolved_after_seconds'] = round(time.monotonic() - before, 3)
|
|
|
|
from scripts.check_jobs import retry_replay
|
|
queue = {}
|
|
|
|
def observe_queue(_):
|
|
observed = time.monotonic()
|
|
eventually(lambda: alert_firing(grafana, 'MessengerQueueLag'), 'Queue lag alert did not fire')
|
|
queue.update(depth=metric(grafana, 'max(messenger_queue_depth_ratio)'),
|
|
oldest_seconds=metric(grafana, 'max(messenger_queue_oldest_seconds)'),
|
|
firing_after_seconds=round(time.monotonic()-observed,3))
|
|
|
|
incident = retry_replay(client, actor, chat, permanent=False, on_queued=observe_queue)
|
|
retried_worker, retry_trace = correlation(grafana, 'messenger-worker', start,
|
|
attachment_id=incident['attachment_id'])
|
|
chain = trace_chain(grafana, retried_worker['trace_id'], retry=True)
|
|
eventually(lambda: not alert_firing(grafana, 'MessengerQueueLag')
|
|
and metric(grafana, 'max(messenger_queue_depth_ratio)') == 0, 'Backlog did not drain')
|
|
queue.update(resolved=True, final_depth=0)
|
|
report['incidents'].append({'scenario':'worker_stopped_then_transient_retry',
|
|
'alert':'MessengerQueueLag', 'queue':queue, 'recovered_status':incident['after']['status'],
|
|
'trace_id':retried_worker['trace_id'], 'chain':chain})
|
|
report['incidents'].append(telemetry_outage(project, client, grafana, actor, chat))
|
|
else:
|
|
retry_trace = {}
|
|
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
|
|
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
|
|
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
|
|
private = [sentinel, *(person['token'] for person in (actor,bob,eve)), *(person['registration']['password'] for person in (actor,bob,eve))]
|
|
private.extend(path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file())
|
|
check(not any(secret and secret in exported for secret in private), 'Private value found in telemetry')
|
|
check(all(record.get('trace_id') and record.get('span_id') and record.get('timestamp') and record.get('severity') for record in logs), 'Incomplete structured logs')
|
|
report['privacy_checked_logs'] = len(logs)
|
|
report['checks'] = {'three_backends':True, 'dashboard_queries':True,'server_consumer_traces':True,'log_trace_correlation':True,'low_cardinality':True,'privacy':True}
|
|
if args.restart_telemetry:
|
|
historical_time = time.time()
|
|
expression = 'sum(messenger_http_requests_total)'
|
|
before = grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result']
|
|
project.run('restart', 'prometheus', 'loki', 'tempo', 'grafana')
|
|
after = eventually(lambda: grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result'], 'Prometheus history lost')
|
|
check(before == after, 'Historical metric changed after storage restart')
|
|
correlation(grafana, 'messenger-api', start, trace_id=api['trace_id'])
|
|
correlation(grafana, 'messenger-worker', start, trace_id=worker['trace_id'])
|
|
report['checks']['history_survives_restart'] = True
|
|
destination = ROOT / args.output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS: telemetry, dashboard, privacy, correlations' + ('; alerts and outage recovery verified' if args.incidents else ''))
|
|
|
|
|
|
def _ready(client):
|
|
return client.call('GET', '/health/ready', expected=[200,503])['status'] == 200
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|