forked from mtusi2026/mtusi2026lab7
128 lines
5.3 KiB
Python
128 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated telemetry-storage outage under load, with baseline/recovery and cgroup data."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from lab7_environment import Environment
|
|
from lab7_http import BASE_URL, open_url
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
for key in ('HTTP_PROXY','HTTPS_PROXY','ALL_PROXY','http_proxy','https_proxy','all_proxy'):
|
|
os.environ.pop(key, None)
|
|
|
|
|
|
def prom(expr):
|
|
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
|
|
with open_url(url, timeout=10, operator=True) as response:
|
|
result = json.loads(response.read()).get('data', {}).get('result', [])
|
|
return result[0]['value'][1] if result else 'MISSING'
|
|
|
|
|
|
def labels(expr, label):
|
|
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
|
|
with open_url(url, timeout=10, operator=True) as response:
|
|
result = json.loads(response.read()).get('data', {}).get('result', [])
|
|
return [r['metric'].get(label, '') for r in result]
|
|
|
|
|
|
def compose(*args, check=True):
|
|
return Environment().compose(*args, check=check)
|
|
|
|
|
|
def cgroup_stats():
|
|
ids = compose('ps', '-q', 'api', 'otel-collector').stdout.split()
|
|
if not ids:
|
|
return []
|
|
output = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}} {{.MemUsage}} {{.MemPerc}}', *ids],
|
|
capture_output=True, text=True, check=True).stdout
|
|
return output.splitlines()
|
|
|
|
|
|
def sample(phase):
|
|
return {
|
|
'phase': phase,
|
|
'utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
'api_memory': prom('max by (service_instance_id) (messenger_process_memory_usage_bytes{service_name="messenger-api"})'),
|
|
'collector_rss': prom('max(otelcol_process_memory_rss)'),
|
|
'collector_heap': prom('max(otelcol_process_runtime_heap_alloc_bytes)'),
|
|
'queue_logs': prom('max(otelcol_exporter_queue_size{data_type="logs"})'),
|
|
'queue_traces': prom('max(otelcol_exporter_queue_size{data_type="traces"})'),
|
|
'sent_spans': prom('sum(rate(otelcol_exporter_sent_spans[5m]))'),
|
|
'send_failed_spans': prom('sum(rate(otelcol_exporter_send_failed_spans[5m]))'),
|
|
'enqueue_failed_spans': prom('sum(rate(otelcol_exporter_enqueue_failed_spans[5m]))'),
|
|
'accepted_logs': prom('sum(rate(otelcol_receiver_accepted_log_records[5m]))'),
|
|
'api_error_5m': prom('slo:api_error_ratio:5m'),
|
|
'api_p99': prom('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))'),
|
|
'api_instances': labels('messenger_process_memory_usage_bytes{service_name="messenger-api"}', 'service_instance_id'),
|
|
'cgroup': cgroup_stats(),
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--baseline', type=int, default=60)
|
|
parser.add_argument('--duration', type=int, default=300)
|
|
parser.add_argument('--recovery', type=int, default=300)
|
|
parser.add_argument('--out', default='evidence/lab6/storage-down-series.json')
|
|
args = parser.parse_args()
|
|
load_path = ROOT/'evidence/lab6/checks/storage-down-load.json'
|
|
load_path.unlink(missing_ok=True)
|
|
load_duration = args.baseline + args.duration + args.recovery + 120
|
|
load = subprocess.Popen([
|
|
'python3', str(ROOT/'scripts/lab6_load.py'), '--base-url', BASE_URL,
|
|
'--rate', '0.2', '--duration', str(load_duration),
|
|
'--upload-ratio', '0.2', '--deadline', '60',
|
|
'--output', str(load_path),
|
|
], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
samples = []
|
|
try:
|
|
for _ in range(args.baseline // 10):
|
|
time.sleep(10)
|
|
samples.append(sample('baseline'))
|
|
compose('stop','loki-tls','tempo-tls','loki','tempo')
|
|
samples.append(sample('down-start'))
|
|
started = time.monotonic()
|
|
while time.monotonic() - started < args.duration:
|
|
time.sleep(10)
|
|
samples.append(sample('down'))
|
|
compose('start','loki','tempo','loki-tls','tempo-tls')
|
|
samples.append(sample('recovery-start'))
|
|
started = time.monotonic()
|
|
zero_streak = 0
|
|
while time.monotonic() - started < args.recovery:
|
|
time.sleep(10)
|
|
samples.append(sample('recovery'))
|
|
logs_zero = prom('max(otelcol_exporter_queue_size{data_type="logs"})') in ('0','0.0')
|
|
traces_zero = prom('max(otelcol_exporter_queue_size{data_type="traces"})') in ('0','0.0')
|
|
if logs_zero and traces_zero:
|
|
zero_streak += 1
|
|
if zero_streak >= 2:
|
|
break
|
|
else:
|
|
zero_streak = 0
|
|
finally:
|
|
compose('start','loki','tempo','loki-tls','tempo-tls')
|
|
try:
|
|
load.wait(timeout=load_duration + 120)
|
|
except subprocess.TimeoutExpired:
|
|
load.kill()
|
|
load.wait()
|
|
print('load_exit=', load.returncode, flush=True)
|
|
if load_path.exists():
|
|
data = json.loads(load_path.read_text())
|
|
print('load_summary=', json.dumps({k: data.get(k) for k in ('counters','failures','failures_by_reason','duration_s')}), flush=True)
|
|
out = ROOT / args.out
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(samples, indent=2) + '\n')
|
|
print(json.dumps(samples, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|