Files
mtusi2026lab7/scripts/lab6_storage_down_series.py

128 lines
5.3 KiB
Python

#!/usr/bin/env python3
"""Isolated telemetry-storage outage under load, with baseline/recovery and cgroup data."""
import argparse
import json
import os
import subprocess
import time
import urllib.parse
import urllib.request
from pathlib import Path
from lab7_environment import Environment
from lab7_http import BASE_URL, open_url
ROOT = Path(__file__).resolve().parents[1]
for key in ('HTTP_PROXY','HTTPS_PROXY','ALL_PROXY','http_proxy','https_proxy','all_proxy'):
os.environ.pop(key, None)
def prom(expr):
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
with open_url(url, timeout=10, operator=True) as response:
result = json.loads(response.read()).get('data', {}).get('result', [])
return result[0]['value'][1] if result else 'MISSING'
def labels(expr, label):
url = 'https://localhost:9090/api/v1/query?' + urllib.parse.urlencode({'query': expr})
with open_url(url, timeout=10, operator=True) as response:
result = json.loads(response.read()).get('data', {}).get('result', [])
return [r['metric'].get(label, '') for r in result]
def compose(*args, check=True):
return Environment().compose(*args, check=check)
def cgroup_stats():
ids = compose('ps', '-q', 'api', 'otel-collector').stdout.split()
if not ids:
return []
output = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}} {{.MemUsage}} {{.MemPerc}}', *ids],
capture_output=True, text=True, check=True).stdout
return output.splitlines()
def sample(phase):
return {
'phase': phase,
'utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'api_memory': prom('max by (service_instance_id) (messenger_process_memory_usage_bytes{service_name="messenger-api"})'),
'collector_rss': prom('max(otelcol_process_memory_rss)'),
'collector_heap': prom('max(otelcol_process_runtime_heap_alloc_bytes)'),
'queue_logs': prom('max(otelcol_exporter_queue_size{data_type="logs"})'),
'queue_traces': prom('max(otelcol_exporter_queue_size{data_type="traces"})'),
'sent_spans': prom('sum(rate(otelcol_exporter_sent_spans[5m]))'),
'send_failed_spans': prom('sum(rate(otelcol_exporter_send_failed_spans[5m]))'),
'enqueue_failed_spans': prom('sum(rate(otelcol_exporter_enqueue_failed_spans[5m]))'),
'accepted_logs': prom('sum(rate(otelcol_receiver_accepted_log_records[5m]))'),
'api_error_5m': prom('slo:api_error_ratio:5m'),
'api_p99': prom('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))'),
'api_instances': labels('messenger_process_memory_usage_bytes{service_name="messenger-api"}', 'service_instance_id'),
'cgroup': cgroup_stats(),
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--baseline', type=int, default=60)
parser.add_argument('--duration', type=int, default=300)
parser.add_argument('--recovery', type=int, default=300)
parser.add_argument('--out', default='evidence/lab6/storage-down-series.json')
args = parser.parse_args()
load_path = ROOT/'evidence/lab6/checks/storage-down-load.json'
load_path.unlink(missing_ok=True)
load_duration = args.baseline + args.duration + args.recovery + 120
load = subprocess.Popen([
'python3', str(ROOT/'scripts/lab6_load.py'), '--base-url', BASE_URL,
'--rate', '0.2', '--duration', str(load_duration),
'--upload-ratio', '0.2', '--deadline', '60',
'--output', str(load_path),
], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
samples = []
try:
for _ in range(args.baseline // 10):
time.sleep(10)
samples.append(sample('baseline'))
compose('stop','loki-tls','tempo-tls','loki','tempo')
samples.append(sample('down-start'))
started = time.monotonic()
while time.monotonic() - started < args.duration:
time.sleep(10)
samples.append(sample('down'))
compose('start','loki','tempo','loki-tls','tempo-tls')
samples.append(sample('recovery-start'))
started = time.monotonic()
zero_streak = 0
while time.monotonic() - started < args.recovery:
time.sleep(10)
samples.append(sample('recovery'))
logs_zero = prom('max(otelcol_exporter_queue_size{data_type="logs"})') in ('0','0.0')
traces_zero = prom('max(otelcol_exporter_queue_size{data_type="traces"})') in ('0','0.0')
if logs_zero and traces_zero:
zero_streak += 1
if zero_streak >= 2:
break
else:
zero_streak = 0
finally:
compose('start','loki','tempo','loki-tls','tempo-tls')
try:
load.wait(timeout=load_duration + 120)
except subprocess.TimeoutExpired:
load.kill()
load.wait()
print('load_exit=', load.returncode, flush=True)
if load_path.exists():
data = json.loads(load_path.read_text())
print('load_summary=', json.dumps({k: data.get(k) for k in ('counters','failures','failures_by_reason','duration_s')}), flush=True)
out = ROOT / args.out
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(samples, indent=2) + '\n')
print(json.dumps(samples, indent=2))
if __name__ == '__main__':
main()