Files
mtusi2026lab6/student-tests/observability_ops.py
T

472 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Проверки наблюдаемости lab6 на Docker-стенде (только по сохранённым наблюдениям).
signals уровень 3–4: метрики контракта есть и с правильными метками; запрос →
трасса (Tempo) → лог с тем же trace_id (Loki); upload → API → relay (producer)
→ worker (consumer) → S3/DB в одной трассе; нет утечек пароля, токена,
текста сообщения; dashboard из provisioning, его запросы возвращают данные
retry-trace S3 недоступен во время обработки: попытки с retry в той же трассе upload
incident-worker инцидент 1: worker остановлен → возраст задач растёт → alert firing →
worker запущен → backlog уходит → resolved; диагностика по метрикам/логам/трассам
incident-db инцидент 2: блокировка таблицы messages (БД «зависла») под нагрузкой →
5xx → MessengerHighErrorRate firing → resolved; трасса показывает, что время
ушло в запрос к БД, а не в обработчик (сравнение с медленным upload)
backend-down Tempo и Loki остановлены под нагрузкой: API без ошибок, память Collector
ограничена, потери считаются; затем остановлен сам Collector
Печатает JSON-итог и PASS/FAIL. Токены и пароли не печатаются.
"""
import json
import re
import statistics
import subprocess
import threading
import time
import uuid
from pathlib import Path
import labutil # noqa: F401 (добавляет tests/ в sys.path)
from client import Client, Failure, arguments, check, run
from labutil import ROOT, compose, psql, sh, wait_status
from loadgen import Load
import obsutil as o
SAMPLE = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
FORBIDDEN_LABELS = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'url', 'url_full',
'http_target', 'filename', 'text', 'username'}
UUID_RE = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
def upload(c, actor, chat, rid=None, data=SAMPLE, name='p.png'):
h = {'Content-Type': 'image/png'}
if rid:
h['X-Request-Id'] = rid
return c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), 202, actor=actor,
raw=data, headers=h, schema='Attachment')['json']
def tree(spans):
"""Краткое дерево трассы: сервис, имя, родитель (по имени), длительность."""
by_id = {s['span_id']: s for s in spans}
out = []
for s in spans:
p = by_id.get(s['parent'])
out.append({'service': s['service'], 'name': s['name'], 'kind': s['kind'], 'ms': s['ms'],
'parent': (p['service'] + ': ' + p['name']) if p else None,
'outcome': s['attrs'].get('messenger.job.outcome'),
'attempt': s['attrs'].get('messenger.job.attempt'),
'status': s['status']})
return out
def chain_ok(spans, child_name, parent_name):
by_id = {s['span_id']: s for s in spans}
for s in spans:
if s['name'] == child_name and by_id.get(s['parent'], {}).get('name') == parent_name:
return True
return False
def wait_alert(name, state, timeout):
t0 = time.time()
got = o.wait_until(lambda: (o.alerts().get(name) == state) if state else (name not in o.alerts()), timeout, 2)
return round(time.time() - t0, 1) if got else None
# ---------------- signals ----------------
METRICS = {
'http requests (counter = _count)': 'http_server_request_duration_seconds_count{job="messenger-api"}',
'http duration histogram (s)': 'http_server_request_duration_seconds_bucket{job="messenger-api"}',
'jobs processed by outcome': 'messenger_jobs_processed_total',
'job duration histogram (s)': 'messenger_job_duration_seconds_bucket',
'time to ready histogram (s)': 'messenger_attachment_time_to_ready_seconds_bucket',
'queue backlog by state': 'messenger_jobs_backlog',
'oldest job age (s)': 'messenger_jobs_oldest_age_seconds',
'outbox pending': 'messenger_outbox_pending',
'rabbitmq queue messages': '{__name__=~"rabbitmq_detailed_queue_messages_ready|rabbitmq_queue_messages_ready"}',
'worker busy': 'messenger_worker_busy',
'worker capacity': 'messenger_worker_capacity',
'db pool acquired': '{__name__=~"pgxpool_acquired_connections.*"}',
'db pool waits': '{__name__=~"pgxpool_empty_acquire.*"}',
'process cpu': '{__name__=~"process_cpu_time.*"}',
'go memory': '{__name__=~"go_memory_used.*"}',
}
def scenario_signals(c):
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
t0 = time.time() - 5
secret_text = 'секрет-текст-' + uuid.uuid4().hex
rid_msg = 'obs-msg-' + uuid.uuid4().hex[:16]
c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': secret_text},
headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': rid_msg})
rid_up = 'obs-upload-' + uuid.uuid4().hex[:16]
att = upload(c, alice, chat, rid_up)
meta = wait_status(c, bob, att['id'], ('ready',))
c.call('GET', '/api/v1/chats/%s/messages' % chat['id'], actor=bob)
res = {}
# 1. Контролируемый запрос → трасса → лог API с тем же trace_id
tid = o.find_trace('{span.request_id="%s"}' % rid_msg, t0)
check(tid, 'trace for controlled request not found in Tempo')
spans, _ = o.tempo_trace(tid)
server = [s for s in spans if s['kind'] in ('SPAN_KIND_SERVER', 2, '2') or s['name'].startswith('POST /api')]
check(server, 'no server span')
logs = o.wait_until(lambda: o.loki('{service_name="messenger-api"} | trace_id="%s"' % tid, t0), 120)
check(logs, 'no API log with trace_id ' + tid)
log_line = logs[0][1]
res['request'] = {'request_id': rid_msg, 'trace_id': tid, 'spans': tree(spans)[:12],
'log': {'line': log_line[:300], 'labels': {k: v for k, v in logs[0][2].items()
if k in ('service_name', 'event', 'route', 'status', 'trace_id', 'span_id',
'detected_level', 'severity_text', 'request_id')}}}
# 2. Upload → relay → worker → S3/DB в одной трассе (W3C через outbox и AMQP)
tid2 = o.find_trace('{span.request_id="%s"}' % rid_up, t0)
check(tid2, 'upload trace not found')
def full():
sp, _ = o.tempo_trace(tid2)
return sp if any(s['service'] == 'messenger-worker' for s in sp) else None
spans2 = o.wait_until(full, 60)
check(spans2, 'worker spans not in the upload trace')
services = sorted({s['service'] for s in spans2})
links = {
'api server → relay producer': chain_ok(spans2, 'attachment.process publish', 'POST /api/v1/chats/{chat_id}/attachments'),
'relay producer → worker consumer': chain_ok(spans2, 'attachment.process process', 'attachment.process publish'),
'worker consumer → S3 PUT': any(s['name'] == 'S3 PUT' for s in spans2),
'worker consumer → image.decode': chain_ok(spans2, 'image.decode', 'attachment.process process'),
}
db_in_worker = sorted({s['name'] for s in spans2 if s['service'] == 'messenger-worker' and s['name'] not in
('attachment.process process', 'image.decode', 'image.variants') and not s['name'].startswith('S3')})
check(all(links.values()), 'broken causal chain: %s' % links)
wlogs = o.wait_until(lambda: o.loki('{service_name="messenger-worker"} | trace_id="%s"' % tid2, t0), 120)
check(wlogs, 'no worker log with the upload trace_id')
res['upload'] = {'request_id': rid_up, 'trace_id': tid2, 'services': services, 'links': links,
'span_count': len(spans2), 'worker_db_spans': db_in_worker[:10], 'tree': tree(spans2)[:40],
'worker_log_events': sorted({l[2].get('event', '') for l in wlogs})}
# 3. Метрики контракта и метки
time.sleep(12) # период экспорта метрик 5 s + scrape 5 s
found = {}
for title, sel in METRICS.items():
r = o.prom(sel)
names = sorted({x['metric'].get('__name__') for x in r})
found[title] = {'series': len(r), 'names': names[:4]}
missing = [k for k, v in found.items() if v['series'] == 0]
check(not missing, 'metrics missing: %s' % missing)
labels = set(o.prom_labels())
bad = sorted(labels & FORBIDDEN_LABELS)
check(not bad, 'forbidden metric labels: %s' % bad)
routes = o.prom_label_values('http_route')
check(not [r for r in routes if UUID_RE.search(r)], 'raw ids in http_route: %s' % routes)
sample_series = o.prom('http_server_request_duration_seconds_count{job="messenger-api"}')[0]['metric']
res['metrics'] = {'found': found, 'labels_total': len(labels), 'forbidden_labels': bad,
'http_route_values': routes, 'example_series_labels': sorted(sample_series)}
# 4. Утечки: пароль, токен, текст сообщения, Authorization — ни в логах (Loki, stdout), ни в трассах
needles = {'password': alice['registration']['password'], 'session_token': alice['token'],
'message_text': secret_text, 'bearer_header': 'Bearer ' + alice['token'][:12]}
leaks = {}
for k, v in needles.items():
hits = o.loki('{service_name=~"messenger-.+"} |= `%s`' % v.replace('`', ''), t0, limit=5)
leaks['loki:' + k] = len(hits)
raw_traces = json.dumps(o.tempo_trace(tid)[1]) + json.dumps(o.tempo_trace(tid2)[1])
for k, v in needles.items():
leaks['tempo:' + k] = raw_traces.count(v)
api_stdout = compose('logs', '--no-color', '--since', '15m', 'api', 'worker', 'relay').stdout
for k, v in needles.items():
leaks['stdout:' + k] = api_stdout.count(v)
check(sum(leaks.values()) == 0, 'telemetry leak: %s' % {k: v for k, v in leaks.items() if v})
res['leak_check'] = leaks
# 5. Provisioning: datasources и dashboard из Git; каждый Prometheus-запрос панели возвращает данные
ds = sorted(d['type'] + ':' + d['uid'] for d in o.grafana('/api/datasources'))
found_d = o.grafana('/api/search', {'query': 'Messenger'})
check(any(d.get('uid') == 'messenger-overview' for d in found_d), 'dashboard not provisioned')
dash = o.grafana('/api/dashboards/uid/messenger-overview')['dashboard']
panel_check = []
for p in dash['panels']:
for t in p.get('targets', []):
if t.get('datasource', {}).get('uid') != 'prometheus':
continue
expr = t['expr'].replace('$__rate_interval', '1m')
try:
n = len(o.prom(expr))
except Exception as e:
n = 'error: %s' % type(e).__name__
panel_check.append({'panel': p['title'], 'legend': t.get('legendFormat', ''), 'series': n})
empty = [x for x in panel_check if x['series'] == 0]
res['grafana'] = {'datasources': ds, 'dashboard': dash['title'], 'panels': len(dash['panels']),
'prometheus_targets': len(panel_check), 'targets_without_data_now': empty,
'targets': panel_check}
errors = [x for x in panel_check if isinstance(x['series'], str)]
check(not errors, 'dashboard queries failed: %s' % errors)
res['attachment'] = {'id': att['id'], 'status': meta['status']}
return res
# ---------------- retry в трассе ----------------
def scenario_retry_trace(c):
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
t0 = time.time() - 5
worker = compose('ps', '-q', 'worker').stdout.split()[0]
s3 = compose('ps', '-q', 's3').stdout.split()[0]
compose('stop', 'worker')
rid = 'obs-retry-' + uuid.uuid4().hex[:16]
att = upload(c, alice, chat, rid)
time.sleep(2)
sh('docker', 'stop', s3)
sh('docker', 'start', worker)
time.sleep(6) # 1–2 попытки упадут на недоступном S3
sh('docker', 'start', s3)
meta = wait_status(c, bob, att['id'], ('ready',), timeout=150)
tid = o.find_trace('{span.request_id="%s"}' % rid, t0)
check(tid, 'trace not found')
def done():
sp, _ = o.tempo_trace(tid)
return sp if any(s['attrs'].get('messenger.job.outcome') == 'done' for s in sp) else None
spans = o.wait_until(done, 90)
check(spans, 'final attempt not in trace')
attempts = [(s['attrs'].get('messenger.job.attempt'), s['attrs'].get('messenger.job.outcome'), s['status'])
for s in spans if s['name'] == 'attachment.process process']
publishes = sum(1 for s in spans if s['name'] == 'attachment.process publish')
check(any(a[1] == 'retry' for a in attempts), 'no retry attempt in trace: %s' % attempts)
ev = psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE job_id = "
"(SELECT job_id FROM jobs WHERE attachment_id = '%s')" % att['id'])
return {'request_id': rid, 'trace_id': tid, 'attempts_in_trace': attempts, 'publish_spans': publishes,
'job_events': ev, 'final_status': meta['status'],
'tree': [x for x in tree(spans) if x['name'].startswith('attachment.') or x['name'].startswith('POST')]}
# ---------------- инцидент 1: worker ----------------
def scenario_incident_worker(c):
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
t_stop = time.time()
compose('stop', 'worker')
ids = [upload(c, alice, chat, 'obs-inc1-%d-%s' % (i, uuid.uuid4().hex[:8]))['id'] for i in range(3)]
t_firing = wait_alert('MessengerQueueLagGrowing', 'firing', 240)
check(t_firing, 'MessengerQueueLagGrowing did not fire')
detect = round(time.time() - t_stop, 1)
al = o.alerts()
# Диагностика «как оператор»: только по сохранённым наблюдениям
diag = {
'alerts_firing': sorted(k for k, v in al.items() if v == 'firing'),
'backlog_queued': o.prom_value('sum(messenger_jobs_backlog{state="queued"})'),
'oldest_age_s': o.prom_value('max(messenger_jobs_oldest_age_seconds)'),
'jobs_processed_rate_1m': o.prom_value('sum(rate(messenger_jobs_processed_total[1m]))', default=0.0),
'worker_capacity_series': len(o.prom('messenger_worker_capacity')),
'rabbitmq_ready': o.prom_value('sum(rabbitmq_detailed_queue_messages_ready{queue="attachment.process"})'),
'api_5xx_ratio': o.prom_value('messenger:http_error_ratio:rate1m', default=0.0),
'producer_spans_since_stop': len(o.tempo_search('{name="attachment.process publish"}', t_stop)),
'consumer_spans_since_stop': len(o.tempo_search('{name="attachment.process process"}', t_stop + 2)),
'last_worker_log_events': sorted({l[2].get('event', '') for l in
o.loki('{service_name="messenger-worker"}', t_stop - 30, limit=50)}),
}
t_start = time.time()
compose('start', 'worker')
for i in ids:
wait_status(c, bob, i, ('ready',), timeout=120)
t_ready = round(time.time() - t_start, 1)
t_resolved = wait_alert('MessengerQueueLagGrowing', None, 180)
check(t_resolved is not None, 'alert did not resolve')
series = o.prom_range('max(messenger_jobs_oldest_age_seconds)', t_stop - 30, time.time(), 10)
return {'detection_seconds_from_worker_stop': detect, 'diagnosis': diag,
'backlog_ready_seconds_after_start': t_ready,
'resolved_seconds_after_worker_start': round(t_ready + t_resolved, 1),
'oldest_age_series': [[round(float(t) - t_stop), round(float(v), 1)] for t, v in (series[0]['values'] if series else [])]}
# ---------------- инцидент 2: БД ----------------
def is_db(s):
return (s['attrs'].get('db.system') == 'postgresql' or s['attrs'].get('db.system.name') == 'postgresql'
or s['name'].split(' ')[0] in ('query', 'acquire', 'batch', 'prepare', 'connect', 'copy_from'))
def breakdown(spans):
"""Доля времени server span, проведённая в дочерних spans БД (pgx) и S3."""
by_parent = {}
for s in spans:
by_parent.setdefault(s['parent'], []).append(s)
srv = next((s for s in spans if s['parent'] == '' or s['name'].startswith(('POST /api', 'GET /api'))), None)
if not srv:
return None
db = sum(s['ms'] for s in spans if s['service'] == srv['service'] and is_db(s))
s3 = sum(s['ms'] for s in spans if s['name'].startswith('S3'))
return {'server_span': srv['name'], 'server_ms': srv['ms'], 'status': srv['attrs'].get('http.response.status_code'),
'db_ms': round(db, 1), 's3_ms': round(s3, 1), 'db_share': round(db / srv['ms'], 3) if srv['ms'] else None,
'db_spans': [(s['name'], s['ms']) for s in spans if is_db(s)][:6]}
def scenario_incident_db(c):
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
load = Load(c.base, None, alice, chat, concurrency=4, timeout=10, prefix='obs-inc2').start()
time.sleep(20)
t_lock = time.time()
lock = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-c', 'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(110); COMMIT;'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
t_firing = wait_alert('MessengerHighErrorRate', 'firing', 150)
detect = round(time.time() - t_lock, 1) if t_firing else None
snap = {
'error_ratio_1m': o.prom_value('messenger:http_error_ratio:rate1m'),
'p99_s': o.prom_value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'),
'p99_by_route': {r['metric'].get('http_route'): round(float(r['value'][1]), 3) for r in o.prom(
'histogram_quantile(0.99, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))')},
'db_pool_acquired': o.prom_value('sum({__name__=~"pgxpool_acquired_connections.*", job="messenger-api"})'),
'health_ready_5xx_rate': o.prom_value('sum(rate(http_server_request_duration_seconds_count{http_route="/health/ready", http_response_status_class="5xx"}[1m]))', default=0.0),
}
lock.wait(timeout=200)
t_unlock = time.time()
t_resolved = wait_alert('MessengerHighErrorRate', None, 180)
load.stop()
# Трассы ошибок: где время? (по сохранённым трассам Tempo)
err_traces = o.tempo_search('{resource.service.name="messenger-api" && span.http.response.status_code >= 500}', t_lock - 1, limit=5)
analysis = []
for t in err_traces[:3]:
sp, _ = o.tempo_trace(t['traceID'])
analysis.append(breakdown(sp))
# Сравнение: медленный upload большого файла — время не в БД
rid = 'obs-slowhandler-' + uuid.uuid4().hex[:8]
from labutil import noise_png
t_up = time.time() - 2
upload(c, alice, chat, rid, data=noise_png(4 * 1024 * 1024), name='big.png')
tid = o.find_trace('{span.request_id="%s"}' % rid, t_up)
slow_handler = breakdown(o.tempo_trace(tid)[0]) if tid else None
err_logs = o.loki('{service_name="messenger-api"} | detected_level=~"error|ERROR"', t_lock, t_unlock + 5, limit=200)
check(detect is not None, 'MessengerHighErrorRate did not fire')
check(t_resolved is not None, 'MessengerHighErrorRate did not resolve')
check(analysis and all(a and a['db_share'] and a['db_share'] > 0.8 for a in analysis),
'error traces do not point to DB: %s' % analysis)
return {'fault': 'LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 110 s under load (4 clients)',
'detection_seconds': detect, 'resolved_seconds_after_unlock': t_resolved,
'metrics_at_firing': snap, 'error_trace_breakdown': analysis,
'slow_handler_comparison': slow_handler,
'error_logs': {'count': len(err_logs),
'events': sorted({l[2].get('event', '') for l in err_logs}),
'routes': sorted({l[2].get('route', '') for l in err_logs})},
'load': load.summary(exclude=[(t_lock, t_unlock)])}
# ---------------- отказ backend телеметрии ----------------
def mem_of(service):
cid = compose('ps', '-q', service).stdout.split()
return round(o.mem_bytes(cid[0]) / 2**20, 1) if cid and o.mem_bytes(cid[0]) else None
def scenario_backend_down(c):
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
load = Load(c.base, None, alice, chat, concurrency=4, timeout=10, prefix='obs-bd').start()
samples = []
stop = threading.Event()
col = compose('ps', '-q', 'otel-collector').stdout.split()[0]
def sampler():
while not stop.is_set():
samples.append({'t': round(time.time() - t0, 1), 'collector_mib': round((o.mem_bytes(col) or 0) / 2**20, 1),
'api_mib': [round((o.mem_bytes(x) or 0) / 2**20, 1) for x in compose('ps', '-q', 'api').stdout.split()]})
time.sleep(5)
t0 = time.time()
threading.Thread(target=sampler, daemon=True).start()
time.sleep(15)
compose('stop', 'tempo', 'loki')
t_down = time.time()
phase = {}
fired = wait_alert('MessengerTelemetryExportFailing', 'firing', 150)
phase['export_failing_alert_seconds'] = round(time.time() - t_down, 1) if fired else None
time.sleep(max(0, 130 - (time.time() - t_down)))
q = lambda e: o.prom_value(e, default=0.0)
phase['during'] = {
'exporter_queue_size_max': q('max_over_time(sum(otelcol_exporter_queue_size)[2m:5s])'),
'exporter_queue_capacity': q('sum(otelcol_exporter_queue_capacity)'),
'send_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[3m]))'),
'send_failed_log_records': q('sum(increase({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[3m]))'),
'enqueue_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[3m]))'),
'refused_spans_memory_limiter': q('sum(increase({__name__=~"otelcol_receiver_refused_spans(_total)?"}[3m]))'),
'metrics_still_scraped': q('sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m]))'),
}
compose('start', 'tempo', 'loki')
compose('up', '-d', '--wait', '--no-deps', 'tempo', 'loki', timeout=180)
t_up = time.time()
rid = 'obs-after-' + uuid.uuid4().hex[:8]
sent0 = q('sum({__name__=~"otelcol_exporter_sent_spans(_total)?", exporter="otlp/tempo"})')
# Новые запросы после возврата Tempo; трасса ищется по ID из заголовка traceresponse
# (минуя поиск) — сначала каждые 5 s новый запрос, пока трасса не появится.
got, tids = None, []
end = time.time() + 240
while time.time() < end and not got:
r = c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': 'after'},
headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': rid})
tid = (r['headers'].get('traceresponse') or '--').split('-')[1]
tids.append(tid)
for t in tids:
try:
if o.tempo_trace(t)[0]:
got = t
break
except Exception:
pass
if not got:
time.sleep(5)
phase['tempo_spans_exported_after_up'] = q('sum({__name__=~"otelcol_exporter_sent_spans(_total)?", exporter="otlp/tempo"})') - sent0
phase['refused_spans_total'] = q('sum({__name__=~"otelcol_receiver_refused_spans(_total)?"})')
phase['new_trace_after_recovery'] = bool(got)
phase['traces_visible_seconds_after_tempo_up'] = round(time.time() - t_up, 1) if got else None
phase['probe_requests_after_up'] = len(tids)
# Фаза 2: остановлен сам Collector — SDK отбрасывает по ограниченной очереди
compose('stop', 'otel-collector')
t_cd = time.time()
time.sleep(40)
compose('start', 'otel-collector')
time.sleep(10)
load.stop()
stop.set()
summary = load.summary()
col_max = max(s['collector_mib'] for s in samples)
api_max = max(max(s['api_mib'] or [0]) for s in samples)
check(summary['errors'] == 0, 'API errors while telemetry backend was down: %s' % summary)
check(col_max < 256, 'collector memory not bounded')
check(phase['new_trace_after_recovery'], 'no traces after backend recovery')
return {'load': summary, 'collector_limit_mib': 256, 'collector_max_mib': col_max,
'api_max_mib': api_max, 'api_limit_mib': 256, 'phases': phase,
'timeline': {'tempo_loki_down_s': round(t_down - t0), 'tempo_loki_up_s': round(t_up - t0),
'collector_down_s': round(t_cd - t0), 'collector_down_for_s': 40},
'memory_samples': samples}
SCENARIOS = {'signals': scenario_signals, 'retry-trace': scenario_retry_trace,
'incident-worker': scenario_incident_worker, 'incident-db': scenario_incident_db,
'backend-down': scenario_backend_down}
def main():
def configure(p):
p.add_argument('scenario', choices=sorted(SCENARIOS))
p.add_argument('--output', default=None)
args = arguments('Lab6 observability scenarios on the Docker stand', configure)
c = Client(args.base_url, args.ca_file, timeout=30)
try:
result = SCENARIOS[args.scenario](c)
finally:
compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 's3', 'api', 'otel-collector', 'tempo', 'loki',
timeout=300, check=False)
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + '\n', encoding='utf-8')
print(text)
print('PASS: ' + args.scenario)
if __name__ == '__main__':
run(main)