472 lines
26 KiB
Python
472 lines
26 KiB
Python
#!/usr/bin/env python3
|
||
"""Проверки наблюдаемости lab6 на Docker-стенде (только по сохранённым наблюдениям).
|
||
|
||
signals уровень 3–4: метрики контракта есть и с правильными метками; запрос →
|
||
трасса (Tempo) → лог с тем же trace_id (Loki); upload → API → relay (producer)
|
||
→ worker (consumer) → S3/DB в одной трассе; нет утечек пароля, токена,
|
||
текста сообщения; dashboard из provisioning, его запросы возвращают данные
|
||
retry-trace S3 недоступен во время обработки: попытки с retry в той же трассе upload
|
||
incident-worker инцидент 1: worker остановлен → возраст задач растёт → alert firing →
|
||
worker запущен → backlog уходит → resolved; диагностика по метрикам/логам/трассам
|
||
incident-db инцидент 2: блокировка таблицы messages (БД «зависла») под нагрузкой →
|
||
5xx → MessengerHighErrorRate firing → resolved; трасса показывает, что время
|
||
ушло в запрос к БД, а не в обработчик (сравнение с медленным upload)
|
||
backend-down Tempo и Loki остановлены под нагрузкой: API без ошибок, память Collector
|
||
ограничена, потери считаются; затем остановлен сам Collector
|
||
|
||
Печатает JSON-итог и PASS/FAIL. Токены и пароли не печатаются.
|
||
"""
|
||
import json
|
||
import re
|
||
import statistics
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
import labutil # noqa: F401 (добавляет tests/ в sys.path)
|
||
from client import Client, Failure, arguments, check, run
|
||
from labutil import ROOT, compose, psql, sh, wait_status
|
||
from loadgen import Load
|
||
import obsutil as o
|
||
|
||
SAMPLE = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
|
||
FORBIDDEN_LABELS = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'url', 'url_full',
|
||
'http_target', 'filename', 'text', 'username'}
|
||
UUID_RE = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
|
||
|
||
|
||
def upload(c, actor, chat, rid=None, data=SAMPLE, name='p.png'):
|
||
h = {'Content-Type': 'image/png'}
|
||
if rid:
|
||
h['X-Request-Id'] = rid
|
||
return c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), 202, actor=actor,
|
||
raw=data, headers=h, schema='Attachment')['json']
|
||
|
||
|
||
def tree(spans):
|
||
"""Краткое дерево трассы: сервис, имя, родитель (по имени), длительность."""
|
||
by_id = {s['span_id']: s for s in spans}
|
||
out = []
|
||
for s in spans:
|
||
p = by_id.get(s['parent'])
|
||
out.append({'service': s['service'], 'name': s['name'], 'kind': s['kind'], 'ms': s['ms'],
|
||
'parent': (p['service'] + ': ' + p['name']) if p else None,
|
||
'outcome': s['attrs'].get('messenger.job.outcome'),
|
||
'attempt': s['attrs'].get('messenger.job.attempt'),
|
||
'status': s['status']})
|
||
return out
|
||
|
||
|
||
def chain_ok(spans, child_name, parent_name):
|
||
by_id = {s['span_id']: s for s in spans}
|
||
for s in spans:
|
||
if s['name'] == child_name and by_id.get(s['parent'], {}).get('name') == parent_name:
|
||
return True
|
||
return False
|
||
|
||
|
||
def wait_alert(name, state, timeout):
|
||
t0 = time.time()
|
||
got = o.wait_until(lambda: (o.alerts().get(name) == state) if state else (name not in o.alerts()), timeout, 2)
|
||
return round(time.time() - t0, 1) if got else None
|
||
|
||
|
||
# ---------------- signals ----------------
|
||
|
||
METRICS = {
|
||
'http requests (counter = _count)': 'http_server_request_duration_seconds_count{job="messenger-api"}',
|
||
'http duration histogram (s)': 'http_server_request_duration_seconds_bucket{job="messenger-api"}',
|
||
'jobs processed by outcome': 'messenger_jobs_processed_total',
|
||
'job duration histogram (s)': 'messenger_job_duration_seconds_bucket',
|
||
'time to ready histogram (s)': 'messenger_attachment_time_to_ready_seconds_bucket',
|
||
'queue backlog by state': 'messenger_jobs_backlog',
|
||
'oldest job age (s)': 'messenger_jobs_oldest_age_seconds',
|
||
'outbox pending': 'messenger_outbox_pending',
|
||
'rabbitmq queue messages': '{__name__=~"rabbitmq_detailed_queue_messages_ready|rabbitmq_queue_messages_ready"}',
|
||
'worker busy': 'messenger_worker_busy',
|
||
'worker capacity': 'messenger_worker_capacity',
|
||
'db pool acquired': '{__name__=~"pgxpool_acquired_connections.*"}',
|
||
'db pool waits': '{__name__=~"pgxpool_empty_acquire.*"}',
|
||
'process cpu': '{__name__=~"process_cpu_time.*"}',
|
||
'go memory': '{__name__=~"go_memory_used.*"}',
|
||
}
|
||
|
||
|
||
def scenario_signals(c):
|
||
o.wait_grafana()
|
||
alice, bob, _, chat = c.bootstrap()
|
||
t0 = time.time() - 5
|
||
secret_text = 'секрет-текст-' + uuid.uuid4().hex
|
||
rid_msg = 'obs-msg-' + uuid.uuid4().hex[:16]
|
||
c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': secret_text},
|
||
headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': rid_msg})
|
||
rid_up = 'obs-upload-' + uuid.uuid4().hex[:16]
|
||
att = upload(c, alice, chat, rid_up)
|
||
meta = wait_status(c, bob, att['id'], ('ready',))
|
||
c.call('GET', '/api/v1/chats/%s/messages' % chat['id'], actor=bob)
|
||
res = {}
|
||
|
||
# 1. Контролируемый запрос → трасса → лог API с тем же trace_id
|
||
tid = o.find_trace('{span.request_id="%s"}' % rid_msg, t0)
|
||
check(tid, 'trace for controlled request not found in Tempo')
|
||
spans, _ = o.tempo_trace(tid)
|
||
server = [s for s in spans if s['kind'] in ('SPAN_KIND_SERVER', 2, '2') or s['name'].startswith('POST /api')]
|
||
check(server, 'no server span')
|
||
logs = o.wait_until(lambda: o.loki('{service_name="messenger-api"} | trace_id="%s"' % tid, t0), 120)
|
||
check(logs, 'no API log with trace_id ' + tid)
|
||
log_line = logs[0][1]
|
||
res['request'] = {'request_id': rid_msg, 'trace_id': tid, 'spans': tree(spans)[:12],
|
||
'log': {'line': log_line[:300], 'labels': {k: v for k, v in logs[0][2].items()
|
||
if k in ('service_name', 'event', 'route', 'status', 'trace_id', 'span_id',
|
||
'detected_level', 'severity_text', 'request_id')}}}
|
||
|
||
# 2. Upload → relay → worker → S3/DB в одной трассе (W3C через outbox и AMQP)
|
||
tid2 = o.find_trace('{span.request_id="%s"}' % rid_up, t0)
|
||
check(tid2, 'upload trace not found')
|
||
|
||
def full():
|
||
sp, _ = o.tempo_trace(tid2)
|
||
return sp if any(s['service'] == 'messenger-worker' for s in sp) else None
|
||
spans2 = o.wait_until(full, 60)
|
||
check(spans2, 'worker spans not in the upload trace')
|
||
services = sorted({s['service'] for s in spans2})
|
||
links = {
|
||
'api server → relay producer': chain_ok(spans2, 'attachment.process publish', 'POST /api/v1/chats/{chat_id}/attachments'),
|
||
'relay producer → worker consumer': chain_ok(spans2, 'attachment.process process', 'attachment.process publish'),
|
||
'worker consumer → S3 PUT': any(s['name'] == 'S3 PUT' for s in spans2),
|
||
'worker consumer → image.decode': chain_ok(spans2, 'image.decode', 'attachment.process process'),
|
||
}
|
||
db_in_worker = sorted({s['name'] for s in spans2 if s['service'] == 'messenger-worker' and s['name'] not in
|
||
('attachment.process process', 'image.decode', 'image.variants') and not s['name'].startswith('S3')})
|
||
check(all(links.values()), 'broken causal chain: %s' % links)
|
||
wlogs = o.wait_until(lambda: o.loki('{service_name="messenger-worker"} | trace_id="%s"' % tid2, t0), 120)
|
||
check(wlogs, 'no worker log with the upload trace_id')
|
||
res['upload'] = {'request_id': rid_up, 'trace_id': tid2, 'services': services, 'links': links,
|
||
'span_count': len(spans2), 'worker_db_spans': db_in_worker[:10], 'tree': tree(spans2)[:40],
|
||
'worker_log_events': sorted({l[2].get('event', '') for l in wlogs})}
|
||
|
||
# 3. Метрики контракта и метки
|
||
time.sleep(12) # период экспорта метрик 5 s + scrape 5 s
|
||
found = {}
|
||
for title, sel in METRICS.items():
|
||
r = o.prom(sel)
|
||
names = sorted({x['metric'].get('__name__') for x in r})
|
||
found[title] = {'series': len(r), 'names': names[:4]}
|
||
missing = [k for k, v in found.items() if v['series'] == 0]
|
||
check(not missing, 'metrics missing: %s' % missing)
|
||
labels = set(o.prom_labels())
|
||
bad = sorted(labels & FORBIDDEN_LABELS)
|
||
check(not bad, 'forbidden metric labels: %s' % bad)
|
||
routes = o.prom_label_values('http_route')
|
||
check(not [r for r in routes if UUID_RE.search(r)], 'raw ids in http_route: %s' % routes)
|
||
sample_series = o.prom('http_server_request_duration_seconds_count{job="messenger-api"}')[0]['metric']
|
||
res['metrics'] = {'found': found, 'labels_total': len(labels), 'forbidden_labels': bad,
|
||
'http_route_values': routes, 'example_series_labels': sorted(sample_series)}
|
||
|
||
# 4. Утечки: пароль, токен, текст сообщения, Authorization — ни в логах (Loki, stdout), ни в трассах
|
||
needles = {'password': alice['registration']['password'], 'session_token': alice['token'],
|
||
'message_text': secret_text, 'bearer_header': 'Bearer ' + alice['token'][:12]}
|
||
leaks = {}
|
||
for k, v in needles.items():
|
||
hits = o.loki('{service_name=~"messenger-.+"} |= `%s`' % v.replace('`', ''), t0, limit=5)
|
||
leaks['loki:' + k] = len(hits)
|
||
raw_traces = json.dumps(o.tempo_trace(tid)[1]) + json.dumps(o.tempo_trace(tid2)[1])
|
||
for k, v in needles.items():
|
||
leaks['tempo:' + k] = raw_traces.count(v)
|
||
api_stdout = compose('logs', '--no-color', '--since', '15m', 'api', 'worker', 'relay').stdout
|
||
for k, v in needles.items():
|
||
leaks['stdout:' + k] = api_stdout.count(v)
|
||
check(sum(leaks.values()) == 0, 'telemetry leak: %s' % {k: v for k, v in leaks.items() if v})
|
||
res['leak_check'] = leaks
|
||
|
||
# 5. Provisioning: datasources и dashboard из Git; каждый Prometheus-запрос панели возвращает данные
|
||
ds = sorted(d['type'] + ':' + d['uid'] for d in o.grafana('/api/datasources'))
|
||
found_d = o.grafana('/api/search', {'query': 'Messenger'})
|
||
check(any(d.get('uid') == 'messenger-overview' for d in found_d), 'dashboard not provisioned')
|
||
dash = o.grafana('/api/dashboards/uid/messenger-overview')['dashboard']
|
||
panel_check = []
|
||
for p in dash['panels']:
|
||
for t in p.get('targets', []):
|
||
if t.get('datasource', {}).get('uid') != 'prometheus':
|
||
continue
|
||
expr = t['expr'].replace('$__rate_interval', '1m')
|
||
try:
|
||
n = len(o.prom(expr))
|
||
except Exception as e:
|
||
n = 'error: %s' % type(e).__name__
|
||
panel_check.append({'panel': p['title'], 'legend': t.get('legendFormat', ''), 'series': n})
|
||
empty = [x for x in panel_check if x['series'] == 0]
|
||
res['grafana'] = {'datasources': ds, 'dashboard': dash['title'], 'panels': len(dash['panels']),
|
||
'prometheus_targets': len(panel_check), 'targets_without_data_now': empty,
|
||
'targets': panel_check}
|
||
errors = [x for x in panel_check if isinstance(x['series'], str)]
|
||
check(not errors, 'dashboard queries failed: %s' % errors)
|
||
res['attachment'] = {'id': att['id'], 'status': meta['status']}
|
||
return res
|
||
|
||
|
||
# ---------------- retry в трассе ----------------
|
||
|
||
def scenario_retry_trace(c):
|
||
o.wait_grafana()
|
||
alice, bob, _, chat = c.bootstrap()
|
||
t0 = time.time() - 5
|
||
worker = compose('ps', '-q', 'worker').stdout.split()[0]
|
||
s3 = compose('ps', '-q', 's3').stdout.split()[0]
|
||
compose('stop', 'worker')
|
||
rid = 'obs-retry-' + uuid.uuid4().hex[:16]
|
||
att = upload(c, alice, chat, rid)
|
||
time.sleep(2)
|
||
sh('docker', 'stop', s3)
|
||
sh('docker', 'start', worker)
|
||
time.sleep(6) # 1–2 попытки упадут на недоступном S3
|
||
sh('docker', 'start', s3)
|
||
meta = wait_status(c, bob, att['id'], ('ready',), timeout=150)
|
||
tid = o.find_trace('{span.request_id="%s"}' % rid, t0)
|
||
check(tid, 'trace not found')
|
||
|
||
def done():
|
||
sp, _ = o.tempo_trace(tid)
|
||
return sp if any(s['attrs'].get('messenger.job.outcome') == 'done' for s in sp) else None
|
||
spans = o.wait_until(done, 90)
|
||
check(spans, 'final attempt not in trace')
|
||
attempts = [(s['attrs'].get('messenger.job.attempt'), s['attrs'].get('messenger.job.outcome'), s['status'])
|
||
for s in spans if s['name'] == 'attachment.process process']
|
||
publishes = sum(1 for s in spans if s['name'] == 'attachment.process publish')
|
||
check(any(a[1] == 'retry' for a in attempts), 'no retry attempt in trace: %s' % attempts)
|
||
ev = psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE job_id = "
|
||
"(SELECT job_id FROM jobs WHERE attachment_id = '%s')" % att['id'])
|
||
return {'request_id': rid, 'trace_id': tid, 'attempts_in_trace': attempts, 'publish_spans': publishes,
|
||
'job_events': ev, 'final_status': meta['status'],
|
||
'tree': [x for x in tree(spans) if x['name'].startswith('attachment.') or x['name'].startswith('POST')]}
|
||
|
||
|
||
# ---------------- инцидент 1: worker ----------------
|
||
|
||
def scenario_incident_worker(c):
|
||
o.wait_grafana()
|
||
alice, bob, _, chat = c.bootstrap()
|
||
t_stop = time.time()
|
||
compose('stop', 'worker')
|
||
ids = [upload(c, alice, chat, 'obs-inc1-%d-%s' % (i, uuid.uuid4().hex[:8]))['id'] for i in range(3)]
|
||
t_firing = wait_alert('MessengerQueueLagGrowing', 'firing', 240)
|
||
check(t_firing, 'MessengerQueueLagGrowing did not fire')
|
||
detect = round(time.time() - t_stop, 1)
|
||
al = o.alerts()
|
||
# Диагностика «как оператор»: только по сохранённым наблюдениям
|
||
diag = {
|
||
'alerts_firing': sorted(k for k, v in al.items() if v == 'firing'),
|
||
'backlog_queued': o.prom_value('sum(messenger_jobs_backlog{state="queued"})'),
|
||
'oldest_age_s': o.prom_value('max(messenger_jobs_oldest_age_seconds)'),
|
||
'jobs_processed_rate_1m': o.prom_value('sum(rate(messenger_jobs_processed_total[1m]))', default=0.0),
|
||
'worker_capacity_series': len(o.prom('messenger_worker_capacity')),
|
||
'rabbitmq_ready': o.prom_value('sum(rabbitmq_detailed_queue_messages_ready{queue="attachment.process"})'),
|
||
'api_5xx_ratio': o.prom_value('messenger:http_error_ratio:rate1m', default=0.0),
|
||
'producer_spans_since_stop': len(o.tempo_search('{name="attachment.process publish"}', t_stop)),
|
||
'consumer_spans_since_stop': len(o.tempo_search('{name="attachment.process process"}', t_stop + 2)),
|
||
'last_worker_log_events': sorted({l[2].get('event', '') for l in
|
||
o.loki('{service_name="messenger-worker"}', t_stop - 30, limit=50)}),
|
||
}
|
||
t_start = time.time()
|
||
compose('start', 'worker')
|
||
for i in ids:
|
||
wait_status(c, bob, i, ('ready',), timeout=120)
|
||
t_ready = round(time.time() - t_start, 1)
|
||
t_resolved = wait_alert('MessengerQueueLagGrowing', None, 180)
|
||
check(t_resolved is not None, 'alert did not resolve')
|
||
series = o.prom_range('max(messenger_jobs_oldest_age_seconds)', t_stop - 30, time.time(), 10)
|
||
return {'detection_seconds_from_worker_stop': detect, 'diagnosis': diag,
|
||
'backlog_ready_seconds_after_start': t_ready,
|
||
'resolved_seconds_after_worker_start': round(t_ready + t_resolved, 1),
|
||
'oldest_age_series': [[round(float(t) - t_stop), round(float(v), 1)] for t, v in (series[0]['values'] if series else [])]}
|
||
|
||
|
||
# ---------------- инцидент 2: БД ----------------
|
||
|
||
def is_db(s):
|
||
return (s['attrs'].get('db.system') == 'postgresql' or s['attrs'].get('db.system.name') == 'postgresql'
|
||
or s['name'].split(' ')[0] in ('query', 'acquire', 'batch', 'prepare', 'connect', 'copy_from'))
|
||
|
||
|
||
def breakdown(spans):
|
||
"""Доля времени server span, проведённая в дочерних spans БД (pgx) и S3."""
|
||
by_parent = {}
|
||
for s in spans:
|
||
by_parent.setdefault(s['parent'], []).append(s)
|
||
srv = next((s for s in spans if s['parent'] == '' or s['name'].startswith(('POST /api', 'GET /api'))), None)
|
||
if not srv:
|
||
return None
|
||
db = sum(s['ms'] for s in spans if s['service'] == srv['service'] and is_db(s))
|
||
s3 = sum(s['ms'] for s in spans if s['name'].startswith('S3'))
|
||
return {'server_span': srv['name'], 'server_ms': srv['ms'], 'status': srv['attrs'].get('http.response.status_code'),
|
||
'db_ms': round(db, 1), 's3_ms': round(s3, 1), 'db_share': round(db / srv['ms'], 3) if srv['ms'] else None,
|
||
'db_spans': [(s['name'], s['ms']) for s in spans if is_db(s)][:6]}
|
||
|
||
|
||
def scenario_incident_db(c):
|
||
o.wait_grafana()
|
||
alice, bob, _, chat = c.bootstrap()
|
||
load = Load(c.base, None, alice, chat, concurrency=4, timeout=10, prefix='obs-inc2').start()
|
||
time.sleep(20)
|
||
t_lock = time.time()
|
||
lock = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||
'-c', 'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(110); COMMIT;'],
|
||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
t_firing = wait_alert('MessengerHighErrorRate', 'firing', 150)
|
||
detect = round(time.time() - t_lock, 1) if t_firing else None
|
||
snap = {
|
||
'error_ratio_1m': o.prom_value('messenger:http_error_ratio:rate1m'),
|
||
'p99_s': o.prom_value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'),
|
||
'p99_by_route': {r['metric'].get('http_route'): round(float(r['value'][1]), 3) for r in o.prom(
|
||
'histogram_quantile(0.99, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))')},
|
||
'db_pool_acquired': o.prom_value('sum({__name__=~"pgxpool_acquired_connections.*", job="messenger-api"})'),
|
||
'health_ready_5xx_rate': o.prom_value('sum(rate(http_server_request_duration_seconds_count{http_route="/health/ready", http_response_status_class="5xx"}[1m]))', default=0.0),
|
||
}
|
||
lock.wait(timeout=200)
|
||
t_unlock = time.time()
|
||
t_resolved = wait_alert('MessengerHighErrorRate', None, 180)
|
||
load.stop()
|
||
# Трассы ошибок: где время? (по сохранённым трассам Tempo)
|
||
err_traces = o.tempo_search('{resource.service.name="messenger-api" && span.http.response.status_code >= 500}', t_lock - 1, limit=5)
|
||
analysis = []
|
||
for t in err_traces[:3]:
|
||
sp, _ = o.tempo_trace(t['traceID'])
|
||
analysis.append(breakdown(sp))
|
||
# Сравнение: медленный upload большого файла — время не в БД
|
||
rid = 'obs-slowhandler-' + uuid.uuid4().hex[:8]
|
||
from labutil import noise_png
|
||
t_up = time.time() - 2
|
||
upload(c, alice, chat, rid, data=noise_png(4 * 1024 * 1024), name='big.png')
|
||
tid = o.find_trace('{span.request_id="%s"}' % rid, t_up)
|
||
slow_handler = breakdown(o.tempo_trace(tid)[0]) if tid else None
|
||
err_logs = o.loki('{service_name="messenger-api"} | detected_level=~"error|ERROR"', t_lock, t_unlock + 5, limit=200)
|
||
check(detect is not None, 'MessengerHighErrorRate did not fire')
|
||
check(t_resolved is not None, 'MessengerHighErrorRate did not resolve')
|
||
check(analysis and all(a and a['db_share'] and a['db_share'] > 0.8 for a in analysis),
|
||
'error traces do not point to DB: %s' % analysis)
|
||
return {'fault': 'LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 110 s under load (4 clients)',
|
||
'detection_seconds': detect, 'resolved_seconds_after_unlock': t_resolved,
|
||
'metrics_at_firing': snap, 'error_trace_breakdown': analysis,
|
||
'slow_handler_comparison': slow_handler,
|
||
'error_logs': {'count': len(err_logs),
|
||
'events': sorted({l[2].get('event', '') for l in err_logs}),
|
||
'routes': sorted({l[2].get('route', '') for l in err_logs})},
|
||
'load': load.summary(exclude=[(t_lock, t_unlock)])}
|
||
|
||
|
||
# ---------------- отказ backend телеметрии ----------------
|
||
|
||
def mem_of(service):
|
||
cid = compose('ps', '-q', service).stdout.split()
|
||
return round(o.mem_bytes(cid[0]) / 2**20, 1) if cid and o.mem_bytes(cid[0]) else None
|
||
|
||
|
||
def scenario_backend_down(c):
|
||
o.wait_grafana()
|
||
alice, bob, _, chat = c.bootstrap()
|
||
load = Load(c.base, None, alice, chat, concurrency=4, timeout=10, prefix='obs-bd').start()
|
||
samples = []
|
||
stop = threading.Event()
|
||
col = compose('ps', '-q', 'otel-collector').stdout.split()[0]
|
||
|
||
def sampler():
|
||
while not stop.is_set():
|
||
samples.append({'t': round(time.time() - t0, 1), 'collector_mib': round((o.mem_bytes(col) or 0) / 2**20, 1),
|
||
'api_mib': [round((o.mem_bytes(x) or 0) / 2**20, 1) for x in compose('ps', '-q', 'api').stdout.split()]})
|
||
time.sleep(5)
|
||
t0 = time.time()
|
||
threading.Thread(target=sampler, daemon=True).start()
|
||
time.sleep(15)
|
||
compose('stop', 'tempo', 'loki')
|
||
t_down = time.time()
|
||
phase = {}
|
||
fired = wait_alert('MessengerTelemetryExportFailing', 'firing', 150)
|
||
phase['export_failing_alert_seconds'] = round(time.time() - t_down, 1) if fired else None
|
||
time.sleep(max(0, 130 - (time.time() - t_down)))
|
||
q = lambda e: o.prom_value(e, default=0.0)
|
||
phase['during'] = {
|
||
'exporter_queue_size_max': q('max_over_time(sum(otelcol_exporter_queue_size)[2m:5s])'),
|
||
'exporter_queue_capacity': q('sum(otelcol_exporter_queue_capacity)'),
|
||
'send_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[3m]))'),
|
||
'send_failed_log_records': q('sum(increase({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[3m]))'),
|
||
'enqueue_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[3m]))'),
|
||
'refused_spans_memory_limiter': q('sum(increase({__name__=~"otelcol_receiver_refused_spans(_total)?"}[3m]))'),
|
||
'metrics_still_scraped': q('sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m]))'),
|
||
}
|
||
compose('start', 'tempo', 'loki')
|
||
compose('up', '-d', '--wait', '--no-deps', 'tempo', 'loki', timeout=180)
|
||
t_up = time.time()
|
||
rid = 'obs-after-' + uuid.uuid4().hex[:8]
|
||
sent0 = q('sum({__name__=~"otelcol_exporter_sent_spans(_total)?", exporter="otlp/tempo"})')
|
||
# Новые запросы после возврата Tempo; трасса ищется по ID из заголовка traceresponse
|
||
# (минуя поиск) — сначала каждые 5 s новый запрос, пока трасса не появится.
|
||
got, tids = None, []
|
||
end = time.time() + 240
|
||
while time.time() < end and not got:
|
||
r = c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': 'after'},
|
||
headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': rid})
|
||
tid = (r['headers'].get('traceresponse') or '--').split('-')[1]
|
||
tids.append(tid)
|
||
for t in tids:
|
||
try:
|
||
if o.tempo_trace(t)[0]:
|
||
got = t
|
||
break
|
||
except Exception:
|
||
pass
|
||
if not got:
|
||
time.sleep(5)
|
||
phase['tempo_spans_exported_after_up'] = q('sum({__name__=~"otelcol_exporter_sent_spans(_total)?", exporter="otlp/tempo"})') - sent0
|
||
phase['refused_spans_total'] = q('sum({__name__=~"otelcol_receiver_refused_spans(_total)?"})')
|
||
phase['new_trace_after_recovery'] = bool(got)
|
||
phase['traces_visible_seconds_after_tempo_up'] = round(time.time() - t_up, 1) if got else None
|
||
phase['probe_requests_after_up'] = len(tids)
|
||
# Фаза 2: остановлен сам Collector — SDK отбрасывает по ограниченной очереди
|
||
compose('stop', 'otel-collector')
|
||
t_cd = time.time()
|
||
time.sleep(40)
|
||
compose('start', 'otel-collector')
|
||
time.sleep(10)
|
||
load.stop()
|
||
stop.set()
|
||
summary = load.summary()
|
||
col_max = max(s['collector_mib'] for s in samples)
|
||
api_max = max(max(s['api_mib'] or [0]) for s in samples)
|
||
check(summary['errors'] == 0, 'API errors while telemetry backend was down: %s' % summary)
|
||
check(col_max < 256, 'collector memory not bounded')
|
||
check(phase['new_trace_after_recovery'], 'no traces after backend recovery')
|
||
return {'load': summary, 'collector_limit_mib': 256, 'collector_max_mib': col_max,
|
||
'api_max_mib': api_max, 'api_limit_mib': 256, 'phases': phase,
|
||
'timeline': {'tempo_loki_down_s': round(t_down - t0), 'tempo_loki_up_s': round(t_up - t0),
|
||
'collector_down_s': round(t_cd - t0), 'collector_down_for_s': 40},
|
||
'memory_samples': samples}
|
||
|
||
|
||
SCENARIOS = {'signals': scenario_signals, 'retry-trace': scenario_retry_trace,
|
||
'incident-worker': scenario_incident_worker, 'incident-db': scenario_incident_db,
|
||
'backend-down': scenario_backend_down}
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('scenario', choices=sorted(SCENARIOS))
|
||
p.add_argument('--output', default=None)
|
||
args = arguments('Lab6 observability scenarios on the Docker stand', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||
try:
|
||
result = SCENARIOS[args.scenario](c)
|
||
finally:
|
||
compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 's3', 'api', 'otel-collector', 'tempo', 'loki',
|
||
timeout=300, check=False)
|
||
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
Path(args.output).write_text(text + '\n', encoding='utf-8')
|
||
print(text)
|
||
print('PASS: ' + args.scenario)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|