lab6: Collector, Prometheus/Loki/Tempo/Grafana, alerts, incidents, track 1 (cost of observability)
This commit is contained in:
@@ -40,6 +40,7 @@ write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)"
|
||||
write_secret s3_api_secret_access_key "$(openssl rand -hex 24)"
|
||||
write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)"
|
||||
write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)"
|
||||
write_secret grafana_admin_password "$(openssl rand -hex 24)"
|
||||
|
||||
if [ -s secrets/broker_worker_password ] && [ -s secrets/broker_worker_url ]; then
|
||||
echo "kept secrets/broker_worker_password secrets/broker_worker_url"
|
||||
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generates the two provisioned Grafana dashboards (stdlib only).
|
||||
|
||||
Output: infra/grafana/dashboards/messenger-overview.json and
|
||||
messenger-telemetry.json. Metric names were verified against the live
|
||||
Prometheus (verify.py inventory). Percentiles are computed from summed buckets,
|
||||
never averaged across replicas; queue/outbox snapshots use max() because every
|
||||
API replica publishes the same PostgreSQL snapshot.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
OUT = Path(__file__).resolve().parents[2] / 'infra/grafana/dashboards'
|
||||
PROM = {'type': 'prometheus', 'uid': 'prometheus'}
|
||||
LOKI = {'type': 'loki', 'uid': 'loki'}
|
||||
BIZ = 'job="messenger-api",http_route!~"/health.*|/internal/.*"'
|
||||
|
||||
|
||||
class Board:
|
||||
def __init__(self, uid, title):
|
||||
self.uid, self.title = uid, title
|
||||
self.panels = []
|
||||
self.y = 0
|
||||
self.x = 0
|
||||
self.row_h = 0
|
||||
self.next_id = 1
|
||||
|
||||
def row(self, title):
|
||||
if self.x:
|
||||
self.y += self.row_h
|
||||
self.x = 0
|
||||
self.panels.append({'type': 'row', 'title': title, 'collapsed': False,
|
||||
'id': self._id(), 'panels': [],
|
||||
'gridPos': {'h': 1, 'w': 24, 'x': 0, 'y': self.y}})
|
||||
self.y += 1
|
||||
self.row_h = 0
|
||||
|
||||
def _id(self):
|
||||
self.next_id += 1
|
||||
return self.next_id - 1
|
||||
|
||||
def _place(self, w, h):
|
||||
if self.x + w > 24:
|
||||
self.y += self.row_h
|
||||
self.x, self.row_h = 0, 0
|
||||
pos = {'h': h, 'w': w, 'x': self.x, 'y': self.y}
|
||||
self.x += w
|
||||
self.row_h = max(self.row_h, h)
|
||||
return pos
|
||||
|
||||
def ts(self, title, queries, unit, description, w=12, h=8, vmin=None, vmax=None):
|
||||
self.panels.append({
|
||||
'type': 'timeseries', 'title': title, 'id': self._id(),
|
||||
'description': description, 'datasource': PROM,
|
||||
'gridPos': self._place(w, h),
|
||||
'fieldConfig': {'defaults': {**{'unit': unit, 'custom': {
|
||||
'lineWidth': 1, 'fillOpacity': 8, 'showPoints': 'never'}},
|
||||
**({'min': vmin} if vmin is not None else {}),
|
||||
**({'max': vmax} if vmax is not None else {})},
|
||||
'overrides': []},
|
||||
'options': {'legend': {'displayMode': 'list', 'placement': 'bottom'},
|
||||
'tooltip': {'mode': 'multi'}},
|
||||
'targets': [{'refId': chr(65 + i), 'datasource': PROM, 'expr': expr,
|
||||
'legendFormat': legend}
|
||||
for i, (expr, legend) in enumerate(queries)]})
|
||||
|
||||
def stat(self, title, expr, unit, description, w=6, h=4, legend=''):
|
||||
self.panels.append({
|
||||
'type': 'stat', 'title': title, 'id': self._id(),
|
||||
'description': description, 'datasource': PROM,
|
||||
'gridPos': self._place(w, h),
|
||||
'fieldConfig': {'defaults': {'unit': unit}, 'overrides': []},
|
||||
'options': {'reduceOptions': {'calcs': ['lastNotNull']},
|
||||
'colorMode': 'value'},
|
||||
'targets': [{'refId': 'A', 'datasource': PROM, 'expr': expr,
|
||||
'legendFormat': legend}]})
|
||||
|
||||
def table(self, title, expr, description, w=12, h=8):
|
||||
self.panels.append({
|
||||
'type': 'table', 'title': title, 'id': self._id(),
|
||||
'description': description, 'datasource': PROM,
|
||||
'gridPos': self._place(w, h),
|
||||
'fieldConfig': {'defaults': {'unit': 'short'}, 'overrides': []},
|
||||
'options': {'showHeader': True},
|
||||
'targets': [{'refId': 'A', 'datasource': PROM, 'expr': expr,
|
||||
'instant': True, 'format': 'table'}],
|
||||
'transformations': [{'id': 'organize', 'options': {
|
||||
'excludeByName': {'Time': True, '__name__': True,
|
||||
'job': True, 'Value': True}}}]})
|
||||
|
||||
def logs(self, title, expr, description, w=12, h=10):
|
||||
self.panels.append({
|
||||
'type': 'logs', 'title': title, 'id': self._id(),
|
||||
'description': description, 'datasource': LOKI,
|
||||
'gridPos': self._place(w, h),
|
||||
'options': {'showTime': True, 'wrapLogMessage': True,
|
||||
'sortOrder': 'Descending', 'enableLogDetails': True},
|
||||
'targets': [{'refId': 'A', 'datasource': LOKI, 'expr': expr,
|
||||
'queryType': 'range'}]})
|
||||
|
||||
def render(self):
|
||||
return {
|
||||
'uid': self.uid, 'title': self.title, 'schemaVersion': 39,
|
||||
'version': 1, 'editable': False, 'refresh': '10s',
|
||||
'time': {'from': 'now-15m', 'to': 'now'}, 'timezone': 'utc',
|
||||
'tags': ['messenger', 'lab6'], 'templating': {'list': []},
|
||||
'annotations': {'list': []}, 'panels': self.panels}
|
||||
|
||||
|
||||
def quantiles(metric, by='', selector='', window='1m'):
|
||||
group = 'le' + (',' + by if by else '')
|
||||
legend = ' {{' + by + '}}' if by else ''
|
||||
return [('histogram_quantile(%s, sum by(%s)(rate(%s_bucket%s[%s])))'
|
||||
% (q, group, metric, selector, window), 'p%d%s' % (round(q * 100), legend))
|
||||
for q in (0.5, 0.95, 0.99)]
|
||||
|
||||
|
||||
def overview():
|
||||
b = Board('messenger-overview', 'Messenger: API и worker')
|
||||
b.row('API (бизнес-запросы; /health* и /internal/* исключены)')
|
||||
b.ts('RPS по маршруту и классу статуса', [(
|
||||
'sum by (http_route, status_class)(label_replace(rate(http_server_request_duration_seconds_count{%s}[1m]), "status_class", "${1}xx", "http_response_status_code", "(.).."))' % BIZ,
|
||||
'{{http_route}} {{status_class}}')], 'reqps',
|
||||
'rate(http_server_request_duration_seconds_count[1m]) по http_route и классу кода; число запросов — _count гистограммы.')
|
||||
b.ts('Доля 5xx (1m)', [('messenger:api_errors:rate1m / messenger:api_requests:rate1m', '5xx / бизнес-запросы')],
|
||||
'percentunit', 'Recording rules: 5xx rate / все бизнес-запросы за 1m; 4xx — не ошибка сервиса. Пусто без трафика (N/A).', vmin=0, vmax=1)
|
||||
b.stat('Availability (5m), цель ≥ 99.5%', 'messenger:api_availability:ratio5m',
|
||||
'percentunit', '1 - 5xx/все бизнес-запросы за 5m (recording rule messenger:api_availability:ratio5m). N/A без трафика.')
|
||||
b.stat('Свежие реплики API', 'count((time() - timestamp(target_info{job="messenger-api"})) < 30) or vector(0)',
|
||||
'short', 'Число API-реплик, чей target_info обновлялся за последние 30 с (OTLP-метрики).')
|
||||
b.stat('HAProxy 5xx, rps', 'sum(rate(haproxy_backend_http_responses_total{proxy="api",code="5xx"}[1m])) or vector(0)',
|
||||
'reqps', 'HAProxy backend api: ответы 5xx (включая 503 балансировщика, когда нет готовых узлов).')
|
||||
b.stat('HAProxy: активные серверы', 'haproxy_backend_active_servers{proxy="api"}', 'short',
|
||||
'Активные серверы backend api по данным HAProxy PROMEX.')
|
||||
b.ts('Задержка HTTP: общая p50/p95/p99', quantiles('http_server_request_duration_seconds', selector='{%s}' % BIZ),
|
||||
's', 'histogram_quantile по суммированным бакетам всех реплик; перцентили реплик не усредняются.')
|
||||
b.ts('Задержка HTTP по маршруту p50/p95/p99', quantiles('http_server_request_duration_seconds', by='http_route', selector='{%s}' % BIZ),
|
||||
's', 'То же, с разбивкой по route template.')
|
||||
b.row('Worker и задачи')
|
||||
b.ts('Задачи по outcome', [('sum by (outcome)(rate(messenger_jobs_processed_total[1m]))', '{{outcome}}')], 'ops',
|
||||
'messenger_jobs_processed_total по фактическому outcome (succeeded, retry, failed, ...), задач/с.')
|
||||
b.ts('Длительность попытки задачи p50/p95/p99', quantiles('messenger_job_duration_seconds'), 's',
|
||||
'От claim до commit одной попытки; не включает ожидание в очереди.')
|
||||
b.ts('Time-to-ready p50/p95/p99', quantiles('messenger_attachment_time_to_ready_seconds'), 's',
|
||||
'От durable acceptance (created_at задачи) до commit ready; включает очередь, ретраи и backoff. Измеряется на commit, до ACK и cleanup.')
|
||||
b.ts('Worker: занято / ёмкость', [('sum(messenger_worker_jobs_active)', 'active'),
|
||||
('max(messenger_worker_jobs_capacity)', 'capacity (WORKER_CONCURRENCY)')],
|
||||
'short', 'Задач в работе против WORKER_CONCURRENCY; показатели не суммируются.')
|
||||
b.row('Очередь и outbox (snapshot PostgreSQL, max по репликам)')
|
||||
b.ts('Глубина очереди по состоянию', [('max by (state)(messenger_jobs_queue_depth)', '{{state}}')], 'short',
|
||||
'Незавершённые задачи по state; обе реплики публикуют одно значение, поэтому max, а не sum.')
|
||||
b.ts('Возраст самой старой задачи', [('max(messenger_jobs_queue_oldest_age_seconds)', 'oldest job age')], 's',
|
||||
'Возраст ожидания backlog (alert MessengerJobsQueueLagGrowing: > 30 с), не время обработки.')
|
||||
b.ts('Outbox: ожидающие и возраст', [('max(messenger_outbox_pending)', 'pending (шт.)'),
|
||||
('max(messenger_outbox_oldest_age_seconds)', 'oldest age (с)')],
|
||||
'short', 'Неопубликованные записи outbox (доступные к публикации) и возраст старейшей.')
|
||||
b.ts('RabbitMQ: ready / unacked', [('sum(rabbitmq_queue_messages_ready)', 'ready'),
|
||||
('sum(rabbitmq_queue_messages_unacked)', 'unacked')], 'short',
|
||||
'Агрегат по очередям, rabbitmq_prometheus (порт 15692, внутри сети).')
|
||||
b.row('PostgreSQL и процессы')
|
||||
b.ts('Пул БД: used / idle / max', [
|
||||
('sum by (db_client_connection_state, instance)(db_client_connection_count{job=~"messenger-.*",db_client_connection_pool_name="main"})', '{{db_client_connection_state}} {{instance}}'),
|
||||
('max by (instance)(db_client_connection_max{job=~"messenger-.*",db_client_connection_pool_name="main"})', 'max {{instance}}')],
|
||||
'short', 'Соединения основного пула по репликам.')
|
||||
b.ts('Пул БД: ожидающие запросы и wait p95', [
|
||||
('sum by (instance)(db_client_connection_pending_requests{job=~"messenger-.*"})', 'pending {{instance}}'),
|
||||
('histogram_quantile(0.95, sum by (le)(rate(db_client_connection_wait_time_seconds_bucket{job=~"messenger-.*"}[1m])))', 'wait p95 (с)')],
|
||||
'short', 'Очередь на получение соединения и p95 ожидания (секунды).')
|
||||
b.ts('Операция БД p95 по db_operation_name', [(
|
||||
'histogram_quantile(0.95, sum by (le, db_operation_name)(rate(db_client_operation_duration_seconds_bucket{job=~"messenger-.*"}[1m])))',
|
||||
'{{db_operation_name}}')], 's', 'p95 длительности операций БД по типу операции.')
|
||||
b.ts('CPU процессов (ядра)', [('sum by (job, instance)(rate(process_cpu_time_seconds_total{job=~"messenger-.*"}[1m]))', '{{job}} {{instance}}')],
|
||||
'short', 'rate(process_cpu_time_seconds_total): 1.0 = одно логическое ядро.')
|
||||
b.ts('RAM процессов', [('process_memory_usage_bytes{job=~"messenger-.*"}', '{{job}} {{instance}}'),
|
||||
('v8js_memory_heap_used_bytes{job=~"messenger-.*"}', 'heap {{job}} {{instance}}')],
|
||||
'bytes', 'Resident memory процесса и использованная куча V8.')
|
||||
b.ts('Event loop delay p99', [('nodejs_eventloop_delay_p99_seconds{job=~"messenger-.*"}', '{{job}} {{instance}}')], 's',
|
||||
'Задержка event loop (p99 окна, секунды).')
|
||||
b.row('Логи (переход в трассу по TraceID)')
|
||||
b.logs('Логи API', '{service_name="messenger-api"}', 'Структурные логи; trace_id — structured metadata, поле TraceID ведёт в Tempo.')
|
||||
b.logs('Логи worker', '{service_name="messenger-worker"}', 'Структурные логи worker; trace_id ведёт в Tempo.')
|
||||
return b.render()
|
||||
|
||||
|
||||
def telemetry():
|
||||
b = Board('messenger-telemetry', 'Messenger: telemetry pipeline')
|
||||
b.row('Collector')
|
||||
b.ts('Collector: принято', [
|
||||
('sum(rate(otelcol_receiver_accepted_spans[1m]))', 'spans'),
|
||||
('sum(rate(otelcol_receiver_accepted_log_records[1m]))', 'log records'),
|
||||
('sum(rate(otelcol_receiver_accepted_metric_points[1m]))', 'metric points')],
|
||||
'suffix:items/s', 'Элементов, принятых OTLP-receiver, в секунду (не байты).')
|
||||
b.ts('Collector: отправлено', [
|
||||
('sum(rate(otelcol_exporter_sent_spans[1m]))', 'spans'),
|
||||
('sum(rate(otelcol_exporter_sent_log_records[1m]))', 'log records'),
|
||||
('sum(rate(otelcol_exporter_sent_metric_points[1m]))', 'metric points')],
|
||||
'suffix:items/s', 'Элементов, успешно отправленных exporter-ами в backend.')
|
||||
b.ts('Collector: ошибки отправки и постановки в очередь', [
|
||||
('sum by (exporter)(rate(otelcol_exporter_send_failed_spans[1m]))', 'send_failed spans {{exporter}}'),
|
||||
('sum by (exporter)(rate(otelcol_exporter_send_failed_log_records[1m]))', 'send_failed logs {{exporter}}'),
|
||||
('sum by (exporter)(rate(otelcol_exporter_send_failed_metric_points[1m]))', 'send_failed metrics {{exporter}}'),
|
||||
('sum by (exporter)(rate(otelcol_exporter_enqueue_failed_spans[1m]))', 'enqueue_failed spans {{exporter}}'),
|
||||
('sum by (exporter)(rate(otelcol_exporter_enqueue_failed_log_records[1m]))', 'enqueue_failed logs {{exporter}}'),
|
||||
('sum by (exporter)(rate(otelcol_exporter_enqueue_failed_metric_points[1m]))', 'enqueue_failed metrics {{exporter}}')],
|
||||
'suffix:items/s', 'Потери на выходе Collector. Нулевые серии могут отсутствовать до первой ошибки.')
|
||||
b.ts('Collector: очередь / ёмкость', [
|
||||
('max by (exporter)(otelcol_exporter_queue_size)', 'size {{exporter}}'),
|
||||
('max by (exporter)(otelcol_exporter_queue_capacity)', 'capacity {{exporter}}')],
|
||||
'short', 'Очередь sending_queue в запросах (batch), не в spans.')
|
||||
b.ts('Collector: RSS', [('otelcol_process_memory_rss', 'RSS')], 'bytes',
|
||||
'Resident set size процесса Collector; лимит контейнера 256 MiB, memory_limiter 204 MiB.')
|
||||
b.ts('Приложение: экспорт телеметрии (items)', [
|
||||
('sum by (signal, outcome)(rate(messenger_telemetry_export_items_total[1m]))', '{{signal}} {{outcome}}')],
|
||||
'suffix:items/s', 'Элементов, дошедших до OTLP-exporter приложения (traces/logs). Не учёт метрик и не все pre-export drops.')
|
||||
b.row('Backend-хранилища')
|
||||
b.ts('Loki / Tempo: up', [('up{job=~"loki|tempo"}', '{{job}}')], 'short', '1 — scrape успешен.')
|
||||
b.ts('Loki / Tempo: ingest', [('sum(rate(loki_distributor_bytes_received_total[1m]))', 'loki'),
|
||||
('sum(rate(tempo_distributor_bytes_received_total[1m]))', 'tempo')],
|
||||
'Bps', 'Байт, принятых distributor-ами, в секунду.')
|
||||
b.ts('Prometheus: размер TSDB', [('prometheus_tsdb_storage_blocks_bytes', 'blocks'),
|
||||
('prometheus_tsdb_wal_storage_size_bytes', 'wal')], 'bytes',
|
||||
'Блоки и WAL локального TSDB (retention 7d).')
|
||||
b.table('Состояние правил (ALERTS)', 'ALERTS{alertname=~"Messenger.*"}',
|
||||
'Pending/firing alerts из метрики ALERTS. Отсутствие строк означает, что нет pending/firing при загруженных rules (правила проверяются через /api/v1/rules).')
|
||||
return b.render()
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
for name, board in (('messenger-overview', overview()),
|
||||
('messenger-telemetry', telemetry())):
|
||||
(OUT / (name + '.json')).write_text(
|
||||
json.dumps(board, indent=2, ensure_ascii=False) + '\n')
|
||||
print('wrote', OUT / (name + '.json'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+676
@@ -0,0 +1,676 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab6 controlled incidents: worker | db | telemetry. Stdlib only.
|
||||
|
||||
Every scenario runs inside obs.app_environment (SDK on, sampling 1,
|
||||
FAULT_INJECTION_ENABLED=1) and always undoes its own faults/locks/stopped
|
||||
services in `finally`. Times are UTC + monotonic offsets; an unobserved
|
||||
transition is null with a reason, never a guess."""
|
||||
import argparse
|
||||
import calendar
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import measurement # noqa: E402
|
||||
import obs # noqa: E402
|
||||
import reconcile # noqa: E402
|
||||
import resources # noqa: E402
|
||||
import verify # noqa: E402
|
||||
import workload # noqa: E402
|
||||
from obs import Failure, stand, utc_now # noqa: E402
|
||||
|
||||
ROOT = obs.ROOT
|
||||
|
||||
|
||||
def utc_epoch(text):
|
||||
return calendar.timegm(time.strptime(text[:19], '%Y-%m-%dT%H:%M:%S'))
|
||||
ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
|
||||
ENV = {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
|
||||
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
|
||||
|
||||
|
||||
class Ctx:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.obs = verify.make_obs(args)
|
||||
self.out = Path(args.output_dir)
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def mono(self):
|
||||
return round(time.monotonic() - self.t0, 2)
|
||||
|
||||
def mark(self, timeline, key, extra=None):
|
||||
timeline[key] = {'utc': utc_now(), 'mono_s': self.mono()}
|
||||
if extra:
|
||||
timeline[key].update(extra)
|
||||
print('[%s] %s' % (timeline[key]['utc'], key), flush=True)
|
||||
|
||||
def actor_chat(self):
|
||||
c = stand.new_client(self.args.base_url, self.args.ca_file)
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
return alice, chat
|
||||
|
||||
|
||||
def alert_state(ctx, name):
|
||||
result = ctx.obs.prom_query('ALERTS{alertname="%s"}' % name)
|
||||
states = {r['metric'].get('alertstate') for r in result}
|
||||
if 'firing' in states:
|
||||
return 'firing'
|
||||
return 'pending' if 'pending' in states else None
|
||||
|
||||
|
||||
def wait_state(ctx, name, want, timeout, timeline, polls):
|
||||
"""want: 'firing' or None (resolved). Returns True when reached."""
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
state = alert_state(ctx, name)
|
||||
polls.append({'utc': utc_now(), 'mono_s': ctx.mono(), 'alert': name, 'state': state})
|
||||
if state == 'pending' and 'pending' not in timeline:
|
||||
ctx.mark(timeline, 'pending')
|
||||
if state == want:
|
||||
return True
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
|
||||
class BackgroundLoad:
|
||||
def __init__(self, ctx, actor, chat, rate, upload_every, phase):
|
||||
self.stop_event = threading.Event()
|
||||
self.result = None
|
||||
self.error = None
|
||||
|
||||
def body():
|
||||
try:
|
||||
self.result = workload.run_window(
|
||||
ctx.args.base_url, ctx.args.ca_file, actor, chat, rate, 1800,
|
||||
phase, upload_every=upload_every, stop_event=self.stop_event,
|
||||
settle=False)
|
||||
except Exception as exc: # reported by stop()
|
||||
self.error = exc
|
||||
self.thread = threading.Thread(target=body, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
self.thread.join(timeout=120)
|
||||
if self.error:
|
||||
raise self.error
|
||||
if self.result is None:
|
||||
raise Failure('background load did not finish')
|
||||
return self.result
|
||||
|
||||
|
||||
def baseline(ctx, actor, chat, rate, upload_every):
|
||||
ctx.obs.require_no_foreign_faults()
|
||||
for name in ALERTS:
|
||||
if alert_state(ctx, name) is not None:
|
||||
raise Failure('alert already active before baseline: ' + name)
|
||||
result = workload.run_window(ctx.args.base_url, ctx.args.ca_file, actor, chat,
|
||||
rate, 60, 'baseline', upload_every=upload_every,
|
||||
settle=False)
|
||||
drain = obs.wait_drained(result['uploads'], obs=ctx.obs)
|
||||
for name in ALERTS:
|
||||
if alert_state(ctx, name) is not None:
|
||||
raise Failure('alert active at the end of baseline: ' + name)
|
||||
if not drain['drained']:
|
||||
raise Failure('baseline jobs did not drain')
|
||||
return {'load': result['load'], 'drain': drain, 'uploads': result['uploads']}
|
||||
|
||||
|
||||
def series(ctx, expr, start, end, step=10):
|
||||
out = []
|
||||
for item in ctx.obs.prom_range(expr, start, end, step):
|
||||
out.append({'labels': item['metric'],
|
||||
'points': [[float(t), v] for t, v in item['values']]})
|
||||
return out
|
||||
|
||||
|
||||
def peak(rows):
|
||||
values = [float(v) for r in rows for _, v in r['points'] if v != 'NaN']
|
||||
return max(values) if values else None
|
||||
|
||||
|
||||
def ready_seconds(ids):
|
||||
"""max/median finished_at - created_at of the jobs of these attachments."""
|
||||
if not ids:
|
||||
return {'n': 0, 'max': None, 'median': None}
|
||||
values = []
|
||||
for i in range(0, len(ids), 200):
|
||||
chunk = ids[i:i + 200]
|
||||
arr = "ARRAY[" + ",".join("'" + x + "'" for x in chunk) + "]::uuid[]"
|
||||
out = obs.psql(
|
||||
"SELECT COALESCE(string_agg(extract(epoch from (finished_at - created_at))::text, ';'),'') "
|
||||
"FROM attachment_jobs WHERE status='succeeded' AND attachment_id = ANY(" + arr + ")")
|
||||
values += [float(x) for x in out.split(';') if x]
|
||||
return {'n': len(values), 'max': max(values) if values else None,
|
||||
'median': measurement.median(values)}
|
||||
|
||||
|
||||
def finish(ctx, name, result):
|
||||
result['passed'] = not result['problems']
|
||||
verify.write_json(ctx.out, name, result)
|
||||
if result['problems']:
|
||||
raise Failure(name + ': ' + '; '.join(result['problems']))
|
||||
print('PASS: ' + name)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- worker
|
||||
|
||||
def scenario_worker(ctx, label=''):
|
||||
alice, chat = ctx.actor_chat()
|
||||
result = {'scenario': 'worker', 'profile': {'rate': 20, 'upload_every': 20},
|
||||
'timeline': {}, 'problems': [], 'polls': []}
|
||||
tl = result['timeline']
|
||||
base = baseline(ctx, alice, chat, 20, 20)
|
||||
result['baseline'] = {'load': base['load'], 'ready_seconds': ready_seconds(base['uploads'])}
|
||||
load = None
|
||||
worker_stopped = False
|
||||
try:
|
||||
load = BackgroundLoad(ctx, alice, chat, 20, 20, 'incident')
|
||||
time.sleep(10)
|
||||
ctx.mark(tl, 'inject')
|
||||
stand.compose('stop', 'worker')
|
||||
worker_stopped = True
|
||||
fired = wait_state(ctx, 'MessengerJobsQueueLagGrowing', 'firing', 150, tl, result['polls'])
|
||||
if fired:
|
||||
ctx.mark(tl, 'firing')
|
||||
else:
|
||||
tl['firing'] = None
|
||||
result['problems'].append('queue-lag alert did not fire within 150 s')
|
||||
# Capture the evidence while the backlog exists.
|
||||
t_now = time.time()
|
||||
result['queue_at_firing'] = {
|
||||
'oldest_age_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_oldest_age_seconds)'),
|
||||
'depth_ready_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_depth{state="ready"})'),
|
||||
'depth_ready_sum_over_replicas': ctx.obs.prom_scalar('sum(messenger_jobs_queue_depth{state="ready"})'),
|
||||
'depth_ready_per_replica': {
|
||||
r['metric']['instance']: float(r['value'][1])
|
||||
for r in ctx.obs.prom_query('messenger_jobs_queue_depth{state="ready"}')},
|
||||
'outbox_pending_max': ctx.obs.prom_scalar('max(messenger_outbox_pending)'),
|
||||
}
|
||||
q = result['queue_at_firing']
|
||||
per = list(q['depth_ready_per_replica'].values())
|
||||
if len(per) >= 2 and q['depth_ready_max'] != max(per):
|
||||
result['problems'].append('max() does not equal the per-replica snapshot')
|
||||
if len(per) >= 2 and q['depth_ready_sum_over_replicas'] == q['depth_ready_max'] and per[0] > 0:
|
||||
result['problems'].append('replicas not both reporting (cannot show non-doubling)')
|
||||
finally:
|
||||
ctx.mark(tl, 'recover_start')
|
||||
if worker_stopped:
|
||||
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
|
||||
ctx.mark(tl, 'worker_back')
|
||||
incident_load = load.stop() if load else None
|
||||
ctx.mark(tl, 'load_stopped')
|
||||
ids = incident_load['uploads']
|
||||
result['incident_load'] = incident_load['load']
|
||||
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
|
||||
result['drain'] = drain
|
||||
ctx.mark(tl, 'backlog_drained' if drain['drained'] else 'drain_deadline')
|
||||
if not drain['drained']:
|
||||
tl['backlog_drained'] = None
|
||||
result['problems'].append('backlog not drained within 180 s')
|
||||
elif any(s != 'ready' for s in drain['states'].values()):
|
||||
result['problems'].append('some accepted attachments are not ready')
|
||||
resolved = wait_state(ctx, 'MessengerJobsQueueLagGrowing', None, 180, tl, result['polls'])
|
||||
if resolved:
|
||||
ctx.mark(tl, 'resolved')
|
||||
else:
|
||||
tl['resolved'] = None
|
||||
result['problems'].append('queue-lag alert not resolved within 180 s')
|
||||
end = time.time()
|
||||
t_inject = utc_epoch(tl['inject']['utc'])
|
||||
window = (t_inject - 60, end)
|
||||
result['queries'] = {
|
||||
'oldest_age': 'max(messenger_jobs_queue_oldest_age_seconds)',
|
||||
'depth': 'max by (state)(messenger_jobs_queue_depth)',
|
||||
'outbox': 'max(messenger_outbox_pending)',
|
||||
'worker_active': 'sum(messenger_worker_jobs_active)'}
|
||||
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
|
||||
peak_age = peak(result['series']['oldest_age'])
|
||||
result['peak_oldest_age_s'] = peak_age
|
||||
if peak_age is None or peak_age < 30:
|
||||
result['problems'].append('queue age did not grow past 30 s')
|
||||
result['incident_ready_seconds'] = ready_seconds(ids)
|
||||
if (result['incident_ready_seconds']['max'] or 0) <= (result['baseline']['ready_seconds']['max'] or 0):
|
||||
result['problems'].append('time-to-ready did not exceed the baseline')
|
||||
http_ok = result['incident_load']['http_5xx'] == 0 and \
|
||||
sum(result['incident_load']['transport_failures'].values()) == 0
|
||||
result['http_acceptance_continued'] = http_ok
|
||||
if not http_ok:
|
||||
result['problems'].append('HTTP acceptance degraded during worker outage')
|
||||
finish(ctx, 'incident-worker.json', result)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------- db
|
||||
|
||||
def trace_gap(spans, server):
|
||||
"""Server duration minus the union of its direct child intervals (ms)."""
|
||||
children = sorted((s['start_ns'], s['end_ns']) for s in spans
|
||||
if s['parent_span_id'] == server['span_id'])
|
||||
# Descendants of the transaction span are covered by their ancestor.
|
||||
covered, last = 0, None
|
||||
for start, end in children:
|
||||
if last is None or start > last:
|
||||
covered += end - start
|
||||
last = end
|
||||
elif end > last:
|
||||
covered += end - last
|
||||
last = end
|
||||
return ((server['end_ns'] - server['start_ns']) - covered) / 1e6
|
||||
|
||||
|
||||
def db_time_ms(spans, server):
|
||||
return sum((s['end_ns'] - s['start_ns']) for s in spans
|
||||
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
|
||||
and s['parent_span_id'] == server['span_id']) / 1e6
|
||||
|
||||
|
||||
def sequential_uploads(ctx, actor, chat, count, delay):
|
||||
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
|
||||
c.timeout = 30
|
||||
key = 'fault:api.after_commit_before_publish'
|
||||
rows = []
|
||||
for _ in range(count):
|
||||
if delay:
|
||||
stand.set_fault(key, 'delay:%d' % delay)
|
||||
started = time.monotonic()
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
||||
range(200, 600), actor=actor, raw=stand.sample_bytes(),
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
elapsed = (time.monotonic() - started) * 1000
|
||||
consumed = True
|
||||
if delay:
|
||||
out = stand.compose('exec', '-T', 'redis', 'redis-cli', 'GET', key).stdout.strip()
|
||||
consumed = out == ''
|
||||
rows.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status'],
|
||||
'elapsed_ms': elapsed, 'consumed': consumed,
|
||||
'attachment_id': r['json']['id'] if r['status'] == 202 else None})
|
||||
return rows
|
||||
|
||||
|
||||
def server_view(ctx, request_id):
|
||||
log = verify.http_log(ctx.obs, request_id)
|
||||
tid, sid = log['labels']['trace_id'], log['labels']['span_id']
|
||||
spans = verify.trace_with(ctx.obs, tid, lambda sp: any(
|
||||
s['span_id'] == sid for s in sp), description='server trace')
|
||||
server = next(s for s in spans if s['span_id'] == sid)
|
||||
return spans, server
|
||||
|
||||
|
||||
def scenario_db(ctx, label=''):
|
||||
alice, chat = ctx.actor_chat()
|
||||
result = {'scenario': 'db', 'profile': {'rate': 2, 'upload_every': 1},
|
||||
'timeline': {}, 'problems': [], 'polls': []}
|
||||
tl = result['timeline']
|
||||
base = baseline(ctx, alice, chat, 2, 1)
|
||||
result['baseline'] = {'load': base['load']}
|
||||
load = None
|
||||
lock_released = False
|
||||
try:
|
||||
load = BackgroundLoad(ctx, alice, chat, 2, 1, 'incident')
|
||||
time.sleep(10)
|
||||
# Own client: 30 s timeout is set inside workload (REQUEST_TIMEOUT).
|
||||
with obs.hold_table_lock('attachments') as pid:
|
||||
ctx.mark(tl, 'inject', {'lock_backend_pid': pid})
|
||||
fired = wait_state(ctx, 'MessengerApiHighErrorRate', 'firing', 120, tl, result['polls'])
|
||||
if fired:
|
||||
ctx.mark(tl, 'firing')
|
||||
else:
|
||||
tl['firing'] = None
|
||||
result['problems'].append('error-rate alert did not fire within 120 s')
|
||||
result['at_firing'] = {
|
||||
'api_5xx_rate_1m': ctx.obs.prom_scalar('messenger:api_errors:rate1m'),
|
||||
'api_request_rate_1m': ctx.obs.prom_scalar('messenger:api_requests:rate1m'),
|
||||
'haproxy_5xx_rate_1m': ctx.obs.prom_scalar(
|
||||
'sum(rate(haproxy_backend_http_responses_total{proxy="api",code="5xx"}[1m]))', 0.0),
|
||||
'db_insert_attachments_p95_s': ctx.obs.prom_scalar(
|
||||
'histogram_quantile(0.95, sum by (le)(rate(db_client_operation_duration_seconds_bucket{db_operation_name="INSERT"}[1m])))'),
|
||||
'pool_pending': ctx.obs.prom_scalar('sum(db_client_connection_pending_requests)'),
|
||||
'pool_used': ctx.obs.prom_scalar('sum(db_client_connection_count{db_client_connection_state="used"})'),
|
||||
}
|
||||
lock_released = True
|
||||
ctx.mark(tl, 'lock_released')
|
||||
finally:
|
||||
if not lock_released:
|
||||
ctx.mark(tl, 'lock_released_in_finally')
|
||||
incident_load = load.stop() if load else None
|
||||
ctx.mark(tl, 'load_stopped')
|
||||
result['incident_load'] = incident_load['load']
|
||||
if incident_load['load']['http_5xx'] == 0:
|
||||
result['problems'].append('no real server 5xx observed in the load')
|
||||
ids = incident_load['uploads']
|
||||
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
|
||||
result['drain'] = drain
|
||||
if drain['drained']:
|
||||
ctx.mark(tl, 'backlog_drained')
|
||||
else:
|
||||
tl['backlog_drained'] = None
|
||||
result['problems'].append('jobs did not drain after recovery')
|
||||
if wait_state(ctx, 'MessengerApiHighErrorRate', None, 120, tl, result['polls']):
|
||||
ctx.mark(tl, 'resolved')
|
||||
else:
|
||||
tl['resolved'] = None
|
||||
result['problems'].append('error-rate alert not resolved within 120 s')
|
||||
# -- localisation: a 5xx trace where the time went to the DB.
|
||||
inject_ns = int(utc_epoch(tl['inject']['utc']) - 5) * 10 ** 9
|
||||
rows = verify.wait_rows(
|
||||
ctx.obs, '{service_name="messenger-api"} | event_name="http.request" '
|
||||
'| http_response_status_code=~"5.."', timeout=60, back=900)
|
||||
rows = [r for r in rows if r['ts'] >= inject_ns]
|
||||
if rows:
|
||||
row = rows[0]
|
||||
spans = verify.trace_with(ctx.obs, row['labels']['trace_id'], lambda sp: any(
|
||||
s['span_id'] == row['labels']['span_id'] for s in sp), description='5xx trace')
|
||||
server = next(s for s in spans if s['span_id'] == row['labels']['span_id'])
|
||||
dur = (server['end_ns'] - server['start_ns']) / 1e6
|
||||
db_ms = db_time_ms(spans, server)
|
||||
result['trace_5xx'] = {
|
||||
'request_id': row['labels'].get('request_id'), 'trace_id': row['labels']['trace_id'],
|
||||
'status': row['labels'].get('http_response_status_code'),
|
||||
'server_ms': dur, 'db_children_ms': db_ms,
|
||||
'handler_gap_ms': trace_gap(spans, server),
|
||||
'db_children': [verify.safe_span(s) | {'duration_ms': (s['end_ns'] - s['start_ns']) / 1e6}
|
||||
for s in spans if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
|
||||
and s['parent_span_id'] == server['span_id']][:6]}
|
||||
if db_ms < 0.5 * dur:
|
||||
result['problems'].append('5xx trace is not dominated by DB spans')
|
||||
else:
|
||||
result['problems'].append('no 5xx http.request log found in Loki')
|
||||
# -- control: handler delay (not DB) shows up as a gap, not as DB time.
|
||||
ctx.obs.require_no_foreign_faults()
|
||||
ctrl_chat = ctx.actor_chat()
|
||||
base_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 0)
|
||||
delay_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 500)
|
||||
stand.clear_fault('fault:api.after_commit_before_publish')
|
||||
|
||||
def gaps(rows_):
|
||||
out = []
|
||||
for r in rows_:
|
||||
spans, server = server_view(ctx, r['request_id'])
|
||||
out.append({'request_id': r['request_id'],
|
||||
'server_ms': (server['end_ns'] - server['start_ns']) / 1e6,
|
||||
'db_children_ms': db_time_ms(spans, server),
|
||||
'handler_gap_ms': trace_gap(spans, server)})
|
||||
return out
|
||||
base_g, delay_g = gaps(base_rows), gaps(delay_rows)
|
||||
med = measurement.median
|
||||
shift = med([g['handler_gap_ms'] for g in delay_g]) - med([g['handler_gap_ms'] for g in base_g])
|
||||
result['control'] = {
|
||||
'baseline_uploads': base_g, 'delayed_uploads': delay_g,
|
||||
'handler_gap_shift_ms_median': shift,
|
||||
'db_children_ms_median_baseline': med([g['db_children_ms'] for g in base_g]),
|
||||
'db_children_ms_median_delayed': med([g['db_children_ms'] for g in delay_g]),
|
||||
'all_keys_consumed': all(r['consumed'] for r in delay_rows),
|
||||
'all_http_202': all(r['status'] == 202 for r in base_rows + delay_rows),
|
||||
'note': 'timer/scheduling noise is part of the shift; the check is ~500 ms +/- 250 ms'}
|
||||
if not result['control']['all_keys_consumed']:
|
||||
result['problems'].append('delay fault key was not consumed')
|
||||
if not result['control']['all_http_202']:
|
||||
result['problems'].append('control uploads were not all 202')
|
||||
if not (250 <= shift <= 750):
|
||||
result['problems'].append('handler gap shift %.0f ms is not ~500 ms' % shift)
|
||||
cdb = result['control']
|
||||
if abs(cdb['db_children_ms_median_delayed'] - cdb['db_children_ms_median_baseline']) > 100:
|
||||
result['problems'].append('DB time changed in the handler-delay control')
|
||||
stand.wait_for(lambda: obs.wait_drained(
|
||||
[r['attachment_id'] for r in base_rows + delay_rows], obs=None, timeout=60,
|
||||
flush_telemetry=False)['drained'], 90, 'control uploads ready')
|
||||
end = time.time()
|
||||
t_inject = utc_epoch(tl['inject']['utc'])
|
||||
window = (t_inject - 60, t_inject + 240)
|
||||
result['queries'] = {
|
||||
'db_operation_p95_by_op': 'histogram_quantile(0.95, sum by (le, db_operation_name)(rate(db_client_operation_duration_seconds_bucket[1m])))',
|
||||
'pool_by_state': 'sum by (db_client_connection_state)(db_client_connection_count{db_client_connection_pool_name="main"})',
|
||||
'pool_pending': 'sum(db_client_connection_pending_requests)',
|
||||
'api_5xx_ratio': 'messenger:api_errors:rate1m / messenger:api_requests:rate1m'}
|
||||
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
|
||||
finish(ctx, 'incident-db.json', result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- telemetry
|
||||
|
||||
def regression_slope(points):
|
||||
n = len(points)
|
||||
if n < 3:
|
||||
return None
|
||||
mx = sum(x for x, _ in points) / n
|
||||
my = sum(y for _, y in points) / n
|
||||
den = sum((x - mx) ** 2 for x, _ in points)
|
||||
return sum((x - mx) * (y - my) for x, y in points) / den if den else None
|
||||
|
||||
|
||||
def rss_trend(samples, service, after_s, start_mono):
|
||||
pts = [(s['mono'] - start_mono, s['rss_bytes']) for s in samples
|
||||
if s['service'] == service and s.get('rss_bytes') is not None
|
||||
and s['mono'] - start_mono >= after_s]
|
||||
by_tick = {}
|
||||
for t, v in pts:
|
||||
by_tick.setdefault(round(t / 5), []).append((t, v))
|
||||
summed = [(sum(t for t, _ in g) / len(g), sum(v for _, v in g)) for g in by_tick.values()]
|
||||
values = [v for _, v in summed]
|
||||
return {'points': len(summed), 'min': min(values) if values else None,
|
||||
'max': max(values) if values else None,
|
||||
'slope_bytes_per_s': regression_slope(summed),
|
||||
'first': values[0] if values else None, 'last': values[-1] if values else None}
|
||||
|
||||
|
||||
def sdk_overflow_logs(since_utc):
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', '--env-file', stand.ENV_FILE, 'logs', '--no-color',
|
||||
'--no-log-prefix', '--since', since_utc, 'api', 'worker'],
|
||||
cwd=ROOT, text=True, capture_output=True, timeout=180)
|
||||
count = 0
|
||||
for line in result.stdout.splitlines():
|
||||
if '"event":"otel.sdk"' in line and ('queue' in line.lower() or 'dropped' in line.lower()):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def resume_probe(ctx, actor, chat, count):
|
||||
"""After recovery: fresh requests must reach Loki (logs) and Tempo (traces)."""
|
||||
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
|
||||
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
|
||||
start_ns = time.time_ns() - 5 * 10 ** 9
|
||||
requests = []
|
||||
for i in range(count):
|
||||
r = c.message(actor, chat, 'lab6-resume-%d' % i)
|
||||
requests.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status']})
|
||||
_, summary = reconcile.reconcile(ctx.obs, 'full', requests, since, start_ns,
|
||||
time.time_ns() + 60 * 10 ** 9)
|
||||
return summary
|
||||
|
||||
|
||||
def telemetry_outage(ctx, service, alice, chat, repeat):
|
||||
out = {'service': service, 'timeline': {}, 'problems': []}
|
||||
tl = out['timeline']
|
||||
ctx.obs.require_no_foreign_faults()
|
||||
base = baseline(ctx, alice, chat, 20, 20)
|
||||
out['baseline'] = {'load': base['load']}
|
||||
state_before = ctx.obs.app_state()
|
||||
ids = resources.container_map()
|
||||
sampler = resources.ResourceSampler(ids, interval=5.0)
|
||||
sampler.start()
|
||||
sampler_start = time.monotonic()
|
||||
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
|
||||
pre_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
|
||||
load = BackgroundLoad(ctx, alice, chat, 20, 20, service + '-outage')
|
||||
stopped = False
|
||||
t_smoke = None
|
||||
try:
|
||||
time.sleep(10)
|
||||
ctx.mark(tl, 'inject')
|
||||
t_inject = time.time()
|
||||
stand.compose('stop', service)
|
||||
stopped = True
|
||||
t_start = time.monotonic()
|
||||
smoke = subprocess.run(['make', 'test', 'BASE_URL=' + ctx.args.base_url], cwd=ROOT,
|
||||
capture_output=True, text=True, timeout=80)
|
||||
t_smoke = time.monotonic() - t_start
|
||||
out['smoke_during_outage'] = {'returncode': smoke.returncode,
|
||||
'seconds': round(t_smoke, 1),
|
||||
'tail': smoke.stdout.strip().splitlines()[-1:] if smoke.stdout else []}
|
||||
if smoke.returncode != 0:
|
||||
out['problems'].append('make test failed during the outage')
|
||||
remaining = 90 - (time.monotonic() - t_start)
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
if service == 'tempo':
|
||||
out['collector_during_outage'] = {
|
||||
'queue_size_max': ctx.obs.prom_scalar('max(max_over_time(otelcol_exporter_queue_size{exporter="otlp/tempo"}[90s]))'),
|
||||
'queue_capacity': ctx.obs.prom_scalar('max(otelcol_exporter_queue_capacity{exporter="otlp/tempo"})'),
|
||||
'send_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_send_failed_spans[90s]))', 0.0),
|
||||
'enqueue_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_enqueue_failed_spans[90s]))', 0.0),
|
||||
'logs_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_log_records[90s]))', 0.0),
|
||||
'metrics_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_metric_points[90s]))', 0.0),
|
||||
'collector_rss_bytes': ctx.obs.prom_scalar('otelcol_process_memory_rss'),
|
||||
'loki_up': ctx.obs.prom_scalar('up{job="loki"}'), 'prometheus_ready': True}
|
||||
ctx.mark(tl, 'outage_end')
|
||||
finally:
|
||||
if stopped:
|
||||
stand.compose('start', service)
|
||||
ctx.mark(tl, 'recover')
|
||||
try:
|
||||
incident_load = load.stop()
|
||||
finally:
|
||||
ctx.mark(tl, 'load_stopped')
|
||||
t_restore = time.time()
|
||||
ctx.obs.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
|
||||
ctx.mark(tl, 'backends_ready')
|
||||
drain = obs.wait_drained(incident_load['uploads'] + base['uploads'], obs=ctx.obs, timeout=180)
|
||||
out['drain'] = drain
|
||||
if drain['drained']:
|
||||
ctx.mark(tl, 'backlog_drained')
|
||||
else:
|
||||
out['problems'].append('jobs did not drain')
|
||||
sampler.stop()
|
||||
out['resume_probe'] = resume_probe(ctx, alice, chat, 10)
|
||||
probe = out['resume_probe']
|
||||
if probe.get('sampled_present') != 10 or probe.get('log_exported') != 10:
|
||||
out['problems'].append('traces/logs did not resume after recovery: %s' % probe)
|
||||
out['incident_load'] = incident_load['load']
|
||||
if incident_load['load']['http_5xx'] or sum(incident_load['load']['transport_failures'].values()):
|
||||
out['problems'].append('business traffic degraded during telemetry outage')
|
||||
# -- no restarts/OOM, same instances.
|
||||
state_after = ctx.obs.app_state()
|
||||
for svc in ('api', 'worker'):
|
||||
if set(state_before[svc]) != set(state_after[svc]):
|
||||
out['problems'].append('instance IDs changed for ' + svc)
|
||||
for cid, info in state_after[svc].items():
|
||||
if info['restarts'] != state_before[svc].get(cid, info)['restarts'] or info['oom_killed']:
|
||||
out['problems'].append('restart/OOM in ' + svc)
|
||||
summary = resources.summarize(sampler.samples)
|
||||
out['resources'] = {k: v for k, v in summary.items() if k != 'sample_errors'}
|
||||
out['sample_errors'] = summary['sample_errors']
|
||||
out['memory_vs_limit'] = {
|
||||
svc: (v['container_memory_bytes']['max'] / v['mem_limit_bytes'])
|
||||
for svc, v in summary.items() if svc != 'sample_errors'
|
||||
and v['mem_limit_bytes'] and v['container_memory_bytes']['max'] is not None}
|
||||
for svc, ratio in out['memory_vs_limit'].items():
|
||||
if ratio >= 1.0:
|
||||
out['problems'].append(svc + ' memory at/over limit')
|
||||
out['rss_trend_after_first_minute'] = {
|
||||
svc: rss_trend(sampler.samples, svc, 60, sampler_start)
|
||||
for svc in (('otel-collector', 'api', 'worker') if service == 'collector' or service == 'otel-collector'
|
||||
else ('otel-collector',))}
|
||||
out['sdk_queue_diagnostic_messages'] = sdk_overflow_logs(since)
|
||||
# -- stdout vs backends for requests accepted during the outage.
|
||||
inject_unix = utc_epoch(tl['inject']['utc'])
|
||||
during = [s for s in incident_load['slots'] if s.get('outcome') == 'http'
|
||||
and s.get('request_id') and s['kind'] == 'message']
|
||||
stdout = reconcile.stdout_http_logs(since)
|
||||
# slots are relative to the load t0; classify by stdout timestamp instead.
|
||||
in_window = []
|
||||
for s in during:
|
||||
row = stdout.get(s['request_id'])
|
||||
if row is None:
|
||||
continue
|
||||
ts = utc_epoch(row['timestamp'])
|
||||
if inject_unix <= ts <= inject_unix + 90:
|
||||
in_window.append((s, row))
|
||||
sampled_window = [(s, r) for s, r in in_window if r.get('trace_sampled') is True]
|
||||
step = max(1, len(sampled_window) // 100)
|
||||
probe = sampled_window[::step][:100]
|
||||
targets = {s['request_id']: (r['trace_id'], r['span_id']) for s, r in probe}
|
||||
found = reconcile.tempo_server_spans(ctx.obs, targets, deadline_s=60)
|
||||
out['stdout_vs_tempo'] = {
|
||||
'requests_in_outage_window_with_stdout': len(in_window),
|
||||
'sampled_in_window_per_stdout': len(sampled_window),
|
||||
'tempo_probe_size': len(probe),
|
||||
'tempo_probe_missing_server_spans': sum(1 for v in found.values() if not v),
|
||||
'note': 'loss is measured for SERVER root spans only, on a systematic probe '
|
||||
'sample; it does not count lost child spans'}
|
||||
if not in_window:
|
||||
out['problems'].append('no stdout http.request lines in the outage window')
|
||||
if any(not r.get('trace_id') or r.get('trace_sampled') is not True for _, r in sampled_window):
|
||||
out['problems'].append('stdout trace fields missing')
|
||||
if service == 'tempo':
|
||||
c = out['collector_during_outage']
|
||||
if (c['queue_size_max'] or 0) > (c['queue_capacity'] or 128):
|
||||
out['problems'].append('Collector queue exceeded capacity')
|
||||
if not (c['send_failed_spans_increase'] or 0) > 0:
|
||||
out['problems'].append('Collector did not count failed span exports')
|
||||
if not (c['logs_sent_increase'] or 0) > 0 or not (c['metrics_sent_increase'] or 0) > 0:
|
||||
out['problems'].append('Loki/Prometheus paths stopped during the Tempo outage')
|
||||
else:
|
||||
post_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
|
||||
out['app_export_failed_items'] = {'before': pre_failed, 'after': post_failed,
|
||||
'delta': post_failed - pre_failed if post_failed >= pre_failed else None,
|
||||
'reason': None if post_failed >= pre_failed else 'counter_reset'}
|
||||
if not (out['app_export_failed_items']['delta'] or 0) > 0:
|
||||
out['problems'].append('app failed-export counter did not grow')
|
||||
resumed = {name: ctx.obs.prom_scalar(expr, 0.0) for name, expr in (
|
||||
('spans_sent_300s', 'sum(increase(otelcol_exporter_sent_spans[300s]))'),
|
||||
('logs_sent_300s', 'sum(increase(otelcol_exporter_sent_log_records[300s]))'),
|
||||
('metrics_sent_300s', 'sum(increase(otelcol_exporter_sent_metric_points[300s]))'))}
|
||||
out['signals_resumed'] = resumed
|
||||
if not all(v > 0 for v in resumed.values()):
|
||||
out['problems'].append('not all three signals resumed')
|
||||
return out
|
||||
|
||||
|
||||
def scenario_telemetry(ctx, label=''):
|
||||
alice, chat = ctx.actor_chat()
|
||||
result = {'scenario': 'telemetry', 'repeat': label, 'problems': [], 'outages': []}
|
||||
for service in ('tempo', 'otel-collector'):
|
||||
outage = telemetry_outage(ctx, service, alice, chat, label)
|
||||
result['outages'].append(outage)
|
||||
result['problems'] += [service + ': ' + p for p in outage['problems']]
|
||||
name = 'incident-telemetry-%s.json' % label
|
||||
finish(ctx, name, result)
|
||||
|
||||
|
||||
SCENARIOS = {'worker': scenario_worker, 'db': scenario_db, 'telemetry': scenario_telemetry}
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--scenario', required=True, choices=sorted(SCENARIOS))
|
||||
parser.add_argument('--repeat', type=int, default=1)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||||
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
||||
parser.add_argument('--output-dir', default='evidence/lab6')
|
||||
args = parser.parse_args(argv)
|
||||
ctx = Ctx(args)
|
||||
failed = False
|
||||
try:
|
||||
with ctx.obs.app_environment(ENV):
|
||||
for repeat in range(1, args.repeat + 1):
|
||||
try:
|
||||
SCENARIOS[args.scenario](ctx, str(repeat))
|
||||
except Failure as error:
|
||||
failed = True
|
||||
print('FAIL: ' + str(error), file=sys.stderr)
|
||||
break
|
||||
except Failure as error:
|
||||
print('FAIL: ' + str(error), file=sys.stderr)
|
||||
return 1
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,337 @@
|
||||
"""Pure measurement logic for lab6 track 1 (no I/O): statistics, counter
|
||||
deltas, cohort classification, run validity and the p25/full selection rules.
|
||||
Everything here is unit-tested in test_measurement.py."""
|
||||
import math
|
||||
|
||||
MODES = {
|
||||
'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER_ARG': '1'},
|
||||
'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1'},
|
||||
'p25': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.25'},
|
||||
'p10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.1'},
|
||||
}
|
||||
SAMPLER_RATIO = {'off': None, 'full': 1.0, 'p25': 0.25, 'p10': 0.1}
|
||||
ORDERS = (('off', 'full', 'p25', 'p10'), ('p25', 'off', 'p10', 'full'),
|
||||
('p10', 'p25', 'full', 'off'))
|
||||
START_LAG_LIMIT_MS = 50.0
|
||||
COMPLETION_LIMIT = 0.99
|
||||
SPAN_RATIO_LIMIT = 0.5
|
||||
LATENCY_RATIO_LIMIT = 1.10
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- statistics
|
||||
|
||||
def nearest_rank(values, q):
|
||||
"""Nearest-rank quantile: sorted[ceil(n*q)-1] (same as tests/load.py)."""
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
return ordered[max(0, math.ceil(len(ordered) * q) - 1)]
|
||||
|
||||
|
||||
def median(values):
|
||||
values = [v for v in values if v is not None]
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
n = len(ordered)
|
||||
mid = n // 2
|
||||
return ordered[mid] if n % 2 else (ordered[mid - 1] + ordered[mid]) / 2
|
||||
|
||||
|
||||
def spread(values):
|
||||
values = [v for v in values if v is not None]
|
||||
if not values:
|
||||
return {'min': None, 'median': None, 'max': None, 'n': 0}
|
||||
return {'min': min(values), 'median': median(values), 'max': max(values),
|
||||
'n': len(values)}
|
||||
|
||||
|
||||
def relative(value, reference):
|
||||
if value is None or reference in (None, 0):
|
||||
return None
|
||||
return value / reference - 1.0
|
||||
|
||||
|
||||
def miss_all_probability(ratio, n):
|
||||
"""(1-r)^N: chance that independent root sampling keeps none of N events.
|
||||
A model assumption, not a measurement."""
|
||||
return (1.0 - ratio) ** n
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- load
|
||||
|
||||
ERROR_OUTCOMES = ('timeout', 'transport_error', 'protocol_error')
|
||||
|
||||
|
||||
def slot_completed(slot):
|
||||
return slot['outcome'] == 'http'
|
||||
|
||||
|
||||
def summarize_load(slots, rate):
|
||||
"""slots: one record per *planned* slot, including client_dropped ones.
|
||||
|
||||
Dropped and late slots stay in the denominator; p95 includes error
|
||||
responses (it is not a latency of successes only)."""
|
||||
scheduled = len(slots)
|
||||
done = [s for s in slots if slot_completed(s)]
|
||||
dropped = sum(1 for s in slots if s['outcome'] == 'client_dropped')
|
||||
failures = {k: sum(1 for s in slots if s['outcome'] == k)
|
||||
for k in ERROR_OUTCOMES}
|
||||
http_5xx = sum(1 for s in done if s['status'] >= 500)
|
||||
http_4xx = sum(1 for s in done if 400 <= s['status'] < 500)
|
||||
successes = [s for s in done if 200 <= s['status'] < 300]
|
||||
started = [s for s in slots if s.get('start_lag_ms') is not None]
|
||||
lag = [s['start_lag_ms'] for s in started]
|
||||
sched_all = [s['scheduled_ms'] for s in done]
|
||||
service_all = [s['service_ms'] for s in done]
|
||||
return {
|
||||
'rate_rps': rate,
|
||||
'scheduled': scheduled,
|
||||
'completed': len(done),
|
||||
'completed_ratio': (len(done) / scheduled) if scheduled else None,
|
||||
'client_dropped': dropped,
|
||||
'transport_failures': failures,
|
||||
'http_5xx': http_5xx,
|
||||
'http_4xx': http_4xx,
|
||||
'http_2xx': len(successes),
|
||||
'start_lag_ms_p95': nearest_rank(lag, 0.95),
|
||||
'start_lag_ms_max': max(lag) if lag else None,
|
||||
'scheduled_ms': {q: nearest_rank(sched_all, v) for q, v in
|
||||
(('p50', .5), ('p95', .95), ('p99', .99))},
|
||||
'service_ms': {q: nearest_rank(service_all, v) for q, v in
|
||||
(('p50', .5), ('p95', .95), ('p99', .99))},
|
||||
'success_scheduled_ms': {q: nearest_rank(
|
||||
[s['scheduled_ms'] for s in successes], v) for q, v in
|
||||
(('p50', .5), ('p95', .95), ('p99', .99))},
|
||||
'success_service_ms': {q: nearest_rank(
|
||||
[s['service_ms'] for s in successes], v) for q, v in
|
||||
(('p50', .5), ('p95', .95), ('p99', .99))},
|
||||
}
|
||||
|
||||
|
||||
def load_errors(load):
|
||||
"""HTTP/protocol errors among delivered healthy traffic."""
|
||||
return (load['http_5xx'] + sum(load['transport_failures'].values()))
|
||||
|
||||
|
||||
def delivered_stream(load):
|
||||
"""Generator delivered the planned stream (protocol validity)."""
|
||||
reasons = []
|
||||
if load['client_dropped']:
|
||||
reasons.append('client_dropped=%d' % load['client_dropped'])
|
||||
lag = load['start_lag_ms_p95']
|
||||
if lag is None or lag > START_LAG_LIMIT_MS:
|
||||
reasons.append('start_lag_p95=%s' % lag)
|
||||
return reasons
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ counters
|
||||
|
||||
def counter_delta(start, end, failure_only=False):
|
||||
"""-> (delta, reason). Never turns a missing positive series into zero and
|
||||
never reports a reset as a negative loss."""
|
||||
if start is None and end is None:
|
||||
if failure_only:
|
||||
return 0.0, 'absent_until_first_error'
|
||||
return None, 'missing_series'
|
||||
if start is None:
|
||||
if failure_only:
|
||||
return end, 'series_created_in_window'
|
||||
return None, 'missing_start'
|
||||
if end is None:
|
||||
return None, 'missing_end'
|
||||
if end < start:
|
||||
return None, 'counter_reset'
|
||||
return end - start, None
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- cohorts
|
||||
|
||||
def classify_record(mode, record):
|
||||
"""Classify one cohort request.
|
||||
|
||||
record: {'stdout': None | {'trace_id','span_id','trace_sampled'} (the keys
|
||||
are absent when the SDK is off or no span context), 'loki': True/False/None
|
||||
(None: backend observation incomplete), 'span': True/False/None}.
|
||||
Returns (trace_class, log_class)."""
|
||||
stdout = record.get('stdout')
|
||||
if stdout is None:
|
||||
return 'unmatched_stdout', 'incomplete_observation'
|
||||
if mode == 'off':
|
||||
if any(k in stdout for k in ('trace_id', 'span_id', 'trace_sampled')):
|
||||
return 'ids_present_with_sdk_off', 'not_applicable'
|
||||
return 'not_applicable', 'not_applicable'
|
||||
sampled = stdout.get('trace_sampled')
|
||||
if not stdout.get('trace_id') or not stdout.get('span_id') or \
|
||||
not isinstance(sampled, bool):
|
||||
return 'unmatched_stdout', 'incomplete_observation'
|
||||
if record.get('loki') is None:
|
||||
log_class = 'incomplete_observation'
|
||||
elif record['loki']:
|
||||
log_class = 'exported'
|
||||
else:
|
||||
log_class = 'missing_exported_logs'
|
||||
if sampled is False:
|
||||
return 'intentional_unsampled', log_class
|
||||
if record.get('span') is None:
|
||||
return 'incomplete_observation', log_class
|
||||
return ('sampled_present' if record['span'] else 'sampled_missing'), log_class
|
||||
|
||||
|
||||
def summarize_cohort(mode, records):
|
||||
counts = {'n': len(records)}
|
||||
for record in records:
|
||||
trace_class, log_class = classify_record(mode, record)
|
||||
counts[trace_class] = counts.get(trace_class, 0) + 1
|
||||
counts['log_' + log_class] = counts.get('log_' + log_class, 0) + 1
|
||||
counts['retained_traces'] = counts.get('sampled_present', 0)
|
||||
return counts
|
||||
|
||||
|
||||
def export_loss(items_failed_delta, send_failed, enqueue_failed):
|
||||
"""Telemetry export losses; a false trace_sampled flag is NOT a loss."""
|
||||
parts = {'app_export_failed_items': items_failed_delta,
|
||||
'collector_send_failed': send_failed,
|
||||
'collector_enqueue_failed': enqueue_failed}
|
||||
known = [v for v in parts.values() if isinstance(v, (int, float))]
|
||||
return {'parts': parts, 'total_known': sum(known),
|
||||
'loss': sum(known) > 0}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ validity
|
||||
|
||||
def evaluate_run(summary):
|
||||
"""-> (valid, invalid_reasons, configuration_failures).
|
||||
|
||||
Invalid = the protocol was not executed as specified; configuration
|
||||
failures are valid adverse outcomes and stay in the matrix."""
|
||||
invalid = list(summary.get('measurement_defects', []))
|
||||
failures = []
|
||||
load = summary['load']
|
||||
invalid += delivered_stream(load)
|
||||
if load['completed_ratio'] is None or load['completed_ratio'] < COMPLETION_LIMIT:
|
||||
invalid.append('completed_ratio=%s' % load['completed_ratio'])
|
||||
if summary.get('effective_mode_ok') is False:
|
||||
invalid.append('effective_env_mismatch')
|
||||
if summary.get('image_ids_ok') is False:
|
||||
invalid.append('image_changed')
|
||||
if summary.get('foreign_faults'):
|
||||
invalid.append('foreign_faults')
|
||||
for name, cohort in (summary.get('cohorts') or {}).items():
|
||||
if cohort.get('invalid'):
|
||||
invalid.append('cohort_invalid:%s' % name)
|
||||
# Counter resets are only explained by an observed restart/OOM.
|
||||
for name, item in (summary.get('ingest') or {}).items():
|
||||
if isinstance(item, dict) and item.get('reason') == 'counter_reset' \
|
||||
and not summary.get('restarts'):
|
||||
invalid.append('unexplained_counter_reset:%s' % name)
|
||||
# --- adverse outcomes of the configuration (valid)
|
||||
if load_errors(load):
|
||||
failures.append('healthy_errors=%d' % load_errors(load))
|
||||
if summary.get('not_ready_at_deadline'):
|
||||
failures.append('not_ready_at_deadline=%d' % summary['not_ready_at_deadline'])
|
||||
if summary.get('integrity_errors'):
|
||||
failures.append('healthy_errors_integrity=%d' % summary['integrity_errors'])
|
||||
if summary.get('restarts'):
|
||||
failures.append('restart_or_oom')
|
||||
loss = summary.get('export_failures') or {}
|
||||
if loss.get('loss'):
|
||||
failures.append('export_loss')
|
||||
for name, cohort in (summary.get('cohorts') or {}).items():
|
||||
if cohort.get('sampled_missing'):
|
||||
failures.append('sampled_missing:%s=%d' % (name, cohort['sampled_missing']))
|
||||
if cohort.get('log_missing_exported_logs'):
|
||||
failures.append('missing_exported_logs:%s=%d' % (
|
||||
name, cohort['log_missing_exported_logs']))
|
||||
return (not invalid), invalid, failures
|
||||
|
||||
|
||||
def matrix_entry(summary):
|
||||
"""Every protocol-run stays in the matrix, valid-adverse included."""
|
||||
return {
|
||||
'mode': summary['mode'], 'repeat': summary.get('repeat'),
|
||||
'attempt': summary.get('attempt', 1), 'valid': summary['valid'],
|
||||
'invalid_reasons': summary['invalid_reasons'],
|
||||
'configuration_failures': summary['configuration_failures'],
|
||||
'spans_per_operation': summary.get('spans_per_operation'),
|
||||
'scheduled_p95_ms': (summary['load']['scheduled_ms'] or {}).get('p95'),
|
||||
'dir': summary.get('dir'),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- selection
|
||||
|
||||
def _clean(run):
|
||||
return (run['valid'] and not any(
|
||||
f.startswith(('healthy_errors', 'not_ready', 'restart', 'export_loss'))
|
||||
for f in run['configuration_failures']))
|
||||
|
||||
|
||||
def _retained(run, name):
|
||||
return (run['cohorts'].get(name) or {}).get('retained_traces', 0)
|
||||
|
||||
|
||||
def _no_missing(run):
|
||||
for cohort in run['cohorts'].values():
|
||||
if cohort.get('sampled_missing') or cohort.get('log_missing_exported_logs') \
|
||||
or cohort.get('log_incomplete_observation') \
|
||||
or cohort.get('unmatched_stdout') or cohort.get('incomplete_observation'):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def full_reference(full_runs):
|
||||
"""Frozen reference: medians of the three full runs; None if any datum is
|
||||
unmeasurable (then p25 cannot be declared a winner)."""
|
||||
spans = [r.get('spans_per_operation') for r in full_runs]
|
||||
p95 = [r['load']['scheduled_ms']['p95'] for r in full_runs]
|
||||
ok = (len(full_runs) == 3 and all(r['valid'] for r in full_runs) and
|
||||
all(v is not None for v in spans + p95))
|
||||
return {'complete': ok, 'spans_per_operation_median': median(spans),
|
||||
'scheduled_p95_ms_median': median(p95),
|
||||
'run_dirs': [r.get('dir') for r in full_runs]}
|
||||
|
||||
|
||||
def select_p25(full_runs, p25_runs):
|
||||
ref = full_reference(full_runs)
|
||||
checks = {'full_reference_complete': ref['complete']}
|
||||
if not ref['complete'] or len(p25_runs) != 3:
|
||||
checks['p25_runs'] = len(p25_runs) == 3
|
||||
return {'passed': False, 'checks': checks, 'reference': ref}
|
||||
checks['all_valid_clean'] = all(_clean(r) for r in p25_runs)
|
||||
checks['retained_error_and_slow_each_run'] = all(
|
||||
_retained(r, 'error') >= 1 and _retained(r, 'slow') >= 1 for r in p25_runs)
|
||||
checks['no_missing_spans_or_logs'] = all(_no_missing(r) for r in p25_runs)
|
||||
spans = median([r.get('spans_per_operation') for r in p25_runs])
|
||||
checks['spans_median_le_50pct_full'] = (
|
||||
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
|
||||
p95 = median([r['load']['scheduled_ms']['p95'] for r in p25_runs])
|
||||
checks['scheduled_p95_le_110pct_full'] = (
|
||||
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
|
||||
return {'passed': all(checks.values()), 'checks': checks, 'reference': ref,
|
||||
'p25_spans_per_operation_median': spans,
|
||||
'p25_scheduled_p95_ms_median': p95}
|
||||
|
||||
|
||||
def validate_single(run, mode, ref):
|
||||
"""One fresh run against the FROZEN full reference (never a 3-run median
|
||||
of the validation itself)."""
|
||||
checks = {'protocol_valid': run['valid'],
|
||||
'zero_errors_notready_loss': _clean(run)}
|
||||
if mode == 'p25':
|
||||
checks['complete_stdout_and_logs'] = _no_missing(run) and all(
|
||||
(run['cohorts'].get(n) or {}).get('n') == 40 for n in ('error', 'slow'))
|
||||
checks['retained_each_class'] = (
|
||||
_retained(run, 'error') >= 1 and _retained(run, 'slow') >= 1)
|
||||
spans = run.get('spans_per_operation')
|
||||
checks['spans_le_50pct_frozen_full'] = (
|
||||
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
|
||||
p95 = run['load']['scheduled_ms']['p95']
|
||||
checks['scheduled_p95_le_110pct_frozen_full'] = (
|
||||
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
|
||||
else:
|
||||
checks['complete_stdout_and_logs'] = _no_missing(run)
|
||||
checks['all_80_server_spans'] = all(
|
||||
(run['cohorts'].get(n) or {}).get('sampled_present') == 40
|
||||
for n in ('error', 'slow'))
|
||||
return {'passed': all(checks.values()), 'checks': checks, 'mode': mode}
|
||||
@@ -0,0 +1,675 @@
|
||||
"""Shared lab6 helpers: Prometheus/Loki/Tempo/Grafana client, controlled app
|
||||
environment, SQL table lock, batch drain. Python 3.9+, standard library only."""
|
||||
import base64
|
||||
import binascii
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from scripts.lab5 import stand # noqa: E402
|
||||
|
||||
Failure = stand.Failure
|
||||
|
||||
HTTP_TIMEOUT = 10
|
||||
SCRAPE_JOBS = ('prometheus', 'otel-collector', 'haproxy', 'rabbitmq', 'loki',
|
||||
'tempo')
|
||||
RULE_ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
|
||||
DASHBOARD_UIDS = ('messenger-overview', 'messenger-telemetry')
|
||||
DATASOURCE_UIDS = ('prometheus', 'loki', 'tempo')
|
||||
LOCKABLE_TABLES = ('messages', 'attachments')
|
||||
|
||||
# Keys the controller may change/restore (identical for api and worker).
|
||||
MUTABLE_KEYS = (
|
||||
'OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG', 'OTEL_METRIC_EXPORT_INTERVAL',
|
||||
'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL',
|
||||
'OTEL_EXPORTER_OTLP_TIMEOUT', 'OTEL_RESOURCE_ATTRIBUTES',
|
||||
'OTEL_BSP_MAX_QUEUE_SIZE', 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE',
|
||||
'OTEL_BSP_SCHEDULE_DELAY', 'OTEL_BLRP_MAX_QUEUE_SIZE',
|
||||
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', 'OTEL_BLRP_SCHEDULE_DELAY',
|
||||
'LOG_LEVEL', 'CACHE_MODE', 'FAULT_INJECTION_ENABLED',
|
||||
)
|
||||
# Recorded as diagnostics, never settable.
|
||||
DIAGNOSTIC_KEYS = ('OTEL_SERVICE_NAME', 'SERVICE_VERSION')
|
||||
|
||||
# Baseline of the SDK knobs (compose defaults) used by track1 and incidents.
|
||||
SDK_DEFAULTS = {
|
||||
'OTEL_SDK_DISABLED': 'false',
|
||||
'OTEL_TRACES_SAMPLER_ARG': '1',
|
||||
'OTEL_METRIC_EXPORT_INTERVAL': '10000',
|
||||
'OTEL_EXPORTER_OTLP_TIMEOUT': '5000',
|
||||
'OTEL_BSP_MAX_QUEUE_SIZE': '2048',
|
||||
'OTEL_BSP_MAX_EXPORT_BATCH_SIZE': '512',
|
||||
'OTEL_BSP_SCHEDULE_DELAY': '1000',
|
||||
'OTEL_BLRP_MAX_QUEUE_SIZE': '2048',
|
||||
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE': '512',
|
||||
'OTEL_BLRP_SCHEDULE_DELAY': '1000',
|
||||
'LOG_LEVEL': 'INFO',
|
||||
}
|
||||
|
||||
|
||||
def utc_now():
|
||||
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}),
|
||||
_NoRedirect())
|
||||
|
||||
|
||||
def http_get(url, headers=None, timeout=HTTP_TIMEOUT):
|
||||
"""GET returning (status, bytes); never prints the URL query/headers."""
|
||||
request = urllib.request.Request(url, headers=headers or {})
|
||||
try:
|
||||
with _OPENER.open(request, timeout=timeout) as response:
|
||||
return response.status, response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
with error:
|
||||
return error.code, error.read()
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise Failure('GET ' + url.split('?')[0] + ': connection failure (' +
|
||||
type(exc).__name__ + ')') from None
|
||||
|
||||
|
||||
def http_json(url, headers=None, timeout=HTTP_TIMEOUT, allow=(200,)):
|
||||
status, data = http_get(url, headers, timeout)
|
||||
if status not in allow:
|
||||
raise Failure('GET ' + url.split('?')[0] + ': status ' + str(status))
|
||||
try:
|
||||
return status, json.loads(data) if data else None
|
||||
except (ValueError, UnicodeError):
|
||||
raise Failure('GET ' + url.split('?')[0] + ': invalid JSON') from None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- trace JSON
|
||||
|
||||
def normalize_id(value, nbytes):
|
||||
"""Hex or base64 ID -> lowercase hex of exactly nbytes; else Failure."""
|
||||
if not isinstance(value, str):
|
||||
raise Failure('trace id is not a string')
|
||||
if value == '':
|
||||
return ''
|
||||
if len(value) == nbytes * 2:
|
||||
try:
|
||||
int(value, 16)
|
||||
return value.lower()
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
raw = base64.b64decode(value, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
raise Failure('unrecognised trace/span id encoding') from None
|
||||
if len(raw) != nbytes:
|
||||
raise Failure('trace/span id has wrong length')
|
||||
return raw.hex()
|
||||
|
||||
|
||||
def _any_value(node):
|
||||
if not isinstance(node, dict):
|
||||
return node
|
||||
for key in ('stringValue', 'boolValue', 'doubleValue'):
|
||||
if key in node:
|
||||
return node[key]
|
||||
if 'intValue' in node:
|
||||
return int(node['intValue'])
|
||||
if 'arrayValue' in node:
|
||||
return [_any_value(v) for v in node['arrayValue'].get('values', [])]
|
||||
return None
|
||||
|
||||
|
||||
def _attributes(items):
|
||||
out = {}
|
||||
for item in items or []:
|
||||
out[item['key']] = _any_value(item.get('value'))
|
||||
return out
|
||||
|
||||
|
||||
_KIND = {0: 'UNSPECIFIED', 1: 'INTERNAL', 2: 'SERVER', 3: 'CLIENT',
|
||||
4: 'PRODUCER', 5: 'CONSUMER'}
|
||||
_STATUS = {0: 'UNSET', 1: 'OK', 2: 'ERROR'}
|
||||
|
||||
|
||||
def normalize_trace(document):
|
||||
"""Tempo JSON (batches or resourceSpans) -> list of span dicts."""
|
||||
if not isinstance(document, dict):
|
||||
raise Failure('unknown Tempo response format')
|
||||
batches = document.get('batches', document.get('resourceSpans'))
|
||||
if batches is None:
|
||||
raise Failure('unknown Tempo response format (no batches/resourceSpans)')
|
||||
spans = []
|
||||
for batch in batches:
|
||||
resource = _attributes(batch.get('resource', {}).get('attributes'))
|
||||
scopes = batch.get('scopeSpans', batch.get('instrumentationLibrarySpans'))
|
||||
if scopes is None:
|
||||
raise Failure('unknown Tempo batch format (no scopeSpans)')
|
||||
for scope in scopes:
|
||||
for span in scope.get('spans', []):
|
||||
kind = span.get('kind', 0)
|
||||
if isinstance(kind, str):
|
||||
kind = kind.replace('SPAN_KIND_', '')
|
||||
else:
|
||||
kind = _KIND.get(kind, str(kind))
|
||||
status = span.get('status', {}).get('code', 0)
|
||||
if isinstance(status, str):
|
||||
status = status.replace('STATUS_CODE_', '')
|
||||
else:
|
||||
status = _STATUS.get(status, str(status))
|
||||
spans.append({
|
||||
'trace_id': normalize_id(span.get('traceId', ''), 16),
|
||||
'span_id': normalize_id(span.get('spanId', ''), 8),
|
||||
'parent_span_id': normalize_id(
|
||||
span.get('parentSpanId', ''), 8),
|
||||
'name': span.get('name', ''),
|
||||
'kind': kind,
|
||||
'start_ns': int(span.get('startTimeUnixNano', 0)),
|
||||
'end_ns': int(span.get('endTimeUnixNano', 0)),
|
||||
'attributes': _attributes(span.get('attributes')),
|
||||
'status': status,
|
||||
'links': [{
|
||||
'trace_id': normalize_id(link.get('traceId', ''), 16),
|
||||
'span_id': normalize_id(link.get('spanId', ''), 8),
|
||||
} for link in span.get('links', [])],
|
||||
'service': resource.get('service.name'),
|
||||
'instance': resource.get('service.instance.id'),
|
||||
})
|
||||
return spans
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- client
|
||||
|
||||
class ObsClient:
|
||||
def __init__(self, prometheus_url, grafana_url, password_file=None,
|
||||
base_url=None, ca_file=None):
|
||||
self.prometheus = prometheus_url.rstrip('/')
|
||||
self.grafana = grafana_url.rstrip('/')
|
||||
self.base_url = base_url
|
||||
self.ca_file = ca_file
|
||||
path = Path(password_file) if password_file else (
|
||||
ROOT / 'secrets/grafana_admin_password')
|
||||
self._auth = None
|
||||
if path.exists():
|
||||
token = base64.b64encode(
|
||||
b'admin:' + path.read_bytes().strip()).decode()
|
||||
self._auth = {'Authorization': 'Basic ' + token}
|
||||
|
||||
# -- Prometheus
|
||||
def prom_query(self, expr, at=None):
|
||||
params = {'query': expr}
|
||||
if at is not None:
|
||||
params['time'] = str(at)
|
||||
_, body = http_json(self.prometheus + '/api/v1/query?' +
|
||||
urllib.parse.urlencode(params))
|
||||
if not isinstance(body, dict) or body.get('status') != 'success':
|
||||
raise Failure('Prometheus query failed')
|
||||
return body['data']['result']
|
||||
|
||||
def prom_scalar(self, expr, default=None):
|
||||
result = self.prom_query(expr)
|
||||
if not result:
|
||||
return default
|
||||
return float(result[0]['value'][1])
|
||||
|
||||
def prom_range(self, expr, start, end, step):
|
||||
params = {'query': expr, 'start': str(start), 'end': str(end),
|
||||
'step': str(step)}
|
||||
_, body = http_json(self.prometheus + '/api/v1/query_range?' +
|
||||
urllib.parse.urlencode(params))
|
||||
if not isinstance(body, dict) or body.get('status') != 'success':
|
||||
raise Failure('Prometheus range query failed')
|
||||
return body['data']['result']
|
||||
|
||||
def prom_names(self):
|
||||
_, body = http_json(self.prometheus + '/api/v1/label/__name__/values')
|
||||
return sorted(body['data'])
|
||||
|
||||
# -- Grafana proxied backends
|
||||
def _grafana_get(self, path, allow=(200,)):
|
||||
if self._auth is None:
|
||||
raise Failure('Grafana password file is missing')
|
||||
return http_get(self.grafana + path, self._auth)
|
||||
|
||||
def grafana_json(self, path, allow=(200,)):
|
||||
status, data = self._grafana_get(path)
|
||||
if status not in allow:
|
||||
raise Failure('Grafana ' + path.split('?')[0] + ': status ' +
|
||||
str(status))
|
||||
try:
|
||||
return json.loads(data)
|
||||
except (ValueError, UnicodeError):
|
||||
raise Failure('Grafana ' + path.split('?')[0] +
|
||||
': invalid JSON') from None
|
||||
|
||||
def loki_rows(self, query, start_ns, end_ns, deadline_s=60):
|
||||
"""All log rows in [start, end]: [{'ts', 'line', 'labels'}]."""
|
||||
limit = 5000
|
||||
deadline = time.monotonic() + deadline_s
|
||||
cursor = int(start_ns)
|
||||
seen = set()
|
||||
rows = []
|
||||
while True:
|
||||
if time.monotonic() > deadline:
|
||||
raise Failure('Loki query deadline exceeded; incomplete window')
|
||||
params = urllib.parse.urlencode({
|
||||
'query': query, 'start': str(cursor), 'end': str(int(end_ns)),
|
||||
'limit': str(limit), 'direction': 'forward'})
|
||||
body = self.grafana_json(
|
||||
'/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' +
|
||||
params)
|
||||
if body.get('status') != 'success':
|
||||
raise Failure('Loki query failed')
|
||||
fresh = 0
|
||||
page = 0
|
||||
last_ts = cursor
|
||||
for stream in body['data']['result']:
|
||||
labels = stream.get('stream', {})
|
||||
key = json.dumps(labels, sort_keys=True)
|
||||
for value in stream.get('values', []):
|
||||
page += 1
|
||||
ts, line = int(value[0]), value[1]
|
||||
last_ts = max(last_ts, ts)
|
||||
ident = (key, ts, line)
|
||||
if ident in seen:
|
||||
continue
|
||||
seen.add(ident)
|
||||
fresh += 1
|
||||
rows.append({'ts': ts, 'line': line, 'labels': labels})
|
||||
if page < limit:
|
||||
return rows
|
||||
if fresh == 0 or last_ts == cursor:
|
||||
raise Failure('Loki pagination made no progress; '
|
||||
'incomplete window')
|
||||
cursor = last_ts # overlap on the last timestamp, deduped above
|
||||
|
||||
def get_trace(self, trace_id, wait=0):
|
||||
"""Normalised spans, or None on 404; waits up to `wait` seconds."""
|
||||
deadline = time.monotonic() + min(wait, 60)
|
||||
path = '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id
|
||||
while True:
|
||||
if self._auth is None:
|
||||
raise Failure('Grafana password file is missing')
|
||||
status, data = http_get(
|
||||
self.grafana + path,
|
||||
{**self._auth, 'Accept': 'application/json'})
|
||||
if status == 200:
|
||||
try:
|
||||
return normalize_trace(json.loads(data))
|
||||
except (ValueError, UnicodeError):
|
||||
raise Failure('Tempo returned invalid JSON') from None
|
||||
if status != 404:
|
||||
raise Failure('Tempo trace lookup: status ' + str(status))
|
||||
if time.monotonic() >= deadline:
|
||||
return None
|
||||
time.sleep(1)
|
||||
|
||||
# -- closed backends from inside the compose network
|
||||
def internal_get(self, service, port, path):
|
||||
script = (
|
||||
"const [h,p,u]=process.argv.slice(1);"
|
||||
"fetch('http://'+h+':'+p+u,{signal:AbortSignal.timeout(5000)})"
|
||||
".then(async r=>{const b=await r.text();"
|
||||
"console.log(JSON.stringify({status:r.status,body:b.slice(0,1048576)}))})"
|
||||
".catch(e=>{console.log(JSON.stringify({status:0,body:String(e.name)}))})")
|
||||
result = stand.compose('exec', '-T', 'api', 'node', '-e', script,
|
||||
service, str(port), path, check_result=False)
|
||||
try:
|
||||
out = json.loads(result.stdout.strip().splitlines()[-1])
|
||||
except (ValueError, IndexError):
|
||||
raise Failure('internal_get ' + service + ': no result') from None
|
||||
return out['status'], out['body']
|
||||
|
||||
# -- readiness
|
||||
def _check(self, problems, name, ok):
|
||||
if not ok:
|
||||
problems.append(name)
|
||||
|
||||
def readiness_problems(self, app_telemetry=True, provisioning=True):
|
||||
problems = []
|
||||
if self.base_url:
|
||||
try:
|
||||
status, _ = http_get(self.base_url.rstrip('/') + '/health/ready')
|
||||
self._check(problems, 'api-ready', status == 200)
|
||||
except Failure:
|
||||
problems.append('api-ready')
|
||||
probes = (('otel-collector', 13133, '/', 'collector-health'),
|
||||
('loki', 3100, '/ready', 'loki-ready'),
|
||||
('tempo', 3200, '/ready', 'tempo-ready'))
|
||||
for service, port, path, name in probes:
|
||||
try:
|
||||
status, _ = self.internal_get(service, port, path)
|
||||
self._check(problems, name, status == 200)
|
||||
except Failure:
|
||||
problems.append(name)
|
||||
try:
|
||||
status, _ = http_get(self.prometheus + '/-/ready')
|
||||
self._check(problems, 'prometheus-ready', status == 200)
|
||||
except Failure:
|
||||
problems.append('prometheus-ready')
|
||||
try:
|
||||
status, _ = http_get(self.grafana + '/api/health')
|
||||
self._check(problems, 'grafana-health', status == 200)
|
||||
except Failure:
|
||||
problems.append('grafana-health')
|
||||
try:
|
||||
up = {r['metric']['job']: r['value'][1]
|
||||
for r in self.prom_query('up')}
|
||||
for job in SCRAPE_JOBS:
|
||||
self._check(problems, 'target-up:' + job, up.get(job) == '1')
|
||||
if app_telemetry:
|
||||
ages = self.prom_query(
|
||||
'time() - timestamp(target_info{job=~"messenger-.*"})')
|
||||
fresh = {}
|
||||
for r in ages:
|
||||
if float(r['value'][1]) < 30:
|
||||
fresh.setdefault(r['metric']['job'], set()).add(
|
||||
r['metric'].get('instance'))
|
||||
self._check(problems, 'api-instances>=2',
|
||||
len(fresh.get('messenger-api', ())) >= 2)
|
||||
self._check(problems, 'worker-instance',
|
||||
len(fresh.get('messenger-worker', ())) >= 1)
|
||||
except Failure:
|
||||
problems.append('prometheus-queries')
|
||||
if provisioning:
|
||||
try:
|
||||
for uid in DATASOURCE_UIDS:
|
||||
if uid == 'tempo':
|
||||
# The Tempo plugin has no health resource; its API echo
|
||||
# through the datasource proxy proves the connection.
|
||||
status, _ = self._grafana_get(
|
||||
'/api/datasources/proxy/uid/tempo/api/echo')
|
||||
self._check(problems, 'datasource:tempo', status == 200)
|
||||
continue
|
||||
body = self.grafana_json(
|
||||
'/api/datasources/uid/' + uid + '/health',
|
||||
allow=(200, 400, 404, 500))
|
||||
self._check(problems, 'datasource:' + uid,
|
||||
isinstance(body, dict) and
|
||||
body.get('status', '').upper() == 'OK')
|
||||
_, rules = http_json(self.prometheus + '/api/v1/rules')
|
||||
names = {r['name'] for g in rules['data']['groups']
|
||||
for r in g['rules']}
|
||||
for alert in RULE_ALERTS:
|
||||
self._check(problems, 'rule:' + alert, alert in names)
|
||||
for uid in DASHBOARD_UIDS:
|
||||
status, _ = self._grafana_get('/api/dashboards/uid/' + uid)
|
||||
self._check(problems, 'dashboard:' + uid, status == 200)
|
||||
except Failure:
|
||||
problems.append('provisioning-queries')
|
||||
return problems
|
||||
|
||||
def wait_ready(self, timeout=120, app_telemetry=True, provisioning=True):
|
||||
deadline = time.monotonic() + timeout
|
||||
problems = ['not-checked']
|
||||
while time.monotonic() < deadline:
|
||||
problems = self.readiness_problems(app_telemetry, provisioning)
|
||||
if not problems:
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure('stack not ready: ' + ', '.join(problems))
|
||||
|
||||
def wait_telemetry_flushed(self, timeout=120):
|
||||
"""Two new Prometheus scrapes of the Collector and empty queues."""
|
||||
deadline = time.monotonic() + timeout
|
||||
stamps = []
|
||||
while time.monotonic() < deadline:
|
||||
result = self.prom_query('timestamp(up{job="otel-collector"})')
|
||||
if result:
|
||||
stamp = float(result[0]['value'][1])
|
||||
if not stamps or stamp > stamps[-1]:
|
||||
stamps.append(stamp)
|
||||
queue = self.prom_query('sum(otelcol_exporter_queue_size)')
|
||||
empty = bool(queue) and float(queue[0]['value'][1]) == 0
|
||||
if len(stamps) >= 3 and empty:
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure('telemetry did not flush before deadline')
|
||||
|
||||
# -- controlled application environment
|
||||
def container_ids(self, service):
|
||||
out = stand.compose('ps', '-q', service).stdout.split()
|
||||
if not out:
|
||||
raise Failure('no running container for ' + service)
|
||||
return out
|
||||
|
||||
def inspect_env(self, service):
|
||||
"""{container_id: {'env': {...}, 'image': id, 'restarts': n}}"""
|
||||
result = {}
|
||||
for cid in self.container_ids(service):
|
||||
data = json.loads(stand.docker('inspect', cid).stdout)[0]
|
||||
env = dict(item.split('=', 1) for item in data['Config']['Env']
|
||||
if '=' in item)
|
||||
result[cid[:12]] = {
|
||||
'env': {k: env.get(k) for k in MUTABLE_KEYS + DIAGNOSTIC_KEYS},
|
||||
'image': data['Image'],
|
||||
'restarts': data['RestartCount'],
|
||||
'oom_killed': data['State'].get('OOMKilled', False),
|
||||
}
|
||||
return result
|
||||
|
||||
def app_state(self):
|
||||
return {'api': self.inspect_env('api'),
|
||||
'worker': self.inspect_env('worker')}
|
||||
|
||||
@contextlib.contextmanager
|
||||
def app_environment(self, initial_overrides=None):
|
||||
saved_state = self.app_state()
|
||||
baseline = {}
|
||||
for service in ('api', 'worker'):
|
||||
for info in saved_state[service].values():
|
||||
for key in MUTABLE_KEYS:
|
||||
baseline.setdefault((service, key), info['env'][key])
|
||||
saved_environ = {k: os.environ.get(k) for k in MUTABLE_KEYS}
|
||||
touched = set()
|
||||
dirty = [False]
|
||||
|
||||
def effective(service):
|
||||
return {cid: info for cid, info in self.inspect_env(service).items()}
|
||||
|
||||
def set_overrides(values, force=False):
|
||||
unknown = set(values) - set(MUTABLE_KEYS)
|
||||
if unknown:
|
||||
raise Failure('not a controllable key: ' + ', '.join(sorted(unknown)))
|
||||
for key in values:
|
||||
api = {i['env'][key] for i in effective('api').values()}
|
||||
worker = {i['env'][key] for i in effective('worker').values()}
|
||||
if len(api | worker) > 1 and api != worker:
|
||||
raise Failure('api/worker disagree on ' + key + ' before change')
|
||||
if not force and all(
|
||||
info['env'][key] == str(value)
|
||||
for service in ('api', 'worker')
|
||||
for info in effective(service).values()
|
||||
for key, value in values.items()):
|
||||
return self.app_state()
|
||||
for key, value in values.items():
|
||||
os.environ[key] = str(value)
|
||||
touched.add(key)
|
||||
dirty[0] = True
|
||||
stand.compose('up', '-d', '--no-deps', '--force-recreate',
|
||||
'--wait', 'api', 'worker')
|
||||
for service in ('api', 'worker'):
|
||||
state = effective(service)
|
||||
for cid, info in state.items():
|
||||
for key, value in values.items():
|
||||
if info['env'][key] != str(value):
|
||||
raise Failure('effective ' + key + ' differs in ' +
|
||||
service + ' ' + cid)
|
||||
sdk_on = os.environ.get('OTEL_SDK_DISABLED', 'false') != 'true'
|
||||
self.wait_ready(app_telemetry=sdk_on, provisioning=True)
|
||||
return self.app_state()
|
||||
|
||||
try:
|
||||
if initial_overrides:
|
||||
set_overrides(initial_overrides)
|
||||
yield set_overrides
|
||||
finally:
|
||||
if dirty[0]:
|
||||
# Restore the original effective values as controller env so the
|
||||
# recreated containers match what was running before.
|
||||
for key in MUTABLE_KEYS:
|
||||
original = baseline.get(('api', key))
|
||||
if original is not None and key in touched:
|
||||
os.environ[key] = original
|
||||
stand.compose('up', '-d', '--no-deps', '--force-recreate',
|
||||
'--wait', 'api', 'worker')
|
||||
for key in touched:
|
||||
if saved_environ[key] is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = saved_environ[key]
|
||||
now = self.app_state()
|
||||
for service in ('api', 'worker'):
|
||||
for cid, info in now[service].items():
|
||||
for key in touched:
|
||||
if info['env'][key] != baseline.get((service, key)):
|
||||
print('WARNING: ' + key + ' not restored on ' +
|
||||
service, file=sys.stderr)
|
||||
with contextlib.suppress(Failure):
|
||||
self.wait_ready(app_telemetry=True, provisioning=True,
|
||||
timeout=120)
|
||||
|
||||
# -- foreign faults
|
||||
def foreign_faults(self):
|
||||
result = stand.compose('exec', '-T', 'redis', 'redis-cli', '--scan',
|
||||
'--pattern', 'fault:*')
|
||||
return sorted(line for line in result.stdout.split() if line)
|
||||
|
||||
def require_no_foreign_faults(self):
|
||||
keys = self.foreign_faults()
|
||||
if keys:
|
||||
raise Failure('foreign fault keys present (finish the other '
|
||||
'experiment first): ' + ', '.join(keys))
|
||||
|
||||
|
||||
# --------------------------------------------------------------- PostgreSQL
|
||||
|
||||
def psql(sql, timeout=30):
|
||||
script = 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 ' \
|
||||
'-At -F , -c "$SQL"'
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
|
||||
'-e', 'SQL=' + sql, 'postgres', 'sh', '-c', script],
|
||||
cwd=ROOT, text=True, capture_output=True, timeout=timeout)
|
||||
if result.returncode != 0:
|
||||
raise Failure('psql failed: ' + ((result.stderr or '').strip()
|
||||
.splitlines() or ['?'])[-1][:200])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hold_table_lock(table):
|
||||
"""ACCESS EXCLUSIVE lock on messages/attachments held until exit."""
|
||||
if table not in LOCKABLE_TABLES:
|
||||
raise Failure('table is not lockable: ' + table)
|
||||
name = 'lab6-lock-' + uuid.uuid4().hex[:12]
|
||||
process = subprocess.Popen(
|
||||
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
|
||||
'postgres', 'sh', '-c',
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -At'],
|
||||
cwd=ROOT, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, bufsize=1)
|
||||
pid = None
|
||||
try:
|
||||
process.stdin.write(
|
||||
"SET application_name='" + name + "';\n"
|
||||
"BEGIN;\n"
|
||||
"SET LOCAL statement_timeout='120s';\n"
|
||||
"SET LOCAL idle_in_transaction_session_timeout='120s';\n"
|
||||
"LOCK TABLE " + table + " IN ACCESS EXCLUSIVE MODE;\n"
|
||||
"SELECT 'LOCKED:'||pg_backend_pid();\n")
|
||||
process.stdin.flush()
|
||||
deadline = time.monotonic() + 30
|
||||
while pid is None:
|
||||
if time.monotonic() > deadline:
|
||||
raise Failure('table lock marker not received')
|
||||
ready, _, _ = select.select([process.stdout], [], [], 1)
|
||||
if not ready:
|
||||
if process.poll() is not None:
|
||||
raise Failure('psql exited before locking')
|
||||
continue
|
||||
line = process.stdout.readline()
|
||||
if line.startswith('LOCKED:'):
|
||||
pid = int(line.split(':')[1])
|
||||
elif line == '' and process.poll() is not None:
|
||||
raise Failure('psql exited before locking')
|
||||
yield pid
|
||||
finally:
|
||||
try:
|
||||
if process.poll() is None:
|
||||
process.stdin.write('ROLLBACK;\n\\q\n')
|
||||
process.stdin.flush()
|
||||
process.stdin.close()
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
if pid is not None:
|
||||
with contextlib.suppress(Failure, subprocess.TimeoutExpired):
|
||||
psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE pid=" + str(int(pid)) +
|
||||
" AND application_name='" + name + "'")
|
||||
for stream in (process.stdout, process.stderr):
|
||||
with contextlib.suppress(Exception):
|
||||
stream.close()
|
||||
|
||||
|
||||
def wait_drained(attachment_ids, obs=None, timeout=120, flush_telemetry=True):
|
||||
"""Batch-check jobs/outbox/attachments in PostgreSQL (no HTTP polling).
|
||||
|
||||
Returns {'drained': bool, 'states': {id: status}, 'queued': n,
|
||||
'outbox': n, 'seconds': t, 'timestamps': ...}. A vanished known ID is a
|
||||
Failure; non-ready at the deadline is reported, not raised.
|
||||
"""
|
||||
ids = list(attachment_ids)
|
||||
for item in ids:
|
||||
uuid.UUID(item)
|
||||
array = "ARRAY[" + ",".join("'" + i + "'" for i in ids) + "]::uuid[]" \
|
||||
if ids else "ARRAY[]::uuid[]"
|
||||
sql = (
|
||||
"SELECT (SELECT count(*) FROM attachment_jobs WHERE status IN "
|
||||
"('queued','processing'))||','||"
|
||||
"(SELECT count(*) FROM job_outbox WHERE published_at IS NULL)||','||"
|
||||
"COALESCE((SELECT string_agg(id::text||':'||status,';') FROM attachments "
|
||||
"WHERE id = ANY(" + array + ")),'')")
|
||||
started = time.monotonic()
|
||||
start_ts = utc_now()
|
||||
states = {}
|
||||
queued = outbox = -1
|
||||
drained = False
|
||||
while time.monotonic() - started < timeout:
|
||||
out = psql(sql)
|
||||
queued_s, outbox_s, rest = (out.split(',', 2) + ['', ''])[:3]
|
||||
queued, outbox = int(queued_s), int(outbox_s)
|
||||
states = dict(part.split(':', 1) for part in rest.split(';') if part)
|
||||
missing = set(ids) - set(states)
|
||||
if missing:
|
||||
raise Failure('known attachment vanished: ' + sorted(missing)[0])
|
||||
terminal = all(s in ('ready', 'failed') for s in states.values())
|
||||
if queued == 0 and outbox == 0 and terminal:
|
||||
drained = True
|
||||
break
|
||||
time.sleep(1)
|
||||
result = {'drained': drained, 'states': states, 'queued_jobs': queued,
|
||||
'unpublished_outbox': outbox,
|
||||
'seconds': round(time.monotonic() - started, 3),
|
||||
'started': start_ts, 'finished': utc_now()}
|
||||
if flush_telemetry and obs is not None:
|
||||
obs.wait_telemetry_flushed(timeout=timeout)
|
||||
result['telemetry_flushed'] = utc_now()
|
||||
return result
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Reconcile what the application wrote to stdout with what reached Loki and
|
||||
Tempo (stdlib only). A record's stdout line is the ground truth for whether a
|
||||
trace was sampled (trace_sampled) and which SERVER span ID it carries."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import measurement # noqa: E402
|
||||
import obs # noqa: E402
|
||||
from obs import Failure, ROOT # noqa: E402
|
||||
|
||||
DEADLINE_S = 60
|
||||
|
||||
|
||||
def stdout_http_logs(since_utc, services=('api',), until_utc=None):
|
||||
"""{request_id: parsed stdout JSON} for event=http.request lines."""
|
||||
args = ['docker', 'compose', '--env-file', obs.stand.ENV_FILE, 'logs',
|
||||
'--no-color', '--no-log-prefix', '--since', since_utc]
|
||||
if until_utc:
|
||||
args += ['--until', until_utc]
|
||||
result = subprocess.run(args + list(services), cwd=ROOT, text=True,
|
||||
capture_output=True, timeout=180)
|
||||
if result.returncode != 0:
|
||||
raise Failure('docker compose logs failed')
|
||||
found = {}
|
||||
for line in result.stdout.splitlines():
|
||||
if not line.startswith('{'):
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if row.get('event') == 'http.request' and row.get('request_id'):
|
||||
found[row['request_id']] = row
|
||||
return found
|
||||
|
||||
|
||||
def loki_by_request_id(client, request_ids, start_ns, end_ns, deadline_s=DEADLINE_S):
|
||||
"""Poll until every id is seen or the deadline passes. Returns
|
||||
({request_id: labels}, complete) where complete means every query returned
|
||||
a full, un-truncated answer (a Failure would propagate otherwise)."""
|
||||
ids = sorted(request_ids)
|
||||
found = {}
|
||||
deadline = time.monotonic() + deadline_s
|
||||
while True:
|
||||
for i in range(0, len(ids), 100):
|
||||
chunk = [r for r in ids[i:i + 100] if r not in found]
|
||||
if not chunk:
|
||||
continue
|
||||
query = ('{service_name="messenger-api"} | event_name="http.request" '
|
||||
'| request_id=~"' + '|'.join(chunk) + '"')
|
||||
for row in client.loki_rows(query, start_ns, end_ns):
|
||||
found[row['labels']['request_id']] = row['labels']
|
||||
if len(found) >= len(ids) or time.monotonic() >= deadline:
|
||||
return found
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def tempo_server_spans(client, targets, deadline_s=DEADLINE_S):
|
||||
"""targets: {request_id: (trace_id, span_id)} -> {request_id: bool}."""
|
||||
found = {}
|
||||
deadline = time.monotonic() + deadline_s
|
||||
while True:
|
||||
for rid, (tid, sid) in targets.items():
|
||||
if found.get(rid):
|
||||
continue
|
||||
spans = client.get_trace(tid)
|
||||
found[rid] = bool(spans) and any(
|
||||
s['span_id'] == sid and s['kind'] == 'SERVER' for s in spans)
|
||||
if all(found.values()) or time.monotonic() >= deadline:
|
||||
return found
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def reconcile(client, mode, requests, since_utc, start_ns, end_ns):
|
||||
"""requests: [{'request_id','status',...}] -> (records, summary).
|
||||
|
||||
off: stdout only (Loki/Tempo not applicable). on: stdout -> Loki and
|
||||
stdout span_id -> Tempo SERVER span."""
|
||||
stdout = stdout_http_logs(since_utc)
|
||||
records = []
|
||||
for req in requests:
|
||||
line = stdout.get(req.get('request_id'))
|
||||
rec = {'request_id': req.get('request_id'), 'status': req.get('status'),
|
||||
'stdout': None, 'loki': None, 'span': None}
|
||||
if line is not None:
|
||||
rec['stdout'] = {k: line[k] for k in
|
||||
('trace_id', 'span_id', 'trace_sampled') if k in line}
|
||||
records.append(rec)
|
||||
if mode != 'off':
|
||||
with_stdout = [r for r in records if r['stdout'] is not None]
|
||||
loki = loki_by_request_id(
|
||||
client, [r['request_id'] for r in with_stdout], start_ns, end_ns)
|
||||
for rec in with_stdout:
|
||||
rec['loki'] = rec['request_id'] in loki
|
||||
targets = {r['request_id']: (r['stdout']['trace_id'], r['stdout']['span_id'])
|
||||
for r in with_stdout
|
||||
if r['stdout'].get('trace_sampled') is True}
|
||||
spans = tempo_server_spans(client, targets)
|
||||
for rec in with_stdout:
|
||||
if rec['request_id'] in spans:
|
||||
rec['span'] = spans[rec['request_id']]
|
||||
return records, measurement.summarize_cohort(mode, records)
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Container resource sampling for lab6: docker stats (container CPU/memory) and
|
||||
docker top (process RSS) every few seconds. Stdlib only.
|
||||
|
||||
MemUsage is *container* memory usage (cgroup, includes page cache); RSS is the
|
||||
sum of process RSS from `docker top` (KiB -> bytes). They are never mixed."""
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import sys # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import measurement # noqa: E402
|
||||
import obs # noqa: E402
|
||||
|
||||
UNITS = {'B': 1, 'kB': 1000, 'MB': 1000 ** 2, 'GB': 1000 ** 3, 'TB': 1000 ** 4,
|
||||
'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3, 'TiB': 1024 ** 4}
|
||||
SIZE = re.compile(r'^\s*([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)\s*$')
|
||||
SERVICES = ('api', 'worker', 'otel-collector', 'prometheus', 'loki', 'tempo',
|
||||
'grafana')
|
||||
|
||||
|
||||
def parse_size(text):
|
||||
"""'147.5MiB' -> bytes; SI and IEC units differ; unknown unit is an error."""
|
||||
match = SIZE.match(text)
|
||||
if not match or match.group(2) not in UNITS:
|
||||
raise ValueError('unparseable size: ' + repr(text))
|
||||
return int(float(match.group(1)) * UNITS[match.group(2)])
|
||||
|
||||
|
||||
def parse_stats_row(row):
|
||||
usage, limit = row['MemUsage'].split('/')
|
||||
cpu = row['CPUPerc'].strip()
|
||||
if not cpu.endswith('%'):
|
||||
raise ValueError('unparseable CPU: ' + repr(cpu))
|
||||
return {'cpu_pct': float(cpu[:-1]), 'mem_usage_bytes': parse_size(usage),
|
||||
'mem_limit_bytes': parse_size(limit)}
|
||||
|
||||
|
||||
def parse_top_rss(text):
|
||||
"""`docker top <id> -eo pid,rss` output -> bytes (KiB x 1024)."""
|
||||
lines = [line.split() for line in text.strip().splitlines()]
|
||||
if not lines or [h.upper() for h in lines[0]] != ['PID', 'RSS']:
|
||||
raise ValueError('unexpected docker top header')
|
||||
return sum(int(row[1]) for row in lines[1:]) * 1024
|
||||
|
||||
|
||||
def container_map():
|
||||
"""{short_id: service} for all observed services."""
|
||||
result = {}
|
||||
for service in SERVICES:
|
||||
ids = obs.stand.compose('ps', '-q', service, check_result=False)
|
||||
for cid in ids.stdout.split():
|
||||
result[cid[:12]] = service
|
||||
return result
|
||||
|
||||
|
||||
class ResourceSampler(threading.Thread):
|
||||
def __init__(self, containers, interval=5.0, path=None):
|
||||
super().__init__(daemon=True)
|
||||
self.containers = dict(containers)
|
||||
self.interval = interval
|
||||
self.path = Path(path) if path else None
|
||||
self.stop_event = threading.Event()
|
||||
self.samples = []
|
||||
self.errors = []
|
||||
|
||||
def one_tick(self, tick):
|
||||
mono = time.monotonic()
|
||||
stamp = obs.utc_now()
|
||||
ids = list(self.containers)
|
||||
stats = subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + ids,
|
||||
capture_output=True, text=True, timeout=60)
|
||||
rows = {}
|
||||
for line in stats.stdout.splitlines():
|
||||
if line.strip():
|
||||
row = json.loads(line)
|
||||
rows[row['ID'][:12]] = row
|
||||
for cid in ids:
|
||||
sample = {'tick': tick, 'utc': stamp, 'mono': mono, 'id': cid,
|
||||
'service': self.containers[cid]}
|
||||
raw = rows.get(cid)
|
||||
if raw is None:
|
||||
sample['error'] = 'no_stats_row'
|
||||
else:
|
||||
sample['raw'] = {k: raw[k] for k in ('CPUPerc', 'MemUsage')}
|
||||
try:
|
||||
sample.update(parse_stats_row(raw))
|
||||
except ValueError as exc:
|
||||
sample['error'] = str(exc)
|
||||
top = subprocess.run(
|
||||
['docker', 'top', cid, '-eo', 'pid,rss'],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if top.returncode == 0:
|
||||
try:
|
||||
sample['rss_bytes'] = parse_top_rss(top.stdout)
|
||||
except ValueError as exc:
|
||||
sample['rss_error'] = str(exc)
|
||||
else:
|
||||
sample['rss_bytes'] = None
|
||||
sample['rss_reason'] = 'not_running'
|
||||
self.samples.append(sample)
|
||||
if self.path is not None:
|
||||
with self.path.open('a') as handle:
|
||||
handle.write(json.dumps(sample) + '\n')
|
||||
if 'error' in sample:
|
||||
self.errors.append(sample['error'])
|
||||
|
||||
def run(self):
|
||||
tick = 0
|
||||
while not self.stop_event.is_set():
|
||||
begun = time.monotonic()
|
||||
try:
|
||||
self.one_tick(tick)
|
||||
except (subprocess.SubprocessError, ValueError, KeyError) as exc:
|
||||
self.errors.append(type(exc).__name__)
|
||||
tick += 1
|
||||
self.stop_event.wait(max(0.0, self.interval - (time.monotonic() - begun)))
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
self.join(timeout=90)
|
||||
|
||||
|
||||
def _summarise(values):
|
||||
values = [v for v in values if v is not None]
|
||||
return {'median': measurement.median(values),
|
||||
'p95': measurement.nearest_rank(values, 0.95),
|
||||
'max': max(values) if values else None, 'n': len(values)}
|
||||
|
||||
|
||||
def summarize(samples, container_ids=None):
|
||||
"""Per-service statistics. For API the replicas are summed on a common
|
||||
tick first; individual values are kept in `per_container`."""
|
||||
ticks = {}
|
||||
for sample in samples:
|
||||
if 'error' in sample:
|
||||
continue
|
||||
ticks.setdefault(sample['tick'], {}).setdefault(
|
||||
sample['service'], []).append(sample)
|
||||
order = sorted(ticks)
|
||||
result = {}
|
||||
services = {s['service'] for s in samples}
|
||||
for service in sorted(services):
|
||||
cpu, mem, rss, times = [], [], [], []
|
||||
for tick in order:
|
||||
group = ticks[tick].get(service)
|
||||
if not group:
|
||||
continue
|
||||
times.append(min(s['mono'] for s in group))
|
||||
cpu.append(sum(s['cpu_pct'] for s in group))
|
||||
mem.append(sum(s['mem_usage_bytes'] for s in group))
|
||||
values = [s.get('rss_bytes') for s in group]
|
||||
rss.append(None if any(v is None for v in values) else sum(values))
|
||||
weights = [times[i + 1] - times[i] for i in range(len(times) - 1)]
|
||||
if times:
|
||||
weights.append(sum(weights) / len(weights) if weights else 1.0)
|
||||
mean_cpu = (sum(c * w for c, w in zip(cpu, weights)) / sum(weights)
|
||||
if weights and sum(weights) > 0 else None)
|
||||
per_container = {}
|
||||
for s in samples:
|
||||
if s['service'] == service and 'error' not in s:
|
||||
per_container.setdefault(s['id'], []).append(s['cpu_pct'])
|
||||
result[service] = {
|
||||
'cpu_pct_time_weighted_mean': mean_cpu,
|
||||
'container_memory_bytes': _summarise(mem),
|
||||
'process_rss_bytes': _summarise(rss),
|
||||
'mem_limit_bytes': max((s['mem_limit_bytes'] for s in samples
|
||||
if s['service'] == service and
|
||||
'mem_limit_bytes' in s), default=None),
|
||||
'ticks': len(times),
|
||||
'per_container_cpu_mean': {k: sum(v) / len(v)
|
||||
for k, v in per_container.items()},
|
||||
}
|
||||
result['sample_errors'] = len(
|
||||
[s for s in samples if 'error' in s])
|
||||
return result
|
||||
@@ -0,0 +1,199 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import measurement as m # noqa: E402
|
||||
|
||||
|
||||
def slot(i, status=201, service=10.0, lag=1.0, outcome='http'):
|
||||
if outcome != 'http':
|
||||
return {'i': i, 'outcome': outcome, 'start_lag_ms': None}
|
||||
return {'i': i, 'outcome': 'http', 'status': status,
|
||||
'service_ms': service, 'scheduled_ms': service + lag,
|
||||
'start_lag_ms': lag}
|
||||
|
||||
|
||||
def cohort(**kw):
|
||||
base = {'n': 40, 'retained_traces': 5, 'sampled_present': 5,
|
||||
'intentional_unsampled': 35}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
def run(mode='p25', spans=10.0, p95=100.0, failures=(), valid=True,
|
||||
error=None, slow=None):
|
||||
return {
|
||||
'mode': mode, 'valid': valid, 'invalid_reasons': [],
|
||||
'configuration_failures': list(failures),
|
||||
'spans_per_operation': spans,
|
||||
'load': {'scheduled_ms': {'p95': p95}},
|
||||
'cohorts': {'error': error or cohort(), 'slow': slow or cohort()},
|
||||
'dir': mode,
|
||||
}
|
||||
|
||||
|
||||
def summary(load_slots, **extra):
|
||||
load = m.summarize_load(load_slots, 10)
|
||||
base = {'mode': 'off', 'load': load, 'cohorts': {}, 'ingest': {},
|
||||
'export_failures': {'loss': False}}
|
||||
base.update(extra)
|
||||
return base
|
||||
|
||||
|
||||
class SamplingIsNotLoss(unittest.TestCase):
|
||||
def test_false_flag_is_not_export_loss_or_missing_span(self):
|
||||
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
||||
'trace_sampled': False}, 'loki': True, 'span': False}
|
||||
self.assertEqual(m.classify_record('p25', rec),
|
||||
('intentional_unsampled', 'exported'))
|
||||
counts = m.summarize_cohort('p25', [rec] * 3)
|
||||
self.assertEqual(counts.get('sampled_missing', 0), 0)
|
||||
self.assertFalse(m.export_loss(0, 0.0, 0.0)['loss'])
|
||||
|
||||
def test_sampled_without_span_is_a_measured_loss(self):
|
||||
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
||||
'trace_sampled': True}, 'loki': True, 'span': False}
|
||||
self.assertEqual(m.classify_record('full', rec)[0], 'sampled_missing')
|
||||
|
||||
def test_missing_stdout_is_not_intentional_sampling(self):
|
||||
self.assertEqual(m.classify_record('p25', {'stdout': None})[0],
|
||||
'unmatched_stdout')
|
||||
no_flag = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16}}
|
||||
self.assertEqual(m.classify_record('p25', no_flag)[0], 'unmatched_stdout')
|
||||
incomplete = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
||||
'trace_sampled': True}, 'loki': None, 'span': None}
|
||||
self.assertEqual(m.classify_record('p25', incomplete),
|
||||
('incomplete_observation', 'incomplete_observation'))
|
||||
|
||||
def test_full_observation_without_log_is_missing_exported_log(self):
|
||||
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
||||
'trace_sampled': False}, 'loki': False}
|
||||
self.assertEqual(m.classify_record('p10', rec)[1], 'missing_exported_logs')
|
||||
|
||||
def test_off_needs_neither_loki_nor_app_counters(self):
|
||||
rec = {'stdout': {'event': 'http.request'}, 'loki': None, 'span': None}
|
||||
self.assertEqual(m.classify_record('off', rec),
|
||||
('not_applicable', 'not_applicable'))
|
||||
leaked = {'stdout': {'trace_id': 'x'}}
|
||||
self.assertEqual(m.classify_record('off', leaked)[0], 'ids_present_with_sdk_off')
|
||||
s = summary([slot(i) for i in range(100)], cohorts={
|
||||
'error': m.summarize_cohort('off', [rec] * 4)})
|
||||
valid, invalid, failures = m.evaluate_run(s)
|
||||
self.assertTrue(valid, invalid)
|
||||
self.assertEqual(failures, [])
|
||||
|
||||
|
||||
class Counters(unittest.TestCase):
|
||||
def test_reset_is_not_a_negative_loss(self):
|
||||
self.assertEqual(m.counter_delta(10.0, 3.0), (None, 'counter_reset'))
|
||||
loss = m.export_loss(None, 0.0, 2.0)
|
||||
self.assertEqual(loss['total_known'], 2.0)
|
||||
self.assertTrue(all(v >= 0 for v in loss['parts'].values() if v is not None))
|
||||
|
||||
def test_missing_positive_series_is_not_zero_failure_series_may_be(self):
|
||||
self.assertEqual(m.counter_delta(None, None), (None, 'missing_series'))
|
||||
self.assertEqual(m.counter_delta(None, None, failure_only=True)[0], 0.0)
|
||||
self.assertEqual(m.counter_delta(5.0, 9.0), (4.0, None))
|
||||
|
||||
def test_unexplained_reset_invalidates_but_restart_explains_it(self):
|
||||
s = summary([slot(i) for i in range(100)],
|
||||
ingest={'spans': {'delta': None, 'reason': 'counter_reset'}})
|
||||
self.assertFalse(m.evaluate_run(s)[0])
|
||||
s['restarts'] = ['collector']
|
||||
valid, _, failures = m.evaluate_run(s)
|
||||
self.assertTrue(valid)
|
||||
self.assertIn('restart_or_oom', failures)
|
||||
|
||||
|
||||
class LoadAccounting(unittest.TestCase):
|
||||
def test_dropped_and_late_slots_stay_in_denominator(self):
|
||||
slots = [slot(i) for i in range(90)] + \
|
||||
[slot(90 + i, outcome='client_dropped') for i in range(5)] + \
|
||||
[slot(95 + i, lag=300.0) for i in range(5)]
|
||||
load = m.summarize_load(slots, 10)
|
||||
self.assertEqual(load['scheduled'], 100)
|
||||
self.assertEqual(load['completed'], 95)
|
||||
self.assertAlmostEqual(load['completed_ratio'], 0.95)
|
||||
valid, invalid, _ = m.evaluate_run(summary(slots))
|
||||
self.assertFalse(valid)
|
||||
self.assertTrue(any('client_dropped' in r for r in invalid))
|
||||
self.assertTrue(any('completed_ratio' in r for r in invalid))
|
||||
|
||||
def test_late_scheduler_fails_start_lag_even_without_drops(self):
|
||||
slots = [slot(i, lag=80.0) for i in range(100)]
|
||||
self.assertFalse(m.evaluate_run(summary(slots))[0])
|
||||
|
||||
def test_overall_p95_includes_error_responses(self):
|
||||
slots = [slot(i, service=10.0) for i in range(90)] + \
|
||||
[slot(90 + i, status=500, service=900.0) for i in range(10)]
|
||||
load = m.summarize_load(slots, 10)
|
||||
self.assertEqual(load['service_ms']['p95'], 900.0)
|
||||
self.assertEqual(load['success_service_ms']['p95'], 10.0)
|
||||
self.assertEqual(m.load_errors(load), 10)
|
||||
|
||||
def test_nearest_rank(self):
|
||||
self.assertEqual(m.nearest_rank(list(range(1, 101)), 0.95), 95)
|
||||
self.assertEqual(m.nearest_rank([7], 0.99), 7)
|
||||
self.assertIsNone(m.nearest_rank([], 0.5))
|
||||
|
||||
|
||||
class MatrixAndSelection(unittest.TestCase):
|
||||
def test_valid_adverse_run_stays_in_matrix(self):
|
||||
s = summary([slot(i) for i in range(100)], mode='p25',
|
||||
export_failures={'loss': True},
|
||||
cohorts={'error': cohort(sampled_missing=2)})
|
||||
valid, invalid, failures = m.evaluate_run(s)
|
||||
self.assertTrue(valid, invalid)
|
||||
self.assertIn('export_loss', failures)
|
||||
self.assertTrue(any(f.startswith('sampled_missing') for f in failures))
|
||||
s.update(valid=valid, invalid_reasons=invalid,
|
||||
configuration_failures=failures, repeat=1)
|
||||
entry = m.matrix_entry(s)
|
||||
self.assertTrue(entry['valid'])
|
||||
self.assertTrue(entry['configuration_failures'])
|
||||
|
||||
def full_runs(self):
|
||||
return [run('full', spans=100.0, p95=100.0) for _ in range(3)]
|
||||
|
||||
def test_p25_selected_only_against_complete_full_reference(self):
|
||||
good = [run('p25', spans=40.0, p95=105.0) for _ in range(3)]
|
||||
self.assertTrue(m.select_p25(self.full_runs(), good)['passed'])
|
||||
broken_full = self.full_runs()
|
||||
broken_full[1]['spans_per_operation'] = None
|
||||
self.assertFalse(m.select_p25(broken_full, good)['passed'])
|
||||
heavy = [run('p25', spans=60.0, p95=105.0) for _ in range(3)]
|
||||
self.assertFalse(m.select_p25(self.full_runs(), heavy)['passed'])
|
||||
slow = [run('p25', spans=40.0, p95=120.0) for _ in range(3)]
|
||||
self.assertFalse(m.select_p25(self.full_runs(), slow)['passed'])
|
||||
|
||||
def test_p25_rejected_when_an_error_class_has_no_retained_trace(self):
|
||||
runs = [run('p25', spans=40.0, p95=100.0) for _ in range(3)]
|
||||
runs[2]['cohorts']['slow'] = cohort(retained_traces=0, sampled_present=0)
|
||||
self.assertFalse(m.select_p25(self.full_runs(), runs)['passed'])
|
||||
|
||||
def test_single_validation_uses_frozen_full_reference(self):
|
||||
ref = m.full_reference(self.full_runs())
|
||||
ok = run('p25', spans=45.0, p95=108.0)
|
||||
self.assertTrue(m.validate_single(ok, 'p25', ref)['passed'])
|
||||
# Would pass a "median of its own" comparison, fails the frozen one.
|
||||
bad = run('p25', spans=55.0, p95=108.0)
|
||||
result = m.validate_single(bad, 'p25', ref)
|
||||
self.assertFalse(result['passed'])
|
||||
self.assertFalse(result['checks']['spans_le_50pct_frozen_full'])
|
||||
|
||||
def test_full_validation_needs_all_eighty_server_spans(self):
|
||||
ref = m.full_reference(self.full_runs())
|
||||
ok = run('full', error=cohort(sampled_present=40, retained_traces=40),
|
||||
slow=cohort(sampled_present=40, retained_traces=40))
|
||||
self.assertTrue(m.validate_single(ok, 'full', ref)['passed'])
|
||||
short = run('full', error=cohort(sampled_present=39, retained_traces=39),
|
||||
slow=cohort(sampled_present=40, retained_traces=40))
|
||||
self.assertFalse(m.validate_single(short, 'full', ref)['passed'])
|
||||
|
||||
def test_miss_all_probability_model(self):
|
||||
self.assertAlmostEqual(m.miss_all_probability(0.25, 40), 0.75 ** 40)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Executable
+457
@@ -0,0 +1,457 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab6 track 1 "cost of observability": off / full / p25 / p10 under the same
|
||||
fixed-rate workload. Stdlib only. The protocol (rate calibration, 3x4 matrix,
|
||||
diagnostic cohorts, p25 acceptance, one independent validation) is frozen in
|
||||
this file; see REPORT.md for the description."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import incidents # noqa: E402
|
||||
import measurement as m # noqa: E402
|
||||
import obs # noqa: E402
|
||||
import reconcile # noqa: E402
|
||||
import resources # noqa: E402
|
||||
import verify # noqa: E402
|
||||
import workload # noqa: E402
|
||||
from obs import Failure, stand, utc_now # noqa: E402
|
||||
|
||||
BASELINE_ENV = dict(obs.SDK_DEFAULTS)
|
||||
BASELINE_ENV.update({'CACHE_MODE': 'protected', 'FAULT_INJECTION_ENABLED': '1',
|
||||
'LOG_LEVEL': 'INFO'})
|
||||
WARMUP_S, MEASURE_S = 60, 180
|
||||
CALIBRATION = ((20, 20, 40), (10, 20, 40), (5, 20, 40)) # rate, warmup, measure
|
||||
N_SLOW, ERR_RATE, ERR_SECONDS = 40, 4, 10
|
||||
|
||||
COUNTERS = (
|
||||
('accepted_spans', 'sum(otelcol_receiver_accepted_spans)', False),
|
||||
('accepted_log_records', 'sum(otelcol_receiver_accepted_log_records)', False),
|
||||
('accepted_metric_points', 'sum(otelcol_receiver_accepted_metric_points)', False),
|
||||
('sent_spans', 'sum(otelcol_exporter_sent_spans)', False),
|
||||
('sent_log_records', 'sum(otelcol_exporter_sent_log_records)', False),
|
||||
('sent_metric_points', 'sum(otelcol_exporter_sent_metric_points)', False),
|
||||
('send_failed_spans', 'sum(otelcol_exporter_send_failed_spans)', True),
|
||||
('send_failed_log_records', 'sum(otelcol_exporter_send_failed_log_records)', True),
|
||||
('send_failed_metric_points', 'sum(otelcol_exporter_send_failed_metric_points)', True),
|
||||
('enqueue_failed_spans', 'sum(otelcol_exporter_enqueue_failed_spans)', True),
|
||||
('enqueue_failed_log_records', 'sum(otelcol_exporter_enqueue_failed_log_records)', True),
|
||||
('enqueue_failed_metric_points', 'sum(otelcol_exporter_enqueue_failed_metric_points)', True),
|
||||
('tempo_bytes_received', 'sum(tempo_distributor_bytes_received_total)', False),
|
||||
('loki_bytes_received', 'sum(loki_distributor_bytes_received_total)', False),
|
||||
)
|
||||
|
||||
|
||||
def write(path, data):
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(path).write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True) + '\n')
|
||||
|
||||
|
||||
def unique_dir(base):
|
||||
base = Path(base)
|
||||
if not base.exists():
|
||||
return base
|
||||
n = 2
|
||||
while Path('%s-attempt-%d' % (base, n)).exists():
|
||||
n += 1
|
||||
return Path('%s-attempt-%d' % (base, n))
|
||||
|
||||
|
||||
def snapshot(client, instance_ids, sdk_on):
|
||||
snap = {'utc': utc_now(), 'values': {}, 'sample_ts': {}}
|
||||
for name, expr, _ in COUNTERS:
|
||||
result = client.prom_query(expr)
|
||||
snap['values'][name] = float(result[0]['value'][1]) if result else None
|
||||
snap['sample_ts'][name] = float(result[0]['value'][0]) if result else None
|
||||
if sdk_on:
|
||||
selector = 'instance=~"%s"' % '|'.join(instance_ids)
|
||||
for outcome in ('failed', 'success'):
|
||||
result = client.prom_query(
|
||||
'sum(messenger_telemetry_export_items_total{outcome="%s",%s})' % (outcome, selector))
|
||||
snap['values']['app_export_' + outcome] = float(result[0]['value'][1]) if result else None
|
||||
return snap
|
||||
|
||||
|
||||
def deltas(a, b, operations, sdk_on):
|
||||
out = {}
|
||||
for name, _, failure_only in COUNTERS:
|
||||
delta, reason = m.counter_delta(a['values'][name], b['values'][name], failure_only)
|
||||
if name.endswith('_bytes_received') and reason == 'missing_series':
|
||||
out[name] = {'status': 'not_exposed'}
|
||||
continue
|
||||
out[name] = {'start': a['values'][name], 'end': b['values'][name],
|
||||
'delta': delta, 'reason': reason,
|
||||
'per_operation': (delta / operations) if delta is not None and operations else None}
|
||||
for outcome in ('failed', 'success'):
|
||||
key = 'app_export_' + outcome
|
||||
if not sdk_on:
|
||||
out[key] = {'status': 'not_applicable'}
|
||||
else:
|
||||
delta, reason = m.counter_delta(a['values'].get(key), b['values'].get(key),
|
||||
failure_only=(outcome == 'failed'))
|
||||
out[key] = {'start': a['values'].get(key), 'end': b['values'].get(key),
|
||||
'delta': delta, 'reason': reason}
|
||||
return out
|
||||
|
||||
|
||||
def container_health(ids):
|
||||
"""{short_id: (service, restarts, oom)} from docker inspect."""
|
||||
out = {}
|
||||
for cid, service in ids.items():
|
||||
data = json.loads(stand.docker('inspect', cid).stdout)[0]
|
||||
out[cid] = (service, data['RestartCount'], bool(data['State'].get('OOMKilled')),
|
||||
data['State'].get('Running'))
|
||||
return out
|
||||
|
||||
|
||||
def rusage():
|
||||
own, kids = resource.getrusage(resource.RUSAGE_SELF), resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
return {'self_cpu_s': own.ru_utime + own.ru_stime, 'children_cpu_s': kids.ru_utime + kids.ru_stime,
|
||||
'loadavg': os.getloadavg()}
|
||||
|
||||
|
||||
class Track:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.obs = verify.make_obs(args)
|
||||
self.out = Path(args.output_dir)
|
||||
self.image_ids = None
|
||||
self.set_env = None
|
||||
|
||||
# -- environment
|
||||
def configure(self, mode, force=True):
|
||||
env = dict(BASELINE_ENV)
|
||||
env.update(m.MODES[mode])
|
||||
state = self.set_env(env, force=force)
|
||||
for service in ('api', 'worker'):
|
||||
for cid, info in state[service].items():
|
||||
for key, value in env.items():
|
||||
if info['env'][key] != value:
|
||||
raise Failure('effective %s=%s in %s %s' % (key, info['env'][key], service, cid))
|
||||
images = {svc: sorted(i['image'] for i in state[svc].values()) for svc in ('api', 'worker')}
|
||||
if self.image_ids is None:
|
||||
self.image_ids = images
|
||||
ok = images == self.image_ids
|
||||
instance_ids = sorted(list(state['api']) + list(state['worker']))
|
||||
return state, instance_ids, ok, {k: env[k] for k in obs.MUTABLE_KEYS if k in env}
|
||||
|
||||
# -- one protocol run
|
||||
def execute_run(self, mode, rate, run_dir, repeat=None, warmup=WARMUP_S, measure=MEASURE_S,
|
||||
n_slow=N_SLOW, err_rate=ERR_RATE, err_seconds=ERR_SECONDS, label=None):
|
||||
run_dir = Path(run_dir)
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
sdk_on = mode != 'off'
|
||||
defects, stamps = [], {}
|
||||
self.obs.require_no_foreign_faults()
|
||||
pre = obs.wait_drained([], obs=self.obs, timeout=180)
|
||||
if not pre['drained']:
|
||||
raise Failure('stack did not drain before the run (%s)' % mode)
|
||||
state, instance_ids, image_ok, effective = self.configure(mode)
|
||||
ids = resources.container_map()
|
||||
health0 = container_health(ids)
|
||||
c = stand.new_client(self.args.base_url, self.args.ca_file)
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
stamps['warmup_start'] = utc_now()
|
||||
warm = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
|
||||
warmup, 'warmup', settle=False)
|
||||
warm_drain = obs.wait_drained(warm['uploads'], obs=self.obs, timeout=180)
|
||||
stamps['warmup_end'] = utc_now()
|
||||
if not warm_drain['drained']:
|
||||
defects.append('warmup_did_not_drain')
|
||||
snap0 = snapshot(self.obs, instance_ids, sdk_on)
|
||||
usage0 = rusage()
|
||||
sampler = resources.ResourceSampler(ids, 5.0, run_dir / 'docker-stats.jsonl')
|
||||
sampler.start()
|
||||
stamps['measure_start'] = utc_now()
|
||||
result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
|
||||
measure, 'measure', settle=False)
|
||||
sampler.stop()
|
||||
stamps['measure_dispatch_end'] = utc_now()
|
||||
usage1 = rusage()
|
||||
drain = obs.wait_drained(result['uploads'], obs=self.obs, timeout=180)
|
||||
snap1 = snapshot(self.obs, instance_ids, sdk_on)
|
||||
stamps['measure_drained'] = utc_now()
|
||||
load = result['load']
|
||||
# -- integrity after the final ingest snapshot (HTTP is allowed now)
|
||||
integrity = 0
|
||||
reference = stand.sha256(stand.sample_bytes())
|
||||
for upload_id in result['uploads']:
|
||||
meta = stand.metadata(c, alice, upload_id)
|
||||
if meta['status'] != 'ready' or meta['sha256'] != reference:
|
||||
integrity += 1
|
||||
not_ready = sum(1 for s in drain['states'].values() if s != 'ready')
|
||||
write(run_dir / 'drain.json', {'warmup': warm_drain, 'measurement': drain})
|
||||
# -- diagnostic cohorts, never mixed with the healthy statistics
|
||||
cohort_since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 2))
|
||||
cohort_start_ns = time.time_ns() - 10 * 10 ** 9
|
||||
stamps['cohorts_start'] = utc_now()
|
||||
slow_rows = incidents.sequential_uploads(self, alice, chat, n_slow, 500)
|
||||
slow_valid = all(r['consumed'] and r['status'] == 202 for r in slow_rows)
|
||||
err_slots = []
|
||||
with obs.hold_table_lock('messages'):
|
||||
err = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat,
|
||||
err_rate, err_seconds, 'error', upload_every=0, settle=False)
|
||||
err_slots = err['slots']
|
||||
err_expected = err_rate * err_seconds
|
||||
err_valid = (len(err_slots) == err_expected and all(
|
||||
s['outcome'] == 'http' and s['status'] >= 500 and s.get('instance_id') and s.get('request_id')
|
||||
for s in err_slots))
|
||||
stamps['cohorts_end'] = utc_now()
|
||||
slow_drain = obs.wait_drained([r['attachment_id'] for r in slow_rows if r['attachment_id']],
|
||||
obs=self.obs, timeout=180)
|
||||
snap2 = snapshot(self.obs, instance_ids, sdk_on)
|
||||
end_ns = time.time_ns() + 60 * 10 ** 9
|
||||
slow_records, slow_summary = reconcile.reconcile(
|
||||
self.obs, mode, [{'request_id': r['request_id'], 'status': r['status']} for r in slow_rows],
|
||||
cohort_since, cohort_start_ns, end_ns)
|
||||
err_records, err_summary = reconcile.reconcile(
|
||||
self.obs, mode, [{'request_id': s.get('request_id'), 'status': s.get('status')}
|
||||
for s in err_slots if s.get('request_id')],
|
||||
cohort_since, cohort_start_ns, end_ns)
|
||||
slow_summary['invalid'] = not slow_valid
|
||||
err_summary['invalid'] = not err_valid
|
||||
err_summary['responses'] = len(err_slots)
|
||||
write(run_dir / 'cohorts.json', {'slow': {'requests': slow_rows, 'records': slow_records,
|
||||
'summary': slow_summary},
|
||||
'error': {'slots': [{k: s.get(k) for k in (
|
||||
'i', 'outcome', 'status', 'request_id', 'instance_id', 'service_ms')}
|
||||
for s in err_slots],
|
||||
'records': err_records, 'summary': err_summary}})
|
||||
health1 = container_health(ids)
|
||||
restarts = [health1[cid][0] for cid in ids
|
||||
if health1[cid][1] != health0[cid][1] or health1[cid][2]]
|
||||
# -- independent proof that SDK-off sends nothing
|
||||
d_final = deltas(snap0, snap2, load['completed'], sdk_on)
|
||||
if not sdk_on:
|
||||
for name in ('accepted_spans', 'accepted_log_records', 'accepted_metric_points'):
|
||||
if (d_final[name]['delta'] or 0) != 0:
|
||||
defects.append('sdk_off_but_collector_ingest:' + name)
|
||||
ingest = deltas(snap0, snap1, load['completed'], sdk_on)
|
||||
loss = m.export_loss(
|
||||
ingest['app_export_failed'].get('delta') if sdk_on else None,
|
||||
sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('send_failed')),
|
||||
sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('enqueue_failed')))
|
||||
res_summary = resources.summarize(sampler.samples)
|
||||
spans_op = ingest['accepted_spans']['per_operation']
|
||||
summary = {
|
||||
'mode': mode, 'sampler_ratio': m.SAMPLER_RATIO[mode], 'repeat': repeat, 'label': label,
|
||||
'rate_rps': rate, 'warmup_s': warmup, 'measurement_s': measure,
|
||||
'effective_env': effective, 'effective_mode_ok': True,
|
||||
'image_ids': self.image_ids, 'image_ids_ok': image_ok, 'instance_ids': instance_ids,
|
||||
'load': load, 'resources': res_summary,
|
||||
'controller_usage': {'start': usage0, 'end': usage1,
|
||||
'cpu_s_during_dispatch': (usage1['self_cpu_s'] + usage1['children_cpu_s']
|
||||
- usage0['self_cpu_s'] - usage0['children_cpu_s'])},
|
||||
'ingest': ingest, 'ingest_through_cohorts': d_final, 'export_failures': loss,
|
||||
'spans_per_operation': spans_op,
|
||||
'log_records_per_operation': ingest['accepted_log_records']['per_operation'],
|
||||
'metric_points_per_operation': ingest['accepted_metric_points']['per_operation'],
|
||||
'cohorts': {'slow': slow_summary, 'error': err_summary},
|
||||
'not_ready_at_deadline': not_ready, 'integrity_errors': integrity,
|
||||
'restarts': restarts, 'measurement_defects': defects, 'timestamps': stamps,
|
||||
'foreign_faults': self.obs.foreign_faults(), 'dir': str(run_dir),
|
||||
'slow_drain': {k: slow_drain[k] for k in ('drained', 'seconds')},
|
||||
}
|
||||
valid, invalid, failures = m.evaluate_run(summary)
|
||||
summary.update(valid=valid, invalid_reasons=invalid, configuration_failures=failures)
|
||||
with (run_dir / 'requests.jsonl').open('w') as handle:
|
||||
for slot in result['slots']:
|
||||
handle.write(json.dumps({k: slot.get(k) for k in (
|
||||
'i', 'kind', 'scheduled', 'start', 'end', 'service_ms', 'scheduled_ms',
|
||||
'start_lag_ms', 'status', 'outcome', 'request_id', 'attachment_id', 'error')}) + '\n')
|
||||
write(run_dir / 'summary.json', summary)
|
||||
print('run %s -> valid=%s invalid=%s failures=%s spans/op=%s p95=%s' % (
|
||||
run_dir, valid, invalid, failures, spans_op, load['scheduled_ms']['p95']), flush=True)
|
||||
return summary
|
||||
|
||||
# -- calibration
|
||||
def calibrate(self):
|
||||
rows = []
|
||||
self.configure('full')
|
||||
for rate, warm, measure in CALIBRATION:
|
||||
ctx_dir = self.out / 'calibration' / ('rate-%d' % rate)
|
||||
ctx_dir.mkdir(parents=True, exist_ok=True)
|
||||
c = stand.new_client(self.args.base_url, self.args.ca_file)
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, warm,
|
||||
'cal-warmup', settle=False)
|
||||
result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
|
||||
measure, 'cal-measure', settle=True, obs_client=self.obs)
|
||||
load = result['load']
|
||||
drain = result['drain']
|
||||
checks = {
|
||||
'zero_client_dropped': load['client_dropped'] == 0,
|
||||
'zero_http_errors': m.load_errors(load) == 0 and load['http_4xx'] == 0,
|
||||
'all_uploads_ready': drain['drained'] and all(s == 'ready' for s in drain['states'].values()),
|
||||
'start_lag_p95_le_50ms': (load['start_lag_ms_p95'] or 1e9) <= m.START_LAG_LIMIT_MS,
|
||||
'completed_ge_99pct': (load['completed_ratio'] or 0) >= m.COMPLETION_LIMIT,
|
||||
}
|
||||
row = {'rate_rps': rate, 'warmup_s': warm, 'measure_s': measure, 'load': load,
|
||||
'drain': drain, 'checks': checks, 'passed': all(checks.values())}
|
||||
rows.append(row)
|
||||
write(ctx_dir / 'calibration.json', row)
|
||||
print('calibration rate=%d passed=%s %s' % (rate, row['passed'], checks), flush=True)
|
||||
if row['passed']:
|
||||
write(self.out / 'calibration.json', {'selected_rate_rps': rate, 'candidates': rows})
|
||||
return rate
|
||||
write(self.out / 'calibration.json', {'selected_rate_rps': None, 'candidates': rows})
|
||||
raise Failure('no candidate rate (20/10/5) passed; fix the stand before the study')
|
||||
|
||||
# -- smoke
|
||||
def smoke(self):
|
||||
out = self.out / 'smoke'
|
||||
results = {}
|
||||
problems = []
|
||||
for mode in ('off', 'full', 'p25', 'p10'):
|
||||
s = self.execute_run(mode, 5, unique_dir(out / mode), warmup=5, measure=15,
|
||||
n_slow=3, err_rate=2, err_seconds=2, label='smoke')
|
||||
results[mode] = {'valid': s['valid'], 'invalid_reasons': s['invalid_reasons'],
|
||||
'spans_per_operation': s['spans_per_operation'],
|
||||
'cohorts': s['cohorts'],
|
||||
'effective_otel': {k: s['effective_env'][k] for k in
|
||||
('OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG')}}
|
||||
if not s['valid']:
|
||||
problems.append('%s: invalid %s' % (mode, s['invalid_reasons']))
|
||||
if mode == 'off' and (s['ingest_through_cohorts']['accepted_spans']['delta'] or 0) != 0:
|
||||
problems.append('off sent spans')
|
||||
if mode == 'full' and not s['cohorts']['error'].get('sampled_present'):
|
||||
problems.append('full has no sampled roots in Tempo')
|
||||
write(out / 'smoke.json', {'note': 'technical smoke only; not a study result',
|
||||
'results': results, 'problems': problems})
|
||||
if problems:
|
||||
raise Failure('smoke: ' + '; '.join(problems))
|
||||
print('PASS: smoke (technical only, not a study result)')
|
||||
|
||||
# -- full protocol
|
||||
def full(self, attempt_label):
|
||||
rate = self.calibrate()
|
||||
base = self.out / 'runs'
|
||||
runs, entries = [], []
|
||||
for repeat, order in enumerate(m.ORDERS, start=1):
|
||||
for mode in order:
|
||||
s = self.execute_run(mode, rate, unique_dir(base / ('%d-%s' % (repeat, mode))),
|
||||
repeat=repeat)
|
||||
runs.append(s)
|
||||
entries.append(m.matrix_entry(s))
|
||||
write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries,
|
||||
'complete': False})
|
||||
if not s['valid']:
|
||||
raise Failure('protocol-invalid run %s: %s (fix the measurement defect and '
|
||||
'rerun the whole block; raw evidence kept)' % (s['dir'], s['invalid_reasons']))
|
||||
by_mode = {mode: [r for r in runs if r['mode'] == mode] for mode in m.MODES}
|
||||
write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries, 'complete': True,
|
||||
'comparisons': comparisons(by_mode)})
|
||||
selection = m.select_p25(by_mode['full'], by_mode['p25'])
|
||||
ref = selection['reference']
|
||||
selected = 'p25' if selection['passed'] else 'full'
|
||||
write(self.out / 'selected.json', {
|
||||
'selected': selected, 'candidate': 'p25', 'candidate_passed': selection['passed'],
|
||||
'selection': selection, 'rate_rps': rate,
|
||||
'rare_event_miss_all_probability_model': rare_events(by_mode)})
|
||||
validations = []
|
||||
verdict = None
|
||||
order = [selected] if selected == 'full' else ['p25', 'full']
|
||||
for mode in order:
|
||||
s = self.execute_run(mode, rate, unique_dir(self.out / 'validation' / mode), label='validation')
|
||||
result = m.validate_single(s, mode, ref) if ref['complete'] else {
|
||||
'passed': False, 'checks': {'frozen_full_reference_complete': False}, 'mode': mode}
|
||||
validations.append({'mode': mode, 'run_dir': s['dir'], **result})
|
||||
if result['passed']:
|
||||
verdict = mode
|
||||
break
|
||||
final = {'selected_before_validation': selected, 'validated_mode': verdict,
|
||||
'validations': validations, 'passed': verdict is not None,
|
||||
'frozen_full_reference': ref}
|
||||
write(self.out / 'selected-validation.json', final)
|
||||
if verdict is None:
|
||||
raise Failure('selected-validation FAIL: neither p25 nor full passed independent validation')
|
||||
print('PASS: track1 selected=%s validated=%s' % (selected, verdict))
|
||||
|
||||
|
||||
def metric_paths(run):
|
||||
r = run['resources']
|
||||
api = r.get('api', {})
|
||||
return {
|
||||
'scheduled_p95_ms': run['load']['scheduled_ms']['p95'],
|
||||
'service_p95_ms': run['load']['service_ms']['p95'],
|
||||
'scheduled_p99_ms': run['load']['scheduled_ms']['p99'],
|
||||
'success_scheduled_p95_ms': run['load']['success_scheduled_ms']['p95'],
|
||||
'api_cpu_pct_sum_replicas': api.get('cpu_pct_time_weighted_mean'),
|
||||
'worker_cpu_pct': r.get('worker', {}).get('cpu_pct_time_weighted_mean'),
|
||||
'collector_cpu_pct': r.get('otel-collector', {}).get('cpu_pct_time_weighted_mean'),
|
||||
'api_container_mem_median': api.get('container_memory_bytes', {}).get('median'),
|
||||
'api_rss_median': api.get('process_rss_bytes', {}).get('median'),
|
||||
'worker_container_mem_median': r.get('worker', {}).get('container_memory_bytes', {}).get('median'),
|
||||
'spans_per_operation': run.get('spans_per_operation'),
|
||||
'log_records_per_operation': run.get('log_records_per_operation'),
|
||||
'metric_points_per_operation': run.get('metric_points_per_operation'),
|
||||
'retained_error_traces': run['cohorts']['error'].get('retained_traces'),
|
||||
'retained_slow_traces': run['cohorts']['slow'].get('retained_traces'),
|
||||
}
|
||||
|
||||
|
||||
def comparisons(by_mode):
|
||||
table = {mode: {} for mode in by_mode}
|
||||
for mode, runs in by_mode.items():
|
||||
keys = metric_paths(runs[0]).keys() if runs else []
|
||||
for key in keys:
|
||||
table[mode][key] = m.spread([metric_paths(r)[key] for r in runs])
|
||||
versus = {}
|
||||
for mode in by_mode:
|
||||
versus[mode] = {}
|
||||
for key in ('scheduled_p95_ms', 'service_p95_ms', 'api_cpu_pct_sum_replicas',
|
||||
'worker_cpu_pct', 'api_container_mem_median', 'api_rss_median'):
|
||||
versus[mode][key + '_vs_off'] = m.relative(
|
||||
table[mode].get(key, {}).get('median'), table['off'].get(key, {}).get('median'))
|
||||
for key in ('spans_per_operation', 'log_records_per_operation', 'metric_points_per_operation'):
|
||||
versus[mode][key + '_vs_full'] = m.relative(
|
||||
table[mode].get(key, {}).get('median'), table['full'].get(key, {}).get('median'))
|
||||
failures = {mode: [r['configuration_failures'] for r in runs] for mode, runs in by_mode.items()}
|
||||
return {'per_mode': table, 'relative': versus, 'configuration_failures': failures}
|
||||
|
||||
|
||||
def rare_events(by_mode):
|
||||
out = {}
|
||||
for mode, runs in by_mode.items():
|
||||
ratio = m.SAMPLER_RATIO[mode]
|
||||
if ratio is None:
|
||||
continue
|
||||
for cls in ('error', 'slow'):
|
||||
kept = sum(r['cohorts'][cls].get('retained_traces', 0) for r in runs)
|
||||
n = sum(r['cohorts'][cls].get('n', 0) for r in runs)
|
||||
out['%s/%s' % (mode, cls)] = {
|
||||
'N_total_over_runs': n, 'observed_kept': kept,
|
||||
'per_run_N': 40, 'miss_all_probability_per_run_model': m.miss_all_probability(ratio, 40),
|
||||
'note': '(1-r)^N assumes independent root sampling; a model, not a measurement'}
|
||||
return out
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||||
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
||||
parser.add_argument('--output-dir', default='evidence/lab6/track1')
|
||||
parser.add_argument('--smoke', action='store_true')
|
||||
args = parser.parse_args(argv)
|
||||
track = Track(args)
|
||||
try:
|
||||
with track.obs.app_environment(BASELINE_ENV) as set_env:
|
||||
track.set_env = set_env
|
||||
if args.smoke:
|
||||
track.smoke()
|
||||
else:
|
||||
track.full('attempt')
|
||||
except Failure as error:
|
||||
print('FAIL: ' + str(error), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+671
@@ -0,0 +1,671 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab6 verification: ready | inventory | signals | correlation | redaction |
|
||||
history. Python 3.9+, standard library only. Every subcommand writes its own
|
||||
evidence file under --output-dir and exits non-zero on any failed criterion."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import obs # noqa: E402
|
||||
from obs import Failure, stand, utc_now # noqa: E402
|
||||
|
||||
FORBIDDEN_LABEL = re.compile(
|
||||
r'(^|_)(user|chat|message|attachment|request|trace|span|job)_id$|'
|
||||
r'(^|_)(url|filename|text|body|username|password|token)($|_)')
|
||||
EXPECTED_SIGNALS = [
|
||||
# logical signal, metric, required label names (normalised), unit
|
||||
('HTTP duration/count', 'http_server_request_duration_seconds_count',
|
||||
('http_request_method', 'http_route', 'http_response_status_code'), 's'),
|
||||
('HTTP duration buckets', 'http_server_request_duration_seconds_bucket', ('le',), 's'),
|
||||
('HTTP duration sum', 'http_server_request_duration_seconds_sum', (), 's'),
|
||||
('Jobs processed', 'messenger_jobs_processed_total', ('job_type', 'outcome'), '{job}'),
|
||||
('Job duration', 'messenger_job_duration_seconds_count', ('job_type', 'outcome'), 's'),
|
||||
('Time to ready', 'messenger_attachment_time_to_ready_seconds_count', ('job_type',), 's'),
|
||||
('Queue depth', 'messenger_jobs_queue_depth', ('state',), '{job}'),
|
||||
('Queue oldest age', 'messenger_jobs_queue_oldest_age_seconds', (), 's'),
|
||||
('Outbox pending', 'messenger_outbox_pending', (), '{message}'),
|
||||
('Outbox oldest age', 'messenger_outbox_oldest_age_seconds', (), 's'),
|
||||
('DB pool connections', 'db_client_connection_count', ('db_client_connection_state',), '{connection}'),
|
||||
('DB pool max', 'db_client_connection_max', (), '{connection}'),
|
||||
('DB pool pending', 'db_client_connection_pending_requests', (), '{request}'),
|
||||
('DB pool wait', 'db_client_connection_wait_time_seconds_count', (), 's'),
|
||||
('DB operation duration', 'db_client_operation_duration_seconds_count', ('db_operation_name',), 's'),
|
||||
('Worker active', 'messenger_worker_jobs_active', (), '{job}'),
|
||||
('Worker capacity', 'messenger_worker_jobs_capacity', (), '{job}'),
|
||||
('Broker published', 'messenger_broker_published_total', ('outcome',), '{message}'),
|
||||
('Telemetry export', 'messenger_telemetry_export_items_total', ('signal', 'outcome'), '{item}'),
|
||||
('Process CPU', 'process_cpu_time_seconds_total', ('cpu_mode',), 's'),
|
||||
('Process RAM', 'process_memory_usage_bytes', (), 'By'),
|
||||
('V8 heap', 'v8js_memory_heap_used_bytes', (), 'By'),
|
||||
('Event loop delay', 'nodejs_eventloop_delay_p99_seconds', (), 's'),
|
||||
]
|
||||
COLLECTOR_SIGNALS = [
|
||||
'otelcol_receiver_accepted_spans', 'otelcol_receiver_accepted_log_records',
|
||||
'otelcol_receiver_accepted_metric_points', 'otelcol_exporter_sent_spans',
|
||||
'otelcol_exporter_sent_log_records', 'otelcol_exporter_sent_metric_points',
|
||||
'otelcol_exporter_queue_size', 'otelcol_exporter_queue_capacity',
|
||||
]
|
||||
COLLECTOR_FAILURE_ONLY = [
|
||||
'otelcol_exporter_send_failed_spans', 'otelcol_exporter_send_failed_log_records',
|
||||
'otelcol_exporter_send_failed_metric_points', 'otelcol_exporter_enqueue_failed_spans',
|
||||
'otelcol_exporter_enqueue_failed_log_records', 'otelcol_exporter_enqueue_failed_metric_points',
|
||||
]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
|
||||
def write_json(out, name, data):
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / name).write_text(json.dumps(data, indent=2, ensure_ascii=False,
|
||||
sort_keys=True) + '\n')
|
||||
|
||||
|
||||
def make_obs(args):
|
||||
return obs.ObsClient(args.prometheus_url, args.grafana_url,
|
||||
base_url=args.base_url, ca_file=args.ca_file)
|
||||
|
||||
|
||||
def ns_window(seconds_back=600, forward=30):
|
||||
now = time.time_ns()
|
||||
return now - seconds_back * 10 ** 9, now + forward * 10 ** 9
|
||||
|
||||
|
||||
def wait_rows(client, query, minimum=1, timeout=60, back=900):
|
||||
deadline = time.monotonic() + timeout
|
||||
rows = []
|
||||
while time.monotonic() < deadline:
|
||||
start, end = ns_window(back)
|
||||
rows = client.loki_rows(query, start, end)
|
||||
if len(rows) >= minimum:
|
||||
return rows
|
||||
time.sleep(2)
|
||||
raise Failure('expected log rows not found in Loki (' + str(len(rows)) +
|
||||
'/' + str(minimum) + ')')
|
||||
|
||||
|
||||
def http_log(client, request_id, timeout=60):
|
||||
rows = wait_rows(
|
||||
client, '{service_name="messenger-api"} | event_name="http.request" '
|
||||
'| request_id="' + request_id + '"', timeout=timeout)
|
||||
if len(rows) != 1:
|
||||
raise Failure('expected exactly one http.request log for the request')
|
||||
return rows[0]
|
||||
|
||||
|
||||
def chain_to_root(spans, span):
|
||||
by_id = {s['span_id']: s for s in spans}
|
||||
path = [span]
|
||||
while path[-1]['parent_span_id']:
|
||||
parent = by_id.get(path[-1]['parent_span_id'])
|
||||
if parent is None:
|
||||
raise Failure('broken parent chain at ' + path[-1]['name'])
|
||||
path.append(parent)
|
||||
if len(path) > 64:
|
||||
raise Failure('parent chain too long')
|
||||
return path
|
||||
|
||||
|
||||
def descendants(spans, root_id):
|
||||
children = {}
|
||||
for s in spans:
|
||||
children.setdefault(s['parent_span_id'], []).append(s)
|
||||
out, stack = [], [root_id]
|
||||
while stack:
|
||||
for child in children.get(stack.pop(), []):
|
||||
out.append(child)
|
||||
stack.append(child['span_id'])
|
||||
return out
|
||||
|
||||
|
||||
def safe_span(s):
|
||||
return {k: s[k] for k in ('span_id', 'parent_span_id', 'name', 'kind',
|
||||
'status', 'service', 'instance')}
|
||||
|
||||
|
||||
def trace_with(client, trace_id, predicate, timeout=60, description='trace'):
|
||||
deadline = time.monotonic() + timeout
|
||||
spans = None
|
||||
while time.monotonic() < deadline:
|
||||
spans = client.get_trace(trace_id)
|
||||
if spans and predicate(spans):
|
||||
return spans
|
||||
time.sleep(2)
|
||||
raise Failure(description + ' did not reach the expected shape in Tempo')
|
||||
|
||||
|
||||
def new_context(base_url, ca_file):
|
||||
c = stand.new_client(base_url, ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
return c, alice, chat
|
||||
|
||||
|
||||
def instrumented_env():
|
||||
return {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
|
||||
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- ready
|
||||
|
||||
def cmd_ready(args):
|
||||
client = make_obs(args)
|
||||
client.wait_ready(app_telemetry=True, provisioning=not args.backend_only)
|
||||
result = {'passed': True, 'backend_only': args.backend_only,
|
||||
'finished': utc_now()}
|
||||
write_json(Path(args.output_dir), 'ready-backend.json' if args.backend_only
|
||||
else 'ready.json', result)
|
||||
print('PASS: ready' + (' (backend only)' if args.backend_only else ''))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- inventory
|
||||
|
||||
def inventory(client):
|
||||
names = client.prom_names()
|
||||
rows = []
|
||||
_, meta = obs.http_json(client.prometheus + '/api/v1/metadata')
|
||||
for name in names:
|
||||
if not re.match(r'(http_server|messenger|db_client|process|nodejs|v8js|target_info)', name):
|
||||
continue
|
||||
_, series = obs.http_json(
|
||||
client.prometheus + '/api/v1/series?' +
|
||||
urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'}))
|
||||
labels = sorted({k for s in series['data'] for k in s} - {'__name__'})
|
||||
base = re.sub(r'_(bucket|count|sum|total)$', '', name)
|
||||
info = (meta['data'].get(name) or meta['data'].get(base) or [{}])[0]
|
||||
rows.append({'metric': name, 'type': info.get('type'),
|
||||
'unit': info.get('unit') or None, 'labels': labels,
|
||||
'series': len(series['data'])})
|
||||
return names, rows
|
||||
|
||||
|
||||
def cmd_inventory(args):
|
||||
client = make_obs(args)
|
||||
names, rows = inventory(client)
|
||||
out = Path(args.output_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / 'metric-names.txt').write_text('\n'.join(names) + '\n')
|
||||
write_json(out, 'inventory.json', {'captured': utc_now(), 'metrics': rows})
|
||||
print('inventory: %d names, %d application/runtime metrics' %
|
||||
(len(names), len(rows)))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ signals
|
||||
|
||||
def series_labels(client, name):
|
||||
_, series = obs.http_json(
|
||||
client.prometheus + '/api/v1/series?' +
|
||||
urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'}))
|
||||
return series['data']
|
||||
|
||||
|
||||
def cmd_signals(args):
|
||||
client = make_obs(args)
|
||||
names, rows = inventory(client)
|
||||
out = Path(args.output_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / 'metric-names.txt').write_text('\n'.join(names) + '\n')
|
||||
problems, table = [], []
|
||||
for logical, metric, required, unit in EXPECTED_SIGNALS:
|
||||
data = series_labels(client, metric)
|
||||
labels = sorted({k for s in data for k in s} - {'__name__'})
|
||||
missing = [r for r in required if r not in labels]
|
||||
table.append({'signal': logical, 'metric': metric, 'labels': labels,
|
||||
'unit': unit, 'series': len(data)})
|
||||
if not data:
|
||||
problems.append('missing metric ' + metric)
|
||||
elif missing:
|
||||
problems.append(metric + ' lacks labels ' + ','.join(missing))
|
||||
# Contract: forbidden labels on any application/runtime metric.
|
||||
for row in rows:
|
||||
for label in row['labels']:
|
||||
if FORBIDDEN_LABEL.search(label):
|
||||
problems.append('forbidden label ' + label + ' on ' + row['metric'])
|
||||
# Business facts: routes, replicas, outcomes.
|
||||
routes = {s.get('http_route') for s in series_labels(
|
||||
client, 'http_server_request_duration_seconds_count')}
|
||||
if not any(r and r.startswith('/api/v1/') and '{' in r for r in routes):
|
||||
problems.append('no route templates with {param} placeholders')
|
||||
if any(r and re.search(r'[0-9a-f]{8}-[0-9a-f]{4}', r) for r in routes):
|
||||
problems.append('raw UUID in http_route')
|
||||
replicas = {s['instance'] for s in series_labels(
|
||||
client, 'http_server_request_duration_seconds_count')
|
||||
if s.get('job') == 'messenger-api'}
|
||||
if len(replicas) < 2:
|
||||
problems.append('fewer than two API replicas in HTTP metrics')
|
||||
outcomes = {s.get('outcome') for s in series_labels(
|
||||
client, 'messenger_jobs_processed_total')}
|
||||
for needed in ('succeeded', 'retry', 'failed'):
|
||||
if needed not in outcomes:
|
||||
problems.append('job outcome not observed: ' + needed +
|
||||
' (run lab6-correlate first)')
|
||||
for job in ('messenger-api', 'messenger-worker'):
|
||||
if not client.prom_query('target_info{job="%s"}' % job):
|
||||
problems.append('no metrics from ' + job)
|
||||
# Queue snapshot is not doubled by the two API replicas.
|
||||
depth_series = client.prom_query('messenger_jobs_queue_depth{state="ready"}')
|
||||
# Collector.
|
||||
collector = {}
|
||||
for metric in COLLECTOR_SIGNALS:
|
||||
data = series_labels(client, metric)
|
||||
collector[metric] = len(data)
|
||||
if not data:
|
||||
problems.append('missing Collector metric ' + metric)
|
||||
for metric in COLLECTOR_FAILURE_ONLY:
|
||||
collector[metric] = len(series_labels(client, metric))
|
||||
if not client.prom_query('otelcol_process_memory_rss'):
|
||||
problems.append('no Collector RSS')
|
||||
# Logs/traces for both services.
|
||||
for service in ('messenger-api', 'messenger-worker'):
|
||||
rows_ = wait_rows(client, '{service_name="%s"}' % service, timeout=30)
|
||||
if not rows_:
|
||||
problems.append('no logs from ' + service)
|
||||
result = {'captured': utc_now(), 'signals': table,
|
||||
'collector_series': collector,
|
||||
'job_outcomes_observed': sorted(o for o in outcomes if o),
|
||||
'api_replicas_with_http_metrics': sorted(replicas),
|
||||
'queue_depth_ready_series': len(depth_series),
|
||||
'route_templates': sorted(r for r in routes if r),
|
||||
'problems': problems, 'passed': not problems}
|
||||
write_json(out, 'signals.json', result)
|
||||
if problems:
|
||||
raise Failure('signals: ' + '; '.join(problems))
|
||||
print('PASS: signals (%d logical signals)' % len(table))
|
||||
|
||||
|
||||
# -------------------------------------------------------------- correlation
|
||||
|
||||
def status_of(attachment_id):
|
||||
code, body = stand.cli('status', attachment_id)
|
||||
if code != 0:
|
||||
raise Failure('jobs.cli status failed')
|
||||
return body
|
||||
|
||||
|
||||
def correlate_http(client, base_url, ca_file, obs_client):
|
||||
c, alice, chat = new_context(base_url, ca_file)
|
||||
key = uuid.uuid4().hex
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
|
||||
actor=alice, body={'text': 'lab6 correlation'},
|
||||
headers={'Idempotency-Key': key}, schema='Message')
|
||||
rid = r['headers']['X-Request-Id']
|
||||
log = http_log(obs_client, rid)
|
||||
labels = log['labels']
|
||||
if labels.get('trace_sampled') != 'true':
|
||||
raise Failure('HTTP log is not marked sampled')
|
||||
spans = trace_with(
|
||||
obs_client, labels['trace_id'],
|
||||
lambda sp: any(s['span_id'] == labels['span_id'] for s in sp) and
|
||||
len(descendants(sp, labels['span_id'])) > 0, description='HTTP trace')
|
||||
server = next(s for s in spans if s['span_id'] == labels['span_id'])
|
||||
if server['kind'] != 'SERVER' or \
|
||||
server['attributes'].get('messenger.request_id') != rid:
|
||||
raise Failure('SERVER span does not match the logged request')
|
||||
db = [s for s in descendants(spans, server['span_id'])
|
||||
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']]
|
||||
if not db:
|
||||
raise Failure('no DB CLIENT child spans under the SERVER span')
|
||||
return {'request_id': rid, 'trace_id': labels['trace_id'],
|
||||
'server_span': safe_span(server),
|
||||
'log_span_id': labels['span_id'],
|
||||
'db_children': [safe_span(s) for s in db]}
|
||||
|
||||
|
||||
def upload_and_trace(c, alice, chat, obs_client, require_attempts=1):
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] +
|
||||
'/attachments?filename=sample.png', 202, actor=alice,
|
||||
raw=stand.sample_bytes(), headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment')
|
||||
return r['json'], r['headers']['X-Request-Id']
|
||||
|
||||
|
||||
def consumer_spans(spans):
|
||||
return sorted((s for s in spans if s['kind'] == 'CONSUMER'),
|
||||
key=lambda s: s['start_ns'])
|
||||
|
||||
|
||||
def correlate_upload(client_args, obs_client):
|
||||
c, alice, chat = new_context(client_args.base_url, client_args.ca_file)
|
||||
att, rid = upload_and_trace(c, alice, chat, obs_client)
|
||||
stand.wait_ready(c, alice, att['id'])
|
||||
log = http_log(obs_client, rid)
|
||||
tid = log['labels']['trace_id']
|
||||
|
||||
def complete(sp):
|
||||
cons = consumer_spans(sp)
|
||||
return bool(cons) and any(s['kind'] == 'SERVER' for s in sp)
|
||||
|
||||
spans = trace_with(obs_client, tid, complete, description='upload trace')
|
||||
consumer = consumer_spans(spans)[0]
|
||||
path = chain_to_root(spans, consumer)
|
||||
kinds = [s['kind'] for s in path]
|
||||
if path[-1]['kind'] != 'SERVER' or 'PRODUCER' not in kinds:
|
||||
raise Failure('consumer is not linked to the HTTP SERVER via PRODUCER')
|
||||
if path[-1]['service'] != 'messenger-api' or consumer['service'] != 'messenger-worker':
|
||||
raise Failure('services along the chain are wrong')
|
||||
below = descendants(spans, consumer['span_id'])
|
||||
s3 = [s for s in below if s['attributes'].get('rpc.system') == 'aws-api']
|
||||
db = [s for s in below if 'db.operation.name' in s['attributes']]
|
||||
if not s3 or not db:
|
||||
raise Failure('consumer has no S3/DB child spans')
|
||||
return {'request_id': rid, 'trace_id': tid, 'attachment_id': att['id'],
|
||||
'chain_consumer_to_root': [safe_span(s) for s in path],
|
||||
'consumer_attempt': consumer['attributes'].get('messenger.job.attempt'),
|
||||
'consumer_outcome': consumer['attributes'].get('messenger.job.outcome'),
|
||||
's3_children': len(s3), 'db_children': len(db),
|
||||
'chain_fields_are_real_parent_child': True}
|
||||
|
||||
|
||||
def start_worker():
|
||||
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
|
||||
|
||||
|
||||
def targeted_job(c, alice, chat, obs_client):
|
||||
"""Stop the worker, accept an upload, return (att, rid, job_id, version)."""
|
||||
stand.compose('stop', 'worker')
|
||||
att, rid = upload_and_trace(c, alice, chat, obs_client)
|
||||
body = status_of(att['id'])
|
||||
jobs = body.get('jobs')
|
||||
if not isinstance(jobs, list) or len(jobs) != 1:
|
||||
raise Failure('expected exactly one job for the new upload')
|
||||
return att, rid, jobs[0]['job_id'], jobs[0]['pipeline_version']
|
||||
|
||||
|
||||
def correlate_fault(client_args, obs_client, kind):
|
||||
c, alice, chat = new_context(client_args.base_url, client_args.ca_file)
|
||||
obs_client.require_no_foreign_faults()
|
||||
key = None
|
||||
try:
|
||||
att, rid, job_id, version = targeted_job(c, alice, chat, obs_client)
|
||||
key = 'fault:worker.before_side_effect:%s:%s' % (att['id'], version)
|
||||
stand.set_fault(key, kind)
|
||||
start_worker()
|
||||
if kind == 'transient':
|
||||
def retried():
|
||||
job = status_of(att['id'])['jobs'][0]
|
||||
return job['attempt'] >= 2 or job['last_error_code']
|
||||
stand.wait_for(retried, 90, 'first retry')
|
||||
stand.clear_fault(key)
|
||||
key = None
|
||||
stand.wait_ready(c, alice, att['id'], timeout=120)
|
||||
else:
|
||||
def failed():
|
||||
return status_of(att['id'])['jobs'][0]['status'] == 'failed'
|
||||
stand.wait_for(failed, 90, 'terminal failed job')
|
||||
meta = stand.wait_ready(c, alice, att['id'], allow_failed=True)
|
||||
if meta['status'] != 'failed':
|
||||
raise Failure('attachment did not become failed')
|
||||
final = status_of(att['id'])
|
||||
log = http_log(obs_client, rid)
|
||||
tid = log['labels']['trace_id']
|
||||
wanted = 2 if kind == 'transient' else 1
|
||||
|
||||
def enough(sp):
|
||||
return len(consumer_spans(sp)) >= wanted
|
||||
|
||||
spans = trace_with(obs_client, tid, enough, timeout=90,
|
||||
description=kind + ' trace')
|
||||
consumers = consumer_spans(spans)
|
||||
ids = {s['span_id'] for s in consumers}
|
||||
if len(ids) != len(consumers):
|
||||
raise Failure('consumer attempts share a span ID')
|
||||
attempts = [s['attributes'].get('messenger.job.attempt') for s in consumers]
|
||||
outcomes = [s['attributes'].get('messenger.job.outcome') for s in consumers]
|
||||
if kind == 'transient':
|
||||
if sorted(attempts)[:2] != [1, 2] or 'retry' not in outcomes:
|
||||
raise Failure('retry attempts not visible in the trace: %s %s' % (attempts, outcomes))
|
||||
else:
|
||||
if 'failed' not in outcomes:
|
||||
raise Failure('failed outcome not visible in the trace')
|
||||
return {'kind': kind, 'request_id': rid, 'trace_id': tid,
|
||||
'attachment_id': att['id'], 'job_id': job_id,
|
||||
'pipeline_version': version,
|
||||
'consumer_spans': [safe_span(s) | {
|
||||
'attempt': s['attributes'].get('messenger.job.attempt'),
|
||||
'outcome': s['attributes'].get('messenger.job.outcome')}
|
||||
for s in consumers],
|
||||
'final_job': {k: final['jobs'][0].get(k) for k in
|
||||
('status', 'attempt', 'last_error_code')},
|
||||
'final_attachment_status': final['attachment']['status']}
|
||||
finally:
|
||||
if key:
|
||||
stand.clear_fault(key)
|
||||
start_worker()
|
||||
|
||||
|
||||
def cmd_correlation(args):
|
||||
client = make_obs(args)
|
||||
out = Path(args.output_dir)
|
||||
with client.app_environment(instrumented_env()):
|
||||
client.require_no_foreign_faults()
|
||||
results = {
|
||||
'correlation-http.json': correlate_http(client, args.base_url, args.ca_file, client),
|
||||
'correlation-upload.json': correlate_upload(args, client),
|
||||
'correlation-retry.json': correlate_fault(args, client, 'transient'),
|
||||
'correlation-failed.json': correlate_fault(args, client, 'permanent'),
|
||||
}
|
||||
for name, data in results.items():
|
||||
data['captured'] = utc_now()
|
||||
write_json(out, name, data)
|
||||
print('PASS: correlation (http, upload, retry, failed)')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- redaction
|
||||
|
||||
PEM = re.compile(r'-----BEGIN [A-Z ]*PRIVATE KEY-----')
|
||||
|
||||
|
||||
def fragments(text, size=48, stride=16):
|
||||
return [text[i:i + size] for i in range(0, max(1, len(text) - size + 1), stride)]
|
||||
|
||||
|
||||
def flatten(value, path=''):
|
||||
if isinstance(value, dict):
|
||||
for k, v in value.items():
|
||||
yield from flatten(v, path + '.' + str(k))
|
||||
yield path + '.key:' + str(k)
|
||||
elif isinstance(value, list):
|
||||
for i, v in enumerate(value):
|
||||
yield from flatten(v, path + '[' + str(i) + ']')
|
||||
else:
|
||||
yield path + '=' + str(value)
|
||||
|
||||
|
||||
def scan(texts, secrets):
|
||||
"""texts: [(source, text)]; secrets: {category: [needle,...]}."""
|
||||
leaks = []
|
||||
for source, text in texts:
|
||||
for category, needles in secrets.items():
|
||||
for needle in needles:
|
||||
if needle and needle in text:
|
||||
leaks.append({'category': category, 'source': source,
|
||||
'hash': hashlib.sha256(needle.encode()).hexdigest()[:12]})
|
||||
break
|
||||
if PEM.search(text):
|
||||
leaks.append({'category': 'private_key_pem', 'source': source, 'hash': ''})
|
||||
return leaks
|
||||
|
||||
|
||||
def cmd_redaction(args):
|
||||
client = make_obs(args)
|
||||
out = Path(args.output_dir)
|
||||
with client.app_environment(instrumented_env()):
|
||||
c = stand.new_client(args.base_url, args.ca_file)
|
||||
actor = c.user('redact')
|
||||
marker = 'LAB6-REDACT-' + uuid.uuid4().hex
|
||||
password = actor['registration']['password']
|
||||
token = actor['token']
|
||||
username = actor['registration']['username']
|
||||
basic = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
signature = 'sig' + uuid.uuid4().hex + uuid.uuid4().hex
|
||||
own_chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
|
||||
body={'title': 'redaction', 'member_ids': [actor['id']]},
|
||||
schema='Chat')['json']
|
||||
requests = {}
|
||||
r = c.call('POST', '/api/v1/chats/' + own_chat['id'] + '/messages', 201,
|
||||
actor=actor, body={'text': 'body ' + marker},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
|
||||
requests['message'] = r['headers']['X-Request-Id']
|
||||
r = c.call('GET', '/api/v1/chats?X-Amz-Signature=' + signature, range(200, 500),
|
||||
actor=actor, headers={'X-Marker': marker})
|
||||
requests['signed_query'] = r['headers']['X-Request-Id']
|
||||
r = c.call('POST', '/api/v1/auth/sessions', 401,
|
||||
headers={'Authorization': 'Basic ' + base64.b64encode(
|
||||
(username + ':wrong-' + marker).encode()).decode()})
|
||||
requests['bad_login'] = r['headers']['X-Request-Id']
|
||||
att, rid = upload_and_trace(c, actor, own_chat, client)
|
||||
requests['upload'] = rid
|
||||
stand.wait_ready(c, actor, att['id'])
|
||||
sample = stand.sample_bytes()
|
||||
b64 = base64.b64encode(sample).decode()
|
||||
hexs = sample.hex()
|
||||
secrets = {
|
||||
'marker': [marker], 'password': [password], 'session_token': [token],
|
||||
'basic_credential': [basic], 'bearer_header': ['Bearer ' + token],
|
||||
'signature': [signature], 'image_base64': [b64],
|
||||
'image_hex': [hexs],
|
||||
'image_base64_fragment': fragments(b64), 'image_hex_fragment': fragments(hexs),
|
||||
'wrong_password_marker': ['wrong-' + marker],
|
||||
}
|
||||
coverage = {'http_logs': {}, 'server_spans': {}, 'worker_chain': False}
|
||||
texts = []
|
||||
trace_ids = {}
|
||||
for name, request_id in requests.items():
|
||||
log = http_log(client, request_id)
|
||||
coverage['http_logs'][name] = True
|
||||
texts.append(('log:' + name, log['line'] + json.dumps(log['labels'], sort_keys=True)))
|
||||
trace_ids[name] = log['labels']['trace_id']
|
||||
for name, tid in trace_ids.items():
|
||||
def has_server(sp, tid=tid, name=name):
|
||||
return any(s['kind'] == 'SERVER' and
|
||||
s['attributes'].get('messenger.request_id') == requests[name]
|
||||
for s in sp)
|
||||
spans = trace_with(client, tid, has_server, description='trace ' + name)
|
||||
coverage['server_spans'][name] = True
|
||||
if name == 'upload':
|
||||
coverage['worker_chain'] = any(s['kind'] == 'CONSUMER' for s in spans) \
|
||||
if any(s['kind'] == 'CONSUMER' for s in spans) else False
|
||||
if not coverage['worker_chain']:
|
||||
spans = trace_with(client, tid, lambda sp: any(
|
||||
s['kind'] == 'CONSUMER' for s in sp), description='worker chain')
|
||||
coverage['worker_chain'] = True
|
||||
for s in spans:
|
||||
for item in flatten({k: s[k] for k in ('name', 'attributes', 'status')}):
|
||||
texts.append(('span:' + name + ':' + s['kind'], item))
|
||||
logs = wait_rows(client, '{service_name=~"messenger-.*"} | trace_id="%s"' % tid)
|
||||
for row in logs:
|
||||
texts.append(('tracelog:' + name, row['line'] + json.dumps(row['labels'], sort_keys=True)))
|
||||
worker_logs = wait_rows(
|
||||
client, '{service_name="messenger-worker"} | messenger_attachment_id="%s"' % att['id'])
|
||||
coverage['worker_logs'] = len(worker_logs)
|
||||
for row in worker_logs:
|
||||
texts.append(('workerlog', row['line'] + json.dumps(row['labels'], sort_keys=True)))
|
||||
leaks = scan(texts, secrets)
|
||||
complete = (all(coverage['http_logs'].get(n) for n in requests) and
|
||||
all(coverage['server_spans'].get(n) for n in requests) and
|
||||
coverage['worker_chain'] and coverage['worker_logs'] > 0)
|
||||
result = {'captured': utc_now(), 'coverage': coverage, 'coverage_complete': complete,
|
||||
'scanned_items': len(texts), 'leak_count': len(leaks), 'leaks': leaks,
|
||||
'passed': complete and not leaks}
|
||||
write_json(out, 'redaction.json', result)
|
||||
(out / 'redaction.txt').write_text(
|
||||
'coverage_complete=%s scanned_items=%d leaks=%d\n' %
|
||||
(complete, len(texts), len(leaks)))
|
||||
if not result['passed']:
|
||||
raise Failure('redaction: coverage_complete=%s leaks=%d' % (complete, len(leaks)))
|
||||
print('PASS: redaction (%d items scanned, 0 leaks)' % len(texts))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ history
|
||||
|
||||
def raw_samples(client, selector, at, window='30s'):
|
||||
result = client.prom_query(selector + '[' + window + ']', at=at)
|
||||
return {json.dumps(r['metric'], sort_keys=True): r['values'] for r in result}
|
||||
|
||||
|
||||
def cmd_history(args):
|
||||
client = make_obs(args)
|
||||
out = Path(args.output_dir)
|
||||
with client.app_environment(instrumented_env()):
|
||||
c, alice, chat = new_context(args.base_url, args.ca_file)
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
|
||||
actor=alice, body={'text': 'history'},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
|
||||
rid_message = r['headers']['X-Request-Id']
|
||||
att, rid = upload_and_trace(c, alice, chat, client)
|
||||
stand.wait_ready(c, alice, att['id'])
|
||||
log = http_log(client, rid)
|
||||
tid = log['labels']['trace_id']
|
||||
spans = trace_with(client, tid, lambda sp: any(s['kind'] == 'CONSUMER' for s in sp),
|
||||
description='history trace')
|
||||
msg_log = http_log(client, rid_message)
|
||||
client.wait_telemetry_flushed()
|
||||
time.sleep(12)
|
||||
client.wait_telemetry_flushed()
|
||||
at = time.time()
|
||||
selector = 'messenger_jobs_processed_total{job="messenger-worker",outcome="succeeded"}'
|
||||
before = raw_samples(client, selector, at)
|
||||
if not before:
|
||||
raise Failure('no Prometheus samples to preserve')
|
||||
series_key, values = next(iter(before.items()))
|
||||
old_sample = {'labels': json.loads(series_key), 'timestamp': values[-1][0],
|
||||
'value': values[-1][1]}
|
||||
old_log = {'ts': log['ts'], 'request_id': rid, 'trace_id': tid,
|
||||
'span_id': log['labels']['span_id']}
|
||||
old_trace = sorted(s['span_id'] for s in spans)
|
||||
old_msg = {'ts': msg_log['ts'], 'trace_id': msg_log['labels']['trace_id']}
|
||||
stand.compose('restart', 'prometheus', 'loki', 'tempo', 'grafana')
|
||||
client.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
|
||||
after_samples = raw_samples(client, selector, at)
|
||||
after = after_samples.get(series_key, [])
|
||||
prom_ok = any(abs(v[0] - old_sample['timestamp']) < 1e-6 and
|
||||
v[1] == old_sample['value'] for v in after)
|
||||
rows = wait_rows(client, '{service_name="messenger-api"} | request_id="%s"' % rid)
|
||||
loki_ok = any(row['ts'] == old_log['ts'] and
|
||||
row['labels'].get('trace_id') == tid for row in rows)
|
||||
spans_after = client.get_trace(tid, wait=30)
|
||||
tempo_ok = bool(spans_after) and \
|
||||
sorted(s['span_id'] for s in spans_after) == old_trace
|
||||
result = {'captured': utc_now(), 'prometheus_old_sample': old_sample,
|
||||
'prometheus_read_back': prom_ok, 'loki_old_log': old_log,
|
||||
'loki_read_back': loki_ok, 'tempo_old_trace_span_ids': old_trace,
|
||||
'tempo_read_back': tempo_ok,
|
||||
'passed': prom_ok and loki_ok and tempo_ok}
|
||||
write_json(out, 'history.json', result)
|
||||
if not result['passed']:
|
||||
raise Failure('history: old data not readable after restart: %s' % {
|
||||
k: result[k] for k in ('prometheus_read_back', 'loki_read_back', 'tempo_read_back')})
|
||||
print('PASS: history (old Prometheus sample, Loki log and Tempo trace survive restart)')
|
||||
|
||||
|
||||
COMMANDS = {'ready': cmd_ready, 'inventory': cmd_inventory, 'signals': cmd_signals,
|
||||
'correlation': cmd_correlation, 'redaction': cmd_redaction,
|
||||
'history': cmd_history}
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=sorted(COMMANDS))
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||||
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
||||
parser.add_argument('--output-dir', default='evidence/lab6')
|
||||
parser.add_argument('--backend-only', action='store_true')
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
COMMANDS[args.command](args)
|
||||
except Failure as error:
|
||||
print('FAIL: ' + str(error), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Fixed-rate open-loop workload for lab6 (stdlib only).
|
||||
|
||||
Slot i is planned at t0 + i/rate on a monotonic clock; the generator never
|
||||
waits for earlier requests and never catches up with a burst. When no
|
||||
in-flight slot is available the slot is recorded as client_dropped and stays
|
||||
in the denominator."""
|
||||
import math
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import measurement # noqa: E402
|
||||
import obs # noqa: E402
|
||||
from obs import Failure, stand # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT = 30
|
||||
MESSAGE_TEXT = 'lab6-track1'
|
||||
|
||||
|
||||
def slot_kind(i, upload_every):
|
||||
if upload_every == 0:
|
||||
return 'message'
|
||||
if upload_every == 1:
|
||||
return 'upload'
|
||||
return 'upload' if (i + 1) % upload_every == 0 else 'message'
|
||||
|
||||
|
||||
class _Local(threading.local):
|
||||
client = None
|
||||
|
||||
|
||||
def run_window(base_url, ca_file, actor, chat, rate, duration, phase_id,
|
||||
upload_every=20, max_inflight=64, stop_event=None, settle=True,
|
||||
obs_client=None):
|
||||
"""Run one window and return {'slots', 'load', 'uploads', 'drain', ...}.
|
||||
|
||||
stop_event stops only new slots; started requests complete. settle=False
|
||||
returns right after HTTP (no job/Collector drain)."""
|
||||
local = _Local()
|
||||
image = stand.sample_bytes()
|
||||
lock = threading.Lock()
|
||||
inflight = [0]
|
||||
planned = []
|
||||
|
||||
def client():
|
||||
if local.client is None:
|
||||
local.client = stand.new_client(base_url, ca_file)
|
||||
local.client.timeout = REQUEST_TIMEOUT
|
||||
return local.client
|
||||
|
||||
def execute(record, kind, t0_mono):
|
||||
record['start'] = time.monotonic() - t0_mono
|
||||
record['start_lag_ms'] = (record['start'] - record['scheduled']) * 1000.0
|
||||
started = time.monotonic()
|
||||
try:
|
||||
c = client()
|
||||
if kind == 'message':
|
||||
result = c.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/messages',
|
||||
range(200, 600), actor=actor, body={'text': MESSAGE_TEXT},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex,
|
||||
'X-Request-Id': uuid.uuid4().hex},
|
||||
schema='Message')
|
||||
else:
|
||||
result = c.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] +
|
||||
'/attachments?filename=sample.png', range(200, 600),
|
||||
actor=actor, raw=image,
|
||||
headers={'Content-Type': 'image/png',
|
||||
'X-Request-Id': uuid.uuid4().hex},
|
||||
schema='Attachment')
|
||||
end = time.monotonic()
|
||||
record['status'] = result['status']
|
||||
record['request_id'] = result['headers'].get('X-Request-Id')
|
||||
record['instance_id'] = result['headers'].get('X-Instance-Id')
|
||||
if kind == 'upload' and result['status'] == 202 and result['json']:
|
||||
record['attachment_id'] = result['json']['id']
|
||||
record['outcome'] = 'http'
|
||||
except Failure as error:
|
||||
end = time.monotonic()
|
||||
text = str(error)
|
||||
record['outcome'] = ('timeout' if 'TimeoutError' in text else
|
||||
'transport_error' if 'connection/TLS' in text
|
||||
else 'protocol_error')
|
||||
record['error'] = text[:160]
|
||||
finally:
|
||||
with lock:
|
||||
inflight[0] -= 1
|
||||
record['end'] = end - t0_mono
|
||||
record['service_ms'] = (end - started) * 1000.0
|
||||
record['scheduled_ms'] = (record['end'] - record['scheduled']) * 1000.0
|
||||
|
||||
pool = ThreadPoolExecutor(max_workers=max_inflight)
|
||||
futures = []
|
||||
t0 = time.monotonic()
|
||||
started_utc = obs.utc_now()
|
||||
total = int(math.floor(rate * duration))
|
||||
try:
|
||||
for i in range(total):
|
||||
due = t0 + i / rate
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
remaining = due - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
time.sleep(min(remaining, 0.05))
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
kind = slot_kind(i, upload_every)
|
||||
record = {'i': i, 'phase': phase_id, 'name': '%s-%d' % (phase_id, i),
|
||||
'kind': kind, 'scheduled': i / rate, 'outcome': None,
|
||||
'start_lag_ms': None}
|
||||
planned.append(record)
|
||||
with lock:
|
||||
free = inflight[0] < max_inflight
|
||||
if free:
|
||||
inflight[0] += 1
|
||||
if not free:
|
||||
record['outcome'] = 'client_dropped'
|
||||
continue
|
||||
futures.append(pool.submit(execute, record, kind, t0))
|
||||
for future in futures:
|
||||
future.result()
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
dispatch_end = time.monotonic() - t0
|
||||
load = measurement.summarize_load(planned, rate)
|
||||
uploads = [s['attachment_id'] for s in planned if s.get('attachment_id')]
|
||||
result = {'phase': phase_id, 'slots': planned, 'load': load,
|
||||
'uploads': uploads, 'started': started_utc,
|
||||
'dispatch_seconds': dispatch_end, 'finished_dispatch': obs.utc_now(),
|
||||
'stopped_early': bool(stop_event is not None and stop_event.is_set())}
|
||||
if settle:
|
||||
result['drain'] = obs.wait_drained(uploads, obs=obs_client)
|
||||
return result
|
||||
+2
-2
@@ -4,5 +4,5 @@
|
||||
set -euo pipefail
|
||||
|
||||
compose=(docker compose --env-file .env)
|
||||
"${compose[@]}" restart postgres redis s3 broker api worker haproxy
|
||||
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy
|
||||
"${compose[@]}" restart postgres redis s3 broker api worker haproxy otel-collector prometheus loki tempo grafana
|
||||
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy otel-collector prometheus loki tempo grafana
|
||||
|
||||
Reference in New Issue
Block a user