lab6: Collector, Prometheus/Loki/Tempo/Grafana, alerts, incidents, track 1 (cost of observability)

This commit is contained in:
lab6
2026-11-10 01:57:47 +03:00
parent 63dba7627c
commit d558148d0d
142 changed files with 94773 additions and 88 deletions
+1
View File
@@ -40,6 +40,7 @@ write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)"
write_secret s3_api_secret_access_key "$(openssl rand -hex 24)"
write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)"
write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)"
write_secret grafana_admin_password "$(openssl rand -hex 24)"
if [ -s secrets/broker_worker_password ] && [ -s secrets/broker_worker_url ]; then
echo "kept secrets/broker_worker_password secrets/broker_worker_url"
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""Generates the two provisioned Grafana dashboards (stdlib only).
Output: infra/grafana/dashboards/messenger-overview.json and
messenger-telemetry.json. Metric names were verified against the live
Prometheus (verify.py inventory). Percentiles are computed from summed buckets,
never averaged across replicas; queue/outbox snapshots use max() because every
API replica publishes the same PostgreSQL snapshot.
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[2] / 'infra/grafana/dashboards'
PROM = {'type': 'prometheus', 'uid': 'prometheus'}
LOKI = {'type': 'loki', 'uid': 'loki'}
BIZ = 'job="messenger-api",http_route!~"/health.*|/internal/.*"'
class Board:
def __init__(self, uid, title):
self.uid, self.title = uid, title
self.panels = []
self.y = 0
self.x = 0
self.row_h = 0
self.next_id = 1
def row(self, title):
if self.x:
self.y += self.row_h
self.x = 0
self.panels.append({'type': 'row', 'title': title, 'collapsed': False,
'id': self._id(), 'panels': [],
'gridPos': {'h': 1, 'w': 24, 'x': 0, 'y': self.y}})
self.y += 1
self.row_h = 0
def _id(self):
self.next_id += 1
return self.next_id - 1
def _place(self, w, h):
if self.x + w > 24:
self.y += self.row_h
self.x, self.row_h = 0, 0
pos = {'h': h, 'w': w, 'x': self.x, 'y': self.y}
self.x += w
self.row_h = max(self.row_h, h)
return pos
def ts(self, title, queries, unit, description, w=12, h=8, vmin=None, vmax=None):
self.panels.append({
'type': 'timeseries', 'title': title, 'id': self._id(),
'description': description, 'datasource': PROM,
'gridPos': self._place(w, h),
'fieldConfig': {'defaults': {**{'unit': unit, 'custom': {
'lineWidth': 1, 'fillOpacity': 8, 'showPoints': 'never'}},
**({'min': vmin} if vmin is not None else {}),
**({'max': vmax} if vmax is not None else {})},
'overrides': []},
'options': {'legend': {'displayMode': 'list', 'placement': 'bottom'},
'tooltip': {'mode': 'multi'}},
'targets': [{'refId': chr(65 + i), 'datasource': PROM, 'expr': expr,
'legendFormat': legend}
for i, (expr, legend) in enumerate(queries)]})
def stat(self, title, expr, unit, description, w=6, h=4, legend=''):
self.panels.append({
'type': 'stat', 'title': title, 'id': self._id(),
'description': description, 'datasource': PROM,
'gridPos': self._place(w, h),
'fieldConfig': {'defaults': {'unit': unit}, 'overrides': []},
'options': {'reduceOptions': {'calcs': ['lastNotNull']},
'colorMode': 'value'},
'targets': [{'refId': 'A', 'datasource': PROM, 'expr': expr,
'legendFormat': legend}]})
def table(self, title, expr, description, w=12, h=8):
self.panels.append({
'type': 'table', 'title': title, 'id': self._id(),
'description': description, 'datasource': PROM,
'gridPos': self._place(w, h),
'fieldConfig': {'defaults': {'unit': 'short'}, 'overrides': []},
'options': {'showHeader': True},
'targets': [{'refId': 'A', 'datasource': PROM, 'expr': expr,
'instant': True, 'format': 'table'}],
'transformations': [{'id': 'organize', 'options': {
'excludeByName': {'Time': True, '__name__': True,
'job': True, 'Value': True}}}]})
def logs(self, title, expr, description, w=12, h=10):
self.panels.append({
'type': 'logs', 'title': title, 'id': self._id(),
'description': description, 'datasource': LOKI,
'gridPos': self._place(w, h),
'options': {'showTime': True, 'wrapLogMessage': True,
'sortOrder': 'Descending', 'enableLogDetails': True},
'targets': [{'refId': 'A', 'datasource': LOKI, 'expr': expr,
'queryType': 'range'}]})
def render(self):
return {
'uid': self.uid, 'title': self.title, 'schemaVersion': 39,
'version': 1, 'editable': False, 'refresh': '10s',
'time': {'from': 'now-15m', 'to': 'now'}, 'timezone': 'utc',
'tags': ['messenger', 'lab6'], 'templating': {'list': []},
'annotations': {'list': []}, 'panels': self.panels}
def quantiles(metric, by='', selector='', window='1m'):
group = 'le' + (',' + by if by else '')
legend = ' {{' + by + '}}' if by else ''
return [('histogram_quantile(%s, sum by(%s)(rate(%s_bucket%s[%s])))'
% (q, group, metric, selector, window), 'p%d%s' % (round(q * 100), legend))
for q in (0.5, 0.95, 0.99)]
def overview():
b = Board('messenger-overview', 'Messenger: API и worker')
b.row('API (бизнес-запросы; /health* и /internal/* исключены)')
b.ts('RPS по маршруту и классу статуса', [(
'sum by (http_route, status_class)(label_replace(rate(http_server_request_duration_seconds_count{%s}[1m]), "status_class", "${1}xx", "http_response_status_code", "(.).."))' % BIZ,
'{{http_route}} {{status_class}}')], 'reqps',
'rate(http_server_request_duration_seconds_count[1m]) по http_route и классу кода; число запросов — _count гистограммы.')
b.ts('Доля 5xx (1m)', [('messenger:api_errors:rate1m / messenger:api_requests:rate1m', '5xx / бизнес-запросы')],
'percentunit', 'Recording rules: 5xx rate / все бизнес-запросы за 1m; 4xx — не ошибка сервиса. Пусто без трафика (N/A).', vmin=0, vmax=1)
b.stat('Availability (5m), цель ≥ 99.5%', 'messenger:api_availability:ratio5m',
'percentunit', '1 - 5xx/все бизнес-запросы за 5m (recording rule messenger:api_availability:ratio5m). N/A без трафика.')
b.stat('Свежие реплики API', 'count((time() - timestamp(target_info{job="messenger-api"})) < 30) or vector(0)',
'short', 'Число API-реплик, чей target_info обновлялся за последние 30 с (OTLP-метрики).')
b.stat('HAProxy 5xx, rps', 'sum(rate(haproxy_backend_http_responses_total{proxy="api",code="5xx"}[1m])) or vector(0)',
'reqps', 'HAProxy backend api: ответы 5xx (включая 503 балансировщика, когда нет готовых узлов).')
b.stat('HAProxy: активные серверы', 'haproxy_backend_active_servers{proxy="api"}', 'short',
'Активные серверы backend api по данным HAProxy PROMEX.')
b.ts('Задержка HTTP: общая p50/p95/p99', quantiles('http_server_request_duration_seconds', selector='{%s}' % BIZ),
's', 'histogram_quantile по суммированным бакетам всех реплик; перцентили реплик не усредняются.')
b.ts('Задержка HTTP по маршруту p50/p95/p99', quantiles('http_server_request_duration_seconds', by='http_route', selector='{%s}' % BIZ),
's', 'То же, с разбивкой по route template.')
b.row('Worker и задачи')
b.ts('Задачи по outcome', [('sum by (outcome)(rate(messenger_jobs_processed_total[1m]))', '{{outcome}}')], 'ops',
'messenger_jobs_processed_total по фактическому outcome (succeeded, retry, failed, ...), задач/с.')
b.ts('Длительность попытки задачи p50/p95/p99', quantiles('messenger_job_duration_seconds'), 's',
'От claim до commit одной попытки; не включает ожидание в очереди.')
b.ts('Time-to-ready p50/p95/p99', quantiles('messenger_attachment_time_to_ready_seconds'), 's',
'От durable acceptance (created_at задачи) до commit ready; включает очередь, ретраи и backoff. Измеряется на commit, до ACK и cleanup.')
b.ts('Worker: занято / ёмкость', [('sum(messenger_worker_jobs_active)', 'active'),
('max(messenger_worker_jobs_capacity)', 'capacity (WORKER_CONCURRENCY)')],
'short', 'Задач в работе против WORKER_CONCURRENCY; показатели не суммируются.')
b.row('Очередь и outbox (snapshot PostgreSQL, max по репликам)')
b.ts('Глубина очереди по состоянию', [('max by (state)(messenger_jobs_queue_depth)', '{{state}}')], 'short',
'Незавершённые задачи по state; обе реплики публикуют одно значение, поэтому max, а не sum.')
b.ts('Возраст самой старой задачи', [('max(messenger_jobs_queue_oldest_age_seconds)', 'oldest job age')], 's',
'Возраст ожидания backlog (alert MessengerJobsQueueLagGrowing: > 30 с), не время обработки.')
b.ts('Outbox: ожидающие и возраст', [('max(messenger_outbox_pending)', 'pending (шт.)'),
('max(messenger_outbox_oldest_age_seconds)', 'oldest age (с)')],
'short', 'Неопубликованные записи outbox (доступные к публикации) и возраст старейшей.')
b.ts('RabbitMQ: ready / unacked', [('sum(rabbitmq_queue_messages_ready)', 'ready'),
('sum(rabbitmq_queue_messages_unacked)', 'unacked')], 'short',
'Агрегат по очередям, rabbitmq_prometheus (порт 15692, внутри сети).')
b.row('PostgreSQL и процессы')
b.ts('Пул БД: used / idle / max', [
('sum by (db_client_connection_state, instance)(db_client_connection_count{job=~"messenger-.*",db_client_connection_pool_name="main"})', '{{db_client_connection_state}} {{instance}}'),
('max by (instance)(db_client_connection_max{job=~"messenger-.*",db_client_connection_pool_name="main"})', 'max {{instance}}')],
'short', 'Соединения основного пула по репликам.')
b.ts('Пул БД: ожидающие запросы и wait p95', [
('sum by (instance)(db_client_connection_pending_requests{job=~"messenger-.*"})', 'pending {{instance}}'),
('histogram_quantile(0.95, sum by (le)(rate(db_client_connection_wait_time_seconds_bucket{job=~"messenger-.*"}[1m])))', 'wait p95 (с)')],
'short', 'Очередь на получение соединения и p95 ожидания (секунды).')
b.ts('Операция БД p95 по db_operation_name', [(
'histogram_quantile(0.95, sum by (le, db_operation_name)(rate(db_client_operation_duration_seconds_bucket{job=~"messenger-.*"}[1m])))',
'{{db_operation_name}}')], 's', 'p95 длительности операций БД по типу операции.')
b.ts('CPU процессов (ядра)', [('sum by (job, instance)(rate(process_cpu_time_seconds_total{job=~"messenger-.*"}[1m]))', '{{job}} {{instance}}')],
'short', 'rate(process_cpu_time_seconds_total): 1.0 = одно логическое ядро.')
b.ts('RAM процессов', [('process_memory_usage_bytes{job=~"messenger-.*"}', '{{job}} {{instance}}'),
('v8js_memory_heap_used_bytes{job=~"messenger-.*"}', 'heap {{job}} {{instance}}')],
'bytes', 'Resident memory процесса и использованная куча V8.')
b.ts('Event loop delay p99', [('nodejs_eventloop_delay_p99_seconds{job=~"messenger-.*"}', '{{job}} {{instance}}')], 's',
'Задержка event loop (p99 окна, секунды).')
b.row('Логи (переход в трассу по TraceID)')
b.logs('Логи API', '{service_name="messenger-api"}', 'Структурные логи; trace_id — structured metadata, поле TraceID ведёт в Tempo.')
b.logs('Логи worker', '{service_name="messenger-worker"}', 'Структурные логи worker; trace_id ведёт в Tempo.')
return b.render()
def telemetry():
b = Board('messenger-telemetry', 'Messenger: telemetry pipeline')
b.row('Collector')
b.ts('Collector: принято', [
('sum(rate(otelcol_receiver_accepted_spans[1m]))', 'spans'),
('sum(rate(otelcol_receiver_accepted_log_records[1m]))', 'log records'),
('sum(rate(otelcol_receiver_accepted_metric_points[1m]))', 'metric points')],
'suffix:items/s', 'Элементов, принятых OTLP-receiver, в секунду (не байты).')
b.ts('Collector: отправлено', [
('sum(rate(otelcol_exporter_sent_spans[1m]))', 'spans'),
('sum(rate(otelcol_exporter_sent_log_records[1m]))', 'log records'),
('sum(rate(otelcol_exporter_sent_metric_points[1m]))', 'metric points')],
'suffix:items/s', 'Элементов, успешно отправленных exporter-ами в backend.')
b.ts('Collector: ошибки отправки и постановки в очередь', [
('sum by (exporter)(rate(otelcol_exporter_send_failed_spans[1m]))', 'send_failed spans {{exporter}}'),
('sum by (exporter)(rate(otelcol_exporter_send_failed_log_records[1m]))', 'send_failed logs {{exporter}}'),
('sum by (exporter)(rate(otelcol_exporter_send_failed_metric_points[1m]))', 'send_failed metrics {{exporter}}'),
('sum by (exporter)(rate(otelcol_exporter_enqueue_failed_spans[1m]))', 'enqueue_failed spans {{exporter}}'),
('sum by (exporter)(rate(otelcol_exporter_enqueue_failed_log_records[1m]))', 'enqueue_failed logs {{exporter}}'),
('sum by (exporter)(rate(otelcol_exporter_enqueue_failed_metric_points[1m]))', 'enqueue_failed metrics {{exporter}}')],
'suffix:items/s', 'Потери на выходе Collector. Нулевые серии могут отсутствовать до первой ошибки.')
b.ts('Collector: очередь / ёмкость', [
('max by (exporter)(otelcol_exporter_queue_size)', 'size {{exporter}}'),
('max by (exporter)(otelcol_exporter_queue_capacity)', 'capacity {{exporter}}')],
'short', 'Очередь sending_queue в запросах (batch), не в spans.')
b.ts('Collector: RSS', [('otelcol_process_memory_rss', 'RSS')], 'bytes',
'Resident set size процесса Collector; лимит контейнера 256 MiB, memory_limiter 204 MiB.')
b.ts('Приложение: экспорт телеметрии (items)', [
('sum by (signal, outcome)(rate(messenger_telemetry_export_items_total[1m]))', '{{signal}} {{outcome}}')],
'suffix:items/s', 'Элементов, дошедших до OTLP-exporter приложения (traces/logs). Не учёт метрик и не все pre-export drops.')
b.row('Backend-хранилища')
b.ts('Loki / Tempo: up', [('up{job=~"loki|tempo"}', '{{job}}')], 'short', '1 — scrape успешен.')
b.ts('Loki / Tempo: ingest', [('sum(rate(loki_distributor_bytes_received_total[1m]))', 'loki'),
('sum(rate(tempo_distributor_bytes_received_total[1m]))', 'tempo')],
'Bps', 'Байт, принятых distributor-ами, в секунду.')
b.ts('Prometheus: размер TSDB', [('prometheus_tsdb_storage_blocks_bytes', 'blocks'),
('prometheus_tsdb_wal_storage_size_bytes', 'wal')], 'bytes',
'Блоки и WAL локального TSDB (retention 7d).')
b.table('Состояние правил (ALERTS)', 'ALERTS{alertname=~"Messenger.*"}',
'Pending/firing alerts из метрики ALERTS. Отсутствие строк означает, что нет pending/firing при загруженных rules (правила проверяются через /api/v1/rules).')
return b.render()
def main():
OUT.mkdir(parents=True, exist_ok=True)
for name, board in (('messenger-overview', overview()),
('messenger-telemetry', telemetry())):
(OUT / (name + '.json')).write_text(
json.dumps(board, indent=2, ensure_ascii=False) + '\n')
print('wrote', OUT / (name + '.json'))
if __name__ == '__main__':
main()
+676
View File
@@ -0,0 +1,676 @@
#!/usr/bin/env python3
"""lab6 controlled incidents: worker | db | telemetry. Stdlib only.
Every scenario runs inside obs.app_environment (SDK on, sampling 1,
FAULT_INJECTION_ENABLED=1) and always undoes its own faults/locks/stopped
services in `finally`. Times are UTC + monotonic offsets; an unobserved
transition is null with a reason, never a guess."""
import argparse
import calendar
import subprocess
import sys
import threading
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
import reconcile # noqa: E402
import resources # noqa: E402
import verify # noqa: E402
import workload # noqa: E402
from obs import Failure, stand, utc_now # noqa: E402
ROOT = obs.ROOT
def utc_epoch(text):
return calendar.timegm(time.strptime(text[:19], '%Y-%m-%dT%H:%M:%S'))
ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
ENV = {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
class Ctx:
def __init__(self, args):
self.args = args
self.obs = verify.make_obs(args)
self.out = Path(args.output_dir)
self.t0 = time.monotonic()
def mono(self):
return round(time.monotonic() - self.t0, 2)
def mark(self, timeline, key, extra=None):
timeline[key] = {'utc': utc_now(), 'mono_s': self.mono()}
if extra:
timeline[key].update(extra)
print('[%s] %s' % (timeline[key]['utc'], key), flush=True)
def actor_chat(self):
c = stand.new_client(self.args.base_url, self.args.ca_file)
alice, _, _, chat = c.bootstrap()
return alice, chat
def alert_state(ctx, name):
result = ctx.obs.prom_query('ALERTS{alertname="%s"}' % name)
states = {r['metric'].get('alertstate') for r in result}
if 'firing' in states:
return 'firing'
return 'pending' if 'pending' in states else None
def wait_state(ctx, name, want, timeout, timeline, polls):
"""want: 'firing' or None (resolved). Returns True when reached."""
end = time.monotonic() + timeout
while time.monotonic() < end:
state = alert_state(ctx, name)
polls.append({'utc': utc_now(), 'mono_s': ctx.mono(), 'alert': name, 'state': state})
if state == 'pending' and 'pending' not in timeline:
ctx.mark(timeline, 'pending')
if state == want:
return True
time.sleep(5)
return False
class BackgroundLoad:
def __init__(self, ctx, actor, chat, rate, upload_every, phase):
self.stop_event = threading.Event()
self.result = None
self.error = None
def body():
try:
self.result = workload.run_window(
ctx.args.base_url, ctx.args.ca_file, actor, chat, rate, 1800,
phase, upload_every=upload_every, stop_event=self.stop_event,
settle=False)
except Exception as exc: # reported by stop()
self.error = exc
self.thread = threading.Thread(target=body, daemon=True)
self.thread.start()
def stop(self):
self.stop_event.set()
self.thread.join(timeout=120)
if self.error:
raise self.error
if self.result is None:
raise Failure('background load did not finish')
return self.result
def baseline(ctx, actor, chat, rate, upload_every):
ctx.obs.require_no_foreign_faults()
for name in ALERTS:
if alert_state(ctx, name) is not None:
raise Failure('alert already active before baseline: ' + name)
result = workload.run_window(ctx.args.base_url, ctx.args.ca_file, actor, chat,
rate, 60, 'baseline', upload_every=upload_every,
settle=False)
drain = obs.wait_drained(result['uploads'], obs=ctx.obs)
for name in ALERTS:
if alert_state(ctx, name) is not None:
raise Failure('alert active at the end of baseline: ' + name)
if not drain['drained']:
raise Failure('baseline jobs did not drain')
return {'load': result['load'], 'drain': drain, 'uploads': result['uploads']}
def series(ctx, expr, start, end, step=10):
out = []
for item in ctx.obs.prom_range(expr, start, end, step):
out.append({'labels': item['metric'],
'points': [[float(t), v] for t, v in item['values']]})
return out
def peak(rows):
values = [float(v) for r in rows for _, v in r['points'] if v != 'NaN']
return max(values) if values else None
def ready_seconds(ids):
"""max/median finished_at - created_at of the jobs of these attachments."""
if not ids:
return {'n': 0, 'max': None, 'median': None}
values = []
for i in range(0, len(ids), 200):
chunk = ids[i:i + 200]
arr = "ARRAY[" + ",".join("'" + x + "'" for x in chunk) + "]::uuid[]"
out = obs.psql(
"SELECT COALESCE(string_agg(extract(epoch from (finished_at - created_at))::text, ';'),'') "
"FROM attachment_jobs WHERE status='succeeded' AND attachment_id = ANY(" + arr + ")")
values += [float(x) for x in out.split(';') if x]
return {'n': len(values), 'max': max(values) if values else None,
'median': measurement.median(values)}
def finish(ctx, name, result):
result['passed'] = not result['problems']
verify.write_json(ctx.out, name, result)
if result['problems']:
raise Failure(name + ': ' + '; '.join(result['problems']))
print('PASS: ' + name)
# ------------------------------------------------------------------- worker
def scenario_worker(ctx, label=''):
alice, chat = ctx.actor_chat()
result = {'scenario': 'worker', 'profile': {'rate': 20, 'upload_every': 20},
'timeline': {}, 'problems': [], 'polls': []}
tl = result['timeline']
base = baseline(ctx, alice, chat, 20, 20)
result['baseline'] = {'load': base['load'], 'ready_seconds': ready_seconds(base['uploads'])}
load = None
worker_stopped = False
try:
load = BackgroundLoad(ctx, alice, chat, 20, 20, 'incident')
time.sleep(10)
ctx.mark(tl, 'inject')
stand.compose('stop', 'worker')
worker_stopped = True
fired = wait_state(ctx, 'MessengerJobsQueueLagGrowing', 'firing', 150, tl, result['polls'])
if fired:
ctx.mark(tl, 'firing')
else:
tl['firing'] = None
result['problems'].append('queue-lag alert did not fire within 150 s')
# Capture the evidence while the backlog exists.
t_now = time.time()
result['queue_at_firing'] = {
'oldest_age_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_oldest_age_seconds)'),
'depth_ready_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_depth{state="ready"})'),
'depth_ready_sum_over_replicas': ctx.obs.prom_scalar('sum(messenger_jobs_queue_depth{state="ready"})'),
'depth_ready_per_replica': {
r['metric']['instance']: float(r['value'][1])
for r in ctx.obs.prom_query('messenger_jobs_queue_depth{state="ready"}')},
'outbox_pending_max': ctx.obs.prom_scalar('max(messenger_outbox_pending)'),
}
q = result['queue_at_firing']
per = list(q['depth_ready_per_replica'].values())
if len(per) >= 2 and q['depth_ready_max'] != max(per):
result['problems'].append('max() does not equal the per-replica snapshot')
if len(per) >= 2 and q['depth_ready_sum_over_replicas'] == q['depth_ready_max'] and per[0] > 0:
result['problems'].append('replicas not both reporting (cannot show non-doubling)')
finally:
ctx.mark(tl, 'recover_start')
if worker_stopped:
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
ctx.mark(tl, 'worker_back')
incident_load = load.stop() if load else None
ctx.mark(tl, 'load_stopped')
ids = incident_load['uploads']
result['incident_load'] = incident_load['load']
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
result['drain'] = drain
ctx.mark(tl, 'backlog_drained' if drain['drained'] else 'drain_deadline')
if not drain['drained']:
tl['backlog_drained'] = None
result['problems'].append('backlog not drained within 180 s')
elif any(s != 'ready' for s in drain['states'].values()):
result['problems'].append('some accepted attachments are not ready')
resolved = wait_state(ctx, 'MessengerJobsQueueLagGrowing', None, 180, tl, result['polls'])
if resolved:
ctx.mark(tl, 'resolved')
else:
tl['resolved'] = None
result['problems'].append('queue-lag alert not resolved within 180 s')
end = time.time()
t_inject = utc_epoch(tl['inject']['utc'])
window = (t_inject - 60, end)
result['queries'] = {
'oldest_age': 'max(messenger_jobs_queue_oldest_age_seconds)',
'depth': 'max by (state)(messenger_jobs_queue_depth)',
'outbox': 'max(messenger_outbox_pending)',
'worker_active': 'sum(messenger_worker_jobs_active)'}
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
peak_age = peak(result['series']['oldest_age'])
result['peak_oldest_age_s'] = peak_age
if peak_age is None or peak_age < 30:
result['problems'].append('queue age did not grow past 30 s')
result['incident_ready_seconds'] = ready_seconds(ids)
if (result['incident_ready_seconds']['max'] or 0) <= (result['baseline']['ready_seconds']['max'] or 0):
result['problems'].append('time-to-ready did not exceed the baseline')
http_ok = result['incident_load']['http_5xx'] == 0 and \
sum(result['incident_load']['transport_failures'].values()) == 0
result['http_acceptance_continued'] = http_ok
if not http_ok:
result['problems'].append('HTTP acceptance degraded during worker outage')
finish(ctx, 'incident-worker.json', result)
# ----------------------------------------------------------------------- db
def trace_gap(spans, server):
"""Server duration minus the union of its direct child intervals (ms)."""
children = sorted((s['start_ns'], s['end_ns']) for s in spans
if s['parent_span_id'] == server['span_id'])
# Descendants of the transaction span are covered by their ancestor.
covered, last = 0, None
for start, end in children:
if last is None or start > last:
covered += end - start
last = end
elif end > last:
covered += end - last
last = end
return ((server['end_ns'] - server['start_ns']) - covered) / 1e6
def db_time_ms(spans, server):
return sum((s['end_ns'] - s['start_ns']) for s in spans
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
and s['parent_span_id'] == server['span_id']) / 1e6
def sequential_uploads(ctx, actor, chat, count, delay):
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
c.timeout = 30
key = 'fault:api.after_commit_before_publish'
rows = []
for _ in range(count):
if delay:
stand.set_fault(key, 'delay:%d' % delay)
started = time.monotonic()
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
range(200, 600), actor=actor, raw=stand.sample_bytes(),
headers={'Content-Type': 'image/png'}, schema='Attachment')
elapsed = (time.monotonic() - started) * 1000
consumed = True
if delay:
out = stand.compose('exec', '-T', 'redis', 'redis-cli', 'GET', key).stdout.strip()
consumed = out == ''
rows.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status'],
'elapsed_ms': elapsed, 'consumed': consumed,
'attachment_id': r['json']['id'] if r['status'] == 202 else None})
return rows
def server_view(ctx, request_id):
log = verify.http_log(ctx.obs, request_id)
tid, sid = log['labels']['trace_id'], log['labels']['span_id']
spans = verify.trace_with(ctx.obs, tid, lambda sp: any(
s['span_id'] == sid for s in sp), description='server trace')
server = next(s for s in spans if s['span_id'] == sid)
return spans, server
def scenario_db(ctx, label=''):
alice, chat = ctx.actor_chat()
result = {'scenario': 'db', 'profile': {'rate': 2, 'upload_every': 1},
'timeline': {}, 'problems': [], 'polls': []}
tl = result['timeline']
base = baseline(ctx, alice, chat, 2, 1)
result['baseline'] = {'load': base['load']}
load = None
lock_released = False
try:
load = BackgroundLoad(ctx, alice, chat, 2, 1, 'incident')
time.sleep(10)
# Own client: 30 s timeout is set inside workload (REQUEST_TIMEOUT).
with obs.hold_table_lock('attachments') as pid:
ctx.mark(tl, 'inject', {'lock_backend_pid': pid})
fired = wait_state(ctx, 'MessengerApiHighErrorRate', 'firing', 120, tl, result['polls'])
if fired:
ctx.mark(tl, 'firing')
else:
tl['firing'] = None
result['problems'].append('error-rate alert did not fire within 120 s')
result['at_firing'] = {
'api_5xx_rate_1m': ctx.obs.prom_scalar('messenger:api_errors:rate1m'),
'api_request_rate_1m': ctx.obs.prom_scalar('messenger:api_requests:rate1m'),
'haproxy_5xx_rate_1m': ctx.obs.prom_scalar(
'sum(rate(haproxy_backend_http_responses_total{proxy="api",code="5xx"}[1m]))', 0.0),
'db_insert_attachments_p95_s': ctx.obs.prom_scalar(
'histogram_quantile(0.95, sum by (le)(rate(db_client_operation_duration_seconds_bucket{db_operation_name="INSERT"}[1m])))'),
'pool_pending': ctx.obs.prom_scalar('sum(db_client_connection_pending_requests)'),
'pool_used': ctx.obs.prom_scalar('sum(db_client_connection_count{db_client_connection_state="used"})'),
}
lock_released = True
ctx.mark(tl, 'lock_released')
finally:
if not lock_released:
ctx.mark(tl, 'lock_released_in_finally')
incident_load = load.stop() if load else None
ctx.mark(tl, 'load_stopped')
result['incident_load'] = incident_load['load']
if incident_load['load']['http_5xx'] == 0:
result['problems'].append('no real server 5xx observed in the load')
ids = incident_load['uploads']
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
result['drain'] = drain
if drain['drained']:
ctx.mark(tl, 'backlog_drained')
else:
tl['backlog_drained'] = None
result['problems'].append('jobs did not drain after recovery')
if wait_state(ctx, 'MessengerApiHighErrorRate', None, 120, tl, result['polls']):
ctx.mark(tl, 'resolved')
else:
tl['resolved'] = None
result['problems'].append('error-rate alert not resolved within 120 s')
# -- localisation: a 5xx trace where the time went to the DB.
inject_ns = int(utc_epoch(tl['inject']['utc']) - 5) * 10 ** 9
rows = verify.wait_rows(
ctx.obs, '{service_name="messenger-api"} | event_name="http.request" '
'| http_response_status_code=~"5.."', timeout=60, back=900)
rows = [r for r in rows if r['ts'] >= inject_ns]
if rows:
row = rows[0]
spans = verify.trace_with(ctx.obs, row['labels']['trace_id'], lambda sp: any(
s['span_id'] == row['labels']['span_id'] for s in sp), description='5xx trace')
server = next(s for s in spans if s['span_id'] == row['labels']['span_id'])
dur = (server['end_ns'] - server['start_ns']) / 1e6
db_ms = db_time_ms(spans, server)
result['trace_5xx'] = {
'request_id': row['labels'].get('request_id'), 'trace_id': row['labels']['trace_id'],
'status': row['labels'].get('http_response_status_code'),
'server_ms': dur, 'db_children_ms': db_ms,
'handler_gap_ms': trace_gap(spans, server),
'db_children': [verify.safe_span(s) | {'duration_ms': (s['end_ns'] - s['start_ns']) / 1e6}
for s in spans if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
and s['parent_span_id'] == server['span_id']][:6]}
if db_ms < 0.5 * dur:
result['problems'].append('5xx trace is not dominated by DB spans')
else:
result['problems'].append('no 5xx http.request log found in Loki')
# -- control: handler delay (not DB) shows up as a gap, not as DB time.
ctx.obs.require_no_foreign_faults()
ctrl_chat = ctx.actor_chat()
base_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 0)
delay_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 500)
stand.clear_fault('fault:api.after_commit_before_publish')
def gaps(rows_):
out = []
for r in rows_:
spans, server = server_view(ctx, r['request_id'])
out.append({'request_id': r['request_id'],
'server_ms': (server['end_ns'] - server['start_ns']) / 1e6,
'db_children_ms': db_time_ms(spans, server),
'handler_gap_ms': trace_gap(spans, server)})
return out
base_g, delay_g = gaps(base_rows), gaps(delay_rows)
med = measurement.median
shift = med([g['handler_gap_ms'] for g in delay_g]) - med([g['handler_gap_ms'] for g in base_g])
result['control'] = {
'baseline_uploads': base_g, 'delayed_uploads': delay_g,
'handler_gap_shift_ms_median': shift,
'db_children_ms_median_baseline': med([g['db_children_ms'] for g in base_g]),
'db_children_ms_median_delayed': med([g['db_children_ms'] for g in delay_g]),
'all_keys_consumed': all(r['consumed'] for r in delay_rows),
'all_http_202': all(r['status'] == 202 for r in base_rows + delay_rows),
'note': 'timer/scheduling noise is part of the shift; the check is ~500 ms +/- 250 ms'}
if not result['control']['all_keys_consumed']:
result['problems'].append('delay fault key was not consumed')
if not result['control']['all_http_202']:
result['problems'].append('control uploads were not all 202')
if not (250 <= shift <= 750):
result['problems'].append('handler gap shift %.0f ms is not ~500 ms' % shift)
cdb = result['control']
if abs(cdb['db_children_ms_median_delayed'] - cdb['db_children_ms_median_baseline']) > 100:
result['problems'].append('DB time changed in the handler-delay control')
stand.wait_for(lambda: obs.wait_drained(
[r['attachment_id'] for r in base_rows + delay_rows], obs=None, timeout=60,
flush_telemetry=False)['drained'], 90, 'control uploads ready')
end = time.time()
t_inject = utc_epoch(tl['inject']['utc'])
window = (t_inject - 60, t_inject + 240)
result['queries'] = {
'db_operation_p95_by_op': 'histogram_quantile(0.95, sum by (le, db_operation_name)(rate(db_client_operation_duration_seconds_bucket[1m])))',
'pool_by_state': 'sum by (db_client_connection_state)(db_client_connection_count{db_client_connection_pool_name="main"})',
'pool_pending': 'sum(db_client_connection_pending_requests)',
'api_5xx_ratio': 'messenger:api_errors:rate1m / messenger:api_requests:rate1m'}
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
finish(ctx, 'incident-db.json', result)
# ---------------------------------------------------------------- telemetry
def regression_slope(points):
n = len(points)
if n < 3:
return None
mx = sum(x for x, _ in points) / n
my = sum(y for _, y in points) / n
den = sum((x - mx) ** 2 for x, _ in points)
return sum((x - mx) * (y - my) for x, y in points) / den if den else None
def rss_trend(samples, service, after_s, start_mono):
pts = [(s['mono'] - start_mono, s['rss_bytes']) for s in samples
if s['service'] == service and s.get('rss_bytes') is not None
and s['mono'] - start_mono >= after_s]
by_tick = {}
for t, v in pts:
by_tick.setdefault(round(t / 5), []).append((t, v))
summed = [(sum(t for t, _ in g) / len(g), sum(v for _, v in g)) for g in by_tick.values()]
values = [v for _, v in summed]
return {'points': len(summed), 'min': min(values) if values else None,
'max': max(values) if values else None,
'slope_bytes_per_s': regression_slope(summed),
'first': values[0] if values else None, 'last': values[-1] if values else None}
def sdk_overflow_logs(since_utc):
result = subprocess.run(
['docker', 'compose', '--env-file', stand.ENV_FILE, 'logs', '--no-color',
'--no-log-prefix', '--since', since_utc, 'api', 'worker'],
cwd=ROOT, text=True, capture_output=True, timeout=180)
count = 0
for line in result.stdout.splitlines():
if '"event":"otel.sdk"' in line and ('queue' in line.lower() or 'dropped' in line.lower()):
count += 1
return count
def resume_probe(ctx, actor, chat, count):
"""After recovery: fresh requests must reach Loki (logs) and Tempo (traces)."""
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
start_ns = time.time_ns() - 5 * 10 ** 9
requests = []
for i in range(count):
r = c.message(actor, chat, 'lab6-resume-%d' % i)
requests.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status']})
_, summary = reconcile.reconcile(ctx.obs, 'full', requests, since, start_ns,
time.time_ns() + 60 * 10 ** 9)
return summary
def telemetry_outage(ctx, service, alice, chat, repeat):
out = {'service': service, 'timeline': {}, 'problems': []}
tl = out['timeline']
ctx.obs.require_no_foreign_faults()
base = baseline(ctx, alice, chat, 20, 20)
out['baseline'] = {'load': base['load']}
state_before = ctx.obs.app_state()
ids = resources.container_map()
sampler = resources.ResourceSampler(ids, interval=5.0)
sampler.start()
sampler_start = time.monotonic()
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
pre_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
load = BackgroundLoad(ctx, alice, chat, 20, 20, service + '-outage')
stopped = False
t_smoke = None
try:
time.sleep(10)
ctx.mark(tl, 'inject')
t_inject = time.time()
stand.compose('stop', service)
stopped = True
t_start = time.monotonic()
smoke = subprocess.run(['make', 'test', 'BASE_URL=' + ctx.args.base_url], cwd=ROOT,
capture_output=True, text=True, timeout=80)
t_smoke = time.monotonic() - t_start
out['smoke_during_outage'] = {'returncode': smoke.returncode,
'seconds': round(t_smoke, 1),
'tail': smoke.stdout.strip().splitlines()[-1:] if smoke.stdout else []}
if smoke.returncode != 0:
out['problems'].append('make test failed during the outage')
remaining = 90 - (time.monotonic() - t_start)
if remaining > 0:
time.sleep(remaining)
if service == 'tempo':
out['collector_during_outage'] = {
'queue_size_max': ctx.obs.prom_scalar('max(max_over_time(otelcol_exporter_queue_size{exporter="otlp/tempo"}[90s]))'),
'queue_capacity': ctx.obs.prom_scalar('max(otelcol_exporter_queue_capacity{exporter="otlp/tempo"})'),
'send_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_send_failed_spans[90s]))', 0.0),
'enqueue_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_enqueue_failed_spans[90s]))', 0.0),
'logs_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_log_records[90s]))', 0.0),
'metrics_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_metric_points[90s]))', 0.0),
'collector_rss_bytes': ctx.obs.prom_scalar('otelcol_process_memory_rss'),
'loki_up': ctx.obs.prom_scalar('up{job="loki"}'), 'prometheus_ready': True}
ctx.mark(tl, 'outage_end')
finally:
if stopped:
stand.compose('start', service)
ctx.mark(tl, 'recover')
try:
incident_load = load.stop()
finally:
ctx.mark(tl, 'load_stopped')
t_restore = time.time()
ctx.obs.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
ctx.mark(tl, 'backends_ready')
drain = obs.wait_drained(incident_load['uploads'] + base['uploads'], obs=ctx.obs, timeout=180)
out['drain'] = drain
if drain['drained']:
ctx.mark(tl, 'backlog_drained')
else:
out['problems'].append('jobs did not drain')
sampler.stop()
out['resume_probe'] = resume_probe(ctx, alice, chat, 10)
probe = out['resume_probe']
if probe.get('sampled_present') != 10 or probe.get('log_exported') != 10:
out['problems'].append('traces/logs did not resume after recovery: %s' % probe)
out['incident_load'] = incident_load['load']
if incident_load['load']['http_5xx'] or sum(incident_load['load']['transport_failures'].values()):
out['problems'].append('business traffic degraded during telemetry outage')
# -- no restarts/OOM, same instances.
state_after = ctx.obs.app_state()
for svc in ('api', 'worker'):
if set(state_before[svc]) != set(state_after[svc]):
out['problems'].append('instance IDs changed for ' + svc)
for cid, info in state_after[svc].items():
if info['restarts'] != state_before[svc].get(cid, info)['restarts'] or info['oom_killed']:
out['problems'].append('restart/OOM in ' + svc)
summary = resources.summarize(sampler.samples)
out['resources'] = {k: v for k, v in summary.items() if k != 'sample_errors'}
out['sample_errors'] = summary['sample_errors']
out['memory_vs_limit'] = {
svc: (v['container_memory_bytes']['max'] / v['mem_limit_bytes'])
for svc, v in summary.items() if svc != 'sample_errors'
and v['mem_limit_bytes'] and v['container_memory_bytes']['max'] is not None}
for svc, ratio in out['memory_vs_limit'].items():
if ratio >= 1.0:
out['problems'].append(svc + ' memory at/over limit')
out['rss_trend_after_first_minute'] = {
svc: rss_trend(sampler.samples, svc, 60, sampler_start)
for svc in (('otel-collector', 'api', 'worker') if service == 'collector' or service == 'otel-collector'
else ('otel-collector',))}
out['sdk_queue_diagnostic_messages'] = sdk_overflow_logs(since)
# -- stdout vs backends for requests accepted during the outage.
inject_unix = utc_epoch(tl['inject']['utc'])
during = [s for s in incident_load['slots'] if s.get('outcome') == 'http'
and s.get('request_id') and s['kind'] == 'message']
stdout = reconcile.stdout_http_logs(since)
# slots are relative to the load t0; classify by stdout timestamp instead.
in_window = []
for s in during:
row = stdout.get(s['request_id'])
if row is None:
continue
ts = utc_epoch(row['timestamp'])
if inject_unix <= ts <= inject_unix + 90:
in_window.append((s, row))
sampled_window = [(s, r) for s, r in in_window if r.get('trace_sampled') is True]
step = max(1, len(sampled_window) // 100)
probe = sampled_window[::step][:100]
targets = {s['request_id']: (r['trace_id'], r['span_id']) for s, r in probe}
found = reconcile.tempo_server_spans(ctx.obs, targets, deadline_s=60)
out['stdout_vs_tempo'] = {
'requests_in_outage_window_with_stdout': len(in_window),
'sampled_in_window_per_stdout': len(sampled_window),
'tempo_probe_size': len(probe),
'tempo_probe_missing_server_spans': sum(1 for v in found.values() if not v),
'note': 'loss is measured for SERVER root spans only, on a systematic probe '
'sample; it does not count lost child spans'}
if not in_window:
out['problems'].append('no stdout http.request lines in the outage window')
if any(not r.get('trace_id') or r.get('trace_sampled') is not True for _, r in sampled_window):
out['problems'].append('stdout trace fields missing')
if service == 'tempo':
c = out['collector_during_outage']
if (c['queue_size_max'] or 0) > (c['queue_capacity'] or 128):
out['problems'].append('Collector queue exceeded capacity')
if not (c['send_failed_spans_increase'] or 0) > 0:
out['problems'].append('Collector did not count failed span exports')
if not (c['logs_sent_increase'] or 0) > 0 or not (c['metrics_sent_increase'] or 0) > 0:
out['problems'].append('Loki/Prometheus paths stopped during the Tempo outage')
else:
post_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
out['app_export_failed_items'] = {'before': pre_failed, 'after': post_failed,
'delta': post_failed - pre_failed if post_failed >= pre_failed else None,
'reason': None if post_failed >= pre_failed else 'counter_reset'}
if not (out['app_export_failed_items']['delta'] or 0) > 0:
out['problems'].append('app failed-export counter did not grow')
resumed = {name: ctx.obs.prom_scalar(expr, 0.0) for name, expr in (
('spans_sent_300s', 'sum(increase(otelcol_exporter_sent_spans[300s]))'),
('logs_sent_300s', 'sum(increase(otelcol_exporter_sent_log_records[300s]))'),
('metrics_sent_300s', 'sum(increase(otelcol_exporter_sent_metric_points[300s]))'))}
out['signals_resumed'] = resumed
if not all(v > 0 for v in resumed.values()):
out['problems'].append('not all three signals resumed')
return out
def scenario_telemetry(ctx, label=''):
alice, chat = ctx.actor_chat()
result = {'scenario': 'telemetry', 'repeat': label, 'problems': [], 'outages': []}
for service in ('tempo', 'otel-collector'):
outage = telemetry_outage(ctx, service, alice, chat, label)
result['outages'].append(outage)
result['problems'] += [service + ': ' + p for p in outage['problems']]
name = 'incident-telemetry-%s.json' % label
finish(ctx, name, result)
SCENARIOS = {'worker': scenario_worker, 'db': scenario_db, 'telemetry': scenario_telemetry}
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--scenario', required=True, choices=sorted(SCENARIOS))
parser.add_argument('--repeat', type=int, default=1)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--ca-file')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output-dir', default='evidence/lab6')
args = parser.parse_args(argv)
ctx = Ctx(args)
failed = False
try:
with ctx.obs.app_environment(ENV):
for repeat in range(1, args.repeat + 1):
try:
SCENARIOS[args.scenario](ctx, str(repeat))
except Failure as error:
failed = True
print('FAIL: ' + str(error), file=sys.stderr)
break
except Failure as error:
print('FAIL: ' + str(error), file=sys.stderr)
return 1
return 1 if failed else 0
if __name__ == '__main__':
sys.exit(main())
+337
View File
@@ -0,0 +1,337 @@
"""Pure measurement logic for lab6 track 1 (no I/O): statistics, counter
deltas, cohort classification, run validity and the p25/full selection rules.
Everything here is unit-tested in test_measurement.py."""
import math
MODES = {
'off': {'OTEL_SDK_DISABLED': 'true', 'OTEL_TRACES_SAMPLER_ARG': '1'},
'full': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1'},
'p25': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.25'},
'p10': {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '0.1'},
}
SAMPLER_RATIO = {'off': None, 'full': 1.0, 'p25': 0.25, 'p10': 0.1}
ORDERS = (('off', 'full', 'p25', 'p10'), ('p25', 'off', 'p10', 'full'),
('p10', 'p25', 'full', 'off'))
START_LAG_LIMIT_MS = 50.0
COMPLETION_LIMIT = 0.99
SPAN_RATIO_LIMIT = 0.5
LATENCY_RATIO_LIMIT = 1.10
# ----------------------------------------------------------------- statistics
def nearest_rank(values, q):
"""Nearest-rank quantile: sorted[ceil(n*q)-1] (same as tests/load.py)."""
if not values:
return None
ordered = sorted(values)
return ordered[max(0, math.ceil(len(ordered) * q) - 1)]
def median(values):
values = [v for v in values if v is not None]
if not values:
return None
ordered = sorted(values)
n = len(ordered)
mid = n // 2
return ordered[mid] if n % 2 else (ordered[mid - 1] + ordered[mid]) / 2
def spread(values):
values = [v for v in values if v is not None]
if not values:
return {'min': None, 'median': None, 'max': None, 'n': 0}
return {'min': min(values), 'median': median(values), 'max': max(values),
'n': len(values)}
def relative(value, reference):
if value is None or reference in (None, 0):
return None
return value / reference - 1.0
def miss_all_probability(ratio, n):
"""(1-r)^N: chance that independent root sampling keeps none of N events.
A model assumption, not a measurement."""
return (1.0 - ratio) ** n
# ---------------------------------------------------------------------- load
ERROR_OUTCOMES = ('timeout', 'transport_error', 'protocol_error')
def slot_completed(slot):
return slot['outcome'] == 'http'
def summarize_load(slots, rate):
"""slots: one record per *planned* slot, including client_dropped ones.
Dropped and late slots stay in the denominator; p95 includes error
responses (it is not a latency of successes only)."""
scheduled = len(slots)
done = [s for s in slots if slot_completed(s)]
dropped = sum(1 for s in slots if s['outcome'] == 'client_dropped')
failures = {k: sum(1 for s in slots if s['outcome'] == k)
for k in ERROR_OUTCOMES}
http_5xx = sum(1 for s in done if s['status'] >= 500)
http_4xx = sum(1 for s in done if 400 <= s['status'] < 500)
successes = [s for s in done if 200 <= s['status'] < 300]
started = [s for s in slots if s.get('start_lag_ms') is not None]
lag = [s['start_lag_ms'] for s in started]
sched_all = [s['scheduled_ms'] for s in done]
service_all = [s['service_ms'] for s in done]
return {
'rate_rps': rate,
'scheduled': scheduled,
'completed': len(done),
'completed_ratio': (len(done) / scheduled) if scheduled else None,
'client_dropped': dropped,
'transport_failures': failures,
'http_5xx': http_5xx,
'http_4xx': http_4xx,
'http_2xx': len(successes),
'start_lag_ms_p95': nearest_rank(lag, 0.95),
'start_lag_ms_max': max(lag) if lag else None,
'scheduled_ms': {q: nearest_rank(sched_all, v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
'service_ms': {q: nearest_rank(service_all, v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
'success_scheduled_ms': {q: nearest_rank(
[s['scheduled_ms'] for s in successes], v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
'success_service_ms': {q: nearest_rank(
[s['service_ms'] for s in successes], v) for q, v in
(('p50', .5), ('p95', .95), ('p99', .99))},
}
def load_errors(load):
"""HTTP/protocol errors among delivered healthy traffic."""
return (load['http_5xx'] + sum(load['transport_failures'].values()))
def delivered_stream(load):
"""Generator delivered the planned stream (protocol validity)."""
reasons = []
if load['client_dropped']:
reasons.append('client_dropped=%d' % load['client_dropped'])
lag = load['start_lag_ms_p95']
if lag is None or lag > START_LAG_LIMIT_MS:
reasons.append('start_lag_p95=%s' % lag)
return reasons
# ------------------------------------------------------------------ counters
def counter_delta(start, end, failure_only=False):
"""-> (delta, reason). Never turns a missing positive series into zero and
never reports a reset as a negative loss."""
if start is None and end is None:
if failure_only:
return 0.0, 'absent_until_first_error'
return None, 'missing_series'
if start is None:
if failure_only:
return end, 'series_created_in_window'
return None, 'missing_start'
if end is None:
return None, 'missing_end'
if end < start:
return None, 'counter_reset'
return end - start, None
# ------------------------------------------------------------------- cohorts
def classify_record(mode, record):
"""Classify one cohort request.
record: {'stdout': None | {'trace_id','span_id','trace_sampled'} (the keys
are absent when the SDK is off or no span context), 'loki': True/False/None
(None: backend observation incomplete), 'span': True/False/None}.
Returns (trace_class, log_class)."""
stdout = record.get('stdout')
if stdout is None:
return 'unmatched_stdout', 'incomplete_observation'
if mode == 'off':
if any(k in stdout for k in ('trace_id', 'span_id', 'trace_sampled')):
return 'ids_present_with_sdk_off', 'not_applicable'
return 'not_applicable', 'not_applicable'
sampled = stdout.get('trace_sampled')
if not stdout.get('trace_id') or not stdout.get('span_id') or \
not isinstance(sampled, bool):
return 'unmatched_stdout', 'incomplete_observation'
if record.get('loki') is None:
log_class = 'incomplete_observation'
elif record['loki']:
log_class = 'exported'
else:
log_class = 'missing_exported_logs'
if sampled is False:
return 'intentional_unsampled', log_class
if record.get('span') is None:
return 'incomplete_observation', log_class
return ('sampled_present' if record['span'] else 'sampled_missing'), log_class
def summarize_cohort(mode, records):
counts = {'n': len(records)}
for record in records:
trace_class, log_class = classify_record(mode, record)
counts[trace_class] = counts.get(trace_class, 0) + 1
counts['log_' + log_class] = counts.get('log_' + log_class, 0) + 1
counts['retained_traces'] = counts.get('sampled_present', 0)
return counts
def export_loss(items_failed_delta, send_failed, enqueue_failed):
"""Telemetry export losses; a false trace_sampled flag is NOT a loss."""
parts = {'app_export_failed_items': items_failed_delta,
'collector_send_failed': send_failed,
'collector_enqueue_failed': enqueue_failed}
known = [v for v in parts.values() if isinstance(v, (int, float))]
return {'parts': parts, 'total_known': sum(known),
'loss': sum(known) > 0}
# ------------------------------------------------------------------ validity
def evaluate_run(summary):
"""-> (valid, invalid_reasons, configuration_failures).
Invalid = the protocol was not executed as specified; configuration
failures are valid adverse outcomes and stay in the matrix."""
invalid = list(summary.get('measurement_defects', []))
failures = []
load = summary['load']
invalid += delivered_stream(load)
if load['completed_ratio'] is None or load['completed_ratio'] < COMPLETION_LIMIT:
invalid.append('completed_ratio=%s' % load['completed_ratio'])
if summary.get('effective_mode_ok') is False:
invalid.append('effective_env_mismatch')
if summary.get('image_ids_ok') is False:
invalid.append('image_changed')
if summary.get('foreign_faults'):
invalid.append('foreign_faults')
for name, cohort in (summary.get('cohorts') or {}).items():
if cohort.get('invalid'):
invalid.append('cohort_invalid:%s' % name)
# Counter resets are only explained by an observed restart/OOM.
for name, item in (summary.get('ingest') or {}).items():
if isinstance(item, dict) and item.get('reason') == 'counter_reset' \
and not summary.get('restarts'):
invalid.append('unexplained_counter_reset:%s' % name)
# --- adverse outcomes of the configuration (valid)
if load_errors(load):
failures.append('healthy_errors=%d' % load_errors(load))
if summary.get('not_ready_at_deadline'):
failures.append('not_ready_at_deadline=%d' % summary['not_ready_at_deadline'])
if summary.get('integrity_errors'):
failures.append('healthy_errors_integrity=%d' % summary['integrity_errors'])
if summary.get('restarts'):
failures.append('restart_or_oom')
loss = summary.get('export_failures') or {}
if loss.get('loss'):
failures.append('export_loss')
for name, cohort in (summary.get('cohorts') or {}).items():
if cohort.get('sampled_missing'):
failures.append('sampled_missing:%s=%d' % (name, cohort['sampled_missing']))
if cohort.get('log_missing_exported_logs'):
failures.append('missing_exported_logs:%s=%d' % (
name, cohort['log_missing_exported_logs']))
return (not invalid), invalid, failures
def matrix_entry(summary):
"""Every protocol-run stays in the matrix, valid-adverse included."""
return {
'mode': summary['mode'], 'repeat': summary.get('repeat'),
'attempt': summary.get('attempt', 1), 'valid': summary['valid'],
'invalid_reasons': summary['invalid_reasons'],
'configuration_failures': summary['configuration_failures'],
'spans_per_operation': summary.get('spans_per_operation'),
'scheduled_p95_ms': (summary['load']['scheduled_ms'] or {}).get('p95'),
'dir': summary.get('dir'),
}
# ---------------------------------------------------------------- selection
def _clean(run):
return (run['valid'] and not any(
f.startswith(('healthy_errors', 'not_ready', 'restart', 'export_loss'))
for f in run['configuration_failures']))
def _retained(run, name):
return (run['cohorts'].get(name) or {}).get('retained_traces', 0)
def _no_missing(run):
for cohort in run['cohorts'].values():
if cohort.get('sampled_missing') or cohort.get('log_missing_exported_logs') \
or cohort.get('log_incomplete_observation') \
or cohort.get('unmatched_stdout') or cohort.get('incomplete_observation'):
return False
return True
def full_reference(full_runs):
"""Frozen reference: medians of the three full runs; None if any datum is
unmeasurable (then p25 cannot be declared a winner)."""
spans = [r.get('spans_per_operation') for r in full_runs]
p95 = [r['load']['scheduled_ms']['p95'] for r in full_runs]
ok = (len(full_runs) == 3 and all(r['valid'] for r in full_runs) and
all(v is not None for v in spans + p95))
return {'complete': ok, 'spans_per_operation_median': median(spans),
'scheduled_p95_ms_median': median(p95),
'run_dirs': [r.get('dir') for r in full_runs]}
def select_p25(full_runs, p25_runs):
ref = full_reference(full_runs)
checks = {'full_reference_complete': ref['complete']}
if not ref['complete'] or len(p25_runs) != 3:
checks['p25_runs'] = len(p25_runs) == 3
return {'passed': False, 'checks': checks, 'reference': ref}
checks['all_valid_clean'] = all(_clean(r) for r in p25_runs)
checks['retained_error_and_slow_each_run'] = all(
_retained(r, 'error') >= 1 and _retained(r, 'slow') >= 1 for r in p25_runs)
checks['no_missing_spans_or_logs'] = all(_no_missing(r) for r in p25_runs)
spans = median([r.get('spans_per_operation') for r in p25_runs])
checks['spans_median_le_50pct_full'] = (
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
p95 = median([r['load']['scheduled_ms']['p95'] for r in p25_runs])
checks['scheduled_p95_le_110pct_full'] = (
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
return {'passed': all(checks.values()), 'checks': checks, 'reference': ref,
'p25_spans_per_operation_median': spans,
'p25_scheduled_p95_ms_median': p95}
def validate_single(run, mode, ref):
"""One fresh run against the FROZEN full reference (never a 3-run median
of the validation itself)."""
checks = {'protocol_valid': run['valid'],
'zero_errors_notready_loss': _clean(run)}
if mode == 'p25':
checks['complete_stdout_and_logs'] = _no_missing(run) and all(
(run['cohorts'].get(n) or {}).get('n') == 40 for n in ('error', 'slow'))
checks['retained_each_class'] = (
_retained(run, 'error') >= 1 and _retained(run, 'slow') >= 1)
spans = run.get('spans_per_operation')
checks['spans_le_50pct_frozen_full'] = (
spans is not None and spans <= SPAN_RATIO_LIMIT * ref['spans_per_operation_median'])
p95 = run['load']['scheduled_ms']['p95']
checks['scheduled_p95_le_110pct_frozen_full'] = (
p95 is not None and p95 <= LATENCY_RATIO_LIMIT * ref['scheduled_p95_ms_median'])
else:
checks['complete_stdout_and_logs'] = _no_missing(run)
checks['all_80_server_spans'] = all(
(run['cohorts'].get(n) or {}).get('sampled_present') == 40
for n in ('error', 'slow'))
return {'passed': all(checks.values()), 'checks': checks, 'mode': mode}
+675
View File
@@ -0,0 +1,675 @@
"""Shared lab6 helpers: Prometheus/Loki/Tempo/Grafana client, controlled app
environment, SQL table lock, batch drain. Python 3.9+, standard library only."""
import base64
import binascii
import contextlib
import json
import os
import select
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.lab5 import stand # noqa: E402
Failure = stand.Failure
HTTP_TIMEOUT = 10
SCRAPE_JOBS = ('prometheus', 'otel-collector', 'haproxy', 'rabbitmq', 'loki',
'tempo')
RULE_ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
DASHBOARD_UIDS = ('messenger-overview', 'messenger-telemetry')
DATASOURCE_UIDS = ('prometheus', 'loki', 'tempo')
LOCKABLE_TABLES = ('messages', 'attachments')
# Keys the controller may change/restore (identical for api and worker).
MUTABLE_KEYS = (
'OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG', 'OTEL_METRIC_EXPORT_INTERVAL',
'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL',
'OTEL_EXPORTER_OTLP_TIMEOUT', 'OTEL_RESOURCE_ATTRIBUTES',
'OTEL_BSP_MAX_QUEUE_SIZE', 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE',
'OTEL_BSP_SCHEDULE_DELAY', 'OTEL_BLRP_MAX_QUEUE_SIZE',
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', 'OTEL_BLRP_SCHEDULE_DELAY',
'LOG_LEVEL', 'CACHE_MODE', 'FAULT_INJECTION_ENABLED',
)
# Recorded as diagnostics, never settable.
DIAGNOSTIC_KEYS = ('OTEL_SERVICE_NAME', 'SERVICE_VERSION')
# Baseline of the SDK knobs (compose defaults) used by track1 and incidents.
SDK_DEFAULTS = {
'OTEL_SDK_DISABLED': 'false',
'OTEL_TRACES_SAMPLER_ARG': '1',
'OTEL_METRIC_EXPORT_INTERVAL': '10000',
'OTEL_EXPORTER_OTLP_TIMEOUT': '5000',
'OTEL_BSP_MAX_QUEUE_SIZE': '2048',
'OTEL_BSP_MAX_EXPORT_BATCH_SIZE': '512',
'OTEL_BSP_SCHEDULE_DELAY': '1000',
'OTEL_BLRP_MAX_QUEUE_SIZE': '2048',
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE': '512',
'OTEL_BLRP_SCHEDULE_DELAY': '1000',
'LOG_LEVEL': 'INFO',
}
def utc_now():
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}),
_NoRedirect())
def http_get(url, headers=None, timeout=HTTP_TIMEOUT):
"""GET returning (status, bytes); never prints the URL query/headers."""
request = urllib.request.Request(url, headers=headers or {})
try:
with _OPENER.open(request, timeout=timeout) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
with error:
return error.code, error.read()
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise Failure('GET ' + url.split('?')[0] + ': connection failure (' +
type(exc).__name__ + ')') from None
def http_json(url, headers=None, timeout=HTTP_TIMEOUT, allow=(200,)):
status, data = http_get(url, headers, timeout)
if status not in allow:
raise Failure('GET ' + url.split('?')[0] + ': status ' + str(status))
try:
return status, json.loads(data) if data else None
except (ValueError, UnicodeError):
raise Failure('GET ' + url.split('?')[0] + ': invalid JSON') from None
# ---------------------------------------------------------------- trace JSON
def normalize_id(value, nbytes):
"""Hex or base64 ID -> lowercase hex of exactly nbytes; else Failure."""
if not isinstance(value, str):
raise Failure('trace id is not a string')
if value == '':
return ''
if len(value) == nbytes * 2:
try:
int(value, 16)
return value.lower()
except ValueError:
pass
try:
raw = base64.b64decode(value, validate=True)
except (binascii.Error, ValueError):
raise Failure('unrecognised trace/span id encoding') from None
if len(raw) != nbytes:
raise Failure('trace/span id has wrong length')
return raw.hex()
def _any_value(node):
if not isinstance(node, dict):
return node
for key in ('stringValue', 'boolValue', 'doubleValue'):
if key in node:
return node[key]
if 'intValue' in node:
return int(node['intValue'])
if 'arrayValue' in node:
return [_any_value(v) for v in node['arrayValue'].get('values', [])]
return None
def _attributes(items):
out = {}
for item in items or []:
out[item['key']] = _any_value(item.get('value'))
return out
_KIND = {0: 'UNSPECIFIED', 1: 'INTERNAL', 2: 'SERVER', 3: 'CLIENT',
4: 'PRODUCER', 5: 'CONSUMER'}
_STATUS = {0: 'UNSET', 1: 'OK', 2: 'ERROR'}
def normalize_trace(document):
"""Tempo JSON (batches or resourceSpans) -> list of span dicts."""
if not isinstance(document, dict):
raise Failure('unknown Tempo response format')
batches = document.get('batches', document.get('resourceSpans'))
if batches is None:
raise Failure('unknown Tempo response format (no batches/resourceSpans)')
spans = []
for batch in batches:
resource = _attributes(batch.get('resource', {}).get('attributes'))
scopes = batch.get('scopeSpans', batch.get('instrumentationLibrarySpans'))
if scopes is None:
raise Failure('unknown Tempo batch format (no scopeSpans)')
for scope in scopes:
for span in scope.get('spans', []):
kind = span.get('kind', 0)
if isinstance(kind, str):
kind = kind.replace('SPAN_KIND_', '')
else:
kind = _KIND.get(kind, str(kind))
status = span.get('status', {}).get('code', 0)
if isinstance(status, str):
status = status.replace('STATUS_CODE_', '')
else:
status = _STATUS.get(status, str(status))
spans.append({
'trace_id': normalize_id(span.get('traceId', ''), 16),
'span_id': normalize_id(span.get('spanId', ''), 8),
'parent_span_id': normalize_id(
span.get('parentSpanId', ''), 8),
'name': span.get('name', ''),
'kind': kind,
'start_ns': int(span.get('startTimeUnixNano', 0)),
'end_ns': int(span.get('endTimeUnixNano', 0)),
'attributes': _attributes(span.get('attributes')),
'status': status,
'links': [{
'trace_id': normalize_id(link.get('traceId', ''), 16),
'span_id': normalize_id(link.get('spanId', ''), 8),
} for link in span.get('links', [])],
'service': resource.get('service.name'),
'instance': resource.get('service.instance.id'),
})
return spans
# ------------------------------------------------------------------- client
class ObsClient:
def __init__(self, prometheus_url, grafana_url, password_file=None,
base_url=None, ca_file=None):
self.prometheus = prometheus_url.rstrip('/')
self.grafana = grafana_url.rstrip('/')
self.base_url = base_url
self.ca_file = ca_file
path = Path(password_file) if password_file else (
ROOT / 'secrets/grafana_admin_password')
self._auth = None
if path.exists():
token = base64.b64encode(
b'admin:' + path.read_bytes().strip()).decode()
self._auth = {'Authorization': 'Basic ' + token}
# -- Prometheus
def prom_query(self, expr, at=None):
params = {'query': expr}
if at is not None:
params['time'] = str(at)
_, body = http_json(self.prometheus + '/api/v1/query?' +
urllib.parse.urlencode(params))
if not isinstance(body, dict) or body.get('status') != 'success':
raise Failure('Prometheus query failed')
return body['data']['result']
def prom_scalar(self, expr, default=None):
result = self.prom_query(expr)
if not result:
return default
return float(result[0]['value'][1])
def prom_range(self, expr, start, end, step):
params = {'query': expr, 'start': str(start), 'end': str(end),
'step': str(step)}
_, body = http_json(self.prometheus + '/api/v1/query_range?' +
urllib.parse.urlencode(params))
if not isinstance(body, dict) or body.get('status') != 'success':
raise Failure('Prometheus range query failed')
return body['data']['result']
def prom_names(self):
_, body = http_json(self.prometheus + '/api/v1/label/__name__/values')
return sorted(body['data'])
# -- Grafana proxied backends
def _grafana_get(self, path, allow=(200,)):
if self._auth is None:
raise Failure('Grafana password file is missing')
return http_get(self.grafana + path, self._auth)
def grafana_json(self, path, allow=(200,)):
status, data = self._grafana_get(path)
if status not in allow:
raise Failure('Grafana ' + path.split('?')[0] + ': status ' +
str(status))
try:
return json.loads(data)
except (ValueError, UnicodeError):
raise Failure('Grafana ' + path.split('?')[0] +
': invalid JSON') from None
def loki_rows(self, query, start_ns, end_ns, deadline_s=60):
"""All log rows in [start, end]: [{'ts', 'line', 'labels'}]."""
limit = 5000
deadline = time.monotonic() + deadline_s
cursor = int(start_ns)
seen = set()
rows = []
while True:
if time.monotonic() > deadline:
raise Failure('Loki query deadline exceeded; incomplete window')
params = urllib.parse.urlencode({
'query': query, 'start': str(cursor), 'end': str(int(end_ns)),
'limit': str(limit), 'direction': 'forward'})
body = self.grafana_json(
'/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' +
params)
if body.get('status') != 'success':
raise Failure('Loki query failed')
fresh = 0
page = 0
last_ts = cursor
for stream in body['data']['result']:
labels = stream.get('stream', {})
key = json.dumps(labels, sort_keys=True)
for value in stream.get('values', []):
page += 1
ts, line = int(value[0]), value[1]
last_ts = max(last_ts, ts)
ident = (key, ts, line)
if ident in seen:
continue
seen.add(ident)
fresh += 1
rows.append({'ts': ts, 'line': line, 'labels': labels})
if page < limit:
return rows
if fresh == 0 or last_ts == cursor:
raise Failure('Loki pagination made no progress; '
'incomplete window')
cursor = last_ts # overlap on the last timestamp, deduped above
def get_trace(self, trace_id, wait=0):
"""Normalised spans, or None on 404; waits up to `wait` seconds."""
deadline = time.monotonic() + min(wait, 60)
path = '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id
while True:
if self._auth is None:
raise Failure('Grafana password file is missing')
status, data = http_get(
self.grafana + path,
{**self._auth, 'Accept': 'application/json'})
if status == 200:
try:
return normalize_trace(json.loads(data))
except (ValueError, UnicodeError):
raise Failure('Tempo returned invalid JSON') from None
if status != 404:
raise Failure('Tempo trace lookup: status ' + str(status))
if time.monotonic() >= deadline:
return None
time.sleep(1)
# -- closed backends from inside the compose network
def internal_get(self, service, port, path):
script = (
"const [h,p,u]=process.argv.slice(1);"
"fetch('http://'+h+':'+p+u,{signal:AbortSignal.timeout(5000)})"
".then(async r=>{const b=await r.text();"
"console.log(JSON.stringify({status:r.status,body:b.slice(0,1048576)}))})"
".catch(e=>{console.log(JSON.stringify({status:0,body:String(e.name)}))})")
result = stand.compose('exec', '-T', 'api', 'node', '-e', script,
service, str(port), path, check_result=False)
try:
out = json.loads(result.stdout.strip().splitlines()[-1])
except (ValueError, IndexError):
raise Failure('internal_get ' + service + ': no result') from None
return out['status'], out['body']
# -- readiness
def _check(self, problems, name, ok):
if not ok:
problems.append(name)
def readiness_problems(self, app_telemetry=True, provisioning=True):
problems = []
if self.base_url:
try:
status, _ = http_get(self.base_url.rstrip('/') + '/health/ready')
self._check(problems, 'api-ready', status == 200)
except Failure:
problems.append('api-ready')
probes = (('otel-collector', 13133, '/', 'collector-health'),
('loki', 3100, '/ready', 'loki-ready'),
('tempo', 3200, '/ready', 'tempo-ready'))
for service, port, path, name in probes:
try:
status, _ = self.internal_get(service, port, path)
self._check(problems, name, status == 200)
except Failure:
problems.append(name)
try:
status, _ = http_get(self.prometheus + '/-/ready')
self._check(problems, 'prometheus-ready', status == 200)
except Failure:
problems.append('prometheus-ready')
try:
status, _ = http_get(self.grafana + '/api/health')
self._check(problems, 'grafana-health', status == 200)
except Failure:
problems.append('grafana-health')
try:
up = {r['metric']['job']: r['value'][1]
for r in self.prom_query('up')}
for job in SCRAPE_JOBS:
self._check(problems, 'target-up:' + job, up.get(job) == '1')
if app_telemetry:
ages = self.prom_query(
'time() - timestamp(target_info{job=~"messenger-.*"})')
fresh = {}
for r in ages:
if float(r['value'][1]) < 30:
fresh.setdefault(r['metric']['job'], set()).add(
r['metric'].get('instance'))
self._check(problems, 'api-instances>=2',
len(fresh.get('messenger-api', ())) >= 2)
self._check(problems, 'worker-instance',
len(fresh.get('messenger-worker', ())) >= 1)
except Failure:
problems.append('prometheus-queries')
if provisioning:
try:
for uid in DATASOURCE_UIDS:
if uid == 'tempo':
# The Tempo plugin has no health resource; its API echo
# through the datasource proxy proves the connection.
status, _ = self._grafana_get(
'/api/datasources/proxy/uid/tempo/api/echo')
self._check(problems, 'datasource:tempo', status == 200)
continue
body = self.grafana_json(
'/api/datasources/uid/' + uid + '/health',
allow=(200, 400, 404, 500))
self._check(problems, 'datasource:' + uid,
isinstance(body, dict) and
body.get('status', '').upper() == 'OK')
_, rules = http_json(self.prometheus + '/api/v1/rules')
names = {r['name'] for g in rules['data']['groups']
for r in g['rules']}
for alert in RULE_ALERTS:
self._check(problems, 'rule:' + alert, alert in names)
for uid in DASHBOARD_UIDS:
status, _ = self._grafana_get('/api/dashboards/uid/' + uid)
self._check(problems, 'dashboard:' + uid, status == 200)
except Failure:
problems.append('provisioning-queries')
return problems
def wait_ready(self, timeout=120, app_telemetry=True, provisioning=True):
deadline = time.monotonic() + timeout
problems = ['not-checked']
while time.monotonic() < deadline:
problems = self.readiness_problems(app_telemetry, provisioning)
if not problems:
return
time.sleep(2)
raise Failure('stack not ready: ' + ', '.join(problems))
def wait_telemetry_flushed(self, timeout=120):
"""Two new Prometheus scrapes of the Collector and empty queues."""
deadline = time.monotonic() + timeout
stamps = []
while time.monotonic() < deadline:
result = self.prom_query('timestamp(up{job="otel-collector"})')
if result:
stamp = float(result[0]['value'][1])
if not stamps or stamp > stamps[-1]:
stamps.append(stamp)
queue = self.prom_query('sum(otelcol_exporter_queue_size)')
empty = bool(queue) and float(queue[0]['value'][1]) == 0
if len(stamps) >= 3 and empty:
return
time.sleep(2)
raise Failure('telemetry did not flush before deadline')
# -- controlled application environment
def container_ids(self, service):
out = stand.compose('ps', '-q', service).stdout.split()
if not out:
raise Failure('no running container for ' + service)
return out
def inspect_env(self, service):
"""{container_id: {'env': {...}, 'image': id, 'restarts': n}}"""
result = {}
for cid in self.container_ids(service):
data = json.loads(stand.docker('inspect', cid).stdout)[0]
env = dict(item.split('=', 1) for item in data['Config']['Env']
if '=' in item)
result[cid[:12]] = {
'env': {k: env.get(k) for k in MUTABLE_KEYS + DIAGNOSTIC_KEYS},
'image': data['Image'],
'restarts': data['RestartCount'],
'oom_killed': data['State'].get('OOMKilled', False),
}
return result
def app_state(self):
return {'api': self.inspect_env('api'),
'worker': self.inspect_env('worker')}
@contextlib.contextmanager
def app_environment(self, initial_overrides=None):
saved_state = self.app_state()
baseline = {}
for service in ('api', 'worker'):
for info in saved_state[service].values():
for key in MUTABLE_KEYS:
baseline.setdefault((service, key), info['env'][key])
saved_environ = {k: os.environ.get(k) for k in MUTABLE_KEYS}
touched = set()
dirty = [False]
def effective(service):
return {cid: info for cid, info in self.inspect_env(service).items()}
def set_overrides(values, force=False):
unknown = set(values) - set(MUTABLE_KEYS)
if unknown:
raise Failure('not a controllable key: ' + ', '.join(sorted(unknown)))
for key in values:
api = {i['env'][key] for i in effective('api').values()}
worker = {i['env'][key] for i in effective('worker').values()}
if len(api | worker) > 1 and api != worker:
raise Failure('api/worker disagree on ' + key + ' before change')
if not force and all(
info['env'][key] == str(value)
for service in ('api', 'worker')
for info in effective(service).values()
for key, value in values.items()):
return self.app_state()
for key, value in values.items():
os.environ[key] = str(value)
touched.add(key)
dirty[0] = True
stand.compose('up', '-d', '--no-deps', '--force-recreate',
'--wait', 'api', 'worker')
for service in ('api', 'worker'):
state = effective(service)
for cid, info in state.items():
for key, value in values.items():
if info['env'][key] != str(value):
raise Failure('effective ' + key + ' differs in ' +
service + ' ' + cid)
sdk_on = os.environ.get('OTEL_SDK_DISABLED', 'false') != 'true'
self.wait_ready(app_telemetry=sdk_on, provisioning=True)
return self.app_state()
try:
if initial_overrides:
set_overrides(initial_overrides)
yield set_overrides
finally:
if dirty[0]:
# Restore the original effective values as controller env so the
# recreated containers match what was running before.
for key in MUTABLE_KEYS:
original = baseline.get(('api', key))
if original is not None and key in touched:
os.environ[key] = original
stand.compose('up', '-d', '--no-deps', '--force-recreate',
'--wait', 'api', 'worker')
for key in touched:
if saved_environ[key] is None:
os.environ.pop(key, None)
else:
os.environ[key] = saved_environ[key]
now = self.app_state()
for service in ('api', 'worker'):
for cid, info in now[service].items():
for key in touched:
if info['env'][key] != baseline.get((service, key)):
print('WARNING: ' + key + ' not restored on ' +
service, file=sys.stderr)
with contextlib.suppress(Failure):
self.wait_ready(app_telemetry=True, provisioning=True,
timeout=120)
# -- foreign faults
def foreign_faults(self):
result = stand.compose('exec', '-T', 'redis', 'redis-cli', '--scan',
'--pattern', 'fault:*')
return sorted(line for line in result.stdout.split() if line)
def require_no_foreign_faults(self):
keys = self.foreign_faults()
if keys:
raise Failure('foreign fault keys present (finish the other '
'experiment first): ' + ', '.join(keys))
# --------------------------------------------------------------- PostgreSQL
def psql(sql, timeout=30):
script = 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 ' \
'-At -F , -c "$SQL"'
result = subprocess.run(
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
'-e', 'SQL=' + sql, 'postgres', 'sh', '-c', script],
cwd=ROOT, text=True, capture_output=True, timeout=timeout)
if result.returncode != 0:
raise Failure('psql failed: ' + ((result.stderr or '').strip()
.splitlines() or ['?'])[-1][:200])
return result.stdout.strip()
@contextlib.contextmanager
def hold_table_lock(table):
"""ACCESS EXCLUSIVE lock on messages/attachments held until exit."""
if table not in LOCKABLE_TABLES:
raise Failure('table is not lockable: ' + table)
name = 'lab6-lock-' + uuid.uuid4().hex[:12]
process = subprocess.Popen(
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
'postgres', 'sh', '-c',
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -At'],
cwd=ROOT, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
pid = None
try:
process.stdin.write(
"SET application_name='" + name + "';\n"
"BEGIN;\n"
"SET LOCAL statement_timeout='120s';\n"
"SET LOCAL idle_in_transaction_session_timeout='120s';\n"
"LOCK TABLE " + table + " IN ACCESS EXCLUSIVE MODE;\n"
"SELECT 'LOCKED:'||pg_backend_pid();\n")
process.stdin.flush()
deadline = time.monotonic() + 30
while pid is None:
if time.monotonic() > deadline:
raise Failure('table lock marker not received')
ready, _, _ = select.select([process.stdout], [], [], 1)
if not ready:
if process.poll() is not None:
raise Failure('psql exited before locking')
continue
line = process.stdout.readline()
if line.startswith('LOCKED:'):
pid = int(line.split(':')[1])
elif line == '' and process.poll() is not None:
raise Failure('psql exited before locking')
yield pid
finally:
try:
if process.poll() is None:
process.stdin.write('ROLLBACK;\n\\q\n')
process.stdin.flush()
process.stdin.close()
except (OSError, ValueError):
pass
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
if pid is not None:
with contextlib.suppress(Failure, subprocess.TimeoutExpired):
psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE pid=" + str(int(pid)) +
" AND application_name='" + name + "'")
for stream in (process.stdout, process.stderr):
with contextlib.suppress(Exception):
stream.close()
def wait_drained(attachment_ids, obs=None, timeout=120, flush_telemetry=True):
"""Batch-check jobs/outbox/attachments in PostgreSQL (no HTTP polling).
Returns {'drained': bool, 'states': {id: status}, 'queued': n,
'outbox': n, 'seconds': t, 'timestamps': ...}. A vanished known ID is a
Failure; non-ready at the deadline is reported, not raised.
"""
ids = list(attachment_ids)
for item in ids:
uuid.UUID(item)
array = "ARRAY[" + ",".join("'" + i + "'" for i in ids) + "]::uuid[]" \
if ids else "ARRAY[]::uuid[]"
sql = (
"SELECT (SELECT count(*) FROM attachment_jobs WHERE status IN "
"('queued','processing'))||','||"
"(SELECT count(*) FROM job_outbox WHERE published_at IS NULL)||','||"
"COALESCE((SELECT string_agg(id::text||':'||status,';') FROM attachments "
"WHERE id = ANY(" + array + ")),'')")
started = time.monotonic()
start_ts = utc_now()
states = {}
queued = outbox = -1
drained = False
while time.monotonic() - started < timeout:
out = psql(sql)
queued_s, outbox_s, rest = (out.split(',', 2) + ['', ''])[:3]
queued, outbox = int(queued_s), int(outbox_s)
states = dict(part.split(':', 1) for part in rest.split(';') if part)
missing = set(ids) - set(states)
if missing:
raise Failure('known attachment vanished: ' + sorted(missing)[0])
terminal = all(s in ('ready', 'failed') for s in states.values())
if queued == 0 and outbox == 0 and terminal:
drained = True
break
time.sleep(1)
result = {'drained': drained, 'states': states, 'queued_jobs': queued,
'unpublished_outbox': outbox,
'seconds': round(time.monotonic() - started, 3),
'started': start_ts, 'finished': utc_now()}
if flush_telemetry and obs is not None:
obs.wait_telemetry_flushed(timeout=timeout)
result['telemetry_flushed'] = utc_now()
return result
+106
View File
@@ -0,0 +1,106 @@
"""Reconcile what the application wrote to stdout with what reached Loki and
Tempo (stdlib only). A record's stdout line is the ground truth for whether a
trace was sampled (trace_sampled) and which SERVER span ID it carries."""
import json
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
from obs import Failure, ROOT # noqa: E402
DEADLINE_S = 60
def stdout_http_logs(since_utc, services=('api',), until_utc=None):
"""{request_id: parsed stdout JSON} for event=http.request lines."""
args = ['docker', 'compose', '--env-file', obs.stand.ENV_FILE, 'logs',
'--no-color', '--no-log-prefix', '--since', since_utc]
if until_utc:
args += ['--until', until_utc]
result = subprocess.run(args + list(services), cwd=ROOT, text=True,
capture_output=True, timeout=180)
if result.returncode != 0:
raise Failure('docker compose logs failed')
found = {}
for line in result.stdout.splitlines():
if not line.startswith('{'):
continue
try:
row = json.loads(line)
except ValueError:
continue
if row.get('event') == 'http.request' and row.get('request_id'):
found[row['request_id']] = row
return found
def loki_by_request_id(client, request_ids, start_ns, end_ns, deadline_s=DEADLINE_S):
"""Poll until every id is seen or the deadline passes. Returns
({request_id: labels}, complete) where complete means every query returned
a full, un-truncated answer (a Failure would propagate otherwise)."""
ids = sorted(request_ids)
found = {}
deadline = time.monotonic() + deadline_s
while True:
for i in range(0, len(ids), 100):
chunk = [r for r in ids[i:i + 100] if r not in found]
if not chunk:
continue
query = ('{service_name="messenger-api"} | event_name="http.request" '
'| request_id=~"' + '|'.join(chunk) + '"')
for row in client.loki_rows(query, start_ns, end_ns):
found[row['labels']['request_id']] = row['labels']
if len(found) >= len(ids) or time.monotonic() >= deadline:
return found
time.sleep(3)
def tempo_server_spans(client, targets, deadline_s=DEADLINE_S):
"""targets: {request_id: (trace_id, span_id)} -> {request_id: bool}."""
found = {}
deadline = time.monotonic() + deadline_s
while True:
for rid, (tid, sid) in targets.items():
if found.get(rid):
continue
spans = client.get_trace(tid)
found[rid] = bool(spans) and any(
s['span_id'] == sid and s['kind'] == 'SERVER' for s in spans)
if all(found.values()) or time.monotonic() >= deadline:
return found
time.sleep(3)
def reconcile(client, mode, requests, since_utc, start_ns, end_ns):
"""requests: [{'request_id','status',...}] -> (records, summary).
off: stdout only (Loki/Tempo not applicable). on: stdout -> Loki and
stdout span_id -> Tempo SERVER span."""
stdout = stdout_http_logs(since_utc)
records = []
for req in requests:
line = stdout.get(req.get('request_id'))
rec = {'request_id': req.get('request_id'), 'status': req.get('status'),
'stdout': None, 'loki': None, 'span': None}
if line is not None:
rec['stdout'] = {k: line[k] for k in
('trace_id', 'span_id', 'trace_sampled') if k in line}
records.append(rec)
if mode != 'off':
with_stdout = [r for r in records if r['stdout'] is not None]
loki = loki_by_request_id(
client, [r['request_id'] for r in with_stdout], start_ns, end_ns)
for rec in with_stdout:
rec['loki'] = rec['request_id'] in loki
targets = {r['request_id']: (r['stdout']['trace_id'], r['stdout']['span_id'])
for r in with_stdout
if r['stdout'].get('trace_sampled') is True}
spans = tempo_server_spans(client, targets)
for rec in with_stdout:
if rec['request_id'] in spans:
rec['span'] = spans[rec['request_id']]
return records, measurement.summarize_cohort(mode, records)
+181
View File
@@ -0,0 +1,181 @@
"""Container resource sampling for lab6: docker stats (container CPU/memory) and
docker top (process RSS) every few seconds. Stdlib only.
MemUsage is *container* memory usage (cgroup, includes page cache); RSS is the
sum of process RSS from `docker top` (KiB -> bytes). They are never mixed."""
import json
import re
import subprocess
import threading
import time
from pathlib import Path
import sys # noqa: E402
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
UNITS = {'B': 1, 'kB': 1000, 'MB': 1000 ** 2, 'GB': 1000 ** 3, 'TB': 1000 ** 4,
'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3, 'TiB': 1024 ** 4}
SIZE = re.compile(r'^\s*([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)\s*$')
SERVICES = ('api', 'worker', 'otel-collector', 'prometheus', 'loki', 'tempo',
'grafana')
def parse_size(text):
"""'147.5MiB' -> bytes; SI and IEC units differ; unknown unit is an error."""
match = SIZE.match(text)
if not match or match.group(2) not in UNITS:
raise ValueError('unparseable size: ' + repr(text))
return int(float(match.group(1)) * UNITS[match.group(2)])
def parse_stats_row(row):
usage, limit = row['MemUsage'].split('/')
cpu = row['CPUPerc'].strip()
if not cpu.endswith('%'):
raise ValueError('unparseable CPU: ' + repr(cpu))
return {'cpu_pct': float(cpu[:-1]), 'mem_usage_bytes': parse_size(usage),
'mem_limit_bytes': parse_size(limit)}
def parse_top_rss(text):
"""`docker top <id> -eo pid,rss` output -> bytes (KiB x 1024)."""
lines = [line.split() for line in text.strip().splitlines()]
if not lines or [h.upper() for h in lines[0]] != ['PID', 'RSS']:
raise ValueError('unexpected docker top header')
return sum(int(row[1]) for row in lines[1:]) * 1024
def container_map():
"""{short_id: service} for all observed services."""
result = {}
for service in SERVICES:
ids = obs.stand.compose('ps', '-q', service, check_result=False)
for cid in ids.stdout.split():
result[cid[:12]] = service
return result
class ResourceSampler(threading.Thread):
def __init__(self, containers, interval=5.0, path=None):
super().__init__(daemon=True)
self.containers = dict(containers)
self.interval = interval
self.path = Path(path) if path else None
self.stop_event = threading.Event()
self.samples = []
self.errors = []
def one_tick(self, tick):
mono = time.monotonic()
stamp = obs.utc_now()
ids = list(self.containers)
stats = subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + ids,
capture_output=True, text=True, timeout=60)
rows = {}
for line in stats.stdout.splitlines():
if line.strip():
row = json.loads(line)
rows[row['ID'][:12]] = row
for cid in ids:
sample = {'tick': tick, 'utc': stamp, 'mono': mono, 'id': cid,
'service': self.containers[cid]}
raw = rows.get(cid)
if raw is None:
sample['error'] = 'no_stats_row'
else:
sample['raw'] = {k: raw[k] for k in ('CPUPerc', 'MemUsage')}
try:
sample.update(parse_stats_row(raw))
except ValueError as exc:
sample['error'] = str(exc)
top = subprocess.run(
['docker', 'top', cid, '-eo', 'pid,rss'],
capture_output=True, text=True, timeout=30)
if top.returncode == 0:
try:
sample['rss_bytes'] = parse_top_rss(top.stdout)
except ValueError as exc:
sample['rss_error'] = str(exc)
else:
sample['rss_bytes'] = None
sample['rss_reason'] = 'not_running'
self.samples.append(sample)
if self.path is not None:
with self.path.open('a') as handle:
handle.write(json.dumps(sample) + '\n')
if 'error' in sample:
self.errors.append(sample['error'])
def run(self):
tick = 0
while not self.stop_event.is_set():
begun = time.monotonic()
try:
self.one_tick(tick)
except (subprocess.SubprocessError, ValueError, KeyError) as exc:
self.errors.append(type(exc).__name__)
tick += 1
self.stop_event.wait(max(0.0, self.interval - (time.monotonic() - begun)))
def stop(self):
self.stop_event.set()
self.join(timeout=90)
def _summarise(values):
values = [v for v in values if v is not None]
return {'median': measurement.median(values),
'p95': measurement.nearest_rank(values, 0.95),
'max': max(values) if values else None, 'n': len(values)}
def summarize(samples, container_ids=None):
"""Per-service statistics. For API the replicas are summed on a common
tick first; individual values are kept in `per_container`."""
ticks = {}
for sample in samples:
if 'error' in sample:
continue
ticks.setdefault(sample['tick'], {}).setdefault(
sample['service'], []).append(sample)
order = sorted(ticks)
result = {}
services = {s['service'] for s in samples}
for service in sorted(services):
cpu, mem, rss, times = [], [], [], []
for tick in order:
group = ticks[tick].get(service)
if not group:
continue
times.append(min(s['mono'] for s in group))
cpu.append(sum(s['cpu_pct'] for s in group))
mem.append(sum(s['mem_usage_bytes'] for s in group))
values = [s.get('rss_bytes') for s in group]
rss.append(None if any(v is None for v in values) else sum(values))
weights = [times[i + 1] - times[i] for i in range(len(times) - 1)]
if times:
weights.append(sum(weights) / len(weights) if weights else 1.0)
mean_cpu = (sum(c * w for c, w in zip(cpu, weights)) / sum(weights)
if weights and sum(weights) > 0 else None)
per_container = {}
for s in samples:
if s['service'] == service and 'error' not in s:
per_container.setdefault(s['id'], []).append(s['cpu_pct'])
result[service] = {
'cpu_pct_time_weighted_mean': mean_cpu,
'container_memory_bytes': _summarise(mem),
'process_rss_bytes': _summarise(rss),
'mem_limit_bytes': max((s['mem_limit_bytes'] for s in samples
if s['service'] == service and
'mem_limit_bytes' in s), default=None),
'ticks': len(times),
'per_container_cpu_mean': {k: sum(v) / len(v)
for k, v in per_container.items()},
}
result['sample_errors'] = len(
[s for s in samples if 'error' in s])
return result
+199
View File
@@ -0,0 +1,199 @@
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement as m # noqa: E402
def slot(i, status=201, service=10.0, lag=1.0, outcome='http'):
if outcome != 'http':
return {'i': i, 'outcome': outcome, 'start_lag_ms': None}
return {'i': i, 'outcome': 'http', 'status': status,
'service_ms': service, 'scheduled_ms': service + lag,
'start_lag_ms': lag}
def cohort(**kw):
base = {'n': 40, 'retained_traces': 5, 'sampled_present': 5,
'intentional_unsampled': 35}
base.update(kw)
return base
def run(mode='p25', spans=10.0, p95=100.0, failures=(), valid=True,
error=None, slow=None):
return {
'mode': mode, 'valid': valid, 'invalid_reasons': [],
'configuration_failures': list(failures),
'spans_per_operation': spans,
'load': {'scheduled_ms': {'p95': p95}},
'cohorts': {'error': error or cohort(), 'slow': slow or cohort()},
'dir': mode,
}
def summary(load_slots, **extra):
load = m.summarize_load(load_slots, 10)
base = {'mode': 'off', 'load': load, 'cohorts': {}, 'ingest': {},
'export_failures': {'loss': False}}
base.update(extra)
return base
class SamplingIsNotLoss(unittest.TestCase):
def test_false_flag_is_not_export_loss_or_missing_span(self):
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': False}, 'loki': True, 'span': False}
self.assertEqual(m.classify_record('p25', rec),
('intentional_unsampled', 'exported'))
counts = m.summarize_cohort('p25', [rec] * 3)
self.assertEqual(counts.get('sampled_missing', 0), 0)
self.assertFalse(m.export_loss(0, 0.0, 0.0)['loss'])
def test_sampled_without_span_is_a_measured_loss(self):
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': True}, 'loki': True, 'span': False}
self.assertEqual(m.classify_record('full', rec)[0], 'sampled_missing')
def test_missing_stdout_is_not_intentional_sampling(self):
self.assertEqual(m.classify_record('p25', {'stdout': None})[0],
'unmatched_stdout')
no_flag = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16}}
self.assertEqual(m.classify_record('p25', no_flag)[0], 'unmatched_stdout')
incomplete = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': True}, 'loki': None, 'span': None}
self.assertEqual(m.classify_record('p25', incomplete),
('incomplete_observation', 'incomplete_observation'))
def test_full_observation_without_log_is_missing_exported_log(self):
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': False}, 'loki': False}
self.assertEqual(m.classify_record('p10', rec)[1], 'missing_exported_logs')
def test_off_needs_neither_loki_nor_app_counters(self):
rec = {'stdout': {'event': 'http.request'}, 'loki': None, 'span': None}
self.assertEqual(m.classify_record('off', rec),
('not_applicable', 'not_applicable'))
leaked = {'stdout': {'trace_id': 'x'}}
self.assertEqual(m.classify_record('off', leaked)[0], 'ids_present_with_sdk_off')
s = summary([slot(i) for i in range(100)], cohorts={
'error': m.summarize_cohort('off', [rec] * 4)})
valid, invalid, failures = m.evaluate_run(s)
self.assertTrue(valid, invalid)
self.assertEqual(failures, [])
class Counters(unittest.TestCase):
def test_reset_is_not_a_negative_loss(self):
self.assertEqual(m.counter_delta(10.0, 3.0), (None, 'counter_reset'))
loss = m.export_loss(None, 0.0, 2.0)
self.assertEqual(loss['total_known'], 2.0)
self.assertTrue(all(v >= 0 for v in loss['parts'].values() if v is not None))
def test_missing_positive_series_is_not_zero_failure_series_may_be(self):
self.assertEqual(m.counter_delta(None, None), (None, 'missing_series'))
self.assertEqual(m.counter_delta(None, None, failure_only=True)[0], 0.0)
self.assertEqual(m.counter_delta(5.0, 9.0), (4.0, None))
def test_unexplained_reset_invalidates_but_restart_explains_it(self):
s = summary([slot(i) for i in range(100)],
ingest={'spans': {'delta': None, 'reason': 'counter_reset'}})
self.assertFalse(m.evaluate_run(s)[0])
s['restarts'] = ['collector']
valid, _, failures = m.evaluate_run(s)
self.assertTrue(valid)
self.assertIn('restart_or_oom', failures)
class LoadAccounting(unittest.TestCase):
def test_dropped_and_late_slots_stay_in_denominator(self):
slots = [slot(i) for i in range(90)] + \
[slot(90 + i, outcome='client_dropped') for i in range(5)] + \
[slot(95 + i, lag=300.0) for i in range(5)]
load = m.summarize_load(slots, 10)
self.assertEqual(load['scheduled'], 100)
self.assertEqual(load['completed'], 95)
self.assertAlmostEqual(load['completed_ratio'], 0.95)
valid, invalid, _ = m.evaluate_run(summary(slots))
self.assertFalse(valid)
self.assertTrue(any('client_dropped' in r for r in invalid))
self.assertTrue(any('completed_ratio' in r for r in invalid))
def test_late_scheduler_fails_start_lag_even_without_drops(self):
slots = [slot(i, lag=80.0) for i in range(100)]
self.assertFalse(m.evaluate_run(summary(slots))[0])
def test_overall_p95_includes_error_responses(self):
slots = [slot(i, service=10.0) for i in range(90)] + \
[slot(90 + i, status=500, service=900.0) for i in range(10)]
load = m.summarize_load(slots, 10)
self.assertEqual(load['service_ms']['p95'], 900.0)
self.assertEqual(load['success_service_ms']['p95'], 10.0)
self.assertEqual(m.load_errors(load), 10)
def test_nearest_rank(self):
self.assertEqual(m.nearest_rank(list(range(1, 101)), 0.95), 95)
self.assertEqual(m.nearest_rank([7], 0.99), 7)
self.assertIsNone(m.nearest_rank([], 0.5))
class MatrixAndSelection(unittest.TestCase):
def test_valid_adverse_run_stays_in_matrix(self):
s = summary([slot(i) for i in range(100)], mode='p25',
export_failures={'loss': True},
cohorts={'error': cohort(sampled_missing=2)})
valid, invalid, failures = m.evaluate_run(s)
self.assertTrue(valid, invalid)
self.assertIn('export_loss', failures)
self.assertTrue(any(f.startswith('sampled_missing') for f in failures))
s.update(valid=valid, invalid_reasons=invalid,
configuration_failures=failures, repeat=1)
entry = m.matrix_entry(s)
self.assertTrue(entry['valid'])
self.assertTrue(entry['configuration_failures'])
def full_runs(self):
return [run('full', spans=100.0, p95=100.0) for _ in range(3)]
def test_p25_selected_only_against_complete_full_reference(self):
good = [run('p25', spans=40.0, p95=105.0) for _ in range(3)]
self.assertTrue(m.select_p25(self.full_runs(), good)['passed'])
broken_full = self.full_runs()
broken_full[1]['spans_per_operation'] = None
self.assertFalse(m.select_p25(broken_full, good)['passed'])
heavy = [run('p25', spans=60.0, p95=105.0) for _ in range(3)]
self.assertFalse(m.select_p25(self.full_runs(), heavy)['passed'])
slow = [run('p25', spans=40.0, p95=120.0) for _ in range(3)]
self.assertFalse(m.select_p25(self.full_runs(), slow)['passed'])
def test_p25_rejected_when_an_error_class_has_no_retained_trace(self):
runs = [run('p25', spans=40.0, p95=100.0) for _ in range(3)]
runs[2]['cohorts']['slow'] = cohort(retained_traces=0, sampled_present=0)
self.assertFalse(m.select_p25(self.full_runs(), runs)['passed'])
def test_single_validation_uses_frozen_full_reference(self):
ref = m.full_reference(self.full_runs())
ok = run('p25', spans=45.0, p95=108.0)
self.assertTrue(m.validate_single(ok, 'p25', ref)['passed'])
# Would pass a "median of its own" comparison, fails the frozen one.
bad = run('p25', spans=55.0, p95=108.0)
result = m.validate_single(bad, 'p25', ref)
self.assertFalse(result['passed'])
self.assertFalse(result['checks']['spans_le_50pct_frozen_full'])
def test_full_validation_needs_all_eighty_server_spans(self):
ref = m.full_reference(self.full_runs())
ok = run('full', error=cohort(sampled_present=40, retained_traces=40),
slow=cohort(sampled_present=40, retained_traces=40))
self.assertTrue(m.validate_single(ok, 'full', ref)['passed'])
short = run('full', error=cohort(sampled_present=39, retained_traces=39),
slow=cohort(sampled_present=40, retained_traces=40))
self.assertFalse(m.validate_single(short, 'full', ref)['passed'])
def test_miss_all_probability_model(self):
self.assertAlmostEqual(m.miss_all_probability(0.25, 40), 0.75 ** 40)
if __name__ == '__main__':
unittest.main()
+457
View File
@@ -0,0 +1,457 @@
#!/usr/bin/env python3
"""lab6 track 1 "cost of observability": off / full / p25 / p10 under the same
fixed-rate workload. Stdlib only. The protocol (rate calibration, 3x4 matrix,
diagnostic cohorts, p25 acceptance, one independent validation) is frozen in
this file; see REPORT.md for the description."""
import argparse
import json
import os
import resource
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import incidents # noqa: E402
import measurement as m # noqa: E402
import obs # noqa: E402
import reconcile # noqa: E402
import resources # noqa: E402
import verify # noqa: E402
import workload # noqa: E402
from obs import Failure, stand, utc_now # noqa: E402
BASELINE_ENV = dict(obs.SDK_DEFAULTS)
BASELINE_ENV.update({'CACHE_MODE': 'protected', 'FAULT_INJECTION_ENABLED': '1',
'LOG_LEVEL': 'INFO'})
WARMUP_S, MEASURE_S = 60, 180
CALIBRATION = ((20, 20, 40), (10, 20, 40), (5, 20, 40)) # rate, warmup, measure
N_SLOW, ERR_RATE, ERR_SECONDS = 40, 4, 10
COUNTERS = (
('accepted_spans', 'sum(otelcol_receiver_accepted_spans)', False),
('accepted_log_records', 'sum(otelcol_receiver_accepted_log_records)', False),
('accepted_metric_points', 'sum(otelcol_receiver_accepted_metric_points)', False),
('sent_spans', 'sum(otelcol_exporter_sent_spans)', False),
('sent_log_records', 'sum(otelcol_exporter_sent_log_records)', False),
('sent_metric_points', 'sum(otelcol_exporter_sent_metric_points)', False),
('send_failed_spans', 'sum(otelcol_exporter_send_failed_spans)', True),
('send_failed_log_records', 'sum(otelcol_exporter_send_failed_log_records)', True),
('send_failed_metric_points', 'sum(otelcol_exporter_send_failed_metric_points)', True),
('enqueue_failed_spans', 'sum(otelcol_exporter_enqueue_failed_spans)', True),
('enqueue_failed_log_records', 'sum(otelcol_exporter_enqueue_failed_log_records)', True),
('enqueue_failed_metric_points', 'sum(otelcol_exporter_enqueue_failed_metric_points)', True),
('tempo_bytes_received', 'sum(tempo_distributor_bytes_received_total)', False),
('loki_bytes_received', 'sum(loki_distributor_bytes_received_total)', False),
)
def write(path, data):
Path(path).parent.mkdir(parents=True, exist_ok=True)
Path(path).write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True) + '\n')
def unique_dir(base):
base = Path(base)
if not base.exists():
return base
n = 2
while Path('%s-attempt-%d' % (base, n)).exists():
n += 1
return Path('%s-attempt-%d' % (base, n))
def snapshot(client, instance_ids, sdk_on):
snap = {'utc': utc_now(), 'values': {}, 'sample_ts': {}}
for name, expr, _ in COUNTERS:
result = client.prom_query(expr)
snap['values'][name] = float(result[0]['value'][1]) if result else None
snap['sample_ts'][name] = float(result[0]['value'][0]) if result else None
if sdk_on:
selector = 'instance=~"%s"' % '|'.join(instance_ids)
for outcome in ('failed', 'success'):
result = client.prom_query(
'sum(messenger_telemetry_export_items_total{outcome="%s",%s})' % (outcome, selector))
snap['values']['app_export_' + outcome] = float(result[0]['value'][1]) if result else None
return snap
def deltas(a, b, operations, sdk_on):
out = {}
for name, _, failure_only in COUNTERS:
delta, reason = m.counter_delta(a['values'][name], b['values'][name], failure_only)
if name.endswith('_bytes_received') and reason == 'missing_series':
out[name] = {'status': 'not_exposed'}
continue
out[name] = {'start': a['values'][name], 'end': b['values'][name],
'delta': delta, 'reason': reason,
'per_operation': (delta / operations) if delta is not None and operations else None}
for outcome in ('failed', 'success'):
key = 'app_export_' + outcome
if not sdk_on:
out[key] = {'status': 'not_applicable'}
else:
delta, reason = m.counter_delta(a['values'].get(key), b['values'].get(key),
failure_only=(outcome == 'failed'))
out[key] = {'start': a['values'].get(key), 'end': b['values'].get(key),
'delta': delta, 'reason': reason}
return out
def container_health(ids):
"""{short_id: (service, restarts, oom)} from docker inspect."""
out = {}
for cid, service in ids.items():
data = json.loads(stand.docker('inspect', cid).stdout)[0]
out[cid] = (service, data['RestartCount'], bool(data['State'].get('OOMKilled')),
data['State'].get('Running'))
return out
def rusage():
own, kids = resource.getrusage(resource.RUSAGE_SELF), resource.getrusage(resource.RUSAGE_CHILDREN)
return {'self_cpu_s': own.ru_utime + own.ru_stime, 'children_cpu_s': kids.ru_utime + kids.ru_stime,
'loadavg': os.getloadavg()}
class Track:
def __init__(self, args):
self.args = args
self.obs = verify.make_obs(args)
self.out = Path(args.output_dir)
self.image_ids = None
self.set_env = None
# -- environment
def configure(self, mode, force=True):
env = dict(BASELINE_ENV)
env.update(m.MODES[mode])
state = self.set_env(env, force=force)
for service in ('api', 'worker'):
for cid, info in state[service].items():
for key, value in env.items():
if info['env'][key] != value:
raise Failure('effective %s=%s in %s %s' % (key, info['env'][key], service, cid))
images = {svc: sorted(i['image'] for i in state[svc].values()) for svc in ('api', 'worker')}
if self.image_ids is None:
self.image_ids = images
ok = images == self.image_ids
instance_ids = sorted(list(state['api']) + list(state['worker']))
return state, instance_ids, ok, {k: env[k] for k in obs.MUTABLE_KEYS if k in env}
# -- one protocol run
def execute_run(self, mode, rate, run_dir, repeat=None, warmup=WARMUP_S, measure=MEASURE_S,
n_slow=N_SLOW, err_rate=ERR_RATE, err_seconds=ERR_SECONDS, label=None):
run_dir = Path(run_dir)
run_dir.mkdir(parents=True, exist_ok=True)
sdk_on = mode != 'off'
defects, stamps = [], {}
self.obs.require_no_foreign_faults()
pre = obs.wait_drained([], obs=self.obs, timeout=180)
if not pre['drained']:
raise Failure('stack did not drain before the run (%s)' % mode)
state, instance_ids, image_ok, effective = self.configure(mode)
ids = resources.container_map()
health0 = container_health(ids)
c = stand.new_client(self.args.base_url, self.args.ca_file)
alice, _, _, chat = c.bootstrap()
stamps['warmup_start'] = utc_now()
warm = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
warmup, 'warmup', settle=False)
warm_drain = obs.wait_drained(warm['uploads'], obs=self.obs, timeout=180)
stamps['warmup_end'] = utc_now()
if not warm_drain['drained']:
defects.append('warmup_did_not_drain')
snap0 = snapshot(self.obs, instance_ids, sdk_on)
usage0 = rusage()
sampler = resources.ResourceSampler(ids, 5.0, run_dir / 'docker-stats.jsonl')
sampler.start()
stamps['measure_start'] = utc_now()
result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
measure, 'measure', settle=False)
sampler.stop()
stamps['measure_dispatch_end'] = utc_now()
usage1 = rusage()
drain = obs.wait_drained(result['uploads'], obs=self.obs, timeout=180)
snap1 = snapshot(self.obs, instance_ids, sdk_on)
stamps['measure_drained'] = utc_now()
load = result['load']
# -- integrity after the final ingest snapshot (HTTP is allowed now)
integrity = 0
reference = stand.sha256(stand.sample_bytes())
for upload_id in result['uploads']:
meta = stand.metadata(c, alice, upload_id)
if meta['status'] != 'ready' or meta['sha256'] != reference:
integrity += 1
not_ready = sum(1 for s in drain['states'].values() if s != 'ready')
write(run_dir / 'drain.json', {'warmup': warm_drain, 'measurement': drain})
# -- diagnostic cohorts, never mixed with the healthy statistics
cohort_since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 2))
cohort_start_ns = time.time_ns() - 10 * 10 ** 9
stamps['cohorts_start'] = utc_now()
slow_rows = incidents.sequential_uploads(self, alice, chat, n_slow, 500)
slow_valid = all(r['consumed'] and r['status'] == 202 for r in slow_rows)
err_slots = []
with obs.hold_table_lock('messages'):
err = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat,
err_rate, err_seconds, 'error', upload_every=0, settle=False)
err_slots = err['slots']
err_expected = err_rate * err_seconds
err_valid = (len(err_slots) == err_expected and all(
s['outcome'] == 'http' and s['status'] >= 500 and s.get('instance_id') and s.get('request_id')
for s in err_slots))
stamps['cohorts_end'] = utc_now()
slow_drain = obs.wait_drained([r['attachment_id'] for r in slow_rows if r['attachment_id']],
obs=self.obs, timeout=180)
snap2 = snapshot(self.obs, instance_ids, sdk_on)
end_ns = time.time_ns() + 60 * 10 ** 9
slow_records, slow_summary = reconcile.reconcile(
self.obs, mode, [{'request_id': r['request_id'], 'status': r['status']} for r in slow_rows],
cohort_since, cohort_start_ns, end_ns)
err_records, err_summary = reconcile.reconcile(
self.obs, mode, [{'request_id': s.get('request_id'), 'status': s.get('status')}
for s in err_slots if s.get('request_id')],
cohort_since, cohort_start_ns, end_ns)
slow_summary['invalid'] = not slow_valid
err_summary['invalid'] = not err_valid
err_summary['responses'] = len(err_slots)
write(run_dir / 'cohorts.json', {'slow': {'requests': slow_rows, 'records': slow_records,
'summary': slow_summary},
'error': {'slots': [{k: s.get(k) for k in (
'i', 'outcome', 'status', 'request_id', 'instance_id', 'service_ms')}
for s in err_slots],
'records': err_records, 'summary': err_summary}})
health1 = container_health(ids)
restarts = [health1[cid][0] for cid in ids
if health1[cid][1] != health0[cid][1] or health1[cid][2]]
# -- independent proof that SDK-off sends nothing
d_final = deltas(snap0, snap2, load['completed'], sdk_on)
if not sdk_on:
for name in ('accepted_spans', 'accepted_log_records', 'accepted_metric_points'):
if (d_final[name]['delta'] or 0) != 0:
defects.append('sdk_off_but_collector_ingest:' + name)
ingest = deltas(snap0, snap1, load['completed'], sdk_on)
loss = m.export_loss(
ingest['app_export_failed'].get('delta') if sdk_on else None,
sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('send_failed')),
sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('enqueue_failed')))
res_summary = resources.summarize(sampler.samples)
spans_op = ingest['accepted_spans']['per_operation']
summary = {
'mode': mode, 'sampler_ratio': m.SAMPLER_RATIO[mode], 'repeat': repeat, 'label': label,
'rate_rps': rate, 'warmup_s': warmup, 'measurement_s': measure,
'effective_env': effective, 'effective_mode_ok': True,
'image_ids': self.image_ids, 'image_ids_ok': image_ok, 'instance_ids': instance_ids,
'load': load, 'resources': res_summary,
'controller_usage': {'start': usage0, 'end': usage1,
'cpu_s_during_dispatch': (usage1['self_cpu_s'] + usage1['children_cpu_s']
- usage0['self_cpu_s'] - usage0['children_cpu_s'])},
'ingest': ingest, 'ingest_through_cohorts': d_final, 'export_failures': loss,
'spans_per_operation': spans_op,
'log_records_per_operation': ingest['accepted_log_records']['per_operation'],
'metric_points_per_operation': ingest['accepted_metric_points']['per_operation'],
'cohorts': {'slow': slow_summary, 'error': err_summary},
'not_ready_at_deadline': not_ready, 'integrity_errors': integrity,
'restarts': restarts, 'measurement_defects': defects, 'timestamps': stamps,
'foreign_faults': self.obs.foreign_faults(), 'dir': str(run_dir),
'slow_drain': {k: slow_drain[k] for k in ('drained', 'seconds')},
}
valid, invalid, failures = m.evaluate_run(summary)
summary.update(valid=valid, invalid_reasons=invalid, configuration_failures=failures)
with (run_dir / 'requests.jsonl').open('w') as handle:
for slot in result['slots']:
handle.write(json.dumps({k: slot.get(k) for k in (
'i', 'kind', 'scheduled', 'start', 'end', 'service_ms', 'scheduled_ms',
'start_lag_ms', 'status', 'outcome', 'request_id', 'attachment_id', 'error')}) + '\n')
write(run_dir / 'summary.json', summary)
print('run %s -> valid=%s invalid=%s failures=%s spans/op=%s p95=%s' % (
run_dir, valid, invalid, failures, spans_op, load['scheduled_ms']['p95']), flush=True)
return summary
# -- calibration
def calibrate(self):
rows = []
self.configure('full')
for rate, warm, measure in CALIBRATION:
ctx_dir = self.out / 'calibration' / ('rate-%d' % rate)
ctx_dir.mkdir(parents=True, exist_ok=True)
c = stand.new_client(self.args.base_url, self.args.ca_file)
alice, _, _, chat = c.bootstrap()
workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, warm,
'cal-warmup', settle=False)
result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
measure, 'cal-measure', settle=True, obs_client=self.obs)
load = result['load']
drain = result['drain']
checks = {
'zero_client_dropped': load['client_dropped'] == 0,
'zero_http_errors': m.load_errors(load) == 0 and load['http_4xx'] == 0,
'all_uploads_ready': drain['drained'] and all(s == 'ready' for s in drain['states'].values()),
'start_lag_p95_le_50ms': (load['start_lag_ms_p95'] or 1e9) <= m.START_LAG_LIMIT_MS,
'completed_ge_99pct': (load['completed_ratio'] or 0) >= m.COMPLETION_LIMIT,
}
row = {'rate_rps': rate, 'warmup_s': warm, 'measure_s': measure, 'load': load,
'drain': drain, 'checks': checks, 'passed': all(checks.values())}
rows.append(row)
write(ctx_dir / 'calibration.json', row)
print('calibration rate=%d passed=%s %s' % (rate, row['passed'], checks), flush=True)
if row['passed']:
write(self.out / 'calibration.json', {'selected_rate_rps': rate, 'candidates': rows})
return rate
write(self.out / 'calibration.json', {'selected_rate_rps': None, 'candidates': rows})
raise Failure('no candidate rate (20/10/5) passed; fix the stand before the study')
# -- smoke
def smoke(self):
out = self.out / 'smoke'
results = {}
problems = []
for mode in ('off', 'full', 'p25', 'p10'):
s = self.execute_run(mode, 5, unique_dir(out / mode), warmup=5, measure=15,
n_slow=3, err_rate=2, err_seconds=2, label='smoke')
results[mode] = {'valid': s['valid'], 'invalid_reasons': s['invalid_reasons'],
'spans_per_operation': s['spans_per_operation'],
'cohorts': s['cohorts'],
'effective_otel': {k: s['effective_env'][k] for k in
('OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG')}}
if not s['valid']:
problems.append('%s: invalid %s' % (mode, s['invalid_reasons']))
if mode == 'off' and (s['ingest_through_cohorts']['accepted_spans']['delta'] or 0) != 0:
problems.append('off sent spans')
if mode == 'full' and not s['cohorts']['error'].get('sampled_present'):
problems.append('full has no sampled roots in Tempo')
write(out / 'smoke.json', {'note': 'technical smoke only; not a study result',
'results': results, 'problems': problems})
if problems:
raise Failure('smoke: ' + '; '.join(problems))
print('PASS: smoke (technical only, not a study result)')
# -- full protocol
def full(self, attempt_label):
rate = self.calibrate()
base = self.out / 'runs'
runs, entries = [], []
for repeat, order in enumerate(m.ORDERS, start=1):
for mode in order:
s = self.execute_run(mode, rate, unique_dir(base / ('%d-%s' % (repeat, mode))),
repeat=repeat)
runs.append(s)
entries.append(m.matrix_entry(s))
write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries,
'complete': False})
if not s['valid']:
raise Failure('protocol-invalid run %s: %s (fix the measurement defect and '
'rerun the whole block; raw evidence kept)' % (s['dir'], s['invalid_reasons']))
by_mode = {mode: [r for r in runs if r['mode'] == mode] for mode in m.MODES}
write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries, 'complete': True,
'comparisons': comparisons(by_mode)})
selection = m.select_p25(by_mode['full'], by_mode['p25'])
ref = selection['reference']
selected = 'p25' if selection['passed'] else 'full'
write(self.out / 'selected.json', {
'selected': selected, 'candidate': 'p25', 'candidate_passed': selection['passed'],
'selection': selection, 'rate_rps': rate,
'rare_event_miss_all_probability_model': rare_events(by_mode)})
validations = []
verdict = None
order = [selected] if selected == 'full' else ['p25', 'full']
for mode in order:
s = self.execute_run(mode, rate, unique_dir(self.out / 'validation' / mode), label='validation')
result = m.validate_single(s, mode, ref) if ref['complete'] else {
'passed': False, 'checks': {'frozen_full_reference_complete': False}, 'mode': mode}
validations.append({'mode': mode, 'run_dir': s['dir'], **result})
if result['passed']:
verdict = mode
break
final = {'selected_before_validation': selected, 'validated_mode': verdict,
'validations': validations, 'passed': verdict is not None,
'frozen_full_reference': ref}
write(self.out / 'selected-validation.json', final)
if verdict is None:
raise Failure('selected-validation FAIL: neither p25 nor full passed independent validation')
print('PASS: track1 selected=%s validated=%s' % (selected, verdict))
def metric_paths(run):
r = run['resources']
api = r.get('api', {})
return {
'scheduled_p95_ms': run['load']['scheduled_ms']['p95'],
'service_p95_ms': run['load']['service_ms']['p95'],
'scheduled_p99_ms': run['load']['scheduled_ms']['p99'],
'success_scheduled_p95_ms': run['load']['success_scheduled_ms']['p95'],
'api_cpu_pct_sum_replicas': api.get('cpu_pct_time_weighted_mean'),
'worker_cpu_pct': r.get('worker', {}).get('cpu_pct_time_weighted_mean'),
'collector_cpu_pct': r.get('otel-collector', {}).get('cpu_pct_time_weighted_mean'),
'api_container_mem_median': api.get('container_memory_bytes', {}).get('median'),
'api_rss_median': api.get('process_rss_bytes', {}).get('median'),
'worker_container_mem_median': r.get('worker', {}).get('container_memory_bytes', {}).get('median'),
'spans_per_operation': run.get('spans_per_operation'),
'log_records_per_operation': run.get('log_records_per_operation'),
'metric_points_per_operation': run.get('metric_points_per_operation'),
'retained_error_traces': run['cohorts']['error'].get('retained_traces'),
'retained_slow_traces': run['cohorts']['slow'].get('retained_traces'),
}
def comparisons(by_mode):
table = {mode: {} for mode in by_mode}
for mode, runs in by_mode.items():
keys = metric_paths(runs[0]).keys() if runs else []
for key in keys:
table[mode][key] = m.spread([metric_paths(r)[key] for r in runs])
versus = {}
for mode in by_mode:
versus[mode] = {}
for key in ('scheduled_p95_ms', 'service_p95_ms', 'api_cpu_pct_sum_replicas',
'worker_cpu_pct', 'api_container_mem_median', 'api_rss_median'):
versus[mode][key + '_vs_off'] = m.relative(
table[mode].get(key, {}).get('median'), table['off'].get(key, {}).get('median'))
for key in ('spans_per_operation', 'log_records_per_operation', 'metric_points_per_operation'):
versus[mode][key + '_vs_full'] = m.relative(
table[mode].get(key, {}).get('median'), table['full'].get(key, {}).get('median'))
failures = {mode: [r['configuration_failures'] for r in runs] for mode, runs in by_mode.items()}
return {'per_mode': table, 'relative': versus, 'configuration_failures': failures}
def rare_events(by_mode):
out = {}
for mode, runs in by_mode.items():
ratio = m.SAMPLER_RATIO[mode]
if ratio is None:
continue
for cls in ('error', 'slow'):
kept = sum(r['cohorts'][cls].get('retained_traces', 0) for r in runs)
n = sum(r['cohorts'][cls].get('n', 0) for r in runs)
out['%s/%s' % (mode, cls)] = {
'N_total_over_runs': n, 'observed_kept': kept,
'per_run_N': 40, 'miss_all_probability_per_run_model': m.miss_all_probability(ratio, 40),
'note': '(1-r)^N assumes independent root sampling; a model, not a measurement'}
return out
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--ca-file')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output-dir', default='evidence/lab6/track1')
parser.add_argument('--smoke', action='store_true')
args = parser.parse_args(argv)
track = Track(args)
try:
with track.obs.app_environment(BASELINE_ENV) as set_env:
track.set_env = set_env
if args.smoke:
track.smoke()
else:
track.full('attempt')
except Failure as error:
print('FAIL: ' + str(error), file=sys.stderr)
return 1
return 0
if __name__ == '__main__':
sys.exit(main())
+671
View File
@@ -0,0 +1,671 @@
#!/usr/bin/env python3
"""lab6 verification: ready | inventory | signals | correlation | redaction |
history. Python 3.9+, standard library only. Every subcommand writes its own
evidence file under --output-dir and exits non-zero on any failed criterion."""
import argparse
import base64
import hashlib
import json
import re
import sys
import time
import urllib.parse
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import obs # noqa: E402
from obs import Failure, stand, utc_now # noqa: E402
FORBIDDEN_LABEL = re.compile(
r'(^|_)(user|chat|message|attachment|request|trace|span|job)_id$|'
r'(^|_)(url|filename|text|body|username|password|token)($|_)')
EXPECTED_SIGNALS = [
# logical signal, metric, required label names (normalised), unit
('HTTP duration/count', 'http_server_request_duration_seconds_count',
('http_request_method', 'http_route', 'http_response_status_code'), 's'),
('HTTP duration buckets', 'http_server_request_duration_seconds_bucket', ('le',), 's'),
('HTTP duration sum', 'http_server_request_duration_seconds_sum', (), 's'),
('Jobs processed', 'messenger_jobs_processed_total', ('job_type', 'outcome'), '{job}'),
('Job duration', 'messenger_job_duration_seconds_count', ('job_type', 'outcome'), 's'),
('Time to ready', 'messenger_attachment_time_to_ready_seconds_count', ('job_type',), 's'),
('Queue depth', 'messenger_jobs_queue_depth', ('state',), '{job}'),
('Queue oldest age', 'messenger_jobs_queue_oldest_age_seconds', (), 's'),
('Outbox pending', 'messenger_outbox_pending', (), '{message}'),
('Outbox oldest age', 'messenger_outbox_oldest_age_seconds', (), 's'),
('DB pool connections', 'db_client_connection_count', ('db_client_connection_state',), '{connection}'),
('DB pool max', 'db_client_connection_max', (), '{connection}'),
('DB pool pending', 'db_client_connection_pending_requests', (), '{request}'),
('DB pool wait', 'db_client_connection_wait_time_seconds_count', (), 's'),
('DB operation duration', 'db_client_operation_duration_seconds_count', ('db_operation_name',), 's'),
('Worker active', 'messenger_worker_jobs_active', (), '{job}'),
('Worker capacity', 'messenger_worker_jobs_capacity', (), '{job}'),
('Broker published', 'messenger_broker_published_total', ('outcome',), '{message}'),
('Telemetry export', 'messenger_telemetry_export_items_total', ('signal', 'outcome'), '{item}'),
('Process CPU', 'process_cpu_time_seconds_total', ('cpu_mode',), 's'),
('Process RAM', 'process_memory_usage_bytes', (), 'By'),
('V8 heap', 'v8js_memory_heap_used_bytes', (), 'By'),
('Event loop delay', 'nodejs_eventloop_delay_p99_seconds', (), 's'),
]
COLLECTOR_SIGNALS = [
'otelcol_receiver_accepted_spans', 'otelcol_receiver_accepted_log_records',
'otelcol_receiver_accepted_metric_points', 'otelcol_exporter_sent_spans',
'otelcol_exporter_sent_log_records', 'otelcol_exporter_sent_metric_points',
'otelcol_exporter_queue_size', 'otelcol_exporter_queue_capacity',
]
COLLECTOR_FAILURE_ONLY = [
'otelcol_exporter_send_failed_spans', 'otelcol_exporter_send_failed_log_records',
'otelcol_exporter_send_failed_metric_points', 'otelcol_exporter_enqueue_failed_spans',
'otelcol_exporter_enqueue_failed_log_records', 'otelcol_exporter_enqueue_failed_metric_points',
]
# ------------------------------------------------------------------ helpers
def write_json(out, name, data):
out.mkdir(parents=True, exist_ok=True)
(out / name).write_text(json.dumps(data, indent=2, ensure_ascii=False,
sort_keys=True) + '\n')
def make_obs(args):
return obs.ObsClient(args.prometheus_url, args.grafana_url,
base_url=args.base_url, ca_file=args.ca_file)
def ns_window(seconds_back=600, forward=30):
now = time.time_ns()
return now - seconds_back * 10 ** 9, now + forward * 10 ** 9
def wait_rows(client, query, minimum=1, timeout=60, back=900):
deadline = time.monotonic() + timeout
rows = []
while time.monotonic() < deadline:
start, end = ns_window(back)
rows = client.loki_rows(query, start, end)
if len(rows) >= minimum:
return rows
time.sleep(2)
raise Failure('expected log rows not found in Loki (' + str(len(rows)) +
'/' + str(minimum) + ')')
def http_log(client, request_id, timeout=60):
rows = wait_rows(
client, '{service_name="messenger-api"} | event_name="http.request" '
'| request_id="' + request_id + '"', timeout=timeout)
if len(rows) != 1:
raise Failure('expected exactly one http.request log for the request')
return rows[0]
def chain_to_root(spans, span):
by_id = {s['span_id']: s for s in spans}
path = [span]
while path[-1]['parent_span_id']:
parent = by_id.get(path[-1]['parent_span_id'])
if parent is None:
raise Failure('broken parent chain at ' + path[-1]['name'])
path.append(parent)
if len(path) > 64:
raise Failure('parent chain too long')
return path
def descendants(spans, root_id):
children = {}
for s in spans:
children.setdefault(s['parent_span_id'], []).append(s)
out, stack = [], [root_id]
while stack:
for child in children.get(stack.pop(), []):
out.append(child)
stack.append(child['span_id'])
return out
def safe_span(s):
return {k: s[k] for k in ('span_id', 'parent_span_id', 'name', 'kind',
'status', 'service', 'instance')}
def trace_with(client, trace_id, predicate, timeout=60, description='trace'):
deadline = time.monotonic() + timeout
spans = None
while time.monotonic() < deadline:
spans = client.get_trace(trace_id)
if spans and predicate(spans):
return spans
time.sleep(2)
raise Failure(description + ' did not reach the expected shape in Tempo')
def new_context(base_url, ca_file):
c = stand.new_client(base_url, ca_file)
alice, bob, eve, chat = c.bootstrap()
return c, alice, chat
def instrumented_env():
return {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
# -------------------------------------------------------------------- ready
def cmd_ready(args):
client = make_obs(args)
client.wait_ready(app_telemetry=True, provisioning=not args.backend_only)
result = {'passed': True, 'backend_only': args.backend_only,
'finished': utc_now()}
write_json(Path(args.output_dir), 'ready-backend.json' if args.backend_only
else 'ready.json', result)
print('PASS: ready' + (' (backend only)' if args.backend_only else ''))
# ---------------------------------------------------------------- inventory
def inventory(client):
names = client.prom_names()
rows = []
_, meta = obs.http_json(client.prometheus + '/api/v1/metadata')
for name in names:
if not re.match(r'(http_server|messenger|db_client|process|nodejs|v8js|target_info)', name):
continue
_, series = obs.http_json(
client.prometheus + '/api/v1/series?' +
urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'}))
labels = sorted({k for s in series['data'] for k in s} - {'__name__'})
base = re.sub(r'_(bucket|count|sum|total)$', '', name)
info = (meta['data'].get(name) or meta['data'].get(base) or [{}])[0]
rows.append({'metric': name, 'type': info.get('type'),
'unit': info.get('unit') or None, 'labels': labels,
'series': len(series['data'])})
return names, rows
def cmd_inventory(args):
client = make_obs(args)
names, rows = inventory(client)
out = Path(args.output_dir)
out.mkdir(parents=True, exist_ok=True)
(out / 'metric-names.txt').write_text('\n'.join(names) + '\n')
write_json(out, 'inventory.json', {'captured': utc_now(), 'metrics': rows})
print('inventory: %d names, %d application/runtime metrics' %
(len(names), len(rows)))
# ------------------------------------------------------------------ signals
def series_labels(client, name):
_, series = obs.http_json(
client.prometheus + '/api/v1/series?' +
urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'}))
return series['data']
def cmd_signals(args):
client = make_obs(args)
names, rows = inventory(client)
out = Path(args.output_dir)
out.mkdir(parents=True, exist_ok=True)
(out / 'metric-names.txt').write_text('\n'.join(names) + '\n')
problems, table = [], []
for logical, metric, required, unit in EXPECTED_SIGNALS:
data = series_labels(client, metric)
labels = sorted({k for s in data for k in s} - {'__name__'})
missing = [r for r in required if r not in labels]
table.append({'signal': logical, 'metric': metric, 'labels': labels,
'unit': unit, 'series': len(data)})
if not data:
problems.append('missing metric ' + metric)
elif missing:
problems.append(metric + ' lacks labels ' + ','.join(missing))
# Contract: forbidden labels on any application/runtime metric.
for row in rows:
for label in row['labels']:
if FORBIDDEN_LABEL.search(label):
problems.append('forbidden label ' + label + ' on ' + row['metric'])
# Business facts: routes, replicas, outcomes.
routes = {s.get('http_route') for s in series_labels(
client, 'http_server_request_duration_seconds_count')}
if not any(r and r.startswith('/api/v1/') and '{' in r for r in routes):
problems.append('no route templates with {param} placeholders')
if any(r and re.search(r'[0-9a-f]{8}-[0-9a-f]{4}', r) for r in routes):
problems.append('raw UUID in http_route')
replicas = {s['instance'] for s in series_labels(
client, 'http_server_request_duration_seconds_count')
if s.get('job') == 'messenger-api'}
if len(replicas) < 2:
problems.append('fewer than two API replicas in HTTP metrics')
outcomes = {s.get('outcome') for s in series_labels(
client, 'messenger_jobs_processed_total')}
for needed in ('succeeded', 'retry', 'failed'):
if needed not in outcomes:
problems.append('job outcome not observed: ' + needed +
' (run lab6-correlate first)')
for job in ('messenger-api', 'messenger-worker'):
if not client.prom_query('target_info{job="%s"}' % job):
problems.append('no metrics from ' + job)
# Queue snapshot is not doubled by the two API replicas.
depth_series = client.prom_query('messenger_jobs_queue_depth{state="ready"}')
# Collector.
collector = {}
for metric in COLLECTOR_SIGNALS:
data = series_labels(client, metric)
collector[metric] = len(data)
if not data:
problems.append('missing Collector metric ' + metric)
for metric in COLLECTOR_FAILURE_ONLY:
collector[metric] = len(series_labels(client, metric))
if not client.prom_query('otelcol_process_memory_rss'):
problems.append('no Collector RSS')
# Logs/traces for both services.
for service in ('messenger-api', 'messenger-worker'):
rows_ = wait_rows(client, '{service_name="%s"}' % service, timeout=30)
if not rows_:
problems.append('no logs from ' + service)
result = {'captured': utc_now(), 'signals': table,
'collector_series': collector,
'job_outcomes_observed': sorted(o for o in outcomes if o),
'api_replicas_with_http_metrics': sorted(replicas),
'queue_depth_ready_series': len(depth_series),
'route_templates': sorted(r for r in routes if r),
'problems': problems, 'passed': not problems}
write_json(out, 'signals.json', result)
if problems:
raise Failure('signals: ' + '; '.join(problems))
print('PASS: signals (%d logical signals)' % len(table))
# -------------------------------------------------------------- correlation
def status_of(attachment_id):
code, body = stand.cli('status', attachment_id)
if code != 0:
raise Failure('jobs.cli status failed')
return body
def correlate_http(client, base_url, ca_file, obs_client):
c, alice, chat = new_context(base_url, ca_file)
key = uuid.uuid4().hex
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
actor=alice, body={'text': 'lab6 correlation'},
headers={'Idempotency-Key': key}, schema='Message')
rid = r['headers']['X-Request-Id']
log = http_log(obs_client, rid)
labels = log['labels']
if labels.get('trace_sampled') != 'true':
raise Failure('HTTP log is not marked sampled')
spans = trace_with(
obs_client, labels['trace_id'],
lambda sp: any(s['span_id'] == labels['span_id'] for s in sp) and
len(descendants(sp, labels['span_id'])) > 0, description='HTTP trace')
server = next(s for s in spans if s['span_id'] == labels['span_id'])
if server['kind'] != 'SERVER' or \
server['attributes'].get('messenger.request_id') != rid:
raise Failure('SERVER span does not match the logged request')
db = [s for s in descendants(spans, server['span_id'])
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']]
if not db:
raise Failure('no DB CLIENT child spans under the SERVER span')
return {'request_id': rid, 'trace_id': labels['trace_id'],
'server_span': safe_span(server),
'log_span_id': labels['span_id'],
'db_children': [safe_span(s) for s in db]}
def upload_and_trace(c, alice, chat, obs_client, require_attempts=1):
r = c.call('POST', '/api/v1/chats/' + chat['id'] +
'/attachments?filename=sample.png', 202, actor=alice,
raw=stand.sample_bytes(), headers={'Content-Type': 'image/png'},
schema='Attachment')
return r['json'], r['headers']['X-Request-Id']
def consumer_spans(spans):
return sorted((s for s in spans if s['kind'] == 'CONSUMER'),
key=lambda s: s['start_ns'])
def correlate_upload(client_args, obs_client):
c, alice, chat = new_context(client_args.base_url, client_args.ca_file)
att, rid = upload_and_trace(c, alice, chat, obs_client)
stand.wait_ready(c, alice, att['id'])
log = http_log(obs_client, rid)
tid = log['labels']['trace_id']
def complete(sp):
cons = consumer_spans(sp)
return bool(cons) and any(s['kind'] == 'SERVER' for s in sp)
spans = trace_with(obs_client, tid, complete, description='upload trace')
consumer = consumer_spans(spans)[0]
path = chain_to_root(spans, consumer)
kinds = [s['kind'] for s in path]
if path[-1]['kind'] != 'SERVER' or 'PRODUCER' not in kinds:
raise Failure('consumer is not linked to the HTTP SERVER via PRODUCER')
if path[-1]['service'] != 'messenger-api' or consumer['service'] != 'messenger-worker':
raise Failure('services along the chain are wrong')
below = descendants(spans, consumer['span_id'])
s3 = [s for s in below if s['attributes'].get('rpc.system') == 'aws-api']
db = [s for s in below if 'db.operation.name' in s['attributes']]
if not s3 or not db:
raise Failure('consumer has no S3/DB child spans')
return {'request_id': rid, 'trace_id': tid, 'attachment_id': att['id'],
'chain_consumer_to_root': [safe_span(s) for s in path],
'consumer_attempt': consumer['attributes'].get('messenger.job.attempt'),
'consumer_outcome': consumer['attributes'].get('messenger.job.outcome'),
's3_children': len(s3), 'db_children': len(db),
'chain_fields_are_real_parent_child': True}
def start_worker():
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
def targeted_job(c, alice, chat, obs_client):
"""Stop the worker, accept an upload, return (att, rid, job_id, version)."""
stand.compose('stop', 'worker')
att, rid = upload_and_trace(c, alice, chat, obs_client)
body = status_of(att['id'])
jobs = body.get('jobs')
if not isinstance(jobs, list) or len(jobs) != 1:
raise Failure('expected exactly one job for the new upload')
return att, rid, jobs[0]['job_id'], jobs[0]['pipeline_version']
def correlate_fault(client_args, obs_client, kind):
c, alice, chat = new_context(client_args.base_url, client_args.ca_file)
obs_client.require_no_foreign_faults()
key = None
try:
att, rid, job_id, version = targeted_job(c, alice, chat, obs_client)
key = 'fault:worker.before_side_effect:%s:%s' % (att['id'], version)
stand.set_fault(key, kind)
start_worker()
if kind == 'transient':
def retried():
job = status_of(att['id'])['jobs'][0]
return job['attempt'] >= 2 or job['last_error_code']
stand.wait_for(retried, 90, 'first retry')
stand.clear_fault(key)
key = None
stand.wait_ready(c, alice, att['id'], timeout=120)
else:
def failed():
return status_of(att['id'])['jobs'][0]['status'] == 'failed'
stand.wait_for(failed, 90, 'terminal failed job')
meta = stand.wait_ready(c, alice, att['id'], allow_failed=True)
if meta['status'] != 'failed':
raise Failure('attachment did not become failed')
final = status_of(att['id'])
log = http_log(obs_client, rid)
tid = log['labels']['trace_id']
wanted = 2 if kind == 'transient' else 1
def enough(sp):
return len(consumer_spans(sp)) >= wanted
spans = trace_with(obs_client, tid, enough, timeout=90,
description=kind + ' trace')
consumers = consumer_spans(spans)
ids = {s['span_id'] for s in consumers}
if len(ids) != len(consumers):
raise Failure('consumer attempts share a span ID')
attempts = [s['attributes'].get('messenger.job.attempt') for s in consumers]
outcomes = [s['attributes'].get('messenger.job.outcome') for s in consumers]
if kind == 'transient':
if sorted(attempts)[:2] != [1, 2] or 'retry' not in outcomes:
raise Failure('retry attempts not visible in the trace: %s %s' % (attempts, outcomes))
else:
if 'failed' not in outcomes:
raise Failure('failed outcome not visible in the trace')
return {'kind': kind, 'request_id': rid, 'trace_id': tid,
'attachment_id': att['id'], 'job_id': job_id,
'pipeline_version': version,
'consumer_spans': [safe_span(s) | {
'attempt': s['attributes'].get('messenger.job.attempt'),
'outcome': s['attributes'].get('messenger.job.outcome')}
for s in consumers],
'final_job': {k: final['jobs'][0].get(k) for k in
('status', 'attempt', 'last_error_code')},
'final_attachment_status': final['attachment']['status']}
finally:
if key:
stand.clear_fault(key)
start_worker()
def cmd_correlation(args):
client = make_obs(args)
out = Path(args.output_dir)
with client.app_environment(instrumented_env()):
client.require_no_foreign_faults()
results = {
'correlation-http.json': correlate_http(client, args.base_url, args.ca_file, client),
'correlation-upload.json': correlate_upload(args, client),
'correlation-retry.json': correlate_fault(args, client, 'transient'),
'correlation-failed.json': correlate_fault(args, client, 'permanent'),
}
for name, data in results.items():
data['captured'] = utc_now()
write_json(out, name, data)
print('PASS: correlation (http, upload, retry, failed)')
# ---------------------------------------------------------------- redaction
PEM = re.compile(r'-----BEGIN [A-Z ]*PRIVATE KEY-----')
def fragments(text, size=48, stride=16):
return [text[i:i + size] for i in range(0, max(1, len(text) - size + 1), stride)]
def flatten(value, path=''):
if isinstance(value, dict):
for k, v in value.items():
yield from flatten(v, path + '.' + str(k))
yield path + '.key:' + str(k)
elif isinstance(value, list):
for i, v in enumerate(value):
yield from flatten(v, path + '[' + str(i) + ']')
else:
yield path + '=' + str(value)
def scan(texts, secrets):
"""texts: [(source, text)]; secrets: {category: [needle,...]}."""
leaks = []
for source, text in texts:
for category, needles in secrets.items():
for needle in needles:
if needle and needle in text:
leaks.append({'category': category, 'source': source,
'hash': hashlib.sha256(needle.encode()).hexdigest()[:12]})
break
if PEM.search(text):
leaks.append({'category': 'private_key_pem', 'source': source, 'hash': ''})
return leaks
def cmd_redaction(args):
client = make_obs(args)
out = Path(args.output_dir)
with client.app_environment(instrumented_env()):
c = stand.new_client(args.base_url, args.ca_file)
actor = c.user('redact')
marker = 'LAB6-REDACT-' + uuid.uuid4().hex
password = actor['registration']['password']
token = actor['token']
username = actor['registration']['username']
basic = base64.b64encode((username + ':' + password).encode()).decode()
signature = 'sig' + uuid.uuid4().hex + uuid.uuid4().hex
own_chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'redaction', 'member_ids': [actor['id']]},
schema='Chat')['json']
requests = {}
r = c.call('POST', '/api/v1/chats/' + own_chat['id'] + '/messages', 201,
actor=actor, body={'text': 'body ' + marker},
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
requests['message'] = r['headers']['X-Request-Id']
r = c.call('GET', '/api/v1/chats?X-Amz-Signature=' + signature, range(200, 500),
actor=actor, headers={'X-Marker': marker})
requests['signed_query'] = r['headers']['X-Request-Id']
r = c.call('POST', '/api/v1/auth/sessions', 401,
headers={'Authorization': 'Basic ' + base64.b64encode(
(username + ':wrong-' + marker).encode()).decode()})
requests['bad_login'] = r['headers']['X-Request-Id']
att, rid = upload_and_trace(c, actor, own_chat, client)
requests['upload'] = rid
stand.wait_ready(c, actor, att['id'])
sample = stand.sample_bytes()
b64 = base64.b64encode(sample).decode()
hexs = sample.hex()
secrets = {
'marker': [marker], 'password': [password], 'session_token': [token],
'basic_credential': [basic], 'bearer_header': ['Bearer ' + token],
'signature': [signature], 'image_base64': [b64],
'image_hex': [hexs],
'image_base64_fragment': fragments(b64), 'image_hex_fragment': fragments(hexs),
'wrong_password_marker': ['wrong-' + marker],
}
coverage = {'http_logs': {}, 'server_spans': {}, 'worker_chain': False}
texts = []
trace_ids = {}
for name, request_id in requests.items():
log = http_log(client, request_id)
coverage['http_logs'][name] = True
texts.append(('log:' + name, log['line'] + json.dumps(log['labels'], sort_keys=True)))
trace_ids[name] = log['labels']['trace_id']
for name, tid in trace_ids.items():
def has_server(sp, tid=tid, name=name):
return any(s['kind'] == 'SERVER' and
s['attributes'].get('messenger.request_id') == requests[name]
for s in sp)
spans = trace_with(client, tid, has_server, description='trace ' + name)
coverage['server_spans'][name] = True
if name == 'upload':
coverage['worker_chain'] = any(s['kind'] == 'CONSUMER' for s in spans) \
if any(s['kind'] == 'CONSUMER' for s in spans) else False
if not coverage['worker_chain']:
spans = trace_with(client, tid, lambda sp: any(
s['kind'] == 'CONSUMER' for s in sp), description='worker chain')
coverage['worker_chain'] = True
for s in spans:
for item in flatten({k: s[k] for k in ('name', 'attributes', 'status')}):
texts.append(('span:' + name + ':' + s['kind'], item))
logs = wait_rows(client, '{service_name=~"messenger-.*"} | trace_id="%s"' % tid)
for row in logs:
texts.append(('tracelog:' + name, row['line'] + json.dumps(row['labels'], sort_keys=True)))
worker_logs = wait_rows(
client, '{service_name="messenger-worker"} | messenger_attachment_id="%s"' % att['id'])
coverage['worker_logs'] = len(worker_logs)
for row in worker_logs:
texts.append(('workerlog', row['line'] + json.dumps(row['labels'], sort_keys=True)))
leaks = scan(texts, secrets)
complete = (all(coverage['http_logs'].get(n) for n in requests) and
all(coverage['server_spans'].get(n) for n in requests) and
coverage['worker_chain'] and coverage['worker_logs'] > 0)
result = {'captured': utc_now(), 'coverage': coverage, 'coverage_complete': complete,
'scanned_items': len(texts), 'leak_count': len(leaks), 'leaks': leaks,
'passed': complete and not leaks}
write_json(out, 'redaction.json', result)
(out / 'redaction.txt').write_text(
'coverage_complete=%s scanned_items=%d leaks=%d\n' %
(complete, len(texts), len(leaks)))
if not result['passed']:
raise Failure('redaction: coverage_complete=%s leaks=%d' % (complete, len(leaks)))
print('PASS: redaction (%d items scanned, 0 leaks)' % len(texts))
# ------------------------------------------------------------------ history
def raw_samples(client, selector, at, window='30s'):
result = client.prom_query(selector + '[' + window + ']', at=at)
return {json.dumps(r['metric'], sort_keys=True): r['values'] for r in result}
def cmd_history(args):
client = make_obs(args)
out = Path(args.output_dir)
with client.app_environment(instrumented_env()):
c, alice, chat = new_context(args.base_url, args.ca_file)
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
actor=alice, body={'text': 'history'},
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
rid_message = r['headers']['X-Request-Id']
att, rid = upload_and_trace(c, alice, chat, client)
stand.wait_ready(c, alice, att['id'])
log = http_log(client, rid)
tid = log['labels']['trace_id']
spans = trace_with(client, tid, lambda sp: any(s['kind'] == 'CONSUMER' for s in sp),
description='history trace')
msg_log = http_log(client, rid_message)
client.wait_telemetry_flushed()
time.sleep(12)
client.wait_telemetry_flushed()
at = time.time()
selector = 'messenger_jobs_processed_total{job="messenger-worker",outcome="succeeded"}'
before = raw_samples(client, selector, at)
if not before:
raise Failure('no Prometheus samples to preserve')
series_key, values = next(iter(before.items()))
old_sample = {'labels': json.loads(series_key), 'timestamp': values[-1][0],
'value': values[-1][1]}
old_log = {'ts': log['ts'], 'request_id': rid, 'trace_id': tid,
'span_id': log['labels']['span_id']}
old_trace = sorted(s['span_id'] for s in spans)
old_msg = {'ts': msg_log['ts'], 'trace_id': msg_log['labels']['trace_id']}
stand.compose('restart', 'prometheus', 'loki', 'tempo', 'grafana')
client.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
after_samples = raw_samples(client, selector, at)
after = after_samples.get(series_key, [])
prom_ok = any(abs(v[0] - old_sample['timestamp']) < 1e-6 and
v[1] == old_sample['value'] for v in after)
rows = wait_rows(client, '{service_name="messenger-api"} | request_id="%s"' % rid)
loki_ok = any(row['ts'] == old_log['ts'] and
row['labels'].get('trace_id') == tid for row in rows)
spans_after = client.get_trace(tid, wait=30)
tempo_ok = bool(spans_after) and \
sorted(s['span_id'] for s in spans_after) == old_trace
result = {'captured': utc_now(), 'prometheus_old_sample': old_sample,
'prometheus_read_back': prom_ok, 'loki_old_log': old_log,
'loki_read_back': loki_ok, 'tempo_old_trace_span_ids': old_trace,
'tempo_read_back': tempo_ok,
'passed': prom_ok and loki_ok and tempo_ok}
write_json(out, 'history.json', result)
if not result['passed']:
raise Failure('history: old data not readable after restart: %s' % {
k: result[k] for k in ('prometheus_read_back', 'loki_read_back', 'tempo_read_back')})
print('PASS: history (old Prometheus sample, Loki log and Tempo trace survive restart)')
COMMANDS = {'ready': cmd_ready, 'inventory': cmd_inventory, 'signals': cmd_signals,
'correlation': cmd_correlation, 'redaction': cmd_redaction,
'history': cmd_history}
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=sorted(COMMANDS))
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--ca-file')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output-dir', default='evidence/lab6')
parser.add_argument('--backend-only', action='store_true')
args = parser.parse_args(argv)
try:
COMMANDS[args.command](args)
except Failure as error:
print('FAIL: ' + str(error), file=sys.stderr)
return 1
return 0
if __name__ == '__main__':
sys.exit(main())
+140
View File
@@ -0,0 +1,140 @@
"""Fixed-rate open-loop workload for lab6 (stdlib only).
Slot i is planned at t0 + i/rate on a monotonic clock; the generator never
waits for earlier requests and never catches up with a burst. When no
in-flight slot is available the slot is recorded as client_dropped and stays
in the denominator."""
import math
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
from obs import Failure, stand # noqa: E402
REQUEST_TIMEOUT = 30
MESSAGE_TEXT = 'lab6-track1'
def slot_kind(i, upload_every):
if upload_every == 0:
return 'message'
if upload_every == 1:
return 'upload'
return 'upload' if (i + 1) % upload_every == 0 else 'message'
class _Local(threading.local):
client = None
def run_window(base_url, ca_file, actor, chat, rate, duration, phase_id,
upload_every=20, max_inflight=64, stop_event=None, settle=True,
obs_client=None):
"""Run one window and return {'slots', 'load', 'uploads', 'drain', ...}.
stop_event stops only new slots; started requests complete. settle=False
returns right after HTTP (no job/Collector drain)."""
local = _Local()
image = stand.sample_bytes()
lock = threading.Lock()
inflight = [0]
planned = []
def client():
if local.client is None:
local.client = stand.new_client(base_url, ca_file)
local.client.timeout = REQUEST_TIMEOUT
return local.client
def execute(record, kind, t0_mono):
record['start'] = time.monotonic() - t0_mono
record['start_lag_ms'] = (record['start'] - record['scheduled']) * 1000.0
started = time.monotonic()
try:
c = client()
if kind == 'message':
result = c.call(
'POST', '/api/v1/chats/' + chat['id'] + '/messages',
range(200, 600), actor=actor, body={'text': MESSAGE_TEXT},
headers={'Idempotency-Key': uuid.uuid4().hex,
'X-Request-Id': uuid.uuid4().hex},
schema='Message')
else:
result = c.call(
'POST', '/api/v1/chats/' + chat['id'] +
'/attachments?filename=sample.png', range(200, 600),
actor=actor, raw=image,
headers={'Content-Type': 'image/png',
'X-Request-Id': uuid.uuid4().hex},
schema='Attachment')
end = time.monotonic()
record['status'] = result['status']
record['request_id'] = result['headers'].get('X-Request-Id')
record['instance_id'] = result['headers'].get('X-Instance-Id')
if kind == 'upload' and result['status'] == 202 and result['json']:
record['attachment_id'] = result['json']['id']
record['outcome'] = 'http'
except Failure as error:
end = time.monotonic()
text = str(error)
record['outcome'] = ('timeout' if 'TimeoutError' in text else
'transport_error' if 'connection/TLS' in text
else 'protocol_error')
record['error'] = text[:160]
finally:
with lock:
inflight[0] -= 1
record['end'] = end - t0_mono
record['service_ms'] = (end - started) * 1000.0
record['scheduled_ms'] = (record['end'] - record['scheduled']) * 1000.0
pool = ThreadPoolExecutor(max_workers=max_inflight)
futures = []
t0 = time.monotonic()
started_utc = obs.utc_now()
total = int(math.floor(rate * duration))
try:
for i in range(total):
due = t0 + i / rate
while True:
if stop_event is not None and stop_event.is_set():
break
remaining = due - time.monotonic()
if remaining <= 0:
break
time.sleep(min(remaining, 0.05))
if stop_event is not None and stop_event.is_set():
break
kind = slot_kind(i, upload_every)
record = {'i': i, 'phase': phase_id, 'name': '%s-%d' % (phase_id, i),
'kind': kind, 'scheduled': i / rate, 'outcome': None,
'start_lag_ms': None}
planned.append(record)
with lock:
free = inflight[0] < max_inflight
if free:
inflight[0] += 1
if not free:
record['outcome'] = 'client_dropped'
continue
futures.append(pool.submit(execute, record, kind, t0))
for future in futures:
future.result()
finally:
pool.shutdown(wait=True)
dispatch_end = time.monotonic() - t0
load = measurement.summarize_load(planned, rate)
uploads = [s['attachment_id'] for s in planned if s.get('attachment_id')]
result = {'phase': phase_id, 'slots': planned, 'load': load,
'uploads': uploads, 'started': started_utc,
'dispatch_seconds': dispatch_end, 'finished_dispatch': obs.utc_now(),
'stopped_early': bool(stop_event is not None and stop_event.is_set())}
if settle:
result['drain'] = obs.wait_drained(uploads, obs=obs_client)
return result
+2 -2
View File
@@ -4,5 +4,5 @@
set -euo pipefail
compose=(docker compose --env-file .env)
"${compose[@]}" restart postgres redis s3 broker api worker haproxy
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy
"${compose[@]}" restart postgres redis s3 broker api worker haproxy otel-collector prometheus loki tempo grafana
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy otel-collector prometheus loki tempo grafana