672 lines
30 KiB
Python
Executable File
672 lines
30 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""lab6 verification: ready | inventory | signals | correlation | redaction |
|
|
history. Python 3.9+, standard library only. Every subcommand writes its own
|
|
evidence file under --output-dir and exits non-zero on any failed criterion."""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import obs # noqa: E402
|
|
from obs import Failure, stand, utc_now # noqa: E402
|
|
|
|
FORBIDDEN_LABEL = re.compile(
|
|
r'(^|_)(user|chat|message|attachment|request|trace|span|job)_id$|'
|
|
r'(^|_)(url|filename|text|body|username|password|token)($|_)')
|
|
EXPECTED_SIGNALS = [
|
|
# logical signal, metric, required label names (normalised), unit
|
|
('HTTP duration/count', 'http_server_request_duration_seconds_count',
|
|
('http_request_method', 'http_route', 'http_response_status_code'), 's'),
|
|
('HTTP duration buckets', 'http_server_request_duration_seconds_bucket', ('le',), 's'),
|
|
('HTTP duration sum', 'http_server_request_duration_seconds_sum', (), 's'),
|
|
('Jobs processed', 'messenger_jobs_processed_total', ('job_type', 'outcome'), '{job}'),
|
|
('Job duration', 'messenger_job_duration_seconds_count', ('job_type', 'outcome'), 's'),
|
|
('Time to ready', 'messenger_attachment_time_to_ready_seconds_count', ('job_type',), 's'),
|
|
('Queue depth', 'messenger_jobs_queue_depth', ('state',), '{job}'),
|
|
('Queue oldest age', 'messenger_jobs_queue_oldest_age_seconds', (), 's'),
|
|
('Outbox pending', 'messenger_outbox_pending', (), '{message}'),
|
|
('Outbox oldest age', 'messenger_outbox_oldest_age_seconds', (), 's'),
|
|
('DB pool connections', 'db_client_connection_count', ('db_client_connection_state',), '{connection}'),
|
|
('DB pool max', 'db_client_connection_max', (), '{connection}'),
|
|
('DB pool pending', 'db_client_connection_pending_requests', (), '{request}'),
|
|
('DB pool wait', 'db_client_connection_wait_time_seconds_count', (), 's'),
|
|
('DB operation duration', 'db_client_operation_duration_seconds_count', ('db_operation_name',), 's'),
|
|
('Worker active', 'messenger_worker_jobs_active', (), '{job}'),
|
|
('Worker capacity', 'messenger_worker_jobs_capacity', (), '{job}'),
|
|
('Broker published', 'messenger_broker_published_total', ('outcome',), '{message}'),
|
|
('Telemetry export', 'messenger_telemetry_export_items_total', ('signal', 'outcome'), '{item}'),
|
|
('Process CPU', 'process_cpu_time_seconds_total', ('cpu_mode',), 's'),
|
|
('Process RAM', 'process_memory_usage_bytes', (), 'By'),
|
|
('V8 heap', 'v8js_memory_heap_used_bytes', (), 'By'),
|
|
('Event loop delay', 'nodejs_eventloop_delay_p99_seconds', (), 's'),
|
|
]
|
|
COLLECTOR_SIGNALS = [
|
|
'otelcol_receiver_accepted_spans', 'otelcol_receiver_accepted_log_records',
|
|
'otelcol_receiver_accepted_metric_points', 'otelcol_exporter_sent_spans',
|
|
'otelcol_exporter_sent_log_records', 'otelcol_exporter_sent_metric_points',
|
|
'otelcol_exporter_queue_size', 'otelcol_exporter_queue_capacity',
|
|
]
|
|
COLLECTOR_FAILURE_ONLY = [
|
|
'otelcol_exporter_send_failed_spans', 'otelcol_exporter_send_failed_log_records',
|
|
'otelcol_exporter_send_failed_metric_points', 'otelcol_exporter_enqueue_failed_spans',
|
|
'otelcol_exporter_enqueue_failed_log_records', 'otelcol_exporter_enqueue_failed_metric_points',
|
|
]
|
|
|
|
|
|
# ------------------------------------------------------------------ helpers
|
|
|
|
def write_json(out, name, data):
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
(out / name).write_text(json.dumps(data, indent=2, ensure_ascii=False,
|
|
sort_keys=True) + '\n')
|
|
|
|
|
|
def make_obs(args):
|
|
return obs.ObsClient(args.prometheus_url, args.grafana_url,
|
|
base_url=args.base_url, ca_file=args.ca_file)
|
|
|
|
|
|
def ns_window(seconds_back=600, forward=30):
|
|
now = time.time_ns()
|
|
return now - seconds_back * 10 ** 9, now + forward * 10 ** 9
|
|
|
|
|
|
def wait_rows(client, query, minimum=1, timeout=60, back=900):
|
|
deadline = time.monotonic() + timeout
|
|
rows = []
|
|
while time.monotonic() < deadline:
|
|
start, end = ns_window(back)
|
|
rows = client.loki_rows(query, start, end)
|
|
if len(rows) >= minimum:
|
|
return rows
|
|
time.sleep(2)
|
|
raise Failure('expected log rows not found in Loki (' + str(len(rows)) +
|
|
'/' + str(minimum) + ')')
|
|
|
|
|
|
def http_log(client, request_id, timeout=60):
|
|
rows = wait_rows(
|
|
client, '{service_name="messenger-api"} | event_name="http.request" '
|
|
'| request_id="' + request_id + '"', timeout=timeout)
|
|
if len(rows) != 1:
|
|
raise Failure('expected exactly one http.request log for the request')
|
|
return rows[0]
|
|
|
|
|
|
def chain_to_root(spans, span):
|
|
by_id = {s['span_id']: s for s in spans}
|
|
path = [span]
|
|
while path[-1]['parent_span_id']:
|
|
parent = by_id.get(path[-1]['parent_span_id'])
|
|
if parent is None:
|
|
raise Failure('broken parent chain at ' + path[-1]['name'])
|
|
path.append(parent)
|
|
if len(path) > 64:
|
|
raise Failure('parent chain too long')
|
|
return path
|
|
|
|
|
|
def descendants(spans, root_id):
|
|
children = {}
|
|
for s in spans:
|
|
children.setdefault(s['parent_span_id'], []).append(s)
|
|
out, stack = [], [root_id]
|
|
while stack:
|
|
for child in children.get(stack.pop(), []):
|
|
out.append(child)
|
|
stack.append(child['span_id'])
|
|
return out
|
|
|
|
|
|
def safe_span(s):
|
|
return {k: s[k] for k in ('span_id', 'parent_span_id', 'name', 'kind',
|
|
'status', 'service', 'instance')}
|
|
|
|
|
|
def trace_with(client, trace_id, predicate, timeout=60, description='trace'):
|
|
deadline = time.monotonic() + timeout
|
|
spans = None
|
|
while time.monotonic() < deadline:
|
|
spans = client.get_trace(trace_id)
|
|
if spans and predicate(spans):
|
|
return spans
|
|
time.sleep(2)
|
|
raise Failure(description + ' did not reach the expected shape in Tempo')
|
|
|
|
|
|
def new_context(base_url, ca_file):
|
|
c = stand.new_client(base_url, ca_file)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
return c, alice, chat
|
|
|
|
|
|
def instrumented_env():
|
|
return {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
|
|
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
|
|
|
|
|
|
# -------------------------------------------------------------------- ready
|
|
|
|
def cmd_ready(args):
|
|
client = make_obs(args)
|
|
client.wait_ready(app_telemetry=True, provisioning=not args.backend_only)
|
|
result = {'passed': True, 'backend_only': args.backend_only,
|
|
'finished': utc_now()}
|
|
write_json(Path(args.output_dir), 'ready-backend.json' if args.backend_only
|
|
else 'ready.json', result)
|
|
print('PASS: ready' + (' (backend only)' if args.backend_only else ''))
|
|
|
|
|
|
# ---------------------------------------------------------------- inventory
|
|
|
|
def inventory(client):
|
|
names = client.prom_names()
|
|
rows = []
|
|
_, meta = obs.http_json(client.prometheus + '/api/v1/metadata')
|
|
for name in names:
|
|
if not re.match(r'(http_server|messenger|db_client|process|nodejs|v8js|target_info)', name):
|
|
continue
|
|
_, series = obs.http_json(
|
|
client.prometheus + '/api/v1/series?' +
|
|
urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'}))
|
|
labels = sorted({k for s in series['data'] for k in s} - {'__name__'})
|
|
base = re.sub(r'_(bucket|count|sum|total)$', '', name)
|
|
info = (meta['data'].get(name) or meta['data'].get(base) or [{}])[0]
|
|
rows.append({'metric': name, 'type': info.get('type'),
|
|
'unit': info.get('unit') or None, 'labels': labels,
|
|
'series': len(series['data'])})
|
|
return names, rows
|
|
|
|
|
|
def cmd_inventory(args):
|
|
client = make_obs(args)
|
|
names, rows = inventory(client)
|
|
out = Path(args.output_dir)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
(out / 'metric-names.txt').write_text('\n'.join(names) + '\n')
|
|
write_json(out, 'inventory.json', {'captured': utc_now(), 'metrics': rows})
|
|
print('inventory: %d names, %d application/runtime metrics' %
|
|
(len(names), len(rows)))
|
|
|
|
|
|
# ------------------------------------------------------------------ signals
|
|
|
|
def series_labels(client, name):
|
|
_, series = obs.http_json(
|
|
client.prometheus + '/api/v1/series?' +
|
|
urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'}))
|
|
return series['data']
|
|
|
|
|
|
def cmd_signals(args):
|
|
client = make_obs(args)
|
|
names, rows = inventory(client)
|
|
out = Path(args.output_dir)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
(out / 'metric-names.txt').write_text('\n'.join(names) + '\n')
|
|
problems, table = [], []
|
|
for logical, metric, required, unit in EXPECTED_SIGNALS:
|
|
data = series_labels(client, metric)
|
|
labels = sorted({k for s in data for k in s} - {'__name__'})
|
|
missing = [r for r in required if r not in labels]
|
|
table.append({'signal': logical, 'metric': metric, 'labels': labels,
|
|
'unit': unit, 'series': len(data)})
|
|
if not data:
|
|
problems.append('missing metric ' + metric)
|
|
elif missing:
|
|
problems.append(metric + ' lacks labels ' + ','.join(missing))
|
|
# Contract: forbidden labels on any application/runtime metric.
|
|
for row in rows:
|
|
for label in row['labels']:
|
|
if FORBIDDEN_LABEL.search(label):
|
|
problems.append('forbidden label ' + label + ' on ' + row['metric'])
|
|
# Business facts: routes, replicas, outcomes.
|
|
routes = {s.get('http_route') for s in series_labels(
|
|
client, 'http_server_request_duration_seconds_count')}
|
|
if not any(r and r.startswith('/api/v1/') and '{' in r for r in routes):
|
|
problems.append('no route templates with {param} placeholders')
|
|
if any(r and re.search(r'[0-9a-f]{8}-[0-9a-f]{4}', r) for r in routes):
|
|
problems.append('raw UUID in http_route')
|
|
replicas = {s['instance'] for s in series_labels(
|
|
client, 'http_server_request_duration_seconds_count')
|
|
if s.get('job') == 'messenger-api'}
|
|
if len(replicas) < 2:
|
|
problems.append('fewer than two API replicas in HTTP metrics')
|
|
outcomes = {s.get('outcome') for s in series_labels(
|
|
client, 'messenger_jobs_processed_total')}
|
|
for needed in ('succeeded', 'retry', 'failed'):
|
|
if needed not in outcomes:
|
|
problems.append('job outcome not observed: ' + needed +
|
|
' (run lab6-correlate first)')
|
|
for job in ('messenger-api', 'messenger-worker'):
|
|
if not client.prom_query('target_info{job="%s"}' % job):
|
|
problems.append('no metrics from ' + job)
|
|
# Queue snapshot is not doubled by the two API replicas.
|
|
depth_series = client.prom_query('messenger_jobs_queue_depth{state="ready"}')
|
|
# Collector.
|
|
collector = {}
|
|
for metric in COLLECTOR_SIGNALS:
|
|
data = series_labels(client, metric)
|
|
collector[metric] = len(data)
|
|
if not data:
|
|
problems.append('missing Collector metric ' + metric)
|
|
for metric in COLLECTOR_FAILURE_ONLY:
|
|
collector[metric] = len(series_labels(client, metric))
|
|
if not client.prom_query('otelcol_process_memory_rss'):
|
|
problems.append('no Collector RSS')
|
|
# Logs/traces for both services.
|
|
for service in ('messenger-api', 'messenger-worker'):
|
|
rows_ = wait_rows(client, '{service_name="%s"}' % service, timeout=30)
|
|
if not rows_:
|
|
problems.append('no logs from ' + service)
|
|
result = {'captured': utc_now(), 'signals': table,
|
|
'collector_series': collector,
|
|
'job_outcomes_observed': sorted(o for o in outcomes if o),
|
|
'api_replicas_with_http_metrics': sorted(replicas),
|
|
'queue_depth_ready_series': len(depth_series),
|
|
'route_templates': sorted(r for r in routes if r),
|
|
'problems': problems, 'passed': not problems}
|
|
write_json(out, 'signals.json', result)
|
|
if problems:
|
|
raise Failure('signals: ' + '; '.join(problems))
|
|
print('PASS: signals (%d logical signals)' % len(table))
|
|
|
|
|
|
# -------------------------------------------------------------- correlation
|
|
|
|
def status_of(attachment_id):
|
|
code, body = stand.cli('status', attachment_id)
|
|
if code != 0:
|
|
raise Failure('jobs.cli status failed')
|
|
return body
|
|
|
|
|
|
def correlate_http(client, base_url, ca_file, obs_client):
|
|
c, alice, chat = new_context(base_url, ca_file)
|
|
key = uuid.uuid4().hex
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
|
|
actor=alice, body={'text': 'lab6 correlation'},
|
|
headers={'Idempotency-Key': key}, schema='Message')
|
|
rid = r['headers']['X-Request-Id']
|
|
log = http_log(obs_client, rid)
|
|
labels = log['labels']
|
|
if labels.get('trace_sampled') != 'true':
|
|
raise Failure('HTTP log is not marked sampled')
|
|
spans = trace_with(
|
|
obs_client, labels['trace_id'],
|
|
lambda sp: any(s['span_id'] == labels['span_id'] for s in sp) and
|
|
len(descendants(sp, labels['span_id'])) > 0, description='HTTP trace')
|
|
server = next(s for s in spans if s['span_id'] == labels['span_id'])
|
|
if server['kind'] != 'SERVER' or \
|
|
server['attributes'].get('messenger.request_id') != rid:
|
|
raise Failure('SERVER span does not match the logged request')
|
|
db = [s for s in descendants(spans, server['span_id'])
|
|
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']]
|
|
if not db:
|
|
raise Failure('no DB CLIENT child spans under the SERVER span')
|
|
return {'request_id': rid, 'trace_id': labels['trace_id'],
|
|
'server_span': safe_span(server),
|
|
'log_span_id': labels['span_id'],
|
|
'db_children': [safe_span(s) for s in db]}
|
|
|
|
|
|
def upload_and_trace(c, alice, chat, obs_client, require_attempts=1):
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] +
|
|
'/attachments?filename=sample.png', 202, actor=alice,
|
|
raw=stand.sample_bytes(), headers={'Content-Type': 'image/png'},
|
|
schema='Attachment')
|
|
return r['json'], r['headers']['X-Request-Id']
|
|
|
|
|
|
def consumer_spans(spans):
|
|
return sorted((s for s in spans if s['kind'] == 'CONSUMER'),
|
|
key=lambda s: s['start_ns'])
|
|
|
|
|
|
def correlate_upload(client_args, obs_client):
|
|
c, alice, chat = new_context(client_args.base_url, client_args.ca_file)
|
|
att, rid = upload_and_trace(c, alice, chat, obs_client)
|
|
stand.wait_ready(c, alice, att['id'])
|
|
log = http_log(obs_client, rid)
|
|
tid = log['labels']['trace_id']
|
|
|
|
def complete(sp):
|
|
cons = consumer_spans(sp)
|
|
return bool(cons) and any(s['kind'] == 'SERVER' for s in sp)
|
|
|
|
spans = trace_with(obs_client, tid, complete, description='upload trace')
|
|
consumer = consumer_spans(spans)[0]
|
|
path = chain_to_root(spans, consumer)
|
|
kinds = [s['kind'] for s in path]
|
|
if path[-1]['kind'] != 'SERVER' or 'PRODUCER' not in kinds:
|
|
raise Failure('consumer is not linked to the HTTP SERVER via PRODUCER')
|
|
if path[-1]['service'] != 'messenger-api' or consumer['service'] != 'messenger-worker':
|
|
raise Failure('services along the chain are wrong')
|
|
below = descendants(spans, consumer['span_id'])
|
|
s3 = [s for s in below if s['attributes'].get('rpc.system') == 'aws-api']
|
|
db = [s for s in below if 'db.operation.name' in s['attributes']]
|
|
if not s3 or not db:
|
|
raise Failure('consumer has no S3/DB child spans')
|
|
return {'request_id': rid, 'trace_id': tid, 'attachment_id': att['id'],
|
|
'chain_consumer_to_root': [safe_span(s) for s in path],
|
|
'consumer_attempt': consumer['attributes'].get('messenger.job.attempt'),
|
|
'consumer_outcome': consumer['attributes'].get('messenger.job.outcome'),
|
|
's3_children': len(s3), 'db_children': len(db),
|
|
'chain_fields_are_real_parent_child': True}
|
|
|
|
|
|
def start_worker():
|
|
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
|
|
|
|
|
|
def targeted_job(c, alice, chat, obs_client):
|
|
"""Stop the worker, accept an upload, return (att, rid, job_id, version)."""
|
|
stand.compose('stop', 'worker')
|
|
att, rid = upload_and_trace(c, alice, chat, obs_client)
|
|
body = status_of(att['id'])
|
|
jobs = body.get('jobs')
|
|
if not isinstance(jobs, list) or len(jobs) != 1:
|
|
raise Failure('expected exactly one job for the new upload')
|
|
return att, rid, jobs[0]['job_id'], jobs[0]['pipeline_version']
|
|
|
|
|
|
def correlate_fault(client_args, obs_client, kind):
|
|
c, alice, chat = new_context(client_args.base_url, client_args.ca_file)
|
|
obs_client.require_no_foreign_faults()
|
|
key = None
|
|
try:
|
|
att, rid, job_id, version = targeted_job(c, alice, chat, obs_client)
|
|
key = 'fault:worker.before_side_effect:%s:%s' % (att['id'], version)
|
|
stand.set_fault(key, kind)
|
|
start_worker()
|
|
if kind == 'transient':
|
|
def retried():
|
|
job = status_of(att['id'])['jobs'][0]
|
|
return job['attempt'] >= 2 or job['last_error_code']
|
|
stand.wait_for(retried, 90, 'first retry')
|
|
stand.clear_fault(key)
|
|
key = None
|
|
stand.wait_ready(c, alice, att['id'], timeout=120)
|
|
else:
|
|
def failed():
|
|
return status_of(att['id'])['jobs'][0]['status'] == 'failed'
|
|
stand.wait_for(failed, 90, 'terminal failed job')
|
|
meta = stand.wait_ready(c, alice, att['id'], allow_failed=True)
|
|
if meta['status'] != 'failed':
|
|
raise Failure('attachment did not become failed')
|
|
final = status_of(att['id'])
|
|
log = http_log(obs_client, rid)
|
|
tid = log['labels']['trace_id']
|
|
wanted = 2 if kind == 'transient' else 1
|
|
|
|
def enough(sp):
|
|
return len(consumer_spans(sp)) >= wanted
|
|
|
|
spans = trace_with(obs_client, tid, enough, timeout=90,
|
|
description=kind + ' trace')
|
|
consumers = consumer_spans(spans)
|
|
ids = {s['span_id'] for s in consumers}
|
|
if len(ids) != len(consumers):
|
|
raise Failure('consumer attempts share a span ID')
|
|
attempts = [s['attributes'].get('messenger.job.attempt') for s in consumers]
|
|
outcomes = [s['attributes'].get('messenger.job.outcome') for s in consumers]
|
|
if kind == 'transient':
|
|
if sorted(attempts)[:2] != [1, 2] or 'retry' not in outcomes:
|
|
raise Failure('retry attempts not visible in the trace: %s %s' % (attempts, outcomes))
|
|
else:
|
|
if 'failed' not in outcomes:
|
|
raise Failure('failed outcome not visible in the trace')
|
|
return {'kind': kind, 'request_id': rid, 'trace_id': tid,
|
|
'attachment_id': att['id'], 'job_id': job_id,
|
|
'pipeline_version': version,
|
|
'consumer_spans': [safe_span(s) | {
|
|
'attempt': s['attributes'].get('messenger.job.attempt'),
|
|
'outcome': s['attributes'].get('messenger.job.outcome')}
|
|
for s in consumers],
|
|
'final_job': {k: final['jobs'][0].get(k) for k in
|
|
('status', 'attempt', 'last_error_code')},
|
|
'final_attachment_status': final['attachment']['status']}
|
|
finally:
|
|
if key:
|
|
stand.clear_fault(key)
|
|
start_worker()
|
|
|
|
|
|
def cmd_correlation(args):
|
|
client = make_obs(args)
|
|
out = Path(args.output_dir)
|
|
with client.app_environment(instrumented_env()):
|
|
client.require_no_foreign_faults()
|
|
results = {
|
|
'correlation-http.json': correlate_http(client, args.base_url, args.ca_file, client),
|
|
'correlation-upload.json': correlate_upload(args, client),
|
|
'correlation-retry.json': correlate_fault(args, client, 'transient'),
|
|
'correlation-failed.json': correlate_fault(args, client, 'permanent'),
|
|
}
|
|
for name, data in results.items():
|
|
data['captured'] = utc_now()
|
|
write_json(out, name, data)
|
|
print('PASS: correlation (http, upload, retry, failed)')
|
|
|
|
|
|
# ---------------------------------------------------------------- redaction
|
|
|
|
PEM = re.compile(r'-----BEGIN [A-Z ]*PRIVATE KEY-----')
|
|
|
|
|
|
def fragments(text, size=48, stride=16):
|
|
return [text[i:i + size] for i in range(0, max(1, len(text) - size + 1), stride)]
|
|
|
|
|
|
def flatten(value, path=''):
|
|
if isinstance(value, dict):
|
|
for k, v in value.items():
|
|
yield from flatten(v, path + '.' + str(k))
|
|
yield path + '.key:' + str(k)
|
|
elif isinstance(value, list):
|
|
for i, v in enumerate(value):
|
|
yield from flatten(v, path + '[' + str(i) + ']')
|
|
else:
|
|
yield path + '=' + str(value)
|
|
|
|
|
|
def scan(texts, secrets):
|
|
"""texts: [(source, text)]; secrets: {category: [needle,...]}."""
|
|
leaks = []
|
|
for source, text in texts:
|
|
for category, needles in secrets.items():
|
|
for needle in needles:
|
|
if needle and needle in text:
|
|
leaks.append({'category': category, 'source': source,
|
|
'hash': hashlib.sha256(needle.encode()).hexdigest()[:12]})
|
|
break
|
|
if PEM.search(text):
|
|
leaks.append({'category': 'private_key_pem', 'source': source, 'hash': ''})
|
|
return leaks
|
|
|
|
|
|
def cmd_redaction(args):
|
|
client = make_obs(args)
|
|
out = Path(args.output_dir)
|
|
with client.app_environment(instrumented_env()):
|
|
c = stand.new_client(args.base_url, args.ca_file)
|
|
actor = c.user('redact')
|
|
marker = 'LAB6-REDACT-' + uuid.uuid4().hex
|
|
password = actor['registration']['password']
|
|
token = actor['token']
|
|
username = actor['registration']['username']
|
|
basic = base64.b64encode((username + ':' + password).encode()).decode()
|
|
signature = 'sig' + uuid.uuid4().hex + uuid.uuid4().hex
|
|
own_chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'redaction', 'member_ids': [actor['id']]},
|
|
schema='Chat')['json']
|
|
requests = {}
|
|
r = c.call('POST', '/api/v1/chats/' + own_chat['id'] + '/messages', 201,
|
|
actor=actor, body={'text': 'body ' + marker},
|
|
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
|
|
requests['message'] = r['headers']['X-Request-Id']
|
|
r = c.call('GET', '/api/v1/chats?X-Amz-Signature=' + signature, range(200, 500),
|
|
actor=actor, headers={'X-Marker': marker})
|
|
requests['signed_query'] = r['headers']['X-Request-Id']
|
|
r = c.call('POST', '/api/v1/auth/sessions', 401,
|
|
headers={'Authorization': 'Basic ' + base64.b64encode(
|
|
(username + ':wrong-' + marker).encode()).decode()})
|
|
requests['bad_login'] = r['headers']['X-Request-Id']
|
|
att, rid = upload_and_trace(c, actor, own_chat, client)
|
|
requests['upload'] = rid
|
|
stand.wait_ready(c, actor, att['id'])
|
|
sample = stand.sample_bytes()
|
|
b64 = base64.b64encode(sample).decode()
|
|
hexs = sample.hex()
|
|
secrets = {
|
|
'marker': [marker], 'password': [password], 'session_token': [token],
|
|
'basic_credential': [basic], 'bearer_header': ['Bearer ' + token],
|
|
'signature': [signature], 'image_base64': [b64],
|
|
'image_hex': [hexs],
|
|
'image_base64_fragment': fragments(b64), 'image_hex_fragment': fragments(hexs),
|
|
'wrong_password_marker': ['wrong-' + marker],
|
|
}
|
|
coverage = {'http_logs': {}, 'server_spans': {}, 'worker_chain': False}
|
|
texts = []
|
|
trace_ids = {}
|
|
for name, request_id in requests.items():
|
|
log = http_log(client, request_id)
|
|
coverage['http_logs'][name] = True
|
|
texts.append(('log:' + name, log['line'] + json.dumps(log['labels'], sort_keys=True)))
|
|
trace_ids[name] = log['labels']['trace_id']
|
|
for name, tid in trace_ids.items():
|
|
def has_server(sp, tid=tid, name=name):
|
|
return any(s['kind'] == 'SERVER' and
|
|
s['attributes'].get('messenger.request_id') == requests[name]
|
|
for s in sp)
|
|
spans = trace_with(client, tid, has_server, description='trace ' + name)
|
|
coverage['server_spans'][name] = True
|
|
if name == 'upload':
|
|
coverage['worker_chain'] = any(s['kind'] == 'CONSUMER' for s in spans) \
|
|
if any(s['kind'] == 'CONSUMER' for s in spans) else False
|
|
if not coverage['worker_chain']:
|
|
spans = trace_with(client, tid, lambda sp: any(
|
|
s['kind'] == 'CONSUMER' for s in sp), description='worker chain')
|
|
coverage['worker_chain'] = True
|
|
for s in spans:
|
|
for item in flatten({k: s[k] for k in ('name', 'attributes', 'status')}):
|
|
texts.append(('span:' + name + ':' + s['kind'], item))
|
|
logs = wait_rows(client, '{service_name=~"messenger-.*"} | trace_id="%s"' % tid)
|
|
for row in logs:
|
|
texts.append(('tracelog:' + name, row['line'] + json.dumps(row['labels'], sort_keys=True)))
|
|
worker_logs = wait_rows(
|
|
client, '{service_name="messenger-worker"} | messenger_attachment_id="%s"' % att['id'])
|
|
coverage['worker_logs'] = len(worker_logs)
|
|
for row in worker_logs:
|
|
texts.append(('workerlog', row['line'] + json.dumps(row['labels'], sort_keys=True)))
|
|
leaks = scan(texts, secrets)
|
|
complete = (all(coverage['http_logs'].get(n) for n in requests) and
|
|
all(coverage['server_spans'].get(n) for n in requests) and
|
|
coverage['worker_chain'] and coverage['worker_logs'] > 0)
|
|
result = {'captured': utc_now(), 'coverage': coverage, 'coverage_complete': complete,
|
|
'scanned_items': len(texts), 'leak_count': len(leaks), 'leaks': leaks,
|
|
'passed': complete and not leaks}
|
|
write_json(out, 'redaction.json', result)
|
|
(out / 'redaction.txt').write_text(
|
|
'coverage_complete=%s scanned_items=%d leaks=%d\n' %
|
|
(complete, len(texts), len(leaks)))
|
|
if not result['passed']:
|
|
raise Failure('redaction: coverage_complete=%s leaks=%d' % (complete, len(leaks)))
|
|
print('PASS: redaction (%d items scanned, 0 leaks)' % len(texts))
|
|
|
|
|
|
# ------------------------------------------------------------------ history
|
|
|
|
def raw_samples(client, selector, at, window='30s'):
|
|
result = client.prom_query(selector + '[' + window + ']', at=at)
|
|
return {json.dumps(r['metric'], sort_keys=True): r['values'] for r in result}
|
|
|
|
|
|
def cmd_history(args):
|
|
client = make_obs(args)
|
|
out = Path(args.output_dir)
|
|
with client.app_environment(instrumented_env()):
|
|
c, alice, chat = new_context(args.base_url, args.ca_file)
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201,
|
|
actor=alice, body={'text': 'history'},
|
|
headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message')
|
|
rid_message = r['headers']['X-Request-Id']
|
|
att, rid = upload_and_trace(c, alice, chat, client)
|
|
stand.wait_ready(c, alice, att['id'])
|
|
log = http_log(client, rid)
|
|
tid = log['labels']['trace_id']
|
|
spans = trace_with(client, tid, lambda sp: any(s['kind'] == 'CONSUMER' for s in sp),
|
|
description='history trace')
|
|
msg_log = http_log(client, rid_message)
|
|
client.wait_telemetry_flushed()
|
|
time.sleep(12)
|
|
client.wait_telemetry_flushed()
|
|
at = time.time()
|
|
selector = 'messenger_jobs_processed_total{job="messenger-worker",outcome="succeeded"}'
|
|
before = raw_samples(client, selector, at)
|
|
if not before:
|
|
raise Failure('no Prometheus samples to preserve')
|
|
series_key, values = next(iter(before.items()))
|
|
old_sample = {'labels': json.loads(series_key), 'timestamp': values[-1][0],
|
|
'value': values[-1][1]}
|
|
old_log = {'ts': log['ts'], 'request_id': rid, 'trace_id': tid,
|
|
'span_id': log['labels']['span_id']}
|
|
old_trace = sorted(s['span_id'] for s in spans)
|
|
old_msg = {'ts': msg_log['ts'], 'trace_id': msg_log['labels']['trace_id']}
|
|
stand.compose('restart', 'prometheus', 'loki', 'tempo', 'grafana')
|
|
client.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
|
|
after_samples = raw_samples(client, selector, at)
|
|
after = after_samples.get(series_key, [])
|
|
prom_ok = any(abs(v[0] - old_sample['timestamp']) < 1e-6 and
|
|
v[1] == old_sample['value'] for v in after)
|
|
rows = wait_rows(client, '{service_name="messenger-api"} | request_id="%s"' % rid)
|
|
loki_ok = any(row['ts'] == old_log['ts'] and
|
|
row['labels'].get('trace_id') == tid for row in rows)
|
|
spans_after = client.get_trace(tid, wait=30)
|
|
tempo_ok = bool(spans_after) and \
|
|
sorted(s['span_id'] for s in spans_after) == old_trace
|
|
result = {'captured': utc_now(), 'prometheus_old_sample': old_sample,
|
|
'prometheus_read_back': prom_ok, 'loki_old_log': old_log,
|
|
'loki_read_back': loki_ok, 'tempo_old_trace_span_ids': old_trace,
|
|
'tempo_read_back': tempo_ok,
|
|
'passed': prom_ok and loki_ok and tempo_ok}
|
|
write_json(out, 'history.json', result)
|
|
if not result['passed']:
|
|
raise Failure('history: old data not readable after restart: %s' % {
|
|
k: result[k] for k in ('prometheus_read_back', 'loki_read_back', 'tempo_read_back')})
|
|
print('PASS: history (old Prometheus sample, Loki log and Tempo trace survive restart)')
|
|
|
|
|
|
COMMANDS = {'ready': cmd_ready, 'inventory': cmd_inventory, 'signals': cmd_signals,
|
|
'correlation': cmd_correlation, 'redaction': cmd_redaction,
|
|
'history': cmd_history}
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('command', choices=sorted(COMMANDS))
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--ca-file')
|
|
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
|
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
|
parser.add_argument('--output-dir', default='evidence/lab6')
|
|
parser.add_argument('--backend-only', action='store_true')
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
COMMANDS[args.command](args)
|
|
except Failure as error:
|
|
print('FAIL: ' + str(error), file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|