#!/usr/bin/env python3 """lab6 verification: ready | inventory | signals | correlation | redaction | history. Python 3.9+, standard library only. Every subcommand writes its own evidence file under --output-dir and exits non-zero on any failed criterion.""" import argparse import base64 import hashlib import json import re import sys import time import urllib.parse import uuid from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import obs # noqa: E402 from obs import Failure, stand, utc_now # noqa: E402 FORBIDDEN_LABEL = re.compile( r'(^|_)(user|chat|message|attachment|request|trace|span|job)_id$|' r'(^|_)(url|filename|text|body|username|password|token)($|_)') EXPECTED_SIGNALS = [ # logical signal, metric, required label names (normalised), unit ('HTTP duration/count', 'http_server_request_duration_seconds_count', ('http_request_method', 'http_route', 'http_response_status_code'), 's'), ('HTTP duration buckets', 'http_server_request_duration_seconds_bucket', ('le',), 's'), ('HTTP duration sum', 'http_server_request_duration_seconds_sum', (), 's'), ('Jobs processed', 'messenger_jobs_processed_total', ('job_type', 'outcome'), '{job}'), ('Job duration', 'messenger_job_duration_seconds_count', ('job_type', 'outcome'), 's'), ('Time to ready', 'messenger_attachment_time_to_ready_seconds_count', ('job_type',), 's'), ('Queue depth', 'messenger_jobs_queue_depth', ('state',), '{job}'), ('Queue oldest age', 'messenger_jobs_queue_oldest_age_seconds', (), 's'), ('Outbox pending', 'messenger_outbox_pending', (), '{message}'), ('Outbox oldest age', 'messenger_outbox_oldest_age_seconds', (), 's'), ('DB pool connections', 'db_client_connection_count', ('db_client_connection_state',), '{connection}'), ('DB pool max', 'db_client_connection_max', (), '{connection}'), ('DB pool pending', 'db_client_connection_pending_requests', (), '{request}'), ('DB pool wait', 'db_client_connection_wait_time_seconds_count', (), 's'), ('DB operation duration', 'db_client_operation_duration_seconds_count', ('db_operation_name',), 's'), ('Worker active', 'messenger_worker_jobs_active', (), '{job}'), ('Worker capacity', 'messenger_worker_jobs_capacity', (), '{job}'), ('Broker published', 'messenger_broker_published_total', ('outcome',), '{message}'), ('Telemetry export', 'messenger_telemetry_export_items_total', ('signal', 'outcome'), '{item}'), ('Process CPU', 'process_cpu_time_seconds_total', ('cpu_mode',), 's'), ('Process RAM', 'process_memory_usage_bytes', (), 'By'), ('V8 heap', 'v8js_memory_heap_used_bytes', (), 'By'), ('Event loop delay', 'nodejs_eventloop_delay_p99_seconds', (), 's'), ] COLLECTOR_SIGNALS = [ 'otelcol_receiver_accepted_spans', 'otelcol_receiver_accepted_log_records', 'otelcol_receiver_accepted_metric_points', 'otelcol_exporter_sent_spans', 'otelcol_exporter_sent_log_records', 'otelcol_exporter_sent_metric_points', 'otelcol_exporter_queue_size', 'otelcol_exporter_queue_capacity', ] COLLECTOR_FAILURE_ONLY = [ 'otelcol_exporter_send_failed_spans', 'otelcol_exporter_send_failed_log_records', 'otelcol_exporter_send_failed_metric_points', 'otelcol_exporter_enqueue_failed_spans', 'otelcol_exporter_enqueue_failed_log_records', 'otelcol_exporter_enqueue_failed_metric_points', ] # ------------------------------------------------------------------ helpers def write_json(out, name, data): out.mkdir(parents=True, exist_ok=True) (out / name).write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True) + '\n') def make_obs(args): return obs.ObsClient(args.prometheus_url, args.grafana_url, base_url=args.base_url, ca_file=args.ca_file) def ns_window(seconds_back=600, forward=30): now = time.time_ns() return now - seconds_back * 10 ** 9, now + forward * 10 ** 9 def wait_rows(client, query, minimum=1, timeout=60, back=900): deadline = time.monotonic() + timeout rows = [] while time.monotonic() < deadline: start, end = ns_window(back) rows = client.loki_rows(query, start, end) if len(rows) >= minimum: return rows time.sleep(2) raise Failure('expected log rows not found in Loki (' + str(len(rows)) + '/' + str(minimum) + ')') def http_log(client, request_id, timeout=60): rows = wait_rows( client, '{service_name="messenger-api"} | event_name="http.request" ' '| request_id="' + request_id + '"', timeout=timeout) if len(rows) != 1: raise Failure('expected exactly one http.request log for the request') return rows[0] def chain_to_root(spans, span): by_id = {s['span_id']: s for s in spans} path = [span] while path[-1]['parent_span_id']: parent = by_id.get(path[-1]['parent_span_id']) if parent is None: raise Failure('broken parent chain at ' + path[-1]['name']) path.append(parent) if len(path) > 64: raise Failure('parent chain too long') return path def descendants(spans, root_id): children = {} for s in spans: children.setdefault(s['parent_span_id'], []).append(s) out, stack = [], [root_id] while stack: for child in children.get(stack.pop(), []): out.append(child) stack.append(child['span_id']) return out def safe_span(s): return {k: s[k] for k in ('span_id', 'parent_span_id', 'name', 'kind', 'status', 'service', 'instance')} def trace_with(client, trace_id, predicate, timeout=60, description='trace'): deadline = time.monotonic() + timeout spans = None while time.monotonic() < deadline: spans = client.get_trace(trace_id) if spans and predicate(spans): return spans time.sleep(2) raise Failure(description + ' did not reach the expected shape in Tempo') def new_context(base_url, ca_file): c = stand.new_client(base_url, ca_file) alice, bob, eve, chat = c.bootstrap() return c, alice, chat def instrumented_env(): return {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1', 'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'} # -------------------------------------------------------------------- ready def cmd_ready(args): client = make_obs(args) client.wait_ready(app_telemetry=True, provisioning=not args.backend_only) result = {'passed': True, 'backend_only': args.backend_only, 'finished': utc_now()} write_json(Path(args.output_dir), 'ready-backend.json' if args.backend_only else 'ready.json', result) print('PASS: ready' + (' (backend only)' if args.backend_only else '')) # ---------------------------------------------------------------- inventory def inventory(client): names = client.prom_names() rows = [] _, meta = obs.http_json(client.prometheus + '/api/v1/metadata') for name in names: if not re.match(r'(http_server|messenger|db_client|process|nodejs|v8js|target_info)', name): continue _, series = obs.http_json( client.prometheus + '/api/v1/series?' + urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'})) labels = sorted({k for s in series['data'] for k in s} - {'__name__'}) base = re.sub(r'_(bucket|count|sum|total)$', '', name) info = (meta['data'].get(name) or meta['data'].get(base) or [{}])[0] rows.append({'metric': name, 'type': info.get('type'), 'unit': info.get('unit') or None, 'labels': labels, 'series': len(series['data'])}) return names, rows def cmd_inventory(args): client = make_obs(args) names, rows = inventory(client) out = Path(args.output_dir) out.mkdir(parents=True, exist_ok=True) (out / 'metric-names.txt').write_text('\n'.join(names) + '\n') write_json(out, 'inventory.json', {'captured': utc_now(), 'metrics': rows}) print('inventory: %d names, %d application/runtime metrics' % (len(names), len(rows))) # ------------------------------------------------------------------ signals def series_labels(client, name): _, series = obs.http_json( client.prometheus + '/api/v1/series?' + urllib.parse.urlencode({'match[]': '{__name__="' + name + '"}'})) return series['data'] def cmd_signals(args): client = make_obs(args) names, rows = inventory(client) out = Path(args.output_dir) out.mkdir(parents=True, exist_ok=True) (out / 'metric-names.txt').write_text('\n'.join(names) + '\n') problems, table = [], [] for logical, metric, required, unit in EXPECTED_SIGNALS: data = series_labels(client, metric) labels = sorted({k for s in data for k in s} - {'__name__'}) missing = [r for r in required if r not in labels] table.append({'signal': logical, 'metric': metric, 'labels': labels, 'unit': unit, 'series': len(data)}) if not data: problems.append('missing metric ' + metric) elif missing: problems.append(metric + ' lacks labels ' + ','.join(missing)) # Contract: forbidden labels on any application/runtime metric. for row in rows: for label in row['labels']: if FORBIDDEN_LABEL.search(label): problems.append('forbidden label ' + label + ' on ' + row['metric']) # Business facts: routes, replicas, outcomes. routes = {s.get('http_route') for s in series_labels( client, 'http_server_request_duration_seconds_count')} if not any(r and r.startswith('/api/v1/') and '{' in r for r in routes): problems.append('no route templates with {param} placeholders') if any(r and re.search(r'[0-9a-f]{8}-[0-9a-f]{4}', r) for r in routes): problems.append('raw UUID in http_route') replicas = {s['instance'] for s in series_labels( client, 'http_server_request_duration_seconds_count') if s.get('job') == 'messenger-api'} if len(replicas) < 2: problems.append('fewer than two API replicas in HTTP metrics') outcomes = {s.get('outcome') for s in series_labels( client, 'messenger_jobs_processed_total')} for needed in ('succeeded', 'retry', 'failed'): if needed not in outcomes: problems.append('job outcome not observed: ' + needed + ' (run lab6-correlate first)') for job in ('messenger-api', 'messenger-worker'): if not client.prom_query('target_info{job="%s"}' % job): problems.append('no metrics from ' + job) # Queue snapshot is not doubled by the two API replicas. depth_series = client.prom_query('messenger_jobs_queue_depth{state="ready"}') # Collector. collector = {} for metric in COLLECTOR_SIGNALS: data = series_labels(client, metric) collector[metric] = len(data) if not data: problems.append('missing Collector metric ' + metric) for metric in COLLECTOR_FAILURE_ONLY: collector[metric] = len(series_labels(client, metric)) if not client.prom_query('otelcol_process_memory_rss'): problems.append('no Collector RSS') # Logs/traces for both services. for service in ('messenger-api', 'messenger-worker'): rows_ = wait_rows(client, '{service_name="%s"}' % service, timeout=30) if not rows_: problems.append('no logs from ' + service) result = {'captured': utc_now(), 'signals': table, 'collector_series': collector, 'job_outcomes_observed': sorted(o for o in outcomes if o), 'api_replicas_with_http_metrics': sorted(replicas), 'queue_depth_ready_series': len(depth_series), 'route_templates': sorted(r for r in routes if r), 'problems': problems, 'passed': not problems} write_json(out, 'signals.json', result) if problems: raise Failure('signals: ' + '; '.join(problems)) print('PASS: signals (%d logical signals)' % len(table)) # -------------------------------------------------------------- correlation def status_of(attachment_id): code, body = stand.cli('status', attachment_id) if code != 0: raise Failure('jobs.cli status failed') return body def correlate_http(client, base_url, ca_file, obs_client): c, alice, chat = new_context(base_url, ca_file) key = uuid.uuid4().hex r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=alice, body={'text': 'lab6 correlation'}, headers={'Idempotency-Key': key}, schema='Message') rid = r['headers']['X-Request-Id'] log = http_log(obs_client, rid) labels = log['labels'] if labels.get('trace_sampled') != 'true': raise Failure('HTTP log is not marked sampled') spans = trace_with( obs_client, labels['trace_id'], lambda sp: any(s['span_id'] == labels['span_id'] for s in sp) and len(descendants(sp, labels['span_id'])) > 0, description='HTTP trace') server = next(s for s in spans if s['span_id'] == labels['span_id']) if server['kind'] != 'SERVER' or \ server['attributes'].get('messenger.request_id') != rid: raise Failure('SERVER span does not match the logged request') db = [s for s in descendants(spans, server['span_id']) if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']] if not db: raise Failure('no DB CLIENT child spans under the SERVER span') return {'request_id': rid, 'trace_id': labels['trace_id'], 'server_span': safe_span(server), 'log_span_id': labels['span_id'], 'db_children': [safe_span(s) for s in db]} def upload_and_trace(c, alice, chat, obs_client, require_attempts=1): r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202, actor=alice, raw=stand.sample_bytes(), headers={'Content-Type': 'image/png'}, schema='Attachment') return r['json'], r['headers']['X-Request-Id'] def consumer_spans(spans): return sorted((s for s in spans if s['kind'] == 'CONSUMER'), key=lambda s: s['start_ns']) def correlate_upload(client_args, obs_client): c, alice, chat = new_context(client_args.base_url, client_args.ca_file) att, rid = upload_and_trace(c, alice, chat, obs_client) stand.wait_ready(c, alice, att['id']) log = http_log(obs_client, rid) tid = log['labels']['trace_id'] def complete(sp): cons = consumer_spans(sp) return bool(cons) and any(s['kind'] == 'SERVER' for s in sp) spans = trace_with(obs_client, tid, complete, description='upload trace') consumer = consumer_spans(spans)[0] path = chain_to_root(spans, consumer) kinds = [s['kind'] for s in path] if path[-1]['kind'] != 'SERVER' or 'PRODUCER' not in kinds: raise Failure('consumer is not linked to the HTTP SERVER via PRODUCER') if path[-1]['service'] != 'messenger-api' or consumer['service'] != 'messenger-worker': raise Failure('services along the chain are wrong') below = descendants(spans, consumer['span_id']) s3 = [s for s in below if s['attributes'].get('rpc.system') == 'aws-api'] db = [s for s in below if 'db.operation.name' in s['attributes']] if not s3 or not db: raise Failure('consumer has no S3/DB child spans') return {'request_id': rid, 'trace_id': tid, 'attachment_id': att['id'], 'chain_consumer_to_root': [safe_span(s) for s in path], 'consumer_attempt': consumer['attributes'].get('messenger.job.attempt'), 'consumer_outcome': consumer['attributes'].get('messenger.job.outcome'), 's3_children': len(s3), 'db_children': len(db), 'chain_fields_are_real_parent_child': True} def start_worker(): stand.compose('up', '-d', '--no-deps', '--wait', 'worker') def targeted_job(c, alice, chat, obs_client): """Stop the worker, accept an upload, return (att, rid, job_id, version).""" stand.compose('stop', 'worker') att, rid = upload_and_trace(c, alice, chat, obs_client) body = status_of(att['id']) jobs = body.get('jobs') if not isinstance(jobs, list) or len(jobs) != 1: raise Failure('expected exactly one job for the new upload') return att, rid, jobs[0]['job_id'], jobs[0]['pipeline_version'] def correlate_fault(client_args, obs_client, kind): c, alice, chat = new_context(client_args.base_url, client_args.ca_file) obs_client.require_no_foreign_faults() key = None try: att, rid, job_id, version = targeted_job(c, alice, chat, obs_client) key = 'fault:worker.before_side_effect:%s:%s' % (att['id'], version) stand.set_fault(key, kind) start_worker() if kind == 'transient': def retried(): job = status_of(att['id'])['jobs'][0] return job['attempt'] >= 2 or job['last_error_code'] stand.wait_for(retried, 90, 'first retry') stand.clear_fault(key) key = None stand.wait_ready(c, alice, att['id'], timeout=120) else: def failed(): return status_of(att['id'])['jobs'][0]['status'] == 'failed' stand.wait_for(failed, 90, 'terminal failed job') meta = stand.wait_ready(c, alice, att['id'], allow_failed=True) if meta['status'] != 'failed': raise Failure('attachment did not become failed') final = status_of(att['id']) log = http_log(obs_client, rid) tid = log['labels']['trace_id'] wanted = 2 if kind == 'transient' else 1 def enough(sp): return len(consumer_spans(sp)) >= wanted spans = trace_with(obs_client, tid, enough, timeout=90, description=kind + ' trace') consumers = consumer_spans(spans) ids = {s['span_id'] for s in consumers} if len(ids) != len(consumers): raise Failure('consumer attempts share a span ID') attempts = [s['attributes'].get('messenger.job.attempt') for s in consumers] outcomes = [s['attributes'].get('messenger.job.outcome') for s in consumers] if kind == 'transient': if sorted(attempts)[:2] != [1, 2] or 'retry' not in outcomes: raise Failure('retry attempts not visible in the trace: %s %s' % (attempts, outcomes)) else: if 'failed' not in outcomes: raise Failure('failed outcome not visible in the trace') return {'kind': kind, 'request_id': rid, 'trace_id': tid, 'attachment_id': att['id'], 'job_id': job_id, 'pipeline_version': version, 'consumer_spans': [safe_span(s) | { 'attempt': s['attributes'].get('messenger.job.attempt'), 'outcome': s['attributes'].get('messenger.job.outcome')} for s in consumers], 'final_job': {k: final['jobs'][0].get(k) for k in ('status', 'attempt', 'last_error_code')}, 'final_attachment_status': final['attachment']['status']} finally: if key: stand.clear_fault(key) start_worker() def cmd_correlation(args): client = make_obs(args) out = Path(args.output_dir) with client.app_environment(instrumented_env()): client.require_no_foreign_faults() results = { 'correlation-http.json': correlate_http(client, args.base_url, args.ca_file, client), 'correlation-upload.json': correlate_upload(args, client), 'correlation-retry.json': correlate_fault(args, client, 'transient'), 'correlation-failed.json': correlate_fault(args, client, 'permanent'), } for name, data in results.items(): data['captured'] = utc_now() write_json(out, name, data) print('PASS: correlation (http, upload, retry, failed)') # ---------------------------------------------------------------- redaction PEM = re.compile(r'-----BEGIN [A-Z ]*PRIVATE KEY-----') def fragments(text, size=48, stride=16): return [text[i:i + size] for i in range(0, max(1, len(text) - size + 1), stride)] def flatten(value, path=''): if isinstance(value, dict): for k, v in value.items(): yield from flatten(v, path + '.' + str(k)) yield path + '.key:' + str(k) elif isinstance(value, list): for i, v in enumerate(value): yield from flatten(v, path + '[' + str(i) + ']') else: yield path + '=' + str(value) def scan(texts, secrets): """texts: [(source, text)]; secrets: {category: [needle,...]}.""" leaks = [] for source, text in texts: for category, needles in secrets.items(): for needle in needles: if needle and needle in text: leaks.append({'category': category, 'source': source, 'hash': hashlib.sha256(needle.encode()).hexdigest()[:12]}) break if PEM.search(text): leaks.append({'category': 'private_key_pem', 'source': source, 'hash': ''}) return leaks def cmd_redaction(args): client = make_obs(args) out = Path(args.output_dir) with client.app_environment(instrumented_env()): c = stand.new_client(args.base_url, args.ca_file) actor = c.user('redact') marker = 'LAB6-REDACT-' + uuid.uuid4().hex password = actor['registration']['password'] token = actor['token'] username = actor['registration']['username'] basic = base64.b64encode((username + ':' + password).encode()).decode() signature = 'sig' + uuid.uuid4().hex + uuid.uuid4().hex own_chat = c.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'redaction', 'member_ids': [actor['id']]}, schema='Chat')['json'] requests = {} r = c.call('POST', '/api/v1/chats/' + own_chat['id'] + '/messages', 201, actor=actor, body={'text': 'body ' + marker}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message') requests['message'] = r['headers']['X-Request-Id'] r = c.call('GET', '/api/v1/chats?X-Amz-Signature=' + signature, range(200, 500), actor=actor, headers={'X-Marker': marker}) requests['signed_query'] = r['headers']['X-Request-Id'] r = c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + base64.b64encode( (username + ':wrong-' + marker).encode()).decode()}) requests['bad_login'] = r['headers']['X-Request-Id'] att, rid = upload_and_trace(c, actor, own_chat, client) requests['upload'] = rid stand.wait_ready(c, actor, att['id']) sample = stand.sample_bytes() b64 = base64.b64encode(sample).decode() hexs = sample.hex() secrets = { 'marker': [marker], 'password': [password], 'session_token': [token], 'basic_credential': [basic], 'bearer_header': ['Bearer ' + token], 'signature': [signature], 'image_base64': [b64], 'image_hex': [hexs], 'image_base64_fragment': fragments(b64), 'image_hex_fragment': fragments(hexs), 'wrong_password_marker': ['wrong-' + marker], } coverage = {'http_logs': {}, 'server_spans': {}, 'worker_chain': False} texts = [] trace_ids = {} for name, request_id in requests.items(): log = http_log(client, request_id) coverage['http_logs'][name] = True texts.append(('log:' + name, log['line'] + json.dumps(log['labels'], sort_keys=True))) trace_ids[name] = log['labels']['trace_id'] for name, tid in trace_ids.items(): def has_server(sp, tid=tid, name=name): return any(s['kind'] == 'SERVER' and s['attributes'].get('messenger.request_id') == requests[name] for s in sp) spans = trace_with(client, tid, has_server, description='trace ' + name) coverage['server_spans'][name] = True if name == 'upload': coverage['worker_chain'] = any(s['kind'] == 'CONSUMER' for s in spans) \ if any(s['kind'] == 'CONSUMER' for s in spans) else False if not coverage['worker_chain']: spans = trace_with(client, tid, lambda sp: any( s['kind'] == 'CONSUMER' for s in sp), description='worker chain') coverage['worker_chain'] = True for s in spans: for item in flatten({k: s[k] for k in ('name', 'attributes', 'status')}): texts.append(('span:' + name + ':' + s['kind'], item)) logs = wait_rows(client, '{service_name=~"messenger-.*"} | trace_id="%s"' % tid) for row in logs: texts.append(('tracelog:' + name, row['line'] + json.dumps(row['labels'], sort_keys=True))) worker_logs = wait_rows( client, '{service_name="messenger-worker"} | messenger_attachment_id="%s"' % att['id']) coverage['worker_logs'] = len(worker_logs) for row in worker_logs: texts.append(('workerlog', row['line'] + json.dumps(row['labels'], sort_keys=True))) leaks = scan(texts, secrets) complete = (all(coverage['http_logs'].get(n) for n in requests) and all(coverage['server_spans'].get(n) for n in requests) and coverage['worker_chain'] and coverage['worker_logs'] > 0) result = {'captured': utc_now(), 'coverage': coverage, 'coverage_complete': complete, 'scanned_items': len(texts), 'leak_count': len(leaks), 'leaks': leaks, 'passed': complete and not leaks} write_json(out, 'redaction.json', result) (out / 'redaction.txt').write_text( 'coverage_complete=%s scanned_items=%d leaks=%d\n' % (complete, len(texts), len(leaks))) if not result['passed']: raise Failure('redaction: coverage_complete=%s leaks=%d' % (complete, len(leaks))) print('PASS: redaction (%d items scanned, 0 leaks)' % len(texts)) # ------------------------------------------------------------------ history def raw_samples(client, selector, at, window='30s'): result = client.prom_query(selector + '[' + window + ']', at=at) return {json.dumps(r['metric'], sort_keys=True): r['values'] for r in result} def cmd_history(args): client = make_obs(args) out = Path(args.output_dir) with client.app_environment(instrumented_env()): c, alice, chat = new_context(args.base_url, args.ca_file) r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=alice, body={'text': 'history'}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message') rid_message = r['headers']['X-Request-Id'] att, rid = upload_and_trace(c, alice, chat, client) stand.wait_ready(c, alice, att['id']) log = http_log(client, rid) tid = log['labels']['trace_id'] spans = trace_with(client, tid, lambda sp: any(s['kind'] == 'CONSUMER' for s in sp), description='history trace') msg_log = http_log(client, rid_message) client.wait_telemetry_flushed() time.sleep(12) client.wait_telemetry_flushed() at = time.time() selector = 'messenger_jobs_processed_total{job="messenger-worker",outcome="succeeded"}' before = raw_samples(client, selector, at) if not before: raise Failure('no Prometheus samples to preserve') series_key, values = next(iter(before.items())) old_sample = {'labels': json.loads(series_key), 'timestamp': values[-1][0], 'value': values[-1][1]} old_log = {'ts': log['ts'], 'request_id': rid, 'trace_id': tid, 'span_id': log['labels']['span_id']} old_trace = sorted(s['span_id'] for s in spans) old_msg = {'ts': msg_log['ts'], 'trace_id': msg_log['labels']['trace_id']} stand.compose('restart', 'prometheus', 'loki', 'tempo', 'grafana') client.wait_ready(timeout=180, app_telemetry=True, provisioning=True) after_samples = raw_samples(client, selector, at) after = after_samples.get(series_key, []) prom_ok = any(abs(v[0] - old_sample['timestamp']) < 1e-6 and v[1] == old_sample['value'] for v in after) rows = wait_rows(client, '{service_name="messenger-api"} | request_id="%s"' % rid) loki_ok = any(row['ts'] == old_log['ts'] and row['labels'].get('trace_id') == tid for row in rows) spans_after = client.get_trace(tid, wait=30) tempo_ok = bool(spans_after) and \ sorted(s['span_id'] for s in spans_after) == old_trace result = {'captured': utc_now(), 'prometheus_old_sample': old_sample, 'prometheus_read_back': prom_ok, 'loki_old_log': old_log, 'loki_read_back': loki_ok, 'tempo_old_trace_span_ids': old_trace, 'tempo_read_back': tempo_ok, 'passed': prom_ok and loki_ok and tempo_ok} write_json(out, 'history.json', result) if not result['passed']: raise Failure('history: old data not readable after restart: %s' % { k: result[k] for k in ('prometheus_read_back', 'loki_read_back', 'tempo_read_back')}) print('PASS: history (old Prometheus sample, Loki log and Tempo trace survive restart)') COMMANDS = {'ready': cmd_ready, 'inventory': cmd_inventory, 'signals': cmd_signals, 'correlation': cmd_correlation, 'redaction': cmd_redaction, 'history': cmd_history} def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('command', choices=sorted(COMMANDS)) parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--ca-file') parser.add_argument('--prometheus-url', default='http://localhost:9090') parser.add_argument('--grafana-url', default='http://localhost:3000') parser.add_argument('--output-dir', default='evidence/lab6') parser.add_argument('--backend-only', action='store_true') args = parser.parse_args(argv) try: COMMANDS[args.command](args) except Failure as error: print('FAIL: ' + str(error), file=sys.stderr) return 1 return 0 if __name__ == '__main__': sys.exit(main())