"""Lab 6 acceptance: real signals, two incidents, retry propagation and bounded outage. Uses only the standard library. Stops services only in this Compose project and always starts them again; never deletes volumes. Run separately from other tests. """ import argparse import json import subprocess import sys import time import urllib.error import urllib.parse import urllib.request import uuid from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check PROM = 'http://127.0.0.1:9090' LOKI = 'http://127.0.0.1:3100' TEMPO = 'http://127.0.0.1:3200' def stamp(): return datetime.now(timezone.utc).isoformat() def get(url, **params): if params: url += '?' + urllib.parse.urlencode(params) with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=10) as r: return json.load(r) def query(expr): data = get(PROM + '/api/v1/query', query=expr) check(data['status'] == 'success', 'PromQL failed') return data['data']['result'] def number(expr): values = query(expr) return float(values[0]['value'][1]) if values else 0.0 def wait(label, fn, timeout=90): end = time.monotonic() + timeout last = None while time.monotonic() < end: try: value = fn() if value: return value except (urllib.error.URLError, TimeoutError, AssertionError) as exc: last = type(exc).__name__ time.sleep(2) raise AssertionError(f'Timeout: {label} ({last})') def compose(*args): subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90) def trace_data(trace_id): data = get(TEMPO + '/api/traces/' + trace_id) spans = [] for batch in data.get('batches', data.get('resourceSpans', [])): resource = {a['key']: a['value'].get('stringValue') for a in batch.get('resource', {}).get('attributes', [])} for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])): for span in scope.get('spans', []): spans.append({**span, 'service': resource.get('service.name')}) return data, spans def logs(trace_id): return get(LOKI + '/loki/api/v1/query_range', query='{service_name=~"messenger-.*"} | trace_id="' + trace_id + '"', start=str(time.time_ns() - 3_600_000_000_000), limit=1000)['data']['result'] def attrs(span): return {a['key']: next(iter(a['value'].values())) for a in span.get('attributes', [])} def pipeline(trace_id, *, retry=False): data, spans = trace_data(trace_id) names = {s['name'] for s in spans} required = {'POST /api/v1/chats/{chat_id}/attachments', 'attachment-jobs publish', 'attachment-jobs process', 'image.process', 's3.GetObject', 's3.PutObject', 'db.UPDATE'} if not required <= names: return None by_id = {s['spanId']: s for s in spans} for name, parent_name in [('attachment-jobs publish', 'POST /api/v1/chats/{chat_id}/attachments'), ('attachment-jobs process', 'attachment-jobs publish'), ('image.process', 'attachment-jobs process')]: for s in (s for s in spans if s['name'] == name): check(by_id.get(s.get('parentSpanId'), {}).get('name') == parent_name, 'Broken trace parent for ' + name) check(s.get('traceState') == 'lab=check', 'W3C tracestate lost at ' + name) attempts = sorted({int(attrs(s)['job.attempt']) for s in spans if s['name'] == 'attachment-jobs process'}) if retry and len(attempts) < 2: return None records = logs(trace_id) services = {r['stream'].get('service_name') for r in records} if not {'messenger-api', 'messenger-worker', 'messenger-dispatcher'} <= services: return None return {'trace_id': trace_id, 'span_count': len(spans), 'span_names': sorted(names), 'log_services': sorted(services), 'attempts': attempts, 'parent_chain_verified': True, 'tracestate_preserved': True, '_raw': json.dumps([data, records])} class Scenario: def __init__(self, base_url): self.client = Client(base_url) self.alice, _, _, self.chat = self.client.bootstrap() self.private_markers = [self.alice['token'], self.alice['registration']['password']] def upload(self): trace_id, request_id = uuid.uuid4().hex, str(uuid.uuid4()) filename = 'private_filename_' + uuid.uuid4().hex + '.png' self.private_markers.append(filename) result = self.client.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=' + filename, 202, actor=self.alice, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png', 'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01', 'tracestate': 'lab=check', 'X-Request-Id': request_id}, schema='Attachment') check(result['headers']['X-Request-Id'] == request_id, 'Request ID changed') return result['json'], trace_id, request_id def check_pipeline(self, trace_id, request_id, retry=False): result = wait('pipeline spans/logs', lambda: pipeline(trace_id, retry=retry)) raw = result.pop('_raw') check(all(value not in raw for value in self.private_markers), 'Private input leaked into telemetry') check(request_id in raw, 'Request ID is missing from trace/log correlation') result['request_id'] = request_id result['private_inputs_absent'] = True return result def business(self, expected=200, trace_id=None): headers = {'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01'} if trace_id else {} return self.client.call('GET', '/api/v1/chats/' + self.chat['id'], expected, actor=self.alice, headers=headers, schema='Chat' if expected == 200 else None) def smoke(self): att, trace_id, request_id = self.upload() self.client.wait_ready(self.alice, att) result = self.check_pipeline(trace_id, request_id) names = ['http_requests_total', 'http_duration_seconds_bucket', 'jobs_processed_total', 'job_duration_seconds_bucket', 'attachment_time_to_ready_seconds_bucket', 'queue_depth', 'queue_oldest_age_seconds', 'db_pool_used', 'db_pool_waiting', 'worker_busy', 'worker_capacity', 'container_cpu_seconds_total', 'container_memory_bytes'] for name in names: wait(name, lambda name=name: query(name)) series = get(PROM + '/api/v1/series', **{'match[]': '{job="messenger"}', 'start': str(time.time() - 300)})['data'] forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url'} check(all(not forbidden.intersection(row) for row in series), 'High-cardinality metric label') check(all(self.chat['id'] not in json.dumps(row) for row in series), 'Raw route ID in metrics') result['metrics_present'] = names return result def queue(self): result = {'started_at': stamp()} compose('stop', 'worker') try: att, trace_id, request_id = self.upload() self.client.message(self.alice, self.chat, 'queued attachment', attachments=[att['id']]) wait('QueueLagHigh firing', lambda: number('ALERTS{alertname="QueueLagHigh",alertstate="firing"}') == 1) result.update(firing_at=stamp(), depth=number('max(queue_depth)'), oldest_age_seconds=number('max(queue_oldest_age_seconds)')) finally: compose('start', 'worker') ready = self.client.wait_ready(self.alice, att) wait('queue alert resolved', lambda: not query('ALERTS{alertname="QueueLagHigh"}')) result.update(resolved_at=stamp(), ready=ready['status'], depth_after=number('max(queue_depth)'), pipeline=self.check_pipeline(trace_id, request_id)) return result def database(self): result = {'started_at': stamp()} trace_id = uuid.uuid4().hex compose('stop', 'postgres') try: errors = 0 def load_until_firing(): nonlocal errors self.business(expected=503, trace_id=trace_id if errors == 0 else None) errors += 1 return number('ALERTS{alertname="ApiHighErrorRate",alertstate="firing"}') == 1 wait('ApiHighErrorRate firing under sustained traffic', load_until_firing, timeout=120) result['failed_requests'] = errors result.update(firing_at=stamp(), error_ratio=number('messenger:http_error_ratio:2m')) def failed_trace(): _, spans = trace_data(trace_id) failed = [s for s in spans if s['name'].startswith('db.') and s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')] return [{'name': s['name'], 'error_type': attrs(s).get('error.type'), 'duration_seconds': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9} for s in failed] result['db_error_spans'] = wait('failed DB span', failed_trace) result['trace_id'] = trace_id result['correlated_log_streams'] = len(wait('DB error logs', lambda: logs(trace_id))) finally: compose('start', 'postgres') wait('API recovered', self.business) for _ in range(10): self.business() wait('API alert resolved', lambda: not query('ALERTS{alertname="ApiHighErrorRate"}'), timeout=160) result['resolved_at'] = stamp() return result def retry(self): failures = number('sum(jobs_processed_total{outcome="failed"})') compose('stop', 'worker') try: att, trace_id, request_id = self.upload() compose('stop', 's3') # `compose start worker` can also start its dependency graph, including S3. compose('up', '-d', '--no-deps', '--no-recreate', 'worker') wait('failed processing attempt', lambda: number('sum(jobs_processed_total{outcome="failed"})') > failures, timeout=40) finally: compose('start', 's3', 'worker') wait('S3/API ready', lambda: self.client.call('GET', '/health/ready', schema='Health')) self.client.wait_ready(self.alice, att) return self.check_pipeline(trace_id, request_id, retry=True) def backend(self): accepted = 'sum(otelcol_receiver_accepted_spans_total)' sent = 'sum(otelcol_exporter_sent_spans_total{exporter="otlphttp/tempo"})' queue = 'sum(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})' # A quiet interval on both ends makes accepted-minus-sent a loss count, # not a count of failed export attempts or currently buffered spans. time.sleep(8) wait('trace queue drained', lambda: number(queue) == 0) before = {'accepted': number(accepted), 'sent': number(sent)} result = {'started_at': stamp(), 'before': before} samples, writes = [], 0 compose('stop', 'tempo') try: end = time.monotonic() + 60 att, _, _ = self.upload() self.client.wait_ready(self.alice, att) while time.monotonic() < end: body = 'private_message_' + uuid.uuid4().hex self.private_markers.append(body) self.client.message(self.alice, self.chat, body) writes += 1 samples.append({'at': stamp(), 'collector_rss_bytes': number('otelcol_process_memory_rss_bytes'), 'queue_batches': number(queue)}) time.sleep(0.3) check(writes > 10, 'Not enough writes during telemetry outage') check(max(s['queue_batches'] for s in samples) <= 32, 'Collector queue is unbounded') check(0 < max(s['collector_rss_bytes'] for s in samples) < 256 * 1024 * 1024, 'Collector exceeded its memory budget') finally: compose('start', 'tempo') wait('trace queue drained', lambda: number(queue) == 0) time.sleep(10) # Include final Prometheus scrape after drain. after = {'accepted': number(accepted), 'sent': number(sent)} lost = (after['accepted'] - before['accepted']) - (after['sent'] - before['sent']) check(lost >= 0, 'Loss window was not quiescent; rerun backend separately') result.update(after=after, lost_accepted_spans=lost, successful_writes=writes, attachment_ready_during_outage=True, samples=samples, recovered_at=stamp()) att, trace_id, request_id = self.upload() self.client.wait_ready(self.alice, att) result['recovered_pipeline'] = self.check_pipeline(trace_id, request_id) return result def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--scenario', choices=['smoke', 'queue', 'database', 'retry', 'backend', 'all'], default='smoke') parser.add_argument('--base-url', default='http://127.0.0.1:8080') args = parser.parse_args() scenario = Scenario(args.base_url) path = ROOT / 'evidence/observability.json' evidence = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {} for name in (['smoke', 'queue', 'database', 'retry', 'backend'] if args.scenario == 'all' else [args.scenario]): print('Running ' + name, flush=True) evidence[name] = {'checked_at': stamp(), **getattr(scenario, name)()} path.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') print('PASS: ' + name, flush=True) if __name__ == '__main__': main()