From 6bb252efbd9ffa54ab44c9ef48edafe777945a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=9D?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8=D1=82=D0=B0?= Date: Mon, 16 Nov 2026 02:54:30 +0300 Subject: [PATCH] Measure sampling, transport volume and bounded exporter losses with reproducible incidents --- compose.observability-fault.yaml | 8 + infra/observability/collector.yaml | 1 + research/common.py | 6 +- research/incident_suite.py | 133 ++++++++++++++++ research/observability_cost.py | 202 +++++++++++++++++++++++++ src/messenger/app.py | 11 +- src/messenger/telemetry.py | 64 ++++++-- student-tests/test_telemetry_bounds.py | 38 +++++ 8 files changed, 447 insertions(+), 16 deletions(-) create mode 100644 compose.observability-fault.yaml create mode 100644 research/incident_suite.py create mode 100644 research/observability_cost.py create mode 100644 student-tests/test_telemetry_bounds.py diff --git a/compose.observability-fault.yaml b/compose.observability-fault.yaml new file mode 100644 index 0000000..4bcf36b --- /dev/null +++ b/compose.observability-fault.yaml @@ -0,0 +1,8 @@ +# Explicit local experiment only; no public fault endpoint. +services: + worker: + environment: + ENABLE_FAULT_INJECTION: "1" + api: + environment: + ENABLE_FAULT_INJECTION: "1" diff --git a/infra/observability/collector.yaml b/infra/observability/collector.yaml index 36d17d0..358d51d 100644 --- a/infra/observability/collector.yaml +++ b/infra/observability/collector.yaml @@ -20,6 +20,7 @@ processors: exporters: prometheus: endpoint: 0.0.0.0:9464 + metric_expiration: 15m resource_to_telemetry_conversion: enabled: true otlp/tempo: diff --git a/research/common.py b/research/common.py index 4695a2c..32ff67a 100644 --- a/research/common.py +++ b/research/common.py @@ -58,7 +58,11 @@ def cgroups(services): for service in services: for container in compose('ps', '-q', service).split(): inspect = json.loads(subprocess.check_output(['docker', 'inspect', container]))[0] - stats = subprocess.check_output(['docker', 'exec', container, 'sh', '-c', 'cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak; cat /sys/fs/cgroup/memory.events'], text=True) + # Read only this container's cgroup through a nonprivileged helper; + # Collector's minimal image intentionally has no shell/python. + pid = str(inspect['State']['Pid']) + script = 'p=$(cut -d: -f3 /proc/' + pid + '/cgroup); cat /sys/fs/cgroup$p/memory.current /sys/fs/cgroup$p/memory.peak; cat /sys/fs/cgroup$p/memory.events' + stats = subprocess.check_output(['docker', 'run', '--rm', '--network=none', '--read-only', '--cap-drop=ALL', '--pid=host', '--cgroupns=host', '--entrypoint', 'sh', 'postgres:17.10-alpine', '-c', script], text=True) lines = stats.splitlines() result.append({'service': service, 'container': container, 'current_bytes': int(lines[0]), 'peak_bytes': int(lines[1]), diff --git a/research/incident_suite.py b/research/incident_suite.py new file mode 100644 index 0000000..04f4385 --- /dev/null +++ b/research/incident_suite.py @@ -0,0 +1,133 @@ +"""Real local failures, bounded restoration, query receipts; never persist credentials.""" +import json +import time +import urllib.parse +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from uuid import uuid4 + +from common import LOKI, ROOT, TEMPO, Client, cgroups, compose, get, query, scalar, wait_for + + +def firing(name): + return bool(query('ALERTS{alertname="' + name + '",alertstate="firing"}')) + + +def trace_spans(trace_id): + payload = get(TEMPO + '/api/traces/' + trace_id) + return [span for batch in payload.get('batches', []) for scope in batch.get('scopeSpans', []) for span in scope.get('spans', [])] + + +def snapshot(): + return {'depth': scalar('max(messenger_queue_depth)'), + 'oldest_age_s': scalar('max(messenger_queue_oldest_age_seconds)'), + 'error_ratio': scalar('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)')} + + +def main(): + report = {'started_at': datetime.now(UTC).isoformat(), 'scenarios': []} + client = Client() + for _ in range(30): + assert client.request('GET', '/api/v1/users/me')[0] == 200 + time.sleep(.1) + time.sleep(8) + # Incoming W3C context propagates durably through outbox and retry. + compose('-f', 'compose.yaml', '-f', 'compose.observability-fault.yaml', 'up', '-d', '--no-deps', 'worker') + compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c', + "INSERT INTO test_faults(point,action) VALUES ('before_effect','error')") + trace_id = uuid4().hex + attachment = client.upload(trace_id) + wait_for(lambda: client.ready(attachment), 45) + def retry_trace(): + spans = trace_spans(trace_id) + consumers = [s for s in spans if s.get('kind') in ('SPAN_KIND_CONSUMER', 5)] + errors = [s for s in consumers if s.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2)] + return spans if len(consumers) >= 2 and errors else None + spans = wait_for(retry_trace, 30) + logs = get(LOKI + '/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': '{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"', 'limit': '100'})) + assert logs['data']['result'] + report['scenarios'].append({'name': 'upload_publish_consume_retry', 'passed': True, + 'trace_id': trace_id, 'attachment_id': attachment, + 'spans': spans, 'log_streams': len(logs['data']['result'])}) + compose('up', '-d', '--no-deps', 'worker') + + # Worker absent: queue gauges must reveal accepted work even with no failures counter. + compose('stop', 'worker') + started = time.monotonic() + try: + attachments = [client.upload() for _ in range(3)] + wait_for(lambda: firing('MessengerQueueLag'), 100) + detected = time.monotonic() - started + state = snapshot() + finally: + compose('start', 'worker') + restored = time.monotonic() + wait_for(lambda: all(client.ready(a) for a in attachments), 45) + wait_for(lambda: not firing('MessengerQueueLag') and scalar('max(messenger_queue_depth)') == 0, 45) + report['scenarios'].append({'name': 'worker_stop_queue_age', 'passed': True, + 'firing_after_s': round(detected, 3), 'firing_observation': state, + 'resolved_after_restore_s': round(time.monotonic()-restored, 3), + 'ready_attachment_count': len(attachments)}) + + # Database pause: client-side deadline must bound calls; application spans show DB error. + compose('pause', 'postgres') + started = time.monotonic() + statuses = [] + try: + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [executor.submit(client.request, 'GET', '/api/v1/users/me') for _ in range(12)] + statuses = [future.result()[0] for future in futures] + wait_for(lambda: firing('MessengerHighErrorRate'), 60) + detected = time.monotonic() - started + state = snapshot() + finally: + compose('unpause', 'postgres') + restored = time.monotonic() + wait_for(lambda: client.request('GET', '/api/v1/users/me')[0] == 200, 25) + for _ in range(60): + assert client.request('GET', '/api/v1/users/me')[0] == 200 + time.sleep(.2) + wait_for(lambda: not firing('MessengerHighErrorRate'), 90) + report['scenarios'].append({'name': 'database_pause', 'passed': True, + 'firing_after_s': round(detected, 3), 'statuses': statuses, + 'firing_observation': state, + 'resolved_after_restore_s': round(time.monotonic()-restored, 3)}) + + before = cgroups(['api', 'worker', 'collector']) + before_failed = scalar('sum(otelcol_exporter_send_failed_spans)') + compose('stop', 'tempo', 'loki') + try: + started = time.monotonic() + statuses = [] + for _ in range(200): + statuses.append(client.request('GET', '/api/v1/users/me')[0]) + time.sleep(.1) + attachment = client.upload() + wait_for(lambda: client.ready(attachment), 30) + time.sleep(10) + during = cgroups(['api', 'worker', 'collector']) + after_failed = scalar('sum(otelcol_exporter_send_failed_spans)') + assert set(statuses) == {200} + assert after_failed > before_failed + assert all(not item['oom_killed'] and int(item['events']['oom_kill']) == 0 for item in during) + finally: + compose('start', 'tempo', 'loki') + wait_for(lambda: get(TEMPO + '/ready') if False else client.request('GET', '/api/v1/users/me')[0] == 200, 30) + new_trace = uuid4().hex + new_attachment = client.upload(new_trace) + wait_for(lambda: client.ready(new_attachment), 30) + wait_for(lambda: len(trace_spans(new_trace)) >= 5, 30) + report['scenarios'].append({'name': 'telemetry_backends_outage', 'passed': True, + 'duration_s': round(time.monotonic()-started, 3), + 'business_successes': len(statuses), + 'failed_span_sends_before': before_failed, 'failed_span_sends_after': after_failed, + 'memory_before': before, 'memory_during': during, + 'restored_trace_id': new_trace}) + report['passed'] = all(x['passed'] for x in report['scenarios']) + report['finished_at'] = datetime.now(UTC).isoformat() + (ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2)) + print(json.dumps({'passed': report['passed'], 'scenarios': len(report['scenarios'])})) + + +if __name__ == '__main__': + main() diff --git a/research/observability_cost.py b/research/observability_cost.py new file mode 100644 index 0000000..90674b5 --- /dev/null +++ b/research/observability_cost.py @@ -0,0 +1,202 @@ +"""Track 1: same synthetic offered workload, off/full/25%/10%, 3 repeats.""" +import concurrent.futures +import json +import math +import os +import re +import statistics +import subprocess +import threading +import time +import urllib.request +from datetime import UTC, datetime + +from common import ROOT, TEMPO, Client, cgroups, compose, get, scalar, wait_for + +PROFILES = [('off', '0', '0'), ('full', '1', '1'), ('sample25', '1', '.25'), ('sample10', '1', '.1')] +DURATION = 10 +OFFERED = 40 + + +def percentile(values, fraction): + return sorted(values)[min(len(values)-1, math.ceil(len(values)*fraction)-1)] + + +def observation(name): + return scalar('sum(messenger_telemetry_export{service_name="messenger-api",observation="'+name+'"})') + + +def telemetry_snapshot(): + # Read Collector endpoints directly after quiescence: no extra Prometheus + # scrape delay may move warmup payload into the measurement baseline. + with urllib.request.urlopen('http://127.0.0.1:19464/metrics', timeout=5) as response: + text = response.read().decode() + counts = {name: 0.0 for name in ('spans_recorded', 'spans_exported', 'spans_export_failed', + 'spans_pending', 'spans_in_flight', 'spans_queue_lost', + 'traces_wire_bytes', 'logs_wire_bytes', 'metrics_wire_bytes')} + for line in text.splitlines(): + if line.startswith('messenger_telemetry_export{') and 'service_name="messenger-api"' in line: + match = re.search(r'observation="([^"]+)"', line) + if match and match[1] in counts: + counts[match[1]] += float(line.rsplit(' ', 1)[1]) + with urllib.request.urlopen('http://127.0.0.1:18888/metrics', timeout=5) as response: + text = response.read().decode() + accepted = [float(line.rsplit(' ', 1)[1]) for line in text.splitlines() + if line.startswith('otelcol_receiver_accepted_spans') and not line.startswith('#')] + counts['collector_accepted_spans'] = sum(accepted) + return counts + + +def exists(trace_id): + if not trace_id or trace_id == '0'*32: + return False + try: + return bool(get(TEMPO + '/api/traces/' + trace_id).get('batches')) + except Exception: + return False + + +class DockerSamples: + def __init__(self): + self.stop = threading.Event() + self.rows = [] + self.ids = compose('ps', '-q').split() + self.thread = threading.Thread(target=self.run, daemon=True) + + def run(self): + while not self.stop.is_set(): + raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *self.ids], text=True) + self.rows.append({'at': time.monotonic(), 'containers': [json.loads(line) for line in raw.splitlines()]}) + self.stop.wait(1) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *args): + self.stop.set() + self.thread.join(timeout=5) + + +def run_profile(client, profile, repeat): + name, enabled, ratio = profile + env = {**os.environ, 'OTEL_ENABLED': enabled, 'OTEL_SAMPLE_RATIO': ratio} + command = ['docker','compose','-f','compose.yaml','-f','compose.observability-fault.yaml', + 'up','-d','--no-deps','--force-recreate','api','worker','dispatcher'] + subprocess.run(command, env=env, cwd=ROOT, check=True, capture_output=True) + wait_for(lambda: client.request('GET','/api/v1/users/me')[0] == 200, 30) + # Establish each process/meter and warm the same endpoint before taking baselines. + for _ in range(40): + assert client.request('GET','/api/v1/users/me')[0] == 200 + time.sleep(.1) + time.sleep(10) + before = telemetry_snapshot() + memory_before = cgroups(['api','worker','collector']) + compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-c', + "INSERT INTO test_faults(point,action) VALUES ('after_upload_commit','error'),('after_upload_commit','error')") + calls = [] + dropped = 0 + lock_future = None + started = time.monotonic() + + def request(index): + begin = time.monotonic() + if index in (80, 240): + result = client.request('POST',f'/api/v1/chats/{client.chat}/attachments?filename=fixture.png', + (ROOT/'tests/fixtures/sample.png').read_bytes(), {'Content-Type':'image/png'}) + else: + result = client.request('GET','/api/v1/users/me') + status, _, headers = result + headers = {key.lower():value for key,value in headers.items()} + return {'index':index,'status':status,'duration_s':time.monotonic()-begin, + 'trace_id':headers.get('x-trace-id'), 'sampled':headers.get('x-trace-sampled')=='true', + 'kind':'controlled_upload_error' if index in (80,240) else 'read'} + + with DockerSamples() as samples, concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + pending = set() + for index in range(DURATION*OFFERED): + target = started + index/OFFERED + delay = target-time.monotonic() + if delay > 0: + time.sleep(delay) + if index == 160: + lock_future = executor.submit(compose,'exec','-T','postgres','psql','-U','messenger','-d','messenger','-c', + 'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(0.35); COMMIT;') + ready = [future for future in pending if future.done()] + for future in ready: + calls.append(future.result()) + pending.remove(future) + if len(pending) >= 15: + dropped += 1 + else: + pending.add(executor.submit(request,index)) + for future in pending: + calls.append(future.result()) + if lock_future: + lock_future.result() + elapsed = time.monotonic()-started + assert sum(x['status']==500 for x in calls)==2, 'Expected exactly two controlled upload errors' + assert all(x['status'] in (200,500) for x in calls), 'Unexpected business failure' + time.sleep(10) + after = telemetry_snapshot() + memory_after = cgroups(['api','worker','collector']) + rare = [row for row in calls if row['status']==500 or (row['kind']=='read' and row['duration_s']>=.15)] + for row in rare: + row['found_in_tempo'] = exists(row['trace_id']) + if enabled=='1' and row['sampled']: + assert row['found_in_tempo'], 'Sampled rare trace failed ingestion' + latency = [row['duration_s'] for row in calls] + deltas = {key: after[key]-before[key] for key in before} + return {'profile':name,'enabled':enabled,'sampling_ratio':float(ratio),'repeat':repeat, + 'offered_rps':OFFERED,'duration_s':DURATION,'elapsed_s':round(elapsed,6), + 'scheduled':DURATION*OFFERED,'not_sent':dropped,'sent':len(calls), + 'successful':sum(row['status']==200 for row in calls),'controlled_500':2, + 'successful_rps':sum(row['status']==200 for row in calls)/elapsed, + 'latency_s':{'p50':percentile(latency,.5),'p95':percentile(latency,.95),'p99':percentile(latency,.99)}, + 'sampled_requests':sum(row['sampled'] for row in calls), + 'rare_probes':rare,'telemetry_before':before,'telemetry_after':after,'telemetry_delta':deltas, + 'cgroups_before':memory_before,'cgroups_after':memory_after, + 'docker_samples':samples.rows,'requests':sorted(calls,key=lambda row:row['index'])} + + +def main(): + report = {'started_at':datetime.now(UTC).isoformat(),'track':'observability cost','runs':[], + 'method':{'profiles':PROFILES,'repeats':3,'offered_rps':OFFERED,'measurement_s':DURATION, + 'latency_clock':'client perf_counter, includes network and queueing', + 'rare_slow':'one 350ms exclusive users-table lock/run', + 'rare_errors':'two actual API500 after durable upload COMMIT/run through private local fault hooks', + 'wire_bytes':'protobuf body length on each SDK HTTP send, retries included; excludes HTTP/TLS headers', + 'sdk_queue_losses':'conservation after 10s quiescence, separate from sampled-out spans', + 'baseline':'40 warmup requests at 10/s, then10s silence; direct Collector endpoints avoid Prometheus scrape delay', + 'post_window':'10s silence before direct Collector counters; bytes include periodic metric exports in observation window', + 'resources':'same image/CPU/RAM; sequential profiles, all services on one Docker host'}} + client = Client() + try: + for repeat in range(1,4): + order = PROFILES[repeat-1:]+PROFILES[:repeat-1] + for profile in order: + row = run_profile(client,profile,repeat) + report['runs'].append(row) + (ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2)) + print(profile[0],repeat,'PASS',round(row['successful_rps'],2),'RPS',flush=True) + finally: + subprocess.run(['docker','compose','up','-d','--no-deps','--force-recreate','api','worker','dispatcher'],cwd=ROOT,check=True,capture_output=True) + report['passed'] = len(report['runs'])==12 + report['finished_at'] = datetime.now(UTC).isoformat() + summary={} + for name,_,_ in PROFILES: + runs=[row for row in report['runs'] if row['profile']==name] + summary[name]={'median_success_rps':statistics.median(row['successful_rps'] for row in runs), + 'median_p95_s':statistics.median(row['latency_s']['p95'] for row in runs), + 'median_trace_wire_bytes':statistics.median(row['telemetry_delta']['traces_wire_bytes'] for row in runs), + 'rare_total':sum(len(row['rare_probes']) for row in runs), + 'rare_sampled':sum(sum(x['sampled'] for x in row['rare_probes']) for row in runs), + 'export_failed_spans':sum(row['telemetry_delta']['spans_export_failed'] for row in runs), + 'queue_lost_spans':sum(row['telemetry_delta']['spans_queue_lost'] for row in runs)} + report['summary']=summary + (ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2)) + print(json.dumps(summary,indent=2)) + + +if __name__=='__main__': + main() diff --git a/src/messenger/app.py b/src/messenger/app.py index 407c182..f978abd 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -100,8 +100,9 @@ async def request_log(request: Request, call_next): response.headers["X-Request-Id"] = request.state.request_id response.headers["X-Instance-Id"] = INSTANCE_ID return response + method = request.method if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"} else "OTHER" context = propagate.extract(dict(request.headers)) - with telemetry.operation("HTTP " + request.method, context=context, kind=SpanKind.SERVER) as span: + with telemetry.operation("HTTP " + method, context=context, kind=SpanKind.SERVER) as span: try: if (request.method == "POST" and request.url.path == "/api/v1/auth/sessions" and await request.body()): @@ -115,12 +116,14 @@ async def request_log(request: Request, call_next): response = error(request, 500, "Внутренняя ошибка") response.headers["X-Request-Id"] = request.state.request_id response.headers["X-Instance-Id"] = INSTANCE_ID + response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x") + response.headers["X-Trace-Sampled"] = "true" if span.is_recording() else "false" route = getattr(request.scope.get("route"), "path", "unknown") elapsed = time.perf_counter() - start - attrs = {"method": request.method, "route": route, + attrs = {"method": method, "route": route, "status_class": str(response.status_code // 100) + "xx"} - span.update_name(request.method + " " + route) - span.set_attributes({"http.request.method": request.method, "http.route": route, + span.update_name(method + " " + route) + span.set_attributes({"http.request.method": method, "http.route": route, "http.response.status_code": response.status_code}) if response.status_code >= 500: span.set_status(Status(StatusCode.ERROR, "server_error")) diff --git a/src/messenger/telemetry.py b/src/messenger/telemetry.py index 9353bb4..426836a 100644 --- a/src/messenger/telemetry.py +++ b/src/messenger/telemetry.py @@ -21,7 +21,7 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased from opentelemetry.trace import SpanKind, Status, StatusCode @@ -32,6 +32,32 @@ INSTANCE_ID = str(uuid4()) _runtime = None +class Accounting(dict): + def __init__(self, keys): + super().__init__((key, 0) for key in keys) + self.lock = threading.Lock() + + def add(self, key, value): + with self.lock: + self[key] += value + + def snapshot(self): + with self.lock: + return dict(self) + + +class RecordedSpans(SpanProcessor): + def __init__(self, stats): + self.stats = stats + + def on_start(self, span, parent_context=None): + pass + + def on_end(self, span): + if span.context.trace_flags.sampled: + self.stats.add("spans_recorded", 1) + + class CountingSession(requests.Session): """Count actual protobuf HTTP sends, including transport retries, without retaining data.""" def __init__(self, stats, signal): @@ -39,8 +65,8 @@ class CountingSession(requests.Session): self.stats, self.signal = stats, signal def send(self, request, **kwargs): - self.stats[self.signal + "_wire_bytes"] += len(request.body or b"") - self.stats[self.signal + "_http_sends"] += 1 + self.stats.add(self.signal + "_wire_bytes", len(request.body or b"")) + self.stats.add(self.signal + "_http_sends", 1) return super().send(request, **kwargs) @@ -50,10 +76,17 @@ class ObservedSpanExporter(OTLPSpanExporter): super().__init__(**kwargs) def export(self, spans): - result = super().export(spans) - key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed" - self.stats[key] += len(spans) - return result + self.stats.add("spans_in_flight", len(spans)) + try: + result = super().export(spans) + key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed" + self.stats.add(key, len(spans)) + return result + except Exception: + self.stats.add("spans_export_failed", len(spans)) + raise + finally: + self.stats.add("spans_in_flight", -len(spans)) class TechnicalLogs(logging.Filter): @@ -86,7 +119,7 @@ class Telemetry: self.enabled = os.getenv("OTEL_ENABLED", "1") == "1" self.busy = 0 self.providers = [] - self.export_stats = {key: 0 for key in ("spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends")} + self.export_stats = Accounting(("spans_recorded", "spans_in_flight", "spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends")) self.stop = threading.Event() self.thread = None resource = Resource.create({ @@ -110,11 +143,13 @@ class Telemetry: # Standard OTLP environment variables also support HTTPS/mTLS in lab7. trace_provider = TracerProvider(resource=resource, sampler=ParentBased( TraceIdRatioBased(float(os.getenv("OTEL_SAMPLE_RATIO", "1"))))) - trace_provider.add_span_processor(BatchSpanProcessor( + trace_provider.add_span_processor(RecordedSpans(self.export_stats)) + self.batch_spans = BatchSpanProcessor( ObservedSpanExporter(self.export_stats, endpoint=endpoint + "/v1/traces", session=CountingSession(self.export_stats, "traces"), **common), max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=1500, - )) + ) + trace_provider.add_span_processor(self.batch_spans) trace.set_tracer_provider(trace_provider) reader = PeriodicExportingMetricReader( OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", session=CountingSession(self.export_stats, "metrics"), **common), @@ -164,7 +199,14 @@ class Telemetry: def collect(): while not self.stop.wait(2): - for name, value in self.export_stats.items(): + counts = self.export_stats.snapshot() + # Read-only queue observation of pinned SDK 1.39.0. Conservation + # is evaluated after quiescence, not during concurrent export. + pending = len(self.batch_spans._batch_processor._queue) + counts["spans_pending"] = pending + counts["span_queue_capacity"] = 256 + counts["spans_queue_lost"] = max(0, counts["spans_recorded"] - counts["spans_exported"] - counts["spans_export_failed"] - counts["spans_in_flight"] - pending) + for name, value in counts.items(): self.export_observation.set(value, {"observation": name}) stats = store.pool.get_stats() self.pool_used.set(max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0))) diff --git a/student-tests/test_telemetry_bounds.py b/student-tests/test_telemetry_bounds.py new file mode 100644 index 0000000..4699e1b --- /dev/null +++ b/student-tests/test_telemetry_bounds.py @@ -0,0 +1,38 @@ +"""Real exporter failure must remain bounded and account for lost sampled spans.""" +import json +import os +import subprocess +import sys +from pathlib import Path + + +def test_span_queue_loss_conservation_when_collector_absent(): + code = ''' +import json, time +from messenger.telemetry import setup +class Pool: + def get_stats(self): return {} +class Store: + pool=Pool() +t=setup("messenger-api") +t.start_background(Store()) +for index in range(1200): + with t.tracer.start_as_current_span("synthetic-safe-span"): + pass +time.sleep(7) +counts=t.export_stats.snapshot() +pending=len(t.batch_spans._batch_processor._queue) +loss=counts["spans_recorded"]-counts["spans_exported"]-counts["spans_export_failed"]-counts["spans_in_flight"]-pending +assert counts["spans_recorded"]==1200 +assert 0<=pending<=256 +assert loss>0 +assert counts["spans_export_failed"]>0 +print(json.dumps({"recorded":1200,"pending":pending,"queue_lost":loss,"export_failed":counts["spans_export_failed"]})) +t.close() +''' + env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]/"src"), + "OTEL_ENABLED": "1", "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1"} + result = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr[-1000:] + payload = json.loads(next(line for line in result.stdout.splitlines() if line.startswith('{'))) + assert payload["queue_lost"] > 0