Measure sampling, transport volume and bounded exporter losses with reproducible incidents
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
# Explicit local experiment only; no public fault endpoint.
|
||||
services:
|
||||
worker:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
api:
|
||||
environment:
|
||||
ENABLE_FAULT_INJECTION: "1"
|
||||
@@ -20,6 +20,7 @@ processors:
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
metric_expiration: 15m
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlp/tempo:
|
||||
|
||||
+5
-1
@@ -58,7 +58,11 @@ def cgroups(services):
|
||||
for service in services:
|
||||
for container in compose('ps', '-q', service).split():
|
||||
inspect = json.loads(subprocess.check_output(['docker', 'inspect', container]))[0]
|
||||
stats = subprocess.check_output(['docker', 'exec', container, 'sh', '-c', 'cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak; cat /sys/fs/cgroup/memory.events'], text=True)
|
||||
# Read only this container's cgroup through a nonprivileged helper;
|
||||
# Collector's minimal image intentionally has no shell/python.
|
||||
pid = str(inspect['State']['Pid'])
|
||||
script = 'p=$(cut -d: -f3 /proc/' + pid + '/cgroup); cat /sys/fs/cgroup$p/memory.current /sys/fs/cgroup$p/memory.peak; cat /sys/fs/cgroup$p/memory.events'
|
||||
stats = subprocess.check_output(['docker', 'run', '--rm', '--network=none', '--read-only', '--cap-drop=ALL', '--pid=host', '--cgroupns=host', '--entrypoint', 'sh', 'postgres:17.10-alpine', '-c', script], text=True)
|
||||
lines = stats.splitlines()
|
||||
result.append({'service': service, 'container': container,
|
||||
'current_bytes': int(lines[0]), 'peak_bytes': int(lines[1]),
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Real local failures, bounded restoration, query receipts; never persist credentials."""
|
||||
import json
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from common import LOKI, ROOT, TEMPO, Client, cgroups, compose, get, query, scalar, wait_for
|
||||
|
||||
|
||||
def firing(name):
|
||||
return bool(query('ALERTS{alertname="' + name + '",alertstate="firing"}'))
|
||||
|
||||
|
||||
def trace_spans(trace_id):
|
||||
payload = get(TEMPO + '/api/traces/' + trace_id)
|
||||
return [span for batch in payload.get('batches', []) for scope in batch.get('scopeSpans', []) for span in scope.get('spans', [])]
|
||||
|
||||
|
||||
def snapshot():
|
||||
return {'depth': scalar('max(messenger_queue_depth)'),
|
||||
'oldest_age_s': scalar('max(messenger_queue_oldest_age_seconds)'),
|
||||
'error_ratio': scalar('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)')}
|
||||
|
||||
|
||||
def main():
|
||||
report = {'started_at': datetime.now(UTC).isoformat(), 'scenarios': []}
|
||||
client = Client()
|
||||
for _ in range(30):
|
||||
assert client.request('GET', '/api/v1/users/me')[0] == 200
|
||||
time.sleep(.1)
|
||||
time.sleep(8)
|
||||
# Incoming W3C context propagates durably through outbox and retry.
|
||||
compose('-f', 'compose.yaml', '-f', 'compose.observability-fault.yaml', 'up', '-d', '--no-deps', 'worker')
|
||||
compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c',
|
||||
"INSERT INTO test_faults(point,action) VALUES ('before_effect','error')")
|
||||
trace_id = uuid4().hex
|
||||
attachment = client.upload(trace_id)
|
||||
wait_for(lambda: client.ready(attachment), 45)
|
||||
def retry_trace():
|
||||
spans = trace_spans(trace_id)
|
||||
consumers = [s for s in spans if s.get('kind') in ('SPAN_KIND_CONSUMER', 5)]
|
||||
errors = [s for s in consumers if s.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2)]
|
||||
return spans if len(consumers) >= 2 and errors else None
|
||||
spans = wait_for(retry_trace, 30)
|
||||
logs = get(LOKI + '/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': '{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"', 'limit': '100'}))
|
||||
assert logs['data']['result']
|
||||
report['scenarios'].append({'name': 'upload_publish_consume_retry', 'passed': True,
|
||||
'trace_id': trace_id, 'attachment_id': attachment,
|
||||
'spans': spans, 'log_streams': len(logs['data']['result'])})
|
||||
compose('up', '-d', '--no-deps', 'worker')
|
||||
|
||||
# Worker absent: queue gauges must reveal accepted work even with no failures counter.
|
||||
compose('stop', 'worker')
|
||||
started = time.monotonic()
|
||||
try:
|
||||
attachments = [client.upload() for _ in range(3)]
|
||||
wait_for(lambda: firing('MessengerQueueLag'), 100)
|
||||
detected = time.monotonic() - started
|
||||
state = snapshot()
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
restored = time.monotonic()
|
||||
wait_for(lambda: all(client.ready(a) for a in attachments), 45)
|
||||
wait_for(lambda: not firing('MessengerQueueLag') and scalar('max(messenger_queue_depth)') == 0, 45)
|
||||
report['scenarios'].append({'name': 'worker_stop_queue_age', 'passed': True,
|
||||
'firing_after_s': round(detected, 3), 'firing_observation': state,
|
||||
'resolved_after_restore_s': round(time.monotonic()-restored, 3),
|
||||
'ready_attachment_count': len(attachments)})
|
||||
|
||||
# Database pause: client-side deadline must bound calls; application spans show DB error.
|
||||
compose('pause', 'postgres')
|
||||
started = time.monotonic()
|
||||
statuses = []
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
futures = [executor.submit(client.request, 'GET', '/api/v1/users/me') for _ in range(12)]
|
||||
statuses = [future.result()[0] for future in futures]
|
||||
wait_for(lambda: firing('MessengerHighErrorRate'), 60)
|
||||
detected = time.monotonic() - started
|
||||
state = snapshot()
|
||||
finally:
|
||||
compose('unpause', 'postgres')
|
||||
restored = time.monotonic()
|
||||
wait_for(lambda: client.request('GET', '/api/v1/users/me')[0] == 200, 25)
|
||||
for _ in range(60):
|
||||
assert client.request('GET', '/api/v1/users/me')[0] == 200
|
||||
time.sleep(.2)
|
||||
wait_for(lambda: not firing('MessengerHighErrorRate'), 90)
|
||||
report['scenarios'].append({'name': 'database_pause', 'passed': True,
|
||||
'firing_after_s': round(detected, 3), 'statuses': statuses,
|
||||
'firing_observation': state,
|
||||
'resolved_after_restore_s': round(time.monotonic()-restored, 3)})
|
||||
|
||||
before = cgroups(['api', 'worker', 'collector'])
|
||||
before_failed = scalar('sum(otelcol_exporter_send_failed_spans)')
|
||||
compose('stop', 'tempo', 'loki')
|
||||
try:
|
||||
started = time.monotonic()
|
||||
statuses = []
|
||||
for _ in range(200):
|
||||
statuses.append(client.request('GET', '/api/v1/users/me')[0])
|
||||
time.sleep(.1)
|
||||
attachment = client.upload()
|
||||
wait_for(lambda: client.ready(attachment), 30)
|
||||
time.sleep(10)
|
||||
during = cgroups(['api', 'worker', 'collector'])
|
||||
after_failed = scalar('sum(otelcol_exporter_send_failed_spans)')
|
||||
assert set(statuses) == {200}
|
||||
assert after_failed > before_failed
|
||||
assert all(not item['oom_killed'] and int(item['events']['oom_kill']) == 0 for item in during)
|
||||
finally:
|
||||
compose('start', 'tempo', 'loki')
|
||||
wait_for(lambda: get(TEMPO + '/ready') if False else client.request('GET', '/api/v1/users/me')[0] == 200, 30)
|
||||
new_trace = uuid4().hex
|
||||
new_attachment = client.upload(new_trace)
|
||||
wait_for(lambda: client.ready(new_attachment), 30)
|
||||
wait_for(lambda: len(trace_spans(new_trace)) >= 5, 30)
|
||||
report['scenarios'].append({'name': 'telemetry_backends_outage', 'passed': True,
|
||||
'duration_s': round(time.monotonic()-started, 3),
|
||||
'business_successes': len(statuses),
|
||||
'failed_span_sends_before': before_failed, 'failed_span_sends_after': after_failed,
|
||||
'memory_before': before, 'memory_during': during,
|
||||
'restored_trace_id': new_trace})
|
||||
report['passed'] = all(x['passed'] for x in report['scenarios'])
|
||||
report['finished_at'] = datetime.now(UTC).isoformat()
|
||||
(ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2))
|
||||
print(json.dumps({'passed': report['passed'], 'scenarios': len(report['scenarios'])}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Track 1: same synthetic offered workload, off/full/25%/10%, 3 repeats."""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from common import ROOT, TEMPO, Client, cgroups, compose, get, scalar, wait_for
|
||||
|
||||
PROFILES = [('off', '0', '0'), ('full', '1', '1'), ('sample25', '1', '.25'), ('sample10', '1', '.1')]
|
||||
DURATION = 10
|
||||
OFFERED = 40
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
return sorted(values)[min(len(values)-1, math.ceil(len(values)*fraction)-1)]
|
||||
|
||||
|
||||
def observation(name):
|
||||
return scalar('sum(messenger_telemetry_export{service_name="messenger-api",observation="'+name+'"})')
|
||||
|
||||
|
||||
def telemetry_snapshot():
|
||||
# Read Collector endpoints directly after quiescence: no extra Prometheus
|
||||
# scrape delay may move warmup payload into the measurement baseline.
|
||||
with urllib.request.urlopen('http://127.0.0.1:19464/metrics', timeout=5) as response:
|
||||
text = response.read().decode()
|
||||
counts = {name: 0.0 for name in ('spans_recorded', 'spans_exported', 'spans_export_failed',
|
||||
'spans_pending', 'spans_in_flight', 'spans_queue_lost',
|
||||
'traces_wire_bytes', 'logs_wire_bytes', 'metrics_wire_bytes')}
|
||||
for line in text.splitlines():
|
||||
if line.startswith('messenger_telemetry_export{') and 'service_name="messenger-api"' in line:
|
||||
match = re.search(r'observation="([^"]+)"', line)
|
||||
if match and match[1] in counts:
|
||||
counts[match[1]] += float(line.rsplit(' ', 1)[1])
|
||||
with urllib.request.urlopen('http://127.0.0.1:18888/metrics', timeout=5) as response:
|
||||
text = response.read().decode()
|
||||
accepted = [float(line.rsplit(' ', 1)[1]) for line in text.splitlines()
|
||||
if line.startswith('otelcol_receiver_accepted_spans') and not line.startswith('#')]
|
||||
counts['collector_accepted_spans'] = sum(accepted)
|
||||
return counts
|
||||
|
||||
|
||||
def exists(trace_id):
|
||||
if not trace_id or trace_id == '0'*32:
|
||||
return False
|
||||
try:
|
||||
return bool(get(TEMPO + '/api/traces/' + trace_id).get('batches'))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class DockerSamples:
|
||||
def __init__(self):
|
||||
self.stop = threading.Event()
|
||||
self.rows = []
|
||||
self.ids = compose('ps', '-q').split()
|
||||
self.thread = threading.Thread(target=self.run, daemon=True)
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *self.ids], text=True)
|
||||
self.rows.append({'at': time.monotonic(), 'containers': [json.loads(line) for line in raw.splitlines()]})
|
||||
self.stop.wait(1)
|
||||
|
||||
def __enter__(self):
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=5)
|
||||
|
||||
|
||||
def run_profile(client, profile, repeat):
|
||||
name, enabled, ratio = profile
|
||||
env = {**os.environ, 'OTEL_ENABLED': enabled, 'OTEL_SAMPLE_RATIO': ratio}
|
||||
command = ['docker','compose','-f','compose.yaml','-f','compose.observability-fault.yaml',
|
||||
'up','-d','--no-deps','--force-recreate','api','worker','dispatcher']
|
||||
subprocess.run(command, env=env, cwd=ROOT, check=True, capture_output=True)
|
||||
wait_for(lambda: client.request('GET','/api/v1/users/me')[0] == 200, 30)
|
||||
# Establish each process/meter and warm the same endpoint before taking baselines.
|
||||
for _ in range(40):
|
||||
assert client.request('GET','/api/v1/users/me')[0] == 200
|
||||
time.sleep(.1)
|
||||
time.sleep(10)
|
||||
before = telemetry_snapshot()
|
||||
memory_before = cgroups(['api','worker','collector'])
|
||||
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',
|
||||
"INSERT INTO test_faults(point,action) VALUES ('after_upload_commit','error'),('after_upload_commit','error')")
|
||||
calls = []
|
||||
dropped = 0
|
||||
lock_future = None
|
||||
started = time.monotonic()
|
||||
|
||||
def request(index):
|
||||
begin = time.monotonic()
|
||||
if index in (80, 240):
|
||||
result = client.request('POST',f'/api/v1/chats/{client.chat}/attachments?filename=fixture.png',
|
||||
(ROOT/'tests/fixtures/sample.png').read_bytes(), {'Content-Type':'image/png'})
|
||||
else:
|
||||
result = client.request('GET','/api/v1/users/me')
|
||||
status, _, headers = result
|
||||
headers = {key.lower():value for key,value in headers.items()}
|
||||
return {'index':index,'status':status,'duration_s':time.monotonic()-begin,
|
||||
'trace_id':headers.get('x-trace-id'), 'sampled':headers.get('x-trace-sampled')=='true',
|
||||
'kind':'controlled_upload_error' if index in (80,240) else 'read'}
|
||||
|
||||
with DockerSamples() as samples, concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
|
||||
pending = set()
|
||||
for index in range(DURATION*OFFERED):
|
||||
target = started + index/OFFERED
|
||||
delay = target-time.monotonic()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
if index == 160:
|
||||
lock_future = executor.submit(compose,'exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',
|
||||
'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(0.35); COMMIT;')
|
||||
ready = [future for future in pending if future.done()]
|
||||
for future in ready:
|
||||
calls.append(future.result())
|
||||
pending.remove(future)
|
||||
if len(pending) >= 15:
|
||||
dropped += 1
|
||||
else:
|
||||
pending.add(executor.submit(request,index))
|
||||
for future in pending:
|
||||
calls.append(future.result())
|
||||
if lock_future:
|
||||
lock_future.result()
|
||||
elapsed = time.monotonic()-started
|
||||
assert sum(x['status']==500 for x in calls)==2, 'Expected exactly two controlled upload errors'
|
||||
assert all(x['status'] in (200,500) for x in calls), 'Unexpected business failure'
|
||||
time.sleep(10)
|
||||
after = telemetry_snapshot()
|
||||
memory_after = cgroups(['api','worker','collector'])
|
||||
rare = [row for row in calls if row['status']==500 or (row['kind']=='read' and row['duration_s']>=.15)]
|
||||
for row in rare:
|
||||
row['found_in_tempo'] = exists(row['trace_id'])
|
||||
if enabled=='1' and row['sampled']:
|
||||
assert row['found_in_tempo'], 'Sampled rare trace failed ingestion'
|
||||
latency = [row['duration_s'] for row in calls]
|
||||
deltas = {key: after[key]-before[key] for key in before}
|
||||
return {'profile':name,'enabled':enabled,'sampling_ratio':float(ratio),'repeat':repeat,
|
||||
'offered_rps':OFFERED,'duration_s':DURATION,'elapsed_s':round(elapsed,6),
|
||||
'scheduled':DURATION*OFFERED,'not_sent':dropped,'sent':len(calls),
|
||||
'successful':sum(row['status']==200 for row in calls),'controlled_500':2,
|
||||
'successful_rps':sum(row['status']==200 for row in calls)/elapsed,
|
||||
'latency_s':{'p50':percentile(latency,.5),'p95':percentile(latency,.95),'p99':percentile(latency,.99)},
|
||||
'sampled_requests':sum(row['sampled'] for row in calls),
|
||||
'rare_probes':rare,'telemetry_before':before,'telemetry_after':after,'telemetry_delta':deltas,
|
||||
'cgroups_before':memory_before,'cgroups_after':memory_after,
|
||||
'docker_samples':samples.rows,'requests':sorted(calls,key=lambda row:row['index'])}
|
||||
|
||||
|
||||
def main():
|
||||
report = {'started_at':datetime.now(UTC).isoformat(),'track':'observability cost','runs':[],
|
||||
'method':{'profiles':PROFILES,'repeats':3,'offered_rps':OFFERED,'measurement_s':DURATION,
|
||||
'latency_clock':'client perf_counter, includes network and queueing',
|
||||
'rare_slow':'one 350ms exclusive users-table lock/run',
|
||||
'rare_errors':'two actual API500 after durable upload COMMIT/run through private local fault hooks',
|
||||
'wire_bytes':'protobuf body length on each SDK HTTP send, retries included; excludes HTTP/TLS headers',
|
||||
'sdk_queue_losses':'conservation after 10s quiescence, separate from sampled-out spans',
|
||||
'baseline':'40 warmup requests at 10/s, then10s silence; direct Collector endpoints avoid Prometheus scrape delay',
|
||||
'post_window':'10s silence before direct Collector counters; bytes include periodic metric exports in observation window',
|
||||
'resources':'same image/CPU/RAM; sequential profiles, all services on one Docker host'}}
|
||||
client = Client()
|
||||
try:
|
||||
for repeat in range(1,4):
|
||||
order = PROFILES[repeat-1:]+PROFILES[:repeat-1]
|
||||
for profile in order:
|
||||
row = run_profile(client,profile,repeat)
|
||||
report['runs'].append(row)
|
||||
(ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2))
|
||||
print(profile[0],repeat,'PASS',round(row['successful_rps'],2),'RPS',flush=True)
|
||||
finally:
|
||||
subprocess.run(['docker','compose','up','-d','--no-deps','--force-recreate','api','worker','dispatcher'],cwd=ROOT,check=True,capture_output=True)
|
||||
report['passed'] = len(report['runs'])==12
|
||||
report['finished_at'] = datetime.now(UTC).isoformat()
|
||||
summary={}
|
||||
for name,_,_ in PROFILES:
|
||||
runs=[row for row in report['runs'] if row['profile']==name]
|
||||
summary[name]={'median_success_rps':statistics.median(row['successful_rps'] for row in runs),
|
||||
'median_p95_s':statistics.median(row['latency_s']['p95'] for row in runs),
|
||||
'median_trace_wire_bytes':statistics.median(row['telemetry_delta']['traces_wire_bytes'] for row in runs),
|
||||
'rare_total':sum(len(row['rare_probes']) for row in runs),
|
||||
'rare_sampled':sum(sum(x['sampled'] for x in row['rare_probes']) for row in runs),
|
||||
'export_failed_spans':sum(row['telemetry_delta']['spans_export_failed'] for row in runs),
|
||||
'queue_lost_spans':sum(row['telemetry_delta']['spans_queue_lost'] for row in runs)}
|
||||
report['summary']=summary
|
||||
(ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2))
|
||||
print(json.dumps(summary,indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -100,8 +100,9 @@ async def request_log(request: Request, call_next):
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
method = request.method if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"} else "OTHER"
|
||||
context = propagate.extract(dict(request.headers))
|
||||
with telemetry.operation("HTTP " + request.method, context=context, kind=SpanKind.SERVER) as span:
|
||||
with telemetry.operation("HTTP " + method, context=context, kind=SpanKind.SERVER) as span:
|
||||
try:
|
||||
if (request.method == "POST" and request.url.path == "/api/v1/auth/sessions"
|
||||
and await request.body()):
|
||||
@@ -115,12 +116,14 @@ async def request_log(request: Request, call_next):
|
||||
response = error(request, 500, "Внутренняя ошибка")
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x")
|
||||
response.headers["X-Trace-Sampled"] = "true" if span.is_recording() else "false"
|
||||
route = getattr(request.scope.get("route"), "path", "unknown")
|
||||
elapsed = time.perf_counter() - start
|
||||
attrs = {"method": request.method, "route": route,
|
||||
attrs = {"method": method, "route": route,
|
||||
"status_class": str(response.status_code // 100) + "xx"}
|
||||
span.update_name(request.method + " " + route)
|
||||
span.set_attributes({"http.request.method": request.method, "http.route": route,
|
||||
span.update_name(method + " " + route)
|
||||
span.set_attributes({"http.request.method": method, "http.route": route,
|
||||
"http.response.status_code": response.status_code})
|
||||
if response.status_code >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR, "server_error"))
|
||||
|
||||
+53
-11
@@ -21,7 +21,7 @@ from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
@@ -32,6 +32,32 @@ INSTANCE_ID = str(uuid4())
|
||||
_runtime = None
|
||||
|
||||
|
||||
class Accounting(dict):
|
||||
def __init__(self, keys):
|
||||
super().__init__((key, 0) for key in keys)
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def add(self, key, value):
|
||||
with self.lock:
|
||||
self[key] += value
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
return dict(self)
|
||||
|
||||
|
||||
class RecordedSpans(SpanProcessor):
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
||||
def on_start(self, span, parent_context=None):
|
||||
pass
|
||||
|
||||
def on_end(self, span):
|
||||
if span.context.trace_flags.sampled:
|
||||
self.stats.add("spans_recorded", 1)
|
||||
|
||||
|
||||
class CountingSession(requests.Session):
|
||||
"""Count actual protobuf HTTP sends, including transport retries, without retaining data."""
|
||||
def __init__(self, stats, signal):
|
||||
@@ -39,8 +65,8 @@ class CountingSession(requests.Session):
|
||||
self.stats, self.signal = stats, signal
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
self.stats[self.signal + "_wire_bytes"] += len(request.body or b"")
|
||||
self.stats[self.signal + "_http_sends"] += 1
|
||||
self.stats.add(self.signal + "_wire_bytes", len(request.body or b""))
|
||||
self.stats.add(self.signal + "_http_sends", 1)
|
||||
return super().send(request, **kwargs)
|
||||
|
||||
|
||||
@@ -50,10 +76,17 @@ class ObservedSpanExporter(OTLPSpanExporter):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def export(self, spans):
|
||||
result = super().export(spans)
|
||||
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
|
||||
self.stats[key] += len(spans)
|
||||
return result
|
||||
self.stats.add("spans_in_flight", len(spans))
|
||||
try:
|
||||
result = super().export(spans)
|
||||
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
|
||||
self.stats.add(key, len(spans))
|
||||
return result
|
||||
except Exception:
|
||||
self.stats.add("spans_export_failed", len(spans))
|
||||
raise
|
||||
finally:
|
||||
self.stats.add("spans_in_flight", -len(spans))
|
||||
|
||||
|
||||
class TechnicalLogs(logging.Filter):
|
||||
@@ -86,7 +119,7 @@ class Telemetry:
|
||||
self.enabled = os.getenv("OTEL_ENABLED", "1") == "1"
|
||||
self.busy = 0
|
||||
self.providers = []
|
||||
self.export_stats = {key: 0 for key in ("spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends")}
|
||||
self.export_stats = Accounting(("spans_recorded", "spans_in_flight", "spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends"))
|
||||
self.stop = threading.Event()
|
||||
self.thread = None
|
||||
resource = Resource.create({
|
||||
@@ -110,11 +143,13 @@ class Telemetry:
|
||||
# Standard OTLP environment variables also support HTTPS/mTLS in lab7.
|
||||
trace_provider = TracerProvider(resource=resource, sampler=ParentBased(
|
||||
TraceIdRatioBased(float(os.getenv("OTEL_SAMPLE_RATIO", "1")))))
|
||||
trace_provider.add_span_processor(BatchSpanProcessor(
|
||||
trace_provider.add_span_processor(RecordedSpans(self.export_stats))
|
||||
self.batch_spans = BatchSpanProcessor(
|
||||
ObservedSpanExporter(self.export_stats, endpoint=endpoint + "/v1/traces", session=CountingSession(self.export_stats, "traces"), **common),
|
||||
max_queue_size=256, max_export_batch_size=64,
|
||||
schedule_delay_millis=500, export_timeout_millis=1500,
|
||||
))
|
||||
)
|
||||
trace_provider.add_span_processor(self.batch_spans)
|
||||
trace.set_tracer_provider(trace_provider)
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", session=CountingSession(self.export_stats, "metrics"), **common),
|
||||
@@ -164,7 +199,14 @@ class Telemetry:
|
||||
|
||||
def collect():
|
||||
while not self.stop.wait(2):
|
||||
for name, value in self.export_stats.items():
|
||||
counts = self.export_stats.snapshot()
|
||||
# Read-only queue observation of pinned SDK 1.39.0. Conservation
|
||||
# is evaluated after quiescence, not during concurrent export.
|
||||
pending = len(self.batch_spans._batch_processor._queue)
|
||||
counts["spans_pending"] = pending
|
||||
counts["span_queue_capacity"] = 256
|
||||
counts["spans_queue_lost"] = max(0, counts["spans_recorded"] - counts["spans_exported"] - counts["spans_export_failed"] - counts["spans_in_flight"] - pending)
|
||||
for name, value in counts.items():
|
||||
self.export_observation.set(value, {"observation": name})
|
||||
stats = store.pool.get_stats()
|
||||
self.pool_used.set(max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)))
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Real exporter failure must remain bounded and account for lost sampled spans."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_span_queue_loss_conservation_when_collector_absent():
|
||||
code = '''
|
||||
import json, time
|
||||
from messenger.telemetry import setup
|
||||
class Pool:
|
||||
def get_stats(self): return {}
|
||||
class Store:
|
||||
pool=Pool()
|
||||
t=setup("messenger-api")
|
||||
t.start_background(Store())
|
||||
for index in range(1200):
|
||||
with t.tracer.start_as_current_span("synthetic-safe-span"):
|
||||
pass
|
||||
time.sleep(7)
|
||||
counts=t.export_stats.snapshot()
|
||||
pending=len(t.batch_spans._batch_processor._queue)
|
||||
loss=counts["spans_recorded"]-counts["spans_exported"]-counts["spans_export_failed"]-counts["spans_in_flight"]-pending
|
||||
assert counts["spans_recorded"]==1200
|
||||
assert 0<=pending<=256
|
||||
assert loss>0
|
||||
assert counts["spans_export_failed"]>0
|
||||
print(json.dumps({"recorded":1200,"pending":pending,"queue_lost":loss,"export_failed":counts["spans_export_failed"]}))
|
||||
t.close()
|
||||
'''
|
||||
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]/"src"),
|
||||
"OTEL_ENABLED": "1", "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1"}
|
||||
result = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=15)
|
||||
assert result.returncode == 0, result.stderr[-1000:]
|
||||
payload = json.loads(next(line for line in result.stdout.splitlines() if line.startswith('{')))
|
||||
assert payload["queue_lost"] > 0
|
||||
Reference in New Issue
Block a user