Measure sampling, transport volume and bounded exporter losses with reproducible incidents

This commit is contained in:
2026-11-16 02:54:30 +03:00
parent 2701db5106
commit 6bb252efbd
8 changed files with 447 additions and 16 deletions
+8
View File
@@ -0,0 +1,8 @@
# Explicit local experiment only; no public fault endpoint.
services:
worker:
environment:
ENABLE_FAULT_INJECTION: "1"
api:
environment:
ENABLE_FAULT_INJECTION: "1"
+1
View File
@@ -20,6 +20,7 @@ processors:
exporters:
prometheus:
endpoint: 0.0.0.0:9464
metric_expiration: 15m
resource_to_telemetry_conversion:
enabled: true
otlp/tempo:
+5 -1
View File
@@ -58,7 +58,11 @@ def cgroups(services):
for service in services:
for container in compose('ps', '-q', service).split():
inspect = json.loads(subprocess.check_output(['docker', 'inspect', container]))[0]
stats = subprocess.check_output(['docker', 'exec', container, 'sh', '-c', 'cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak; cat /sys/fs/cgroup/memory.events'], text=True)
# Read only this container's cgroup through a nonprivileged helper;
# Collector's minimal image intentionally has no shell/python.
pid = str(inspect['State']['Pid'])
script = 'p=$(cut -d: -f3 /proc/' + pid + '/cgroup); cat /sys/fs/cgroup$p/memory.current /sys/fs/cgroup$p/memory.peak; cat /sys/fs/cgroup$p/memory.events'
stats = subprocess.check_output(['docker', 'run', '--rm', '--network=none', '--read-only', '--cap-drop=ALL', '--pid=host', '--cgroupns=host', '--entrypoint', 'sh', 'postgres:17.10-alpine', '-c', script], text=True)
lines = stats.splitlines()
result.append({'service': service, 'container': container,
'current_bytes': int(lines[0]), 'peak_bytes': int(lines[1]),
+133
View File
@@ -0,0 +1,133 @@
"""Real local failures, bounded restoration, query receipts; never persist credentials."""
import json
import time
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from uuid import uuid4
from common import LOKI, ROOT, TEMPO, Client, cgroups, compose, get, query, scalar, wait_for
def firing(name):
return bool(query('ALERTS{alertname="' + name + '",alertstate="firing"}'))
def trace_spans(trace_id):
payload = get(TEMPO + '/api/traces/' + trace_id)
return [span for batch in payload.get('batches', []) for scope in batch.get('scopeSpans', []) for span in scope.get('spans', [])]
def snapshot():
return {'depth': scalar('max(messenger_queue_depth)'),
'oldest_age_s': scalar('max(messenger_queue_oldest_age_seconds)'),
'error_ratio': scalar('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)')}
def main():
report = {'started_at': datetime.now(UTC).isoformat(), 'scenarios': []}
client = Client()
for _ in range(30):
assert client.request('GET', '/api/v1/users/me')[0] == 200
time.sleep(.1)
time.sleep(8)
# Incoming W3C context propagates durably through outbox and retry.
compose('-f', 'compose.yaml', '-f', 'compose.observability-fault.yaml', 'up', '-d', '--no-deps', 'worker')
compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c',
"INSERT INTO test_faults(point,action) VALUES ('before_effect','error')")
trace_id = uuid4().hex
attachment = client.upload(trace_id)
wait_for(lambda: client.ready(attachment), 45)
def retry_trace():
spans = trace_spans(trace_id)
consumers = [s for s in spans if s.get('kind') in ('SPAN_KIND_CONSUMER', 5)]
errors = [s for s in consumers if s.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2)]
return spans if len(consumers) >= 2 and errors else None
spans = wait_for(retry_trace, 30)
logs = get(LOKI + '/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': '{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"', 'limit': '100'}))
assert logs['data']['result']
report['scenarios'].append({'name': 'upload_publish_consume_retry', 'passed': True,
'trace_id': trace_id, 'attachment_id': attachment,
'spans': spans, 'log_streams': len(logs['data']['result'])})
compose('up', '-d', '--no-deps', 'worker')
# Worker absent: queue gauges must reveal accepted work even with no failures counter.
compose('stop', 'worker')
started = time.monotonic()
try:
attachments = [client.upload() for _ in range(3)]
wait_for(lambda: firing('MessengerQueueLag'), 100)
detected = time.monotonic() - started
state = snapshot()
finally:
compose('start', 'worker')
restored = time.monotonic()
wait_for(lambda: all(client.ready(a) for a in attachments), 45)
wait_for(lambda: not firing('MessengerQueueLag') and scalar('max(messenger_queue_depth)') == 0, 45)
report['scenarios'].append({'name': 'worker_stop_queue_age', 'passed': True,
'firing_after_s': round(detected, 3), 'firing_observation': state,
'resolved_after_restore_s': round(time.monotonic()-restored, 3),
'ready_attachment_count': len(attachments)})
# Database pause: client-side deadline must bound calls; application spans show DB error.
compose('pause', 'postgres')
started = time.monotonic()
statuses = []
try:
with ThreadPoolExecutor(max_workers=4) as executor:
futures = [executor.submit(client.request, 'GET', '/api/v1/users/me') for _ in range(12)]
statuses = [future.result()[0] for future in futures]
wait_for(lambda: firing('MessengerHighErrorRate'), 60)
detected = time.monotonic() - started
state = snapshot()
finally:
compose('unpause', 'postgres')
restored = time.monotonic()
wait_for(lambda: client.request('GET', '/api/v1/users/me')[0] == 200, 25)
for _ in range(60):
assert client.request('GET', '/api/v1/users/me')[0] == 200
time.sleep(.2)
wait_for(lambda: not firing('MessengerHighErrorRate'), 90)
report['scenarios'].append({'name': 'database_pause', 'passed': True,
'firing_after_s': round(detected, 3), 'statuses': statuses,
'firing_observation': state,
'resolved_after_restore_s': round(time.monotonic()-restored, 3)})
before = cgroups(['api', 'worker', 'collector'])
before_failed = scalar('sum(otelcol_exporter_send_failed_spans)')
compose('stop', 'tempo', 'loki')
try:
started = time.monotonic()
statuses = []
for _ in range(200):
statuses.append(client.request('GET', '/api/v1/users/me')[0])
time.sleep(.1)
attachment = client.upload()
wait_for(lambda: client.ready(attachment), 30)
time.sleep(10)
during = cgroups(['api', 'worker', 'collector'])
after_failed = scalar('sum(otelcol_exporter_send_failed_spans)')
assert set(statuses) == {200}
assert after_failed > before_failed
assert all(not item['oom_killed'] and int(item['events']['oom_kill']) == 0 for item in during)
finally:
compose('start', 'tempo', 'loki')
wait_for(lambda: get(TEMPO + '/ready') if False else client.request('GET', '/api/v1/users/me')[0] == 200, 30)
new_trace = uuid4().hex
new_attachment = client.upload(new_trace)
wait_for(lambda: client.ready(new_attachment), 30)
wait_for(lambda: len(trace_spans(new_trace)) >= 5, 30)
report['scenarios'].append({'name': 'telemetry_backends_outage', 'passed': True,
'duration_s': round(time.monotonic()-started, 3),
'business_successes': len(statuses),
'failed_span_sends_before': before_failed, 'failed_span_sends_after': after_failed,
'memory_before': before, 'memory_during': during,
'restored_trace_id': new_trace})
report['passed'] = all(x['passed'] for x in report['scenarios'])
report['finished_at'] = datetime.now(UTC).isoformat()
(ROOT/'evidence/incidents.json').write_text(json.dumps(report, indent=2))
print(json.dumps({'passed': report['passed'], 'scenarios': len(report['scenarios'])}))
if __name__ == '__main__':
main()
+202
View File
@@ -0,0 +1,202 @@
"""Track 1: same synthetic offered workload, off/full/25%/10%, 3 repeats."""
import concurrent.futures
import json
import math
import os
import re
import statistics
import subprocess
import threading
import time
import urllib.request
from datetime import UTC, datetime
from common import ROOT, TEMPO, Client, cgroups, compose, get, scalar, wait_for
PROFILES = [('off', '0', '0'), ('full', '1', '1'), ('sample25', '1', '.25'), ('sample10', '1', '.1')]
DURATION = 10
OFFERED = 40
def percentile(values, fraction):
return sorted(values)[min(len(values)-1, math.ceil(len(values)*fraction)-1)]
def observation(name):
return scalar('sum(messenger_telemetry_export{service_name="messenger-api",observation="'+name+'"})')
def telemetry_snapshot():
# Read Collector endpoints directly after quiescence: no extra Prometheus
# scrape delay may move warmup payload into the measurement baseline.
with urllib.request.urlopen('http://127.0.0.1:19464/metrics', timeout=5) as response:
text = response.read().decode()
counts = {name: 0.0 for name in ('spans_recorded', 'spans_exported', 'spans_export_failed',
'spans_pending', 'spans_in_flight', 'spans_queue_lost',
'traces_wire_bytes', 'logs_wire_bytes', 'metrics_wire_bytes')}
for line in text.splitlines():
if line.startswith('messenger_telemetry_export{') and 'service_name="messenger-api"' in line:
match = re.search(r'observation="([^"]+)"', line)
if match and match[1] in counts:
counts[match[1]] += float(line.rsplit(' ', 1)[1])
with urllib.request.urlopen('http://127.0.0.1:18888/metrics', timeout=5) as response:
text = response.read().decode()
accepted = [float(line.rsplit(' ', 1)[1]) for line in text.splitlines()
if line.startswith('otelcol_receiver_accepted_spans') and not line.startswith('#')]
counts['collector_accepted_spans'] = sum(accepted)
return counts
def exists(trace_id):
if not trace_id or trace_id == '0'*32:
return False
try:
return bool(get(TEMPO + '/api/traces/' + trace_id).get('batches'))
except Exception:
return False
class DockerSamples:
def __init__(self):
self.stop = threading.Event()
self.rows = []
self.ids = compose('ps', '-q').split()
self.thread = threading.Thread(target=self.run, daemon=True)
def run(self):
while not self.stop.is_set():
raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *self.ids], text=True)
self.rows.append({'at': time.monotonic(), 'containers': [json.loads(line) for line in raw.splitlines()]})
self.stop.wait(1)
def __enter__(self):
self.thread.start()
return self
def __exit__(self, *args):
self.stop.set()
self.thread.join(timeout=5)
def run_profile(client, profile, repeat):
name, enabled, ratio = profile
env = {**os.environ, 'OTEL_ENABLED': enabled, 'OTEL_SAMPLE_RATIO': ratio}
command = ['docker','compose','-f','compose.yaml','-f','compose.observability-fault.yaml',
'up','-d','--no-deps','--force-recreate','api','worker','dispatcher']
subprocess.run(command, env=env, cwd=ROOT, check=True, capture_output=True)
wait_for(lambda: client.request('GET','/api/v1/users/me')[0] == 200, 30)
# Establish each process/meter and warm the same endpoint before taking baselines.
for _ in range(40):
assert client.request('GET','/api/v1/users/me')[0] == 200
time.sleep(.1)
time.sleep(10)
before = telemetry_snapshot()
memory_before = cgroups(['api','worker','collector'])
compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',
"INSERT INTO test_faults(point,action) VALUES ('after_upload_commit','error'),('after_upload_commit','error')")
calls = []
dropped = 0
lock_future = None
started = time.monotonic()
def request(index):
begin = time.monotonic()
if index in (80, 240):
result = client.request('POST',f'/api/v1/chats/{client.chat}/attachments?filename=fixture.png',
(ROOT/'tests/fixtures/sample.png').read_bytes(), {'Content-Type':'image/png'})
else:
result = client.request('GET','/api/v1/users/me')
status, _, headers = result
headers = {key.lower():value for key,value in headers.items()}
return {'index':index,'status':status,'duration_s':time.monotonic()-begin,
'trace_id':headers.get('x-trace-id'), 'sampled':headers.get('x-trace-sampled')=='true',
'kind':'controlled_upload_error' if index in (80,240) else 'read'}
with DockerSamples() as samples, concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
pending = set()
for index in range(DURATION*OFFERED):
target = started + index/OFFERED
delay = target-time.monotonic()
if delay > 0:
time.sleep(delay)
if index == 160:
lock_future = executor.submit(compose,'exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',
'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(0.35); COMMIT;')
ready = [future for future in pending if future.done()]
for future in ready:
calls.append(future.result())
pending.remove(future)
if len(pending) >= 15:
dropped += 1
else:
pending.add(executor.submit(request,index))
for future in pending:
calls.append(future.result())
if lock_future:
lock_future.result()
elapsed = time.monotonic()-started
assert sum(x['status']==500 for x in calls)==2, 'Expected exactly two controlled upload errors'
assert all(x['status'] in (200,500) for x in calls), 'Unexpected business failure'
time.sleep(10)
after = telemetry_snapshot()
memory_after = cgroups(['api','worker','collector'])
rare = [row for row in calls if row['status']==500 or (row['kind']=='read' and row['duration_s']>=.15)]
for row in rare:
row['found_in_tempo'] = exists(row['trace_id'])
if enabled=='1' and row['sampled']:
assert row['found_in_tempo'], 'Sampled rare trace failed ingestion'
latency = [row['duration_s'] for row in calls]
deltas = {key: after[key]-before[key] for key in before}
return {'profile':name,'enabled':enabled,'sampling_ratio':float(ratio),'repeat':repeat,
'offered_rps':OFFERED,'duration_s':DURATION,'elapsed_s':round(elapsed,6),
'scheduled':DURATION*OFFERED,'not_sent':dropped,'sent':len(calls),
'successful':sum(row['status']==200 for row in calls),'controlled_500':2,
'successful_rps':sum(row['status']==200 for row in calls)/elapsed,
'latency_s':{'p50':percentile(latency,.5),'p95':percentile(latency,.95),'p99':percentile(latency,.99)},
'sampled_requests':sum(row['sampled'] for row in calls),
'rare_probes':rare,'telemetry_before':before,'telemetry_after':after,'telemetry_delta':deltas,
'cgroups_before':memory_before,'cgroups_after':memory_after,
'docker_samples':samples.rows,'requests':sorted(calls,key=lambda row:row['index'])}
def main():
report = {'started_at':datetime.now(UTC).isoformat(),'track':'observability cost','runs':[],
'method':{'profiles':PROFILES,'repeats':3,'offered_rps':OFFERED,'measurement_s':DURATION,
'latency_clock':'client perf_counter, includes network and queueing',
'rare_slow':'one 350ms exclusive users-table lock/run',
'rare_errors':'two actual API500 after durable upload COMMIT/run through private local fault hooks',
'wire_bytes':'protobuf body length on each SDK HTTP send, retries included; excludes HTTP/TLS headers',
'sdk_queue_losses':'conservation after 10s quiescence, separate from sampled-out spans',
'baseline':'40 warmup requests at 10/s, then10s silence; direct Collector endpoints avoid Prometheus scrape delay',
'post_window':'10s silence before direct Collector counters; bytes include periodic metric exports in observation window',
'resources':'same image/CPU/RAM; sequential profiles, all services on one Docker host'}}
client = Client()
try:
for repeat in range(1,4):
order = PROFILES[repeat-1:]+PROFILES[:repeat-1]
for profile in order:
row = run_profile(client,profile,repeat)
report['runs'].append(row)
(ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2))
print(profile[0],repeat,'PASS',round(row['successful_rps'],2),'RPS',flush=True)
finally:
subprocess.run(['docker','compose','up','-d','--no-deps','--force-recreate','api','worker','dispatcher'],cwd=ROOT,check=True,capture_output=True)
report['passed'] = len(report['runs'])==12
report['finished_at'] = datetime.now(UTC).isoformat()
summary={}
for name,_,_ in PROFILES:
runs=[row for row in report['runs'] if row['profile']==name]
summary[name]={'median_success_rps':statistics.median(row['successful_rps'] for row in runs),
'median_p95_s':statistics.median(row['latency_s']['p95'] for row in runs),
'median_trace_wire_bytes':statistics.median(row['telemetry_delta']['traces_wire_bytes'] for row in runs),
'rare_total':sum(len(row['rare_probes']) for row in runs),
'rare_sampled':sum(sum(x['sampled'] for x in row['rare_probes']) for row in runs),
'export_failed_spans':sum(row['telemetry_delta']['spans_export_failed'] for row in runs),
'queue_lost_spans':sum(row['telemetry_delta']['spans_queue_lost'] for row in runs)}
report['summary']=summary
(ROOT/'evidence/observability-cost.json').write_text(json.dumps(report,indent=2))
print(json.dumps(summary,indent=2))
if __name__=='__main__':
main()
+7 -4
View File
@@ -100,8 +100,9 @@ async def request_log(request: Request, call_next):
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
method = request.method if request.method in {"GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"} else "OTHER"
context = propagate.extract(dict(request.headers))
with telemetry.operation("HTTP " + request.method, context=context, kind=SpanKind.SERVER) as span:
with telemetry.operation("HTTP " + method, context=context, kind=SpanKind.SERVER) as span:
try:
if (request.method == "POST" and request.url.path == "/api/v1/auth/sessions"
and await request.body()):
@@ -115,12 +116,14 @@ async def request_log(request: Request, call_next):
response = error(request, 500, "Внутренняя ошибка")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
response.headers["X-Trace-Id"] = format(span.get_span_context().trace_id, "032x")
response.headers["X-Trace-Sampled"] = "true" if span.is_recording() else "false"
route = getattr(request.scope.get("route"), "path", "unknown")
elapsed = time.perf_counter() - start
attrs = {"method": request.method, "route": route,
attrs = {"method": method, "route": route,
"status_class": str(response.status_code // 100) + "xx"}
span.update_name(request.method + " " + route)
span.set_attributes({"http.request.method": request.method, "http.route": route,
span.update_name(method + " " + route)
span.set_attributes({"http.request.method": method, "http.route": route,
"http.response.status_code": response.status_code})
if response.status_code >= 500:
span.set_status(Status(StatusCode.ERROR, "server_error"))
+53 -11
View File
@@ -21,7 +21,7 @@ from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
@@ -32,6 +32,32 @@ INSTANCE_ID = str(uuid4())
_runtime = None
class Accounting(dict):
def __init__(self, keys):
super().__init__((key, 0) for key in keys)
self.lock = threading.Lock()
def add(self, key, value):
with self.lock:
self[key] += value
def snapshot(self):
with self.lock:
return dict(self)
class RecordedSpans(SpanProcessor):
def __init__(self, stats):
self.stats = stats
def on_start(self, span, parent_context=None):
pass
def on_end(self, span):
if span.context.trace_flags.sampled:
self.stats.add("spans_recorded", 1)
class CountingSession(requests.Session):
"""Count actual protobuf HTTP sends, including transport retries, without retaining data."""
def __init__(self, stats, signal):
@@ -39,8 +65,8 @@ class CountingSession(requests.Session):
self.stats, self.signal = stats, signal
def send(self, request, **kwargs):
self.stats[self.signal + "_wire_bytes"] += len(request.body or b"")
self.stats[self.signal + "_http_sends"] += 1
self.stats.add(self.signal + "_wire_bytes", len(request.body or b""))
self.stats.add(self.signal + "_http_sends", 1)
return super().send(request, **kwargs)
@@ -50,10 +76,17 @@ class ObservedSpanExporter(OTLPSpanExporter):
super().__init__(**kwargs)
def export(self, spans):
result = super().export(spans)
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
self.stats[key] += len(spans)
return result
self.stats.add("spans_in_flight", len(spans))
try:
result = super().export(spans)
key = "spans_exported" if result == SpanExportResult.SUCCESS else "spans_export_failed"
self.stats.add(key, len(spans))
return result
except Exception:
self.stats.add("spans_export_failed", len(spans))
raise
finally:
self.stats.add("spans_in_flight", -len(spans))
class TechnicalLogs(logging.Filter):
@@ -86,7 +119,7 @@ class Telemetry:
self.enabled = os.getenv("OTEL_ENABLED", "1") == "1"
self.busy = 0
self.providers = []
self.export_stats = {key: 0 for key in ("spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends")}
self.export_stats = Accounting(("spans_recorded", "spans_in_flight", "spans_exported", "spans_export_failed", "traces_wire_bytes", "traces_http_sends", "metrics_wire_bytes", "metrics_http_sends", "logs_wire_bytes", "logs_http_sends"))
self.stop = threading.Event()
self.thread = None
resource = Resource.create({
@@ -110,11 +143,13 @@ class Telemetry:
# Standard OTLP environment variables also support HTTPS/mTLS in lab7.
trace_provider = TracerProvider(resource=resource, sampler=ParentBased(
TraceIdRatioBased(float(os.getenv("OTEL_SAMPLE_RATIO", "1")))))
trace_provider.add_span_processor(BatchSpanProcessor(
trace_provider.add_span_processor(RecordedSpans(self.export_stats))
self.batch_spans = BatchSpanProcessor(
ObservedSpanExporter(self.export_stats, endpoint=endpoint + "/v1/traces", session=CountingSession(self.export_stats, "traces"), **common),
max_queue_size=256, max_export_batch_size=64,
schedule_delay_millis=500, export_timeout_millis=1500,
))
)
trace_provider.add_span_processor(self.batch_spans)
trace.set_tracer_provider(trace_provider)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", session=CountingSession(self.export_stats, "metrics"), **common),
@@ -164,7 +199,14 @@ class Telemetry:
def collect():
while not self.stop.wait(2):
for name, value in self.export_stats.items():
counts = self.export_stats.snapshot()
# Read-only queue observation of pinned SDK 1.39.0. Conservation
# is evaluated after quiescence, not during concurrent export.
pending = len(self.batch_spans._batch_processor._queue)
counts["spans_pending"] = pending
counts["span_queue_capacity"] = 256
counts["spans_queue_lost"] = max(0, counts["spans_recorded"] - counts["spans_exported"] - counts["spans_export_failed"] - counts["spans_in_flight"] - pending)
for name, value in counts.items():
self.export_observation.set(value, {"observation": name})
stats = store.pool.get_stats()
self.pool_used.set(max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)))
+38
View File
@@ -0,0 +1,38 @@
"""Real exporter failure must remain bounded and account for lost sampled spans."""
import json
import os
import subprocess
import sys
from pathlib import Path
def test_span_queue_loss_conservation_when_collector_absent():
code = '''
import json, time
from messenger.telemetry import setup
class Pool:
def get_stats(self): return {}
class Store:
pool=Pool()
t=setup("messenger-api")
t.start_background(Store())
for index in range(1200):
with t.tracer.start_as_current_span("synthetic-safe-span"):
pass
time.sleep(7)
counts=t.export_stats.snapshot()
pending=len(t.batch_spans._batch_processor._queue)
loss=counts["spans_recorded"]-counts["spans_exported"]-counts["spans_export_failed"]-counts["spans_in_flight"]-pending
assert counts["spans_recorded"]==1200
assert 0<=pending<=256
assert loss>0
assert counts["spans_export_failed"]>0
print(json.dumps({"recorded":1200,"pending":pending,"queue_lost":loss,"export_failed":counts["spans_export_failed"]}))
t.close()
'''
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]/"src"),
"OTEL_ENABLED": "1", "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1"}
result = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=15)
assert result.returncode == 0, result.stderr[-1000:]
payload = json.loads(next(line for line in result.stdout.splitlines() if line.startswith('{')))
assert payload["queue_lost"] > 0