Implement lab6 observability with OpenTelemetry and Grafana

This commit is contained in:
ferraronp
2026-11-17 19:14:43 +03:00
parent 470d9d6f1e
commit e636d4cc76
68 changed files with 5863 additions and 47 deletions
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion.
Admission metrics measure upload/enqueue requests, not worker throughput.
Completion is checked separately so an accepted-but-stuck job cannot pass.
"""
import argparse
import concurrent.futures
import json
import statistics
import time
from pathlib import Path
from client import Client, check, run
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--output', default='evidence/attachments-concurrent.json')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--wait-seconds', type=int, default=180)
args = parser.parse_args()
check(args.concurrency > 0, 'concurrency must be positive')
check(args.wait_seconds > 0, 'wait-seconds must be positive')
client = Client(args.base_url)
alice, bob, eve, chat = client.bootstrap()
source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png'
def upload(_):
cli = Client(args.base_url)
response = cli.call('POST', path, 202, actor=alice, raw=source,
headers={'Content-Type': 'image/png'}, schema='Attachment')
att = response['json']
check(att['status'] == 'queued', '202 must return a queued attachment')
check(att['size_bytes'] == len(source), 'size mismatch')
check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'],
'incorrect attachment ownership')
return att['id'], response['elapsed']
started = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(upload, range(args.concurrency)))
accepted_seconds = time.monotonic() - started
ids = [att_id for att_id, _ in results]
timings = [duration for _, duration in results]
check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs')
# A 202 response alone only proves acceptance, not successful processing.
pending = set(ids)
deadline = time.monotonic() + args.wait_seconds
while pending and time.monotonic() < deadline:
for att_id in list(pending):
att = client.call('GET', '/api/v1/attachments/' + att_id,
actor=alice, schema='Attachment')['json']
check(att['status'] != 'failed', 'attachment reached failed: ' + att_id)
if att['status'] == 'ready':
variants = [v['name'] for v in att['variants']]
check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'},
'missing/duplicate image variants: ' + att_id)
pending.remove(att_id)
if pending:
time.sleep(0.5)
check(not pending, 'attachments not ready before deadline: ' + str(len(pending)))
complete_seconds = time.monotonic() - started
data = {
'concurrency': args.concurrency,
'accepted_uploads': len(results),
'ready_attachments': args.concurrency,
'failed_requests': 0,
'acceptance_elapsed_seconds': round(accepted_seconds, 3),
'acceptance_rps': round(len(results) / accepted_seconds, 3),
'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1),
'upload_latency_max_ms': round(max(timings) * 1000, 1),
'all_ready_elapsed_seconds': round(complete_seconds, 3),
'bytes_per_file': len(source),
'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.',
}
output = Path(args.output)
output.parent.mkdir(exist_ok=True, parents=True)
output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency)
print(json.dumps(data, ensure_ascii=False, indent=2))
if __name__ == '__main__':
run(main)
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Lab5 negative API tests with isolated synthetic users (requires live stack)."""
from urllib.parse import quote
import uuid
from client import Client, arguments, run, check
from pathlib import Path
def main():
args = arguments('Lab5 negative file/ACL/security tests')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
base = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'})
c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'})
c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'})
c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'})
c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'})
c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'})
uploaded = c.attachment(alice,chat)[0]
ready = c.wait_ready(alice, uploaded)
unknown = str(uuid.uuid4())
c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice)
c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve)
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve)
# A ready Lab5 attachment has a downloadable thumbnail for chat members.
# A non-member still gets 404, and an unknown variant must not resolve.
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve)
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice)
check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant')
thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice)
check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME')
other_chat = c.call('POST','/api/v1/chats',201,actor=alice,
body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
path = '/api/v1/chats/'+other_chat['id']+'/messages'
c.call('POST',path,404,actor=alice,
body={'text':'cross-chat','attachment_ids':[uploaded['id']]},
headers={'Idempotency-Key':uuid.uuid4().hex})
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice,
body={'text':'unknown','attachment_ids':[unknown]},
headers={'Idempotency-Key':uuid.uuid4().hex})
ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json']
check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked')
print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks')
if __name__=='__main__':
run(main)
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas."""
import concurrent.futures
import sys
import uuid
from client import Client, Failure, arguments, check, run
def main():
args = arguments("Check atomic idempotency across API replicas")
c = Client(args.base_url, args.ca_file)
alice, bob, _, chat = c.bootstrap()
key = uuid.uuid4().hex
body = {"text": "Concurrent replay " + key}
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(_):
# Independent request client; one shared bearer token and idempotency key.
client = Client(args.base_url, args.ca_file)
return client.call("POST", path, expected=201, actor=alice, body=body,
headers={"Idempotency-Key": key}, schema="Message")
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
results = list(executor.map(worker, range(30)))
ids = {result["json"]["id"] for result in results}
instances = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel replay created different messages")
check(len(instances) >= 2, "LB did not serve requests via both API replicas")
check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result")
c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key})
c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message")
history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history")
print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409")
if __name__ == "__main__":
run(main)
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Prevent a mismatch between SQL placeholders and parameter tuple in worker result persistence."""
import ast
from pathlib import Path
import re
source = (Path(__file__).resolve().parents[1] / "app" / "jobs.py").read_text(encoding="utf-8")
tree = ast.parse(source)
fn = next(x for x in tree.body if isinstance(x, ast.FunctionDef) and x.name == "publish_result")
matches = []
for node in ast.walk(fn):
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
continue
if node.func.attr != "execute" or len(node.args) < 2:
continue
if not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str):
continue
sql = node.args[0].value
if "INSERT INTO attachment_variants" not in sql:
continue
assert isinstance(node.args[1], ast.Tuple), "Expected literal parameters tuple"
placeholders = sql.count("%s")
params = len(node.args[1].elts)
columns_match = re.search(r"INSERT INTO attachment_variants\s*\(([^)]*)\)", sql, re.I | re.S)
assert columns_match, "Columns missing in INSERT"
columns = len(columns_match.group(1).split(","))
assert placeholders == columns == params, f"SQL mismatch: {columns} columns, {placeholders} placeholders, {params} parameters"
matches.append((columns, placeholders, params))
assert len(matches) == 1, "Expected exactly one attachment_variants INSERT"
print("PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments")
+341
View File
@@ -0,0 +1,341 @@
#!/usr/bin/env python3
"""Lab6 live evidence. Only synthetic test stack; no volume removal.
Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads.
"""
import argparse
import base64
import math
import concurrent.futures
import json
import os
import subprocess
import time
import urllib.parse
import urllib.request
import urllib.error
from pathlib import Path
from client import Client, Failure, check, run
from pipeline_extended import compose, pg, scalar, wait_for
from pipeline_resilience import verify_ready
ROOT=Path(__file__).resolve().parents[1]
PROM='http://127.0.0.1:9090'
LOKI='http://127.0.0.1:3100'
TEMPO='http://127.0.0.1:3200'
GRAFANA='http://127.0.0.1:3000'
ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)'
LAG='max(lab_queue_oldest_age_seconds)'
DEPTH='max(lab_queue_depth)'
FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
# Explicit Prometheus readers can append unit suffixes to Collector self-metrics.
# Missing gauges are not zero: absent telemetry must fail the outage test.
QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})'
QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})'
RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})'
def get(url, params=None, *, as_json=True):
try:
with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r:
raw=r.read(32*1024*1024)
return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace')
except (urllib.error.URLError,TimeoutError,ValueError) as exc:
raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None
def prom(expr):
data=get(PROM+'/api/v1/query',{'query':expr})
check(data.get('status')=='success','PromQL query failed')
return data['data']['result']
def val(expr):
rows=prom(expr)
return float(rows[0]['value'][1]) if rows else None
def alert_state(name):
rules=get(PROM+'/api/v1/rules')['data']['groups']
for group in rules:
for rule in group['rules']:
if rule['name']==name:
check(rule.get('health')=='ok','alert rule is not healthy: '+name)
return rule['state']
raise Failure('alert rule missing: '+name)
def wait_alert(name,state,seconds=100):
return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2)
def ready():
for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'):
wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2)
wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100)
wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40)
wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40)
def trace_spans(tid):
raw=get(TEMPO+'/api/traces/'+tid)
spans=[]
def walk(obj):
if isinstance(obj,dict):
if {'spanId','traceId','name'}<=set(obj): spans.append(obj)
else:
for v in obj.values():walk(v)
elif isinstance(obj,list):
for v in obj:walk(v)
walk(raw)
return spans
def logs(tid,service,request_id=None):
expr='{service_name="%s"} | trace_id="%s"' % (service,tid)
if request_id: expr+=' | request_id="%s"'%request_id
data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100})
return data['data']['result']
def upload(c,alice,chat):
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
tid=r['headers'].get('X-Trace-Id','')
check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement')
return r['json'],raw,tid,r['headers']['X-Request-Id']
def span_attrs(s):
attrs={}
for a in s.get('attributes',[]):
value=a.get('value',{})
attrs[a['key']]=next(iter(value.values()),None)
return attrs
def chain_ok(spans):
names={s['name'] for s in spans}
required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'}
if not required<=names:return False
producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'}
consumers=[s for s in spans if s['name']=='rabbitmq.consume']
outcomes={span_attrs(s).get('job.outcome') for s in consumers}
return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers)
def trace_retry(c):
alice,bob,_,chat=c.bootstrap()
compose('stop','worker')
try:
att,raw,tid,rid=upload(c,alice,chat)
# Override only worker, never a public endpoint; reset in finally.
env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'}
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180)
verify_ready(c,bob,att,raw,seconds=180)
spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3)
wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
def worker_events(streams):
return {v[1] for row in streams for v in row.get('values',[])}
wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90)
# Keep only allowlisted names/IDs, not arbitrary span values or log payloads.
summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans]
result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True}
print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid)
return result
finally:
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180)
def queue_lag(c):
alice,bob,_,chat=c.bootstrap()
wait_alert('LabQueueLag','inactive')
compose('stop','worker')
try:
att,raw,tid,rid=upload(c,alice,chat)
t=time.monotonic()
wait_alert('LabQueueLag','firing')
lag=val(LAG);depth=val(DEPTH)
check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics')
detected=time.monotonic()-t
print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True)
compose('up','-d','worker')
verify_ready(c,bob,att,raw,seconds=180)
wait_alert('LabQueueLag','inactive',seconds=120)
wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2)
print('PASS queue-lag: firing -> ready -> resolved')
return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)}
finally:compose('up','-d','worker')
def db_error(c):
alice,_,_,_=c.bootstrap()
wait_alert('LabAPIHighErrorRate','inactive')
user=compose('exec','-T','postgres','printenv','POSTGRES_USER')
db=compose('exec','-T','postgres','printenv','POSTGRES_DB')
# Bounded, isolated table lock. Health SELECT 1 works, authenticated business
# read times out at statement_timeout. Never stop DB (LB would hide API 5xx).
cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;']
lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15)
start=time.monotonic(); traces=[]; codes=[]
while time.monotonic()-start < 43:
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
def get_user(_):
r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User')
return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id')
for code,tid,rid in pool.map(get_user,range(3)):
codes.append(code)
if code==503 and tid:traces.append((tid,rid))
if traces and alert_state('LabAPIHighErrorRate')=='firing':break
check(bool(traces),'no real HTTP 503 from DB lock')
wait_alert('LabAPIHighErrorRate','firing',seconds=20)
ratio=val(ERR)
check(ratio is not None and ratio>0.05,'error ratio below threshold')
tid,rid=traces[0]
spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2)
wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60)
print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True)
finally:
# SQL transaction has server-side 60s bound; wait for normal COMMIT.
try:lock.wait(timeout=75)
except subprocess.TimeoutExpired:
lock.terminate();lock.wait(timeout=10)
# Keep successful business traffic in the evaluation window to prove recovery.
t=time.monotonic()
while time.monotonic()-t<65:
c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break
time.sleep(1)
wait_alert('LabAPIHighErrorRate','inactive',seconds=40)
print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved')
return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]}
def collector_sample():
check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down')
rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY)
check(rss is not None and math.isfinite(rss) and rss>0,
'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus')
check(queue is not None and math.isfinite(queue) and queue>=0,
'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics')
check(capacity is not None and math.isfinite(capacity) and capacity==64,
'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity))
return rss,queue,capacity
def backend_down(c):
alice,bob,_,chat=c.bootstrap()
baseline=val(FAILED) or 0
cid=compose('ps','-q','otel-collector')
before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip())
check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit))
rss,queue,capacity=collector_sample()
print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True)
compose('stop','tempo')
try:
start=time.monotonic();n=0;memory=[];queues=[];failures=[]
att,raw,tid,_=upload(c,alice,chat)
while time.monotonic()-start<40:
c.message(alice,chat,'telemetry-backend-offline')
n+=1
rss,queue,capacity=collector_sample()
memory.append(rss);queues.append(queue)
x=val(FAILED)
if x is not None:failures.append(x)
time.sleep(.3)
verify_ready(c,bob,att,raw,seconds=180)
check(n>0,'no successful writes during backend outage')
check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics')
rss,queue,capacity=collector_sample()
memory.append(rss);queues.append(queue)
print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True)
after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after)
check(max(memory)<memory_limit,f'Collector RSS {max(memory):g} bytes exceeds/equal to container limit {memory_limit} bytes')
check(max(queues)<=capacity,'collector queue exceeds configured cap')
result={'successful_messages_during_tempo_outage':n,'trace_id':tid,'max_sampled_collector_rss_bytes':max(memory),'collector_memory_limit_bytes':memory_limit,'rss_sample_count':len(memory),'max_export_queue':max(queues),'export_queue_capacity':capacity,'failed_span_counter_delta':max(failures)-baseline,'restart_state':after}
finally:compose('up','-d','tempo')
ready()
# Confirm a fresh post-recovery trace can be retrieved.
r=c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
tid2=r['headers']['X-Trace-Id']
wait_for('post-recovery trace',lambda:trace_spans(tid2),bool,seconds=90)
result['recovered_trace_id']=tid2
print('PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered')
return result
def metrics_snapshot(c):
alice,bob,_,chat=c.bootstrap()
att,raw,_,_=upload(c,alice,chat)
verify_ready(c,bob,att,raw,seconds=150)
until=time.monotonic()+12
while time.monotonic()<until:
c.message(alice,chat,'synthetic-metric-check')
time.sleep(5)
# Read the real provisioned Grafana dashboard/datasource list, without
# printing credentials or copying the response into the evidence.
secret=compose('exec','-T','grafana','printenv','GF_SECURITY_ADMIN_PASSWORD')
auth=base64.b64encode(('admin:'+secret).encode()).decode()
def grafana_json(path):
r=urllib.request.Request(GRAFANA+path,headers={'Authorization':'Basic '+auth})
try:
with urllib.request.urlopen(r,timeout=8) as f:return json.load(f)
except (urllib.error.URLError,TimeoutError,ValueError):
raise Failure('Grafana authenticated provisioning check failed; verify admin password/volume') from None
dashboard=grafana_json('/api/dashboards/uid/lab6-messenger')['dashboard']
check(len(dashboard['panels'])>=15,'provisioned dashboard incomplete')
check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing')
queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR,
'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count',
'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting',
'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'}
output={}
for key,expr in queries.items():
rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2)
if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load')
output[key]={'query':expr,'result':rows}
forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'}
for item in output.values():
check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels')
print('PASS metrics snapshot: all expected series present, no forbidden ID labels')
return output
def history(c):
alice,bob,_,chat=c.bootstrap()
att,raw,tid,rid=upload(c,alice,chat)
verify_ready(c,bob,att,raw,seconds=150)
wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90)
wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
time.sleep(6)
historic_time=time.time()
expr='sum(lab_http_requests_total{route=~"/api/.*"})'
def historic():
rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result']
return float(rows[0]['value'][1]) if rows else None
before=historic();check(before is not None and before>0,'no historic metric')
compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180)
ready()
wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100)
wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100)
after=historic();check(after==before,'historic metric changed or disappeared')
print('PASS history: saved metrics, logs and traces survive backend restart')
return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after}
def main():
p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args()
check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack')
ready();c=Client(timeout=12)
fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario]
result=fun(c)
file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
print('Saved',file.relative_to(ROOT))
if __name__=='__main__':run(main)
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Fast non-Docker checks for input contracts and cursor scoping."""
import sys
from pathlib import Path
from datetime import datetime, timezone
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.core import make_cursor, read_cursor, parse_limit, APIError
from app.models import CreateUser, CreateChat, CreateMessage
from pydantic import ValidationError
def reject(fn):
try:
fn()
except (APIError, ValidationError, ValueError):
return
raise AssertionError("Expected invalid input")
def main():
token = str(uuid.uuid4())
dt = datetime.now(timezone.utc)
cursor = make_cursor("messages", token, dt, str(uuid.uuid4()))
assert read_cursor(cursor, "messages", token)[0] == dt
reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4())))
reject(lambda: read_cursor(cursor, "chats", token))
assert parse_limit(None) == 50
reject(lambda: parse_limit("0"))
reject(lambda: parse_limit("101"))
reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret"))
reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3))
assert CreateMessage(text="A\u0000B").text == "A\u0000B"
reject(lambda: CreateMessage(text=" \t"))
reject(lambda: CreateChat(title="title", member_ids=[token] * 2))
reject(lambda: CreateChat(title="title", member_ids=[token] * 101))
print("PASS: models, cursor scoping, limits, Unicode U+0000")
if __name__ == "__main__":
main()
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
broker restart without a running dispatcher or worker. This differs from the
outbox-restoration tests where dispatcher publishes *after* broker recovery.
failed-replay: safely inject a deterministic checksum mismatch into one fresh
synthetic attachment's DB metadata, verify permanent failure, restore the
original hash, replay via the operator-only CLI, and verify success plus audit.
staging-cleanup: verify the staged S3 object is absent after ready (with a
matching cleared staging_key in the DB).
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
run only against a disposable LAB5 environment and never concurrently.
"""
import argparse
import hashlib
import os
import re
import subprocess
import time
import uuid
from client import Client, Failure, check, run
from pipeline_resilience import verify_ready, wait_api
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,), timeout=240):
proc = subprocess.run(
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
)
if proc.returncode not in expected:
# Don't print env vars, tokens or command stdout/stderr by default.
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
(' '.join(args[:2]), proc.returncode))
return proc.stdout.strip()
def pg(sql):
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
def scalar(sql):
return pg(sql).splitlines()[-1].strip()
def wait_for(label, action, predicate, seconds=30, interval=0.5):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
last = action()
if predicate(last):
return last
except Failure:
pass
time.sleep(interval)
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
def job_query(att_id, fields):
uuid.UUID(att_id)
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
(fields, att_id))
def queue_ready():
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
'name', 'messages_ready', 'messages_unacknowledged')
for line in out.splitlines():
parts = re.split(r'\s+', line.strip())
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
return int(parts[1]), int(parts[2])
raise Failure('RabbitMQ durable queue missing')
def make_upload(c):
alice, bob, _eve, chat = c.bootstrap()
att, original = c.attachment(alice, chat)
check(att['status'] == 'queued', 'upload must return 202 queued')
uuid.UUID(att['id'])
return alice, bob, att, original
def wait_staging_cleared(att_id, seconds=20):
return wait_for('staging_key cleared after ready',
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
lambda value: value == '<NULL>', seconds=seconds)
def verify_staging_absent(att_id):
# This executes *inside* the worker container with its normal S3 credentials.
# Never expose S3 secrets to the host or write them to evidence files.
code = '''import sys
from botocore.exceptions import ClientError
from app.attachments import s3_client, BUCKET
key='staging/'+sys.argv[1]+'/original'
try:
s3_client().head_object(Bucket=BUCKET,Key=key)
except ClientError as exc:
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
print('ABSENT')
else:
raise
else:
print('STILL_PRESENT')'''
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
def broker_persistence(c):
compose('stop', 'worker')
compose('up', '-d', 'dispatcher')
baseline = queue_ready()[0]
alice, _bob, att, original = make_upload(c)
print('202 queued:', att['id'], flush=True)
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
lambda n: n >= baseline + 1, seconds=15)
compose('stop', 'dispatcher')
before = queue_ready()[0]
check(before >= baseline + 1, 'published message disappeared before restart')
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
compose('restart', 'broker')
compose('up', '-d', '--wait', 'broker')
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
lambda n: n >= before, seconds=40, interval=1)
check(scalar(sql) == 't', 'outbox publish confirmation lost')
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
# can deliver this job to the worker.
compose('up', '-d', 'worker')
verify_ready(c, alice, att, original, seconds=180)
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
def failed_replay(c):
compose('stop', 'worker', 'dispatcher')
alice, _bob, att, original = make_upload(c)
ident = att['id']
original_sha = hashlib.sha256(original).hexdigest()
check(original_sha == att['sha256'], 'fixture hash mismatch')
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
'DB hash unexpected before controlled fault')
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
changed = False
try:
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
(fault_sha, ident))
check(ident in updated, 'failed to inject synthetic checksum mismatch')
changed = True
compose('up', '-d', 'dispatcher', 'worker')
def get_state():
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
failed = wait_for('permanent failed status', get_state,
lambda a: a['status'] == 'failed', seconds=90)
check(failed['error_code'] == 'invalid_image',
'expected permanent invalid_image, got ' + str(failed['error_code']))
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
'failed job created variants')
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
# Repair only the generated synthetic row before explicit operator replay.
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
(original_sha, ident))
check(ident in restored, 'synthetic metadata could not be restored')
changed = False
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
verify_ready(c, alice, att, original, seconds=180)
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
attempt = job_query(ident, 'attempt')
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
'replay', '--id', ident, expected=(1,))
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
finally:
if changed:
# Never leave the synthetic attachment with altered metadata.
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
def staging_cleanup(c):
compose('up', '-d', 'dispatcher', 'worker')
alice, _bob, att, original = make_upload(c)
verify_ready(c, alice, att, original, seconds=180)
wait_staging_cleared(att['id'], seconds=30)
verify_staging_absent(att['id'])
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--scenario', required=True,
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
args = parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
c = Client('http://localhost:8080')
try:
wait_api()
if args.scenario == 'broker-persistence':
broker_persistence(c)
elif args.scenario == 'failed-replay':
failed_replay(c)
else:
staging_cleanup(c)
finally:
# Restore normal services even when an assertion fails. NO volume deletion.
try:
compose('up', '-d', '--wait', 'broker')
compose('up', '-d', 'dispatcher', 'worker')
except Failure:
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
if __name__ == '__main__':
run(main)
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""Explicit operator evidence for Lab5 queue durability and failure windows.
Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test
preserves user/session secrets strictly in process memory and prints no tokens.
"""
import argparse
import hashlib
import os
import subprocess
import sys
import time
from pathlib import Path
from client import Client, Failure, check, run
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,)):
p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240)
if p.returncode not in expected:
raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs'
% (' '.join(args[:2]), p.returncode))
return p
def wait_api(timeout=60):
import urllib.request
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as r:
if r.status==200:
return
except Exception:
pass
time.sleep(.5)
raise Failure('API did not become ready')
def make_queued(client):
alice,bob,eve,chat=client.bootstrap()
att,original=client.attachment(alice,chat)
check(att['status']=='queued','202 must return queued snapshot')
c=client.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
attached=client.message(alice,chat,'',attachments=[att['id']])['json']
check(attached['attachment_ids']==[att['id']],'queued attachment cannot be linked')
return alice,bob,att,original
def verify_ready(client,actor,att,original,seconds=180):
r=client.wait_ready(actor,att,wait_seconds=seconds)
check(len(r['variants']) == 3, 'expected exactly three variants, no duplicate entries')
variants={v['name']:v for v in r['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'missing/extra variants')
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==(256,160), 'thumbnail dimensions')
check((variants['crop']['width'],variants['crop']['height'])==(128,128), 'crop dimensions')
raw=client.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
check(raw['bytes']==original,'original changed across asynchronous pipeline')
for name in ('thumbnail','crop','optimized'):
v=variants[name]
response=client.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],
'variant checksum mismatch: '+name)
return r
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--scenario',choices=['paused','outbox-crash','worker-crash'],required=True)
args=parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST')!='yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes in PowerShell to run destructive failure test')
c=Client('http://localhost:8080')
# All tests make a fresh attachment while dispatcher/worker are stopped.
compose('stop','worker','dispatcher')
try:
alice,bob,att,original=make_queued(c)
print('Created durable queued attachment',att['id'],flush=True)
if args.scenario=='paused':
compose('restart','api1','api2','broker')
compose('up','-d','--wait','broker')
wait_api()
elif args.scenario=='outbox-crash':
# Transaction committed, dispatcher crashes before publish.
compose('run','--rm','-e','LAB5_FAULT_BEFORE_PUBLISH=once',
'dispatcher',expected=(78,))
print('Injected dispatcher exit 78 after committed outbox, before publish')
else:
compose('up','-d','dispatcher')
# Fault after all S3 writes and before PostgreSQL result COMMIT and ack.
compose('run','--rm','-e','LAB5_FAULT_AFTER_S3=once',
'worker',expected=(77,))
print('Injected worker exit 77 after S3 effects, before result/ack')
compose('up','-d','dispatcher','worker')
wait_api()
ready=verify_ready(c,bob,att,original)
check(ready['status']=='ready','attachment not ready')
print('PASS:',args.scenario,'durable queued->ready; original/3 variants verified; no duplicates')
finally:
# Restore the normal topology even after an assertion failure.
compose('up','-d','dispatcher','worker')
if __name__=='__main__':
run(main)
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Real Python OTLP SDK wire-format test against a local in-process receiver.
Requires requirements.txt (does not require Docker or external services).
This is not a Collector/backend integration test.
"""
import os
import sys
import time
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from opentelemetry.trace import SpanKind
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
from app import telemetry as ot
received={k:[] for k in ('/v1/traces','/v1/logs','/v1/metrics')}
class Receiver(BaseHTTPRequestHandler):
def do_POST(self):
data=self.rfile.read(int(self.headers['Content-Length']))
received[self.path].append(data)
self.send_response(200);self.send_header('Content-Type','application/x-protobuf');self.send_header('Content-Length','0');self.end_headers()
def log_message(self,*args): pass
def main():
server=ThreadingHTTPServer(('127.0.0.1',0),Receiver)
threading.Thread(target=server.serve_forever,daemon=True).start()
os.environ['OTEL_ENABLED']='yes'
os.environ['OTEL_EXPORTER_OTLP_ENDPOINT']='http://127.0.0.1:'+str(server.server_port)
ot.init('worker')
with ot.span('POST /api/v1/chats/{chat_id}/attachments',kind=SpanKind.SERVER) as root:
stored=ot.inject(); trace_id=ot.trace_id()
with ot.span('rabbitmq.publish',kind=SpanKind.PRODUCER,context=ot.extract(stored)) as producer:
sent=ot.inject()
with ot.span('rabbitmq.consume',kind=SpanKind.CONSUMER,context=ot.extract(sent)) as consumer:
ot.event('job.ready',job_id='synthetic-id',attempt=2)
ot.count('lab_jobs',{'job_type':'attachment.process.v1','outcome':'ready'})
ot.observe('lab_job_duration',2.5,{'job_type':'attachment.process.v1','outcome':'ready'})
ot.observe('lab_ready_duration',5.0,{'job_type':'attachment.process.v1'})
class FakeS3:
def put_object(self,**kw): return True
ot.S3Tracing(FakeS3()).put_object(Key='SECRET_FILENAME_SENTINEL',Body=b'SECRET_CONTENT_SENTINEL')
ot.flush()
for provider in reversed(ot.providers): provider.shutdown()
server.shutdown()
spans=[s for raw in received['/v1/traces'] for r in ExportTraceServiceRequest.FromString(raw).resource_spans for scope in r.scope_spans for s in scope.spans]
byname={s.name:s for s in spans}
assert byname['rabbitmq.publish'].parent_span_id==root.get_span_context().span_id.to_bytes(8,'big')
assert byname['rabbitmq.consume'].parent_span_id==producer.get_span_context().span_id.to_bytes(8,'big')
assert all(s.trace_id.hex()==trace_id for s in spans)
assert 's3.put_object' in byname
logs=[l for raw in received['/v1/logs'] for r in ExportLogsServiceRequest.FromString(raw).resource_logs for scope in r.scope_logs for l in scope.log_records]
assert any(l.trace_id.hex()==trace_id and l.span_id==consumer.get_span_context().span_id.to_bytes(8,'big') and l.body.string_value=='job.ready' for l in logs)
points=[m for raw in received['/v1/metrics'] for r in ExportMetricsServiceRequest.FromString(raw).resource_metrics for scope in r.scope_metrics for m in scope.metrics]
assert any(m.name=='lab_jobs' and m.sum.data_points[0].as_int==1 for m in points)
assert any(m.name=='lab_job_duration' and list(m.histogram.data_points[0].explicit_bounds)==list(ot.BUCKETS) for m in points)
for raw in sum(received.values(),[]): assert b'SECRET_FILENAME_SENTINEL' not in raw and b'SECRET_CONTENT_SENTINEL' not in raw
print('PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes')
if __name__=='__main__': main()