Files
mtusi2026lab6/scripts/prove-correlation.py
T
2026-11-17 13:08:13 +03:00

69 lines
4.5 KiB
Python
Executable File

#!/usr/bin/env python3
"""Reproduce ordinary and retried uploads and save real trace/log relationships."""
import json,subprocess,sys,time,uuid,os
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client,check
from observations import get
C=[str(ROOT/'scripts/compose.sh')]
def compose(*args):subprocess.run(C+list(args),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=120)
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True).strip()
def wait(fn,seconds=60):
end=time.monotonic()+seconds
while time.monotonic()<end:
try:
x=fn()
if x:return x
except Exception:pass
time.sleep(1)
raise RuntimeError('Timed out waiting for correlation evidence')
def trace_summary(d):
spans=[]
for b in d.get('batches',d.get('resourceSpans',[])):
attrs={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
for s in scope.get('spans',[]):
spans.append({k:s.get(k) for k in ['name','kind','traceId','spanId','parentSpanId','startTimeUnixNano','endTimeUnixNano','status'] }|{'service':attrs.get('service.name')})
return spans
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
results=[];faults=ROOT/'.lab6-faults';faults.mkdir(exist_ok=True);faults.chmod(0o777)
for retry in [False,True]:
traceid=uuid.uuid4().hex
marker=None
try:
if retry:compose('stop','worker')
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=a,raw=data,headers={'Content-Type':'image/png','traceparent':'00-'+traceid+'-'+uuid.uuid4().hex[:16]+'-01'},schema='Attachment')['json']
jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%str(uuid.UUID(att['id'])))
if retry:
marker=faults/('before_objects.'+jid);marker.write_text('temporary');marker.chmod(0o644)
compose('-f','compose.yaml','-f','compose.faults.yaml','up','-d','--no-deps','worker')
wait(lambda:int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid))>=2)
marker.unlink()
ready=c.wait_ready(a,att)
def complete():
spans=trace_summary(get('tempo','/api/traces/'+traceid))
return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None
spans=wait(complete)
def logs():
d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'})
rows=d['data']['result']
return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None
rows=wait(logs)
events=[]
for r in rows:
stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values'])
check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing')
check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing')
producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]}
consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]]
check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer')
check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost')
results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events})
print('PASS: correlated upload, retry='+str(retry),flush=True)
finally:
if marker:marker.unlink(missing_ok=True)
if retry:compose('up','-d','--no-deps','worker')
(ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n')