69 lines
4.5 KiB
Python
Executable File
69 lines
4.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reproduce ordinary and retried uploads and save real trace/log relationships."""
|
|
import json,subprocess,sys,time,uuid,os
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
|
from client import Client,check
|
|
from observations import get
|
|
C=[str(ROOT/'scripts/compose.sh')]
|
|
def compose(*args):subprocess.run(C+list(args),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=120)
|
|
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True).strip()
|
|
def wait(fn,seconds=60):
|
|
end=time.monotonic()+seconds
|
|
while time.monotonic()<end:
|
|
try:
|
|
x=fn()
|
|
if x:return x
|
|
except Exception:pass
|
|
time.sleep(1)
|
|
raise RuntimeError('Timed out waiting for correlation evidence')
|
|
def trace_summary(d):
|
|
spans=[]
|
|
for b in d.get('batches',d.get('resourceSpans',[])):
|
|
attrs={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
|
|
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
|
|
for s in scope.get('spans',[]):
|
|
spans.append({k:s.get(k) for k in ['name','kind','traceId','spanId','parentSpanId','startTimeUnixNano','endTimeUnixNano','status'] }|{'service':attrs.get('service.name')})
|
|
return spans
|
|
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
|
results=[];faults=ROOT/'.lab6-faults';faults.mkdir(exist_ok=True);faults.chmod(0o777)
|
|
for retry in [False,True]:
|
|
traceid=uuid.uuid4().hex
|
|
marker=None
|
|
try:
|
|
if retry:compose('stop','worker')
|
|
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=a,raw=data,headers={'Content-Type':'image/png','traceparent':'00-'+traceid+'-'+uuid.uuid4().hex[:16]+'-01'},schema='Attachment')['json']
|
|
jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%str(uuid.UUID(att['id'])))
|
|
if retry:
|
|
marker=faults/('before_objects.'+jid);marker.write_text('temporary');marker.chmod(0o644)
|
|
compose('-f','compose.yaml','-f','compose.faults.yaml','up','-d','--no-deps','worker')
|
|
wait(lambda:int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid))>=2)
|
|
marker.unlink()
|
|
ready=c.wait_ready(a,att)
|
|
def complete():
|
|
spans=trace_summary(get('tempo','/api/traces/'+traceid))
|
|
return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None
|
|
spans=wait(complete)
|
|
def logs():
|
|
d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'})
|
|
rows=d['data']['result']
|
|
return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None
|
|
rows=wait(logs)
|
|
events=[]
|
|
for r in rows:
|
|
stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values'])
|
|
check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing')
|
|
check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing')
|
|
producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]}
|
|
consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]]
|
|
check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer')
|
|
check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost')
|
|
results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events})
|
|
print('PASS: correlated upload, retry='+str(retry),flush=True)
|
|
finally:
|
|
if marker:marker.unlink(missing_ok=True)
|
|
if retry:compose('up','-d','--no-deps','worker')
|
|
(ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n')
|