#!/usr/bin/env python3 """Reproduce ordinary and retried uploads and save real trace/log relationships.""" import json,subprocess,sys,time,uuid,os from pathlib import Path ROOT=Path(__file__).resolve().parents[1] sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')] from client import Client,check from observations import get C=[str(ROOT/'scripts/compose.sh')] def compose(*args):subprocess.run(C+list(args),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=120) def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True).strip() def wait(fn,seconds=60): end=time.monotonic()+seconds while time.monotonic()=2) marker.unlink() ready=c.wait_ready(a,att) def complete(): spans=trace_summary(get('tempo','/api/traces/'+traceid)) return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None spans=wait(complete) def logs(): d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'}) rows=d['data']['result'] return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None rows=wait(logs) events=[] for r in rows: stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values']) check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing') check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing') producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]} consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]] check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer') check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost') results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events}) print('PASS: correlated upload, retry='+str(retry),flush=True) finally: if marker:marker.unlink(missing_ok=True) if retry:compose('up','-d','--no-deps','worker') (ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n')