forked from mtusi2026/mtusi2026lab6
Реализация на 5
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Save bounded-buffer and loss observations for the actual Tempo outage."""
|
||||
import datetime as dt,json,re,subprocess,sys
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
|
||||
from observations import get
|
||||
x=json.loads((ROOT/'evidence/slo-experiment.json').read_text())
|
||||
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
|
||||
p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain')
|
||||
metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity']
|
||||
def at(name,t):
|
||||
rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result']
|
||||
return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo']
|
||||
before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics}
|
||||
def total(rows):return sum(float(r['value'][1]) for r in rows)
|
||||
deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')}
|
||||
samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result']
|
||||
peak=max((float(v[1]) for r in samples for v in r['values']),default=0)
|
||||
raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True)
|
||||
drops=[]
|
||||
for line in raw.splitlines():
|
||||
try:d=json.loads(line[line.index('{'):])
|
||||
except (ValueError,KeyError):continue
|
||||
if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d):
|
||||
drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d})
|
||||
mem=[]
|
||||
for record in x['resources']:
|
||||
if p['start']<=record['at']<=finish:
|
||||
for c in record['containers']:
|
||||
if 'otel-collector' in c['Name']:
|
||||
m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale)
|
||||
out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')}
|
||||
(ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n')
|
||||
print('Saved backend outage balance, queue, memory and business persistence evidence')
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create synthetic local broker credentials, preserving existing settings."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
from urllib.parse import quote
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
env=root/'.env.lab6'
|
||||
values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#'))
|
||||
password=values.get('BROKER_PASSWORD')
|
||||
if not password:
|
||||
password=secrets.token_hex(24)
|
||||
with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n')
|
||||
p=root/'secrets/lab6/broker_url'
|
||||
if p.exists(): p.chmod(0o600)
|
||||
p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n')
|
||||
p.chmod(0o444)
|
||||
print('Local broker credentials ready.')
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local synthetic storage credentials without rotating existing keys."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root=Path(__file__).resolve().parents[1]/'secrets'/'lab6'
|
||||
root.mkdir(parents=True,exist_ok=True)
|
||||
for name in ('s3_admin_access','s3_admin_secret','s3_api_access','s3_api_secret'):
|
||||
p=root/name
|
||||
if not p.exists():
|
||||
p.write_text(secrets.token_hex(24)+'\n')
|
||||
p.chmod(0o444) # Non-root containers must read these local synthetic secrets.
|
||||
bucket=os.environ.get('S3_BUCKET','messenger-attachments')
|
||||
identities=[]
|
||||
for role,actions in [('admin',['Admin','Read','Write','List','Tagging']),('api',['Read:'+bucket,'Write:'+bucket,'List:'+bucket])]:
|
||||
identities.append({'name':role,'credentials':[{'accessKey':(root/('s3_'+role+'_access')).read_text().strip(),'secretKey':(root/('s3_'+role+'_secret')).read_text().strip()}],'actions':actions})
|
||||
p=root/'s3.json'
|
||||
content=json.dumps({'identities':identities})+'\n'
|
||||
if not p.exists() or p.read_text()!=content:
|
||||
if p.exists(): p.chmod(0o600)
|
||||
p.write_text(content)
|
||||
p.chmod(0o444)
|
||||
print('Local storage credentials ready; existing keys preserved.')
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets/lab6
|
||||
if [ ! -f .env.lab6 ]; then
|
||||
{ printf 'POSTGRES_PASSWORD='; openssl rand -hex 24
|
||||
printf 'REDIS_PASSWORD='; openssl rand -hex 24
|
||||
printf 'HOST_PORT=18086\nAPI_REPLICAS=2\nSESSION_TTL_SECONDS=14400\n'
|
||||
} > .env.lab6
|
||||
fi
|
||||
if [ ! -f secrets/lab6/private_key.pem ]; then
|
||||
openssl genrsa -out secrets/lab6/private_key.pem 2048 2>/dev/null
|
||||
fi
|
||||
if [ ! -f secrets/lab6/public_key.pem ]; then
|
||||
openssl rsa -in secrets/lab6/private_key.pem -pubout -out secrets/lab6/public_key.pem 2>/dev/null
|
||||
fi
|
||||
# Only synthetic stand keys, readable by the non-root application inside containers.
|
||||
chmod 755 secrets secrets/lab6
|
||||
chmod 444 secrets/lab6/*.pem
|
||||
printf '%s\n' 'Local lab6 settings and synthetic keys ready; existing files preserved.'
|
||||
|
||||
python3 scripts/bootstrap-storage.py
|
||||
|
||||
python3 scripts/bootstrap-broker.py
|
||||
|
||||
if [ ! -f secrets/lab6/grafana_password ]; then
|
||||
openssl rand -hex 24 > secrets/lab6/grafana_password
|
||||
fi
|
||||
chmod 444 secrets/lab6/grafana_password
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Record reproducibility metadata, with an allowlist excluding all secrets."""
|
||||
import hashlib,json,platform,subprocess,time
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
def cmd(*args):return subprocess.check_output(args,cwd=root,text=True).strip()
|
||||
paths=[p for d in ['service','infra','scripts','tests','contracts','student-tests'] for p in (root/d).rglob('*') if p.is_file() and '__pycache__' not in p.parts]
|
||||
paths += [root/x for x in ['Dockerfile','compose.yaml','compose.faults.yaml','Makefile']]
|
||||
h=hashlib.sha256()
|
||||
for p in sorted(paths):h.update(str(p.relative_to(root)).encode());h.update(p.read_bytes())
|
||||
containers=[json.loads(x) for x in cmd('docker','compose','--env-file','.env.lab6','ps','--format','json').splitlines()]
|
||||
ids=[c['ID'] for c in containers]
|
||||
inspect=json.loads(cmd('docker','inspect',*ids))
|
||||
rows=[{'service':c['Config']['Labels']['com.docker.compose.service'],'image':c['Config']['Image'],'image_id':c['Image'],'started_at':c['State']['StartedAt'],'memory_limit':c['HostConfig']['Memory'],'nano_cpus':c['HostConfig']['NanoCpus'],'mounts':[{'type':m['Type'],'name':m.get('Name'),'destination':m['Destination']} for m in c['Mounts'] if m['Type']=='volume']} for c in inspect]
|
||||
info=json.loads(cmd('docker','info','--format','{{json .}}'))
|
||||
out={'timestamp':time.time(),'revision':cmd('git','rev-parse','HEAD'),'working_tree_dirty':bool(cmd('git','status','--porcelain')),'source_sha256':h.hexdigest(),'platform':platform.platform(),'go':cmd('go','version'),'docker':{'server_version':info['ServerVersion'],'cpus':info['NCPU'],'memory':info['MemTotal'],'architecture':info['Architecture']},'containers':rows}
|
||||
if platform.system()=='Darwin':out['host']={'cpu':cmd('sysctl','-n','machdep.cpu.brand_string'),'memory':cmd('sysctl','-n','hw.memsize')}
|
||||
(root/'evidence/environment.json').write_text(json.dumps(out,indent=2)+'\n')
|
||||
print('Saved sanitized environment and source fingerprint')
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Passively save stored log/trace evidence during the running SLO experiment."""
|
||||
import json,sys,time
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
|
||||
from observations import get
|
||||
source=ROOT/'evidence/slo-experiment.json';target=ROOT/'evidence/incident-signals.json'
|
||||
result={};errors=[]
|
||||
def logs(query,start,end):
|
||||
return get('loki','/loki/api/v1/query_range',{'query':query,'start':str(int(start*1e9)),'end':str(int(end*1e9)),'limit':100})['data']['result']
|
||||
def summary(d):
|
||||
rows=[]
|
||||
for b in d.get('batches',[]):
|
||||
resource={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
|
||||
for scope in b.get('scopeSpans',[]):
|
||||
for s in scope.get('spans',[]):
|
||||
rows.append({'service':resource.get('service.name'),'name':s['name'],'kind':s.get('kind'),'span_id':s.get('spanId'),'parent_span_id':s.get('parentSpanId'),'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,'status':s.get('status')})
|
||||
return rows
|
||||
end=time.time()+8000
|
||||
while time.time()<end:
|
||||
try:
|
||||
x=json.loads(source.read_text());last=x['observations'][-1];phase=last['phase'];event=next(e for e in reversed(x['events']) if e['phase']==phase)
|
||||
if phase.startswith('db_long_') and time.time()-event['at']>60 and phase not in result:
|
||||
rows=logs('{service_name="messenger-api"} |= "http_error"',event['at'],time.time()-3)
|
||||
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
|
||||
if ids:
|
||||
traceid=ids[-1];spans=summary(get('tempo','/api/traces/'+traceid))
|
||||
if any(s['name'].startswith('postgresql ') and s['duration_ms']>1000 for s in spans):result[phase]={'captured':time.time(),'observed':last,'trace_id':traceid,'spans':spans,'error_log_events':sum(len(r['values']) for r in rows),'diagnosis':'DB wait dominates server span; HTTP 5xx, not image processing'}
|
||||
if phase.startswith('worker_long_') and time.time()-event['at']>70 and phase not in result:
|
||||
worker=logs('{service_name="messenger-worker"} |= "job_received"',event['at'],time.time()-3)
|
||||
uploads=logs('{service_name="messenger-api"} |= "http_request" | route="/api/v1/chats/{chat_id}/attachments" | status="202"',event['at'],time.time()-3)
|
||||
result[phase]={'captured':time.time(),'observed':last,'worker_received_events':sum(len(r['values']) for r in worker),'accepted_upload_events':sum(len(r['values']) for r in uploads),'diagnosis':'Queue age and ready burn increase while API accepts uploads and no worker consumption is observed'}
|
||||
if phase.startswith('worker_recovery_') and phase not in result and time.time()-event['at']>20:
|
||||
rows=logs('{service_name="messenger-worker"} |= "job_ready"',event['at'],time.time()-3)
|
||||
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
|
||||
if ids:
|
||||
traceid=ids[0];spans=summary(get('tempo','/api/traces/'+traceid))
|
||||
if spans:result[phase]={'captured':time.time(),'trace_id':traceid,'spans':spans,'diagnosis':'Worker resumes consumption and publishes S3/DB results'}
|
||||
target.write_text(json.dumps({'evidence':result,'collection_errors':errors[-20:]},indent=2)+'\n')
|
||||
if x.get('completed') is not None:break
|
||||
except Exception as e:errors.append({'at':time.time(),'type':type(e).__name__})
|
||||
time.sleep(5)
|
||||
print('Saved incident observations:',len(result))
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec docker compose --env-file .env.lab6 "$@"
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded SLO experiment on the isolated lab6 stand, with raw observations.
|
||||
No synthetic metrics or time acceleration. Restores every injected fault in finally.
|
||||
"""
|
||||
import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
||||
from client import Client
|
||||
from observations import get,query,scalar
|
||||
C=[str(ROOT/'scripts/compose.sh')]
|
||||
def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90)
|
||||
def start_service(name):
|
||||
ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split()
|
||||
if not ids:raise RuntimeError('Missing existing '+name+' container')
|
||||
subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
|
||||
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
|
||||
def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None
|
||||
class Traffic:
|
||||
def __init__(self):
|
||||
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
||||
self.chat=chat;self.token=a['token'];self.actor=a;self.client=c
|
||||
self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64)
|
||||
def request(self,index,scheduled,upload=False):
|
||||
try:
|
||||
path='/api/v1/chats/'+self.chat['id']
|
||||
headers={'Authorization':'Bearer '+self.token};body=None
|
||||
if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png'
|
||||
elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex})
|
||||
else:method='GET';path+='/messages?limit=50'
|
||||
r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method)
|
||||
status=0;att=None;message_id=None
|
||||
try:
|
||||
try:response=urllib.request.urlopen(r,timeout=6)
|
||||
except urllib.error.HTTPError as e:response=e
|
||||
with response:
|
||||
status=response.status;data=response.read()
|
||||
if upload and status==202:att=json.loads(data)['id']
|
||||
if not upload and method=='POST' and status==201:message_id=json.loads(data)['id']
|
||||
except (OSError,TimeoutError):pass
|
||||
with self.lock:
|
||||
self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000))
|
||||
if att:self.attachments.append(att)
|
||||
if message_id:self.messages.append(message_id)
|
||||
finally:self.slots.release()
|
||||
def run(self):
|
||||
start=time.monotonic();i=0
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
|
||||
while not self.stop.is_set():
|
||||
scheduled=start+i*.1
|
||||
if self.stop.wait(max(0,scheduled-time.monotonic())):break
|
||||
if self.slots.acquire(False):pool.submit(self.request,i,scheduled)
|
||||
else:
|
||||
with self.lock:self.dropped+=1
|
||||
if i%20==0:
|
||||
if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True)
|
||||
else:
|
||||
with self.lock:self.dropped+=1
|
||||
i+=1
|
||||
def summary(self):
|
||||
with self.lock:rows=list(self.rows);dropped=self.dropped
|
||||
xs=[r[3] for r in rows if 200<=r[2]<300]
|
||||
return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}}
|
||||
|
||||
def main():
|
||||
def interrupted(*_):raise KeyboardInterrupt()
|
||||
signal.signal(signal.SIGTERM,interrupted)
|
||||
p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args()
|
||||
out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]}
|
||||
out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
|
||||
target=ROOT/a.output
|
||||
def save():target.write_text(json.dumps(out,indent=2)+'\n')
|
||||
traffic=Traffic();t=threading.Thread(target=traffic.run);t.start()
|
||||
def phase(name,seconds):
|
||||
at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save()
|
||||
before=traffic.summary();nextstats=0;nextprint=0
|
||||
while time.time()<at+seconds:
|
||||
now=time.time()
|
||||
row={'at':now,'phase':name}
|
||||
try:
|
||||
row.update({'alerts':get('prometheus','/api/v1/alerts')['data']['alerts'],'pending':scalar('sum(messenger_jobs_pending)'), 'db_lock_waiting':scalar('sum(messenger_db_locks_waiting)'), 'queue_age':scalar('max(messenger_queue_oldest_age_seconds)'), 'api_burn':scalar('messenger:api_error_ratio:1m / 0.01'), 'ready_burn':scalar('messenger:ready_error_ratio:1m / 0.01')})
|
||||
except Exception as e:row['collection_error']=type(e).__name__
|
||||
out['observations'].append(row)
|
||||
if now>=nextstats:
|
||||
ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split()
|
||||
if ids:
|
||||
raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20)
|
||||
out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]})
|
||||
nextstats=now+30
|
||||
if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60
|
||||
save();time.sleep(min(5,max(0,at+seconds-time.time())))
|
||||
with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()]
|
||||
lat=[r[3] for r in rows if 200<=r[2]<300]
|
||||
elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows)
|
||||
out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save()
|
||||
def worker(duration,name):
|
||||
compose('stop','worker')
|
||||
try:phase(name,duration)
|
||||
finally:start_service('worker')
|
||||
def database(duration,name):
|
||||
# A separate bounded transaction blocks only the local message table.
|
||||
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
deadline=time.time()+10
|
||||
while time.time()<deadline:
|
||||
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_slo_injection' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
|
||||
time.sleep(.1)
|
||||
else:raise RuntimeError('DB injection did not acquire lock')
|
||||
phase(name,duration)
|
||||
finally:
|
||||
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'")
|
||||
proc.wait(timeout=10)
|
||||
try:
|
||||
phase('baseline',a.warmup)
|
||||
for i in range(1,a.repeats+1):
|
||||
worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60)
|
||||
database(5,f'db_short_{i}');phase(f'db_short_recovery_{i}',60)
|
||||
worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown)
|
||||
database(a.long,f'db_long_{i}');phase(f'db_recovery_{i}',a.cooldown)
|
||||
if a.backend:
|
||||
out['backend_before']={n:query(n) for n in ['otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_queue_size','otelcol_receiver_accepted_spans_total']}
|
||||
compose('stop','tempo')
|
||||
try:phase('tempo_unavailable',a.backend)
|
||||
finally:start_service('tempo')
|
||||
phase('tempo_recovery',60)
|
||||
out['backend_after']={n:query(n) for n in out['backend_before']}
|
||||
traffic.stop.set();t.join(timeout=15)
|
||||
phase('final_drain',30)
|
||||
# Every accepted upload must have a published result after recovery.
|
||||
with traffic.lock:ids=list(traffic.attachments)
|
||||
out['accepted_ids_count']=len(ids)
|
||||
counts={}
|
||||
for id in ids:
|
||||
state=traffic.client.call('GET','/api/v1/attachments/'+id,actor=traffic.actor,schema='Attachment')['json']['status'];counts[state]=counts.get(state,0)+1
|
||||
out['attachment_states']=counts
|
||||
stored=set(sql("SELECT id FROM messages WHERE chat_id='%s'"%str(uuid.UUID(traffic.chat['id']))).splitlines())
|
||||
with traffic.lock:confirmed=set(traffic.messages)
|
||||
out['message_persistence']={'confirmed':len(confirmed),'stored':len(stored),'confirmed_missing':len(confirmed-stored)}
|
||||
if confirmed-stored or counts.get('ready',0)!=len(ids):raise RuntimeError('Confirmed business data not preserved')
|
||||
out['completed']=True
|
||||
except BaseException as e:
|
||||
out['completed']=False;out['failure_type']=type(e).__name__;raise
|
||||
finally:
|
||||
traffic.stop.set();t.join(timeout=15)
|
||||
out['finished']=time.time();out['traffic']=traffic.summary();save()
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert a temporary generation view to the oapi-codegen 3.0 subset.
|
||||
Runtime handlers enforce anyOf and required fields; normative contract is untouched.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
s=json.loads(Path('contracts/openapi.json').read_text())
|
||||
for schema in s['components']['schemas'].values():
|
||||
schema.pop('anyOf',None)
|
||||
for prop in schema.get('properties',{}).values():
|
||||
if prop.get('type')==['string','null']:
|
||||
prop.update(type='string',nullable=True)
|
||||
# Preserve pointers so handlers can distinguish missing fields from empty values.
|
||||
for name,field in [('CreateUser','password'),('CreateMessage','text')]:
|
||||
s['components']['schemas'][name]['properties'][field]['x-go-type-skip-optional-pointer']=False
|
||||
s['components']['schemas'][name]['required'].remove(field)
|
||||
print(json.dumps(s))
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read local telemetry through authenticated Grafana proxies; never print credentials."""
|
||||
import math,base64,json,urllib.request,urllib.parse,time
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
def grafana(path,params=None):
|
||||
password=(ROOT/'secrets/lab6/grafana_password').read_text().strip()
|
||||
token=base64.b64encode(('admin:'+password).encode()).decode()
|
||||
url='http://127.0.0.1:13006'+path
|
||||
if params:url+='?'+urllib.parse.urlencode(params)
|
||||
r=urllib.request.Request(url,headers={'Authorization':'Basic '+token,'Accept':'application/json'})
|
||||
with urllib.request.urlopen(r,timeout=20) as response:return json.load(response)
|
||||
def get(source,path,params=None):return grafana('/api/datasources/proxy/uid/'+source+path,params)
|
||||
def query(expr):return get('prometheus','/api/v1/query',{'query':expr})['data']['result']
|
||||
def scalar(expr,default=None):
|
||||
x=query(expr)
|
||||
v=float(x[0]['value'][1]) if x else default
|
||||
return v if v is not None and math.isfinite(v) else default
|
||||
|
||||
def collect():
|
||||
names=['messenger_http_requests_total','messenger_jobs_total','messenger_job_duration_seconds_count','messenger_job_ready_duration_seconds_count','messenger_db_pool_used','messenger_worker_busy','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total','messenger_observation_timestamp','ALERTS','otelcol_exporter_send_failed_spans_total','otelcol_exporter_sent_spans_total','otelcol_receiver_accepted_spans_total']
|
||||
return {'timestamp':time.time(),'queries':{n:query(n) for n in names}}
|
||||
if __name__=='__main__':
|
||||
import argparse
|
||||
p=argparse.ArgumentParser();p.add_argument('--output',default='evidence/signals.json');a=p.parse_args()
|
||||
result=collect();Path(a.output).write_text(json.dumps(result,indent=2)+'\n')
|
||||
print('Saved signal snapshot:',a.output)
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproduce ordinary and retried uploads and save real trace/log relationships."""
|
||||
import json,subprocess,sys,time,uuid,os
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
||||
from client import Client,check
|
||||
from observations import get
|
||||
C=[str(ROOT/'scripts/compose.sh')]
|
||||
def compose(*args):subprocess.run(C+list(args),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=120)
|
||||
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True).strip()
|
||||
def wait(fn,seconds=60):
|
||||
end=time.monotonic()+seconds
|
||||
while time.monotonic()<end:
|
||||
try:
|
||||
x=fn()
|
||||
if x:return x
|
||||
except Exception:pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError('Timed out waiting for correlation evidence')
|
||||
def trace_summary(d):
|
||||
spans=[]
|
||||
for b in d.get('batches',d.get('resourceSpans',[])):
|
||||
attrs={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
|
||||
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
|
||||
for s in scope.get('spans',[]):
|
||||
spans.append({k:s.get(k) for k in ['name','kind','traceId','spanId','parentSpanId','startTimeUnixNano','endTimeUnixNano','status'] }|{'service':attrs.get('service.name')})
|
||||
return spans
|
||||
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
||||
results=[];faults=ROOT/'.lab6-faults';faults.mkdir(exist_ok=True);faults.chmod(0o777)
|
||||
for retry in [False,True]:
|
||||
traceid=uuid.uuid4().hex
|
||||
marker=None
|
||||
try:
|
||||
if retry:compose('stop','worker')
|
||||
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=a,raw=data,headers={'Content-Type':'image/png','traceparent':'00-'+traceid+'-'+uuid.uuid4().hex[:16]+'-01'},schema='Attachment')['json']
|
||||
jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%str(uuid.UUID(att['id'])))
|
||||
if retry:
|
||||
marker=faults/('before_objects.'+jid);marker.write_text('temporary');marker.chmod(0o644)
|
||||
compose('-f','compose.yaml','-f','compose.faults.yaml','up','-d','--no-deps','worker')
|
||||
wait(lambda:int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid))>=2)
|
||||
marker.unlink()
|
||||
ready=c.wait_ready(a,att)
|
||||
def complete():
|
||||
spans=trace_summary(get('tempo','/api/traces/'+traceid))
|
||||
return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None
|
||||
spans=wait(complete)
|
||||
def logs():
|
||||
d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'})
|
||||
rows=d['data']['result']
|
||||
return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None
|
||||
rows=wait(logs)
|
||||
events=[]
|
||||
for r in rows:
|
||||
stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values'])
|
||||
check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing')
|
||||
check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing')
|
||||
producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]}
|
||||
consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]]
|
||||
check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer')
|
||||
check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost')
|
||||
results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events})
|
||||
print('PASS: correlated upload, retry='+str(retry),flush=True)
|
||||
finally:
|
||||
if marker:marker.unlink(missing_ok=True)
|
||||
if retry:compose('up','-d','--no-deps','worker')
|
||||
(ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n')
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Confirm that Prometheus sees the first 5xx spike after zero-series initialization."""
|
||||
import concurrent.futures,json,subprocess,sys,time
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
||||
from client import Client,check
|
||||
from observations import query,scalar
|
||||
C=[str(ROOT/'scripts/compose.sh')]
|
||||
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
|
||||
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
||||
before=query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}')
|
||||
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',"SET application_name='lab6_counter_probe'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(5); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
def request():return c.call('GET','/api/v1/chats/'+chat['id']+'/messages',expected=[200,500,503],actor=a)['status']
|
||||
try:
|
||||
end=time.time()+5
|
||||
while time.time()<end:
|
||||
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_counter_probe' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
|
||||
time.sleep(.05)
|
||||
futures=[]
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
|
||||
start=time.monotonic()
|
||||
for i in range(50):
|
||||
time.sleep(max(0,start+i*.1-time.monotonic()));futures.append(pool.submit(request))
|
||||
statuses=[f.result() for f in futures]
|
||||
finally:
|
||||
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_counter_probe'");proc.wait(timeout=10)
|
||||
time.sleep(15)
|
||||
increase=scalar('sum(increase(messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}[1m]))')
|
||||
check(increase is not None and increase>0,'first spike missing from rate/increase')
|
||||
out={'at':time.time(),'before':before,'after':query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'),'increase_1m':increase,'statuses':{str(s):statuses.count(s) for s in set(statuses)},'passed':True}
|
||||
(ROOT/'evidence/first-spike.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: first 5xx spike appears in Prometheus increase:',round(increase,2))
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach container/volume evidence to the existing public persistence scenario."""
|
||||
import json,subprocess,time
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
compose=[str(root/'scripts/compose.sh')]
|
||||
def snapshot():
|
||||
ids=subprocess.check_output(compose+['ps','-q'],text=True).split()
|
||||
rows=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
|
||||
return {r['Name'].lstrip('/'):{'service':r['Config']['Labels']['com.docker.compose.service'],'started_at':r['State']['StartedAt'],'volumes':sorted(m['Name'] for m in r['Mounts'] if m['Type']=='volume')} for r in rows}
|
||||
before=snapshot();start=time.time()
|
||||
p=subprocess.run(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],cwd=root,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
||||
(root/'evidence/persistence.txt').write_text(p.stdout)
|
||||
after=snapshot();stores=['postgres','redis','s3','broker','prometheus','loki','tempo','grafana']
|
||||
checks={}
|
||||
for service in stores:
|
||||
old=next(v for v in before.values() if v['service']==service);new=next(v for v in after.values() if v['service']==service)
|
||||
checks[service]={'restarted':old['started_at']!=new['started_at'],'same_volumes':old['volumes']==new['volumes']}
|
||||
out={'started':start,'finished':time.time(),'command':['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],'returncode':p.returncode,'before':before,'after':after,'checks':checks}
|
||||
(root/'evidence/persistence-processes.json').write_text(json.dumps(out,indent=2)+'\n')
|
||||
if p.returncode or not all(v['restarted'] and v['same_volumes'] for v in checks.values()):raise SystemExit('FAIL: persistence or actual store restart evidence')
|
||||
print('PASS: persistence, real restart of all stores, unchanged volume identities')
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect real stored signals, label cardinality and synthetic privacy canaries."""
|
||||
import json,sys,time,uuid
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1];sys.path[:0]=[str(root/'tests'),str(root/'scripts')]
|
||||
from client import Client,check
|
||||
from observations import get,grafana,query,collect
|
||||
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
|
||||
marker='private_canary_'+uuid.uuid4().hex
|
||||
c.message(a,chat,marker)
|
||||
c.call('POST','/api/v1/chats',400,actor=a,body={marker:marker})
|
||||
att,_=c.attachment(a,chat);c.wait_ready(a,att)
|
||||
time.sleep(12)
|
||||
series=query('{__name__=~"messenger_.*"}')
|
||||
for row in series:
|
||||
labels=row['metric']
|
||||
check(not set(labels).intersection({'user_id','chat_id','message_id','attachment_id','request_id','filename','url'}),'high-cardinality metric label')
|
||||
check(not __import__('re').search(r'[0-9a-f]{8}-[0-9a-f]{4}-',labels.get('route','')),'raw route contains UUID')
|
||||
required=['messenger_http_requests_total','messenger_http_duration_seconds_bucket','messenger_jobs_total','messenger_job_duration_seconds_bucket','messenger_job_ready_duration_seconds_bucket','messenger_db_pool_used','messenger_db_pool_waiting','messenger_worker_busy','messenger_worker_capacity','messenger_process_memory','messenger_process_cpu','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total']
|
||||
found={r['metric']['__name__'] for r in series}
|
||||
check(set(required)<=found,'missing required metrics: '+str(set(required)-found))
|
||||
logs=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} |= "'+marker+'"','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))})
|
||||
check(not logs['data']['result'],'canary leaked into logs')
|
||||
# Search trace attributes; HTTP bodies and headers are never recorded.
|
||||
traces=get('tempo','/api/search',{'q':'{ resource.service.name = "messenger-api" }','limit':10})
|
||||
trace_rows=[]
|
||||
for tr in traces.get('traces',[]):
|
||||
d=get('tempo','/api/traces/'+tr['traceID']);check(marker not in json.dumps(d),'canary leaked into traces');trace_rows.append(tr['traceID'])
|
||||
check(bool(trace_rows),'trace search returned no observations')
|
||||
# Verify that the imported dashboard references valid expressions.
|
||||
dashboard=grafana('/api/dashboards/uid/messenger-lab6')['dashboard']
|
||||
check(len(dashboard['panels'])>=18,'dashboard not provisioned')
|
||||
provisioned=grafana('/api/datasources');check({'prometheus','loki','tempo'}<=set(d['uid'] for d in provisioned),'datasources not provisioned')
|
||||
for panel in dashboard['panels']:
|
||||
for target in panel.get('targets',[]):query(target['expr'])
|
||||
out=collect();out.update({'checks':{'required_metrics_present':True,'forbidden_labels_absent':True,'raw_uuid_routes_absent':True,'synthetic_canary_absent_in_logs':True,'synthetic_canary_absent_in_traces':True,'dashboard_expressions_valid':True},'series_count':len(series),'trace_ids_checked':trace_rows})
|
||||
(root/'evidence/signals.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: stored metrics, safe labels, privacy canaries, trace search and dashboard queries')
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
./scripts/compose.sh stop api lb worker dispatcher
|
||||
./scripts/compose.sh restart postgres redis s3 broker prometheus loki tempo otel-collector grafana
|
||||
./scripts/compose.sh run --rm -T s3-init
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drain and replace API processes individually, retaining other container IDs."""
|
||||
import json,os,subprocess,time
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
def run(args,input=None):
|
||||
p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100)
|
||||
if p.returncode:raise RuntimeError(p.stderr.strip()[:300])
|
||||
return p.stdout.strip()
|
||||
def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n')
|
||||
def ids():return run(['./scripts/compose.sh','ps','-q','api']).split()
|
||||
def states(lb):
|
||||
lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split()
|
||||
return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()]
|
||||
def main():
|
||||
initial=ids()
|
||||
if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas')
|
||||
lb=run(['./scripts/compose.sh','ps','-q','lb'])
|
||||
for old in initial:
|
||||
info=json.loads(run(['docker','inspect',old]))[0]
|
||||
ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress']
|
||||
backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api')
|
||||
runtime(lb,f'set server api/{backend} state drain')
|
||||
try:
|
||||
deadline=time.monotonic()+10
|
||||
while True:
|
||||
# Existing transactions get a chance to finish before SIGTERM.
|
||||
import csv,io
|
||||
raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv'])
|
||||
row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend)
|
||||
if int(row['scur'])==0:break
|
||||
if time.monotonic()>deadline:raise RuntimeError('backend did not drain')
|
||||
time.sleep(.1)
|
||||
before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old])
|
||||
run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api'])
|
||||
after=set(ids())
|
||||
if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly')
|
||||
new=(after-before).pop();deadline=time.monotonic()+60
|
||||
while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy':
|
||||
if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness')
|
||||
time.sleep(.5)
|
||||
print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True)
|
||||
finally:runtime(lb,f'set server api/{backend} state ready')
|
||||
time.sleep(3) # allow DNS refresh and a successful active readiness check
|
||||
if __name__=='__main__':main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/rolling-update.py
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?missing target alias}"
|
||||
for id in $(./scripts/compose.sh ps -q api); do
|
||||
current=$(docker exec "$id" wget -T 2 -qO- http://127.0.0.1:8080/internal/metrics | python3 -c 'import json,sys; print(json.load(sys.stdin)["instance_id"])')
|
||||
if [ "$current" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 8 "$id" >/dev/null
|
||||
python3 - "$id" "$TARGET_INSTANCE_ID" <<'PYLOG'
|
||||
import json,sys,time
|
||||
from pathlib import Path
|
||||
with Path('evidence/failover-action.jsonl').open('a') as f:
|
||||
f.write(json.dumps({'at':time.time(),'container':sys.argv[1],'instance_id':sys.argv[2],'action':'docker stop','project':'mtusi2026lab6'})+'\n')
|
||||
PYLOG
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf '%s\n' 'Target does not match a running lab6 API process' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Derive incident timing from recorded observations, without inventing missing data."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
x=json.loads((root/'evidence/slo-experiment.json').read_text())
|
||||
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
|
||||
rows=x['observations'];events=x['events'];results=[]
|
||||
for i,event in enumerate(events):
|
||||
name=event['phase']
|
||||
if not any(name.startswith(s) for s in ['worker_short_','db_short_','worker_long_','db_long_']):continue
|
||||
if 'recovery' in name:continue
|
||||
prefix=name.rsplit('_',1)[0];start=event['at']
|
||||
phase=next(p for p in x['phases'] if p['name']==name);recovered=phase['end']
|
||||
nextfault=next((e['at'] for e in events[i+1:] if ('_short_' in e['phase'] or '_long_' in e['phase']) and 'recovery' not in e['phase']),x.get('finished',float('inf')))
|
||||
observed=[r for r in rows if start<=r['at']<nextfault]
|
||||
expected=['QueueLag','ReadyBurnRateFast','ReadyBurnRateSlow'] if name.startswith('worker') else ['ApiBurnRateFast','ApiBurnRateSlow']
|
||||
alerts={}
|
||||
for alert in expected:
|
||||
def firing(r):return any(a['labels']['alertname']==alert and a['state']=='firing' for a in r.get('alerts',[]))
|
||||
first=next((r['at'] for r in observed if firing(r)),None)
|
||||
resolved=next((r['at'] for r in observed if first is not None and r['at']>max(first,recovered) and not firing(r)),None)
|
||||
alerts[alert]={'detection_seconds':None if first is None else first-start,'resolved_after_restore_seconds':None if resolved is None else resolved-recovered,'fired':first is not None}
|
||||
results.append({'phase':name,'duration':recovered-start,'alerts':alerts,'successful_rps':phase['successful_rps'],'error_ratio':phase['error_ratio'],'latency_success_ms':phase['latency_success_ms']})
|
||||
summary={'completed':x.get('completed',False),'duration_seconds':x.get('finished',0)-x['started'],'parameters':x['parameters'],'incidents':results,'traffic':x.get('traffic'),'attachment_states':x.get('attachment_states'),'image_id':x.get('image_id')}
|
||||
(root/'evidence/slo-summary.json').write_text(json.dumps(summary,indent=2)+'\n')
|
||||
print(json.dumps(summary,indent=2))
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p bin
|
||||
(cd service && GOOS=linux GOARCH=$(docker info --format '{{.Architecture}}' | sed 's/aarch64/arm64/;s/x86_64/amd64/') CGO_ENABLED=0 go test -c -o ../bin/storage.test ./internal/adapter/repository/storage)
|
||||
database="lab5_atomic_$(date +%s)_$$"
|
||||
./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" >/dev/null
|
||||
trap './scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "DROP DATABASE IF EXISTS $database WITH (FORCE)" >/dev/null' EXIT
|
||||
id=$(./scripts/compose.sh ps -q api | head -n 1)
|
||||
docker cp bin/storage.test "$id:/tmp/storage.test" >/dev/null
|
||||
docker exec -e "TEST_DATABASE_NAME=$database" "$id" /tmp/storage.test -test.v
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wait for the business endpoint and each telemetry backend, not only running processes."""
|
||||
import subprocess,time,urllib.request
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
checks=['http://otel-collector:13133/','http://prometheus:9090/-/ready','http://loki:3100/ready','http://tempo:3200/ready','http://grafana:3000/api/health']
|
||||
end=time.monotonic()+120
|
||||
while time.monotonic()<end:
|
||||
try:
|
||||
with urllib.request.urlopen('http://127.0.0.1:18086/health/ready',timeout=2) as r:
|
||||
if r.status!=200:raise RuntimeError()
|
||||
for url in checks:
|
||||
subprocess.run([str(root/'scripts/compose.sh'),'exec','-T','--index','1','api','wget','-q','-O','/dev/null',url],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=5)
|
||||
print('PASS: business, Collector, Prometheus, Loki, Tempo and Grafana ready');break
|
||||
except Exception:time.sleep(2)
|
||||
else:raise SystemExit('FAIL: stand readiness timeout')
|
||||
Reference in New Issue
Block a user