Реализация на 5

This commit is contained in:
Grbisba
2026-11-17 13:08:13 +03:00
parent 470d9d6f1e
commit 60f48c0337
229 changed files with 74928 additions and 47 deletions
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
"""Save bounded-buffer and loss observations for the actual Tempo outage."""
import datetime as dt,json,re,subprocess,sys
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
from observations import get
x=json.loads((ROOT/'evidence/slo-experiment.json').read_text())
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain')
metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity']
def at(name,t):
rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result']
return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo']
before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics}
def total(rows):return sum(float(r['value'][1]) for r in rows)
deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')}
samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result']
peak=max((float(v[1]) for r in samples for v in r['values']),default=0)
raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True)
drops=[]
for line in raw.splitlines():
try:d=json.loads(line[line.index('{'):])
except (ValueError,KeyError):continue
if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d):
drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d})
mem=[]
for record in x['resources']:
if p['start']<=record['at']<=finish:
for c in record['containers']:
if 'otel-collector' in c['Name']:
m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale)
out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')}
(ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n')
print('Saved backend outage balance, queue, memory and business persistence evidence')
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""Create synthetic local broker credentials, preserving existing settings."""
from pathlib import Path
import secrets
from urllib.parse import quote
root=Path(__file__).resolve().parents[1]
env=root/'.env.lab6'
values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#'))
password=values.get('BROKER_PASSWORD')
if not password:
password=secrets.token_hex(24)
with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n')
p=root/'secrets/lab6/broker_url'
if p.exists(): p.chmod(0o600)
p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n')
p.chmod(0o444)
print('Local broker credentials ready.')
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Generate local synthetic storage credentials without rotating existing keys."""
import json
import os
from pathlib import Path
import secrets
root=Path(__file__).resolve().parents[1]/'secrets'/'lab6'
root.mkdir(parents=True,exist_ok=True)
for name in ('s3_admin_access','s3_admin_secret','s3_api_access','s3_api_secret'):
p=root/name
if not p.exists():
p.write_text(secrets.token_hex(24)+'\n')
p.chmod(0o444) # Non-root containers must read these local synthetic secrets.
bucket=os.environ.get('S3_BUCKET','messenger-attachments')
identities=[]
for role,actions in [('admin',['Admin','Read','Write','List','Tagging']),('api',['Read:'+bucket,'Write:'+bucket,'List:'+bucket])]:
identities.append({'name':role,'credentials':[{'accessKey':(root/('s3_'+role+'_access')).read_text().strip(),'secretKey':(root/('s3_'+role+'_secret')).read_text().strip()}],'actions':actions})
p=root/'s3.json'
content=json.dumps({'identities':identities})+'\n'
if not p.exists() or p.read_text()!=content:
if p.exists(): p.chmod(0o600)
p.write_text(content)
p.chmod(0o444)
print('Local storage credentials ready; existing keys preserved.')
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets/lab6
if [ ! -f .env.lab6 ]; then
{ printf 'POSTGRES_PASSWORD='; openssl rand -hex 24
printf 'REDIS_PASSWORD='; openssl rand -hex 24
printf 'HOST_PORT=18086\nAPI_REPLICAS=2\nSESSION_TTL_SECONDS=14400\n'
} > .env.lab6
fi
if [ ! -f secrets/lab6/private_key.pem ]; then
openssl genrsa -out secrets/lab6/private_key.pem 2048 2>/dev/null
fi
if [ ! -f secrets/lab6/public_key.pem ]; then
openssl rsa -in secrets/lab6/private_key.pem -pubout -out secrets/lab6/public_key.pem 2>/dev/null
fi
# Only synthetic stand keys, readable by the non-root application inside containers.
chmod 755 secrets secrets/lab6
chmod 444 secrets/lab6/*.pem
printf '%s\n' 'Local lab6 settings and synthetic keys ready; existing files preserved.'
python3 scripts/bootstrap-storage.py
python3 scripts/bootstrap-broker.py
if [ ! -f secrets/lab6/grafana_password ]; then
openssl rand -hex 24 > secrets/lab6/grafana_password
fi
chmod 444 secrets/lab6/grafana_password
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
"""Record reproducibility metadata, with an allowlist excluding all secrets."""
import hashlib,json,platform,subprocess,time
from pathlib import Path
root=Path(__file__).resolve().parents[1]
def cmd(*args):return subprocess.check_output(args,cwd=root,text=True).strip()
paths=[p for d in ['service','infra','scripts','tests','contracts','student-tests'] for p in (root/d).rglob('*') if p.is_file() and '__pycache__' not in p.parts]
paths += [root/x for x in ['Dockerfile','compose.yaml','compose.faults.yaml','Makefile']]
h=hashlib.sha256()
for p in sorted(paths):h.update(str(p.relative_to(root)).encode());h.update(p.read_bytes())
containers=[json.loads(x) for x in cmd('docker','compose','--env-file','.env.lab6','ps','--format','json').splitlines()]
ids=[c['ID'] for c in containers]
inspect=json.loads(cmd('docker','inspect',*ids))
rows=[{'service':c['Config']['Labels']['com.docker.compose.service'],'image':c['Config']['Image'],'image_id':c['Image'],'started_at':c['State']['StartedAt'],'memory_limit':c['HostConfig']['Memory'],'nano_cpus':c['HostConfig']['NanoCpus'],'mounts':[{'type':m['Type'],'name':m.get('Name'),'destination':m['Destination']} for m in c['Mounts'] if m['Type']=='volume']} for c in inspect]
info=json.loads(cmd('docker','info','--format','{{json .}}'))
out={'timestamp':time.time(),'revision':cmd('git','rev-parse','HEAD'),'working_tree_dirty':bool(cmd('git','status','--porcelain')),'source_sha256':h.hexdigest(),'platform':platform.platform(),'go':cmd('go','version'),'docker':{'server_version':info['ServerVersion'],'cpus':info['NCPU'],'memory':info['MemTotal'],'architecture':info['Architecture']},'containers':rows}
if platform.system()=='Darwin':out['host']={'cpu':cmd('sysctl','-n','machdep.cpu.brand_string'),'memory':cmd('sysctl','-n','hw.memsize')}
(root/'evidence/environment.json').write_text(json.dumps(out,indent=2)+'\n')
print('Saved sanitized environment and source fingerprint')
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""Passively save stored log/trace evidence during the running SLO experiment."""
import json,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'))
from observations import get
source=ROOT/'evidence/slo-experiment.json';target=ROOT/'evidence/incident-signals.json'
result={};errors=[]
def logs(query,start,end):
return get('loki','/loki/api/v1/query_range',{'query':query,'start':str(int(start*1e9)),'end':str(int(end*1e9)),'limit':100})['data']['result']
def summary(d):
rows=[]
for b in d.get('batches',[]):
resource={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',[]):
for s in scope.get('spans',[]):
rows.append({'service':resource.get('service.name'),'name':s['name'],'kind':s.get('kind'),'span_id':s.get('spanId'),'parent_span_id':s.get('parentSpanId'),'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,'status':s.get('status')})
return rows
end=time.time()+8000
while time.time()<end:
try:
x=json.loads(source.read_text());last=x['observations'][-1];phase=last['phase'];event=next(e for e in reversed(x['events']) if e['phase']==phase)
if phase.startswith('db_long_') and time.time()-event['at']>60 and phase not in result:
rows=logs('{service_name="messenger-api"} |= "http_error"',event['at'],time.time()-3)
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
if ids:
traceid=ids[-1];spans=summary(get('tempo','/api/traces/'+traceid))
if any(s['name'].startswith('postgresql ') and s['duration_ms']>1000 for s in spans):result[phase]={'captured':time.time(),'observed':last,'trace_id':traceid,'spans':spans,'error_log_events':sum(len(r['values']) for r in rows),'diagnosis':'DB wait dominates server span; HTTP 5xx, not image processing'}
if phase.startswith('worker_long_') and time.time()-event['at']>70 and phase not in result:
worker=logs('{service_name="messenger-worker"} |= "job_received"',event['at'],time.time()-3)
uploads=logs('{service_name="messenger-api"} |= "http_request" | route="/api/v1/chats/{chat_id}/attachments" | status="202"',event['at'],time.time()-3)
result[phase]={'captured':time.time(),'observed':last,'worker_received_events':sum(len(r['values']) for r in worker),'accepted_upload_events':sum(len(r['values']) for r in uploads),'diagnosis':'Queue age and ready burn increase while API accepts uploads and no worker consumption is observed'}
if phase.startswith('worker_recovery_') and phase not in result and time.time()-event['at']>20:
rows=logs('{service_name="messenger-worker"} |= "job_ready"',event['at'],time.time()-3)
ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')]
if ids:
traceid=ids[0];spans=summary(get('tempo','/api/traces/'+traceid))
if spans:result[phase]={'captured':time.time(),'trace_id':traceid,'spans':spans,'diagnosis':'Worker resumes consumption and publishes S3/DB results'}
target.write_text(json.dumps({'evidence':result,'collection_errors':errors[-20:]},indent=2)+'\n')
if x.get('completed') is not None:break
except Exception as e:errors.append({'at':time.time(),'type':type(e).__name__})
time.sleep(5)
print('Saved incident observations:',len(result))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec docker compose --env-file .env.lab6 "$@"
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Bounded SLO experiment on the isolated lab6 stand, with raw observations.
No synthetic metrics or time acceleration. Restores every injected fault in finally.
"""
import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client
from observations import get,query,scalar
C=[str(ROOT/'scripts/compose.sh')]
def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90)
def start_service(name):
ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split()
if not ids:raise RuntimeError('Missing existing '+name+' container')
subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None
class Traffic:
def __init__(self):
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
self.chat=chat;self.token=a['token'];self.actor=a;self.client=c
self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes()
self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64)
def request(self,index,scheduled,upload=False):
try:
path='/api/v1/chats/'+self.chat['id']
headers={'Authorization':'Bearer '+self.token};body=None
if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png'
elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex})
else:method='GET';path+='/messages?limit=50'
r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method)
status=0;att=None;message_id=None
try:
try:response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as e:response=e
with response:
status=response.status;data=response.read()
if upload and status==202:att=json.loads(data)['id']
if not upload and method=='POST' and status==201:message_id=json.loads(data)['id']
except (OSError,TimeoutError):pass
with self.lock:
self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000))
if att:self.attachments.append(att)
if message_id:self.messages.append(message_id)
finally:self.slots.release()
def run(self):
start=time.monotonic();i=0
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
while not self.stop.is_set():
scheduled=start+i*.1
if self.stop.wait(max(0,scheduled-time.monotonic())):break
if self.slots.acquire(False):pool.submit(self.request,i,scheduled)
else:
with self.lock:self.dropped+=1
if i%20==0:
if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True)
else:
with self.lock:self.dropped+=1
i+=1
def summary(self):
with self.lock:rows=list(self.rows);dropped=self.dropped
xs=[r[3] for r in rows if 200<=r[2]<300]
return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}}
def main():
def interrupted(*_):raise KeyboardInterrupt()
signal.signal(signal.SIGTERM,interrupted)
p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args()
out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]}
out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip()
target=ROOT/a.output
def save():target.write_text(json.dumps(out,indent=2)+'\n')
traffic=Traffic();t=threading.Thread(target=traffic.run);t.start()
def phase(name,seconds):
at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save()
before=traffic.summary();nextstats=0;nextprint=0
while time.time()<at+seconds:
now=time.time()
row={'at':now,'phase':name}
try:
row.update({'alerts':get('prometheus','/api/v1/alerts')['data']['alerts'],'pending':scalar('sum(messenger_jobs_pending)'), 'db_lock_waiting':scalar('sum(messenger_db_locks_waiting)'), 'queue_age':scalar('max(messenger_queue_oldest_age_seconds)'), 'api_burn':scalar('messenger:api_error_ratio:1m / 0.01'), 'ready_burn':scalar('messenger:ready_error_ratio:1m / 0.01')})
except Exception as e:row['collection_error']=type(e).__name__
out['observations'].append(row)
if now>=nextstats:
ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split()
if ids:
raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20)
out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]})
nextstats=now+30
if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60
save();time.sleep(min(5,max(0,at+seconds-time.time())))
with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()]
lat=[r[3] for r in rows if 200<=r[2]<300]
elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows)
out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save()
def worker(duration,name):
compose('stop','worker')
try:phase(name,duration)
finally:start_service('worker')
def database(duration,name):
# A separate bounded transaction blocks only the local message table.
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
deadline=time.time()+10
while time.time()<deadline:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_slo_injection' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
time.sleep(.1)
else:raise RuntimeError('DB injection did not acquire lock')
phase(name,duration)
finally:
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'")
proc.wait(timeout=10)
try:
phase('baseline',a.warmup)
for i in range(1,a.repeats+1):
worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60)
database(5,f'db_short_{i}');phase(f'db_short_recovery_{i}',60)
worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown)
database(a.long,f'db_long_{i}');phase(f'db_recovery_{i}',a.cooldown)
if a.backend:
out['backend_before']={n:query(n) for n in ['otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_queue_size','otelcol_receiver_accepted_spans_total']}
compose('stop','tempo')
try:phase('tempo_unavailable',a.backend)
finally:start_service('tempo')
phase('tempo_recovery',60)
out['backend_after']={n:query(n) for n in out['backend_before']}
traffic.stop.set();t.join(timeout=15)
phase('final_drain',30)
# Every accepted upload must have a published result after recovery.
with traffic.lock:ids=list(traffic.attachments)
out['accepted_ids_count']=len(ids)
counts={}
for id in ids:
state=traffic.client.call('GET','/api/v1/attachments/'+id,actor=traffic.actor,schema='Attachment')['json']['status'];counts[state]=counts.get(state,0)+1
out['attachment_states']=counts
stored=set(sql("SELECT id FROM messages WHERE chat_id='%s'"%str(uuid.UUID(traffic.chat['id']))).splitlines())
with traffic.lock:confirmed=set(traffic.messages)
out['message_persistence']={'confirmed':len(confirmed),'stored':len(stored),'confirmed_missing':len(confirmed-stored)}
if confirmed-stored or counts.get('ready',0)!=len(ids):raise RuntimeError('Confirmed business data not preserved')
out['completed']=True
except BaseException as e:
out['completed']=False;out['failure_type']=type(e).__name__;raise
finally:
traffic.stop.set();t.join(timeout=15)
out['finished']=time.time();out['traffic']=traffic.summary();save()
if __name__=='__main__':main()
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""Convert a temporary generation view to the oapi-codegen 3.0 subset.
Runtime handlers enforce anyOf and required fields; normative contract is untouched.
"""
import json
from pathlib import Path
s=json.loads(Path('contracts/openapi.json').read_text())
for schema in s['components']['schemas'].values():
schema.pop('anyOf',None)
for prop in schema.get('properties',{}).values():
if prop.get('type')==['string','null']:
prop.update(type='string',nullable=True)
# Preserve pointers so handlers can distinguish missing fields from empty values.
for name,field in [('CreateUser','password'),('CreateMessage','text')]:
s['components']['schemas'][name]['properties'][field]['x-go-type-skip-optional-pointer']=False
s['components']['schemas'][name]['required'].remove(field)
print(json.dumps(s))
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Read local telemetry through authenticated Grafana proxies; never print credentials."""
import math,base64,json,urllib.request,urllib.parse,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def grafana(path,params=None):
password=(ROOT/'secrets/lab6/grafana_password').read_text().strip()
token=base64.b64encode(('admin:'+password).encode()).decode()
url='http://127.0.0.1:13006'+path
if params:url+='?'+urllib.parse.urlencode(params)
r=urllib.request.Request(url,headers={'Authorization':'Basic '+token,'Accept':'application/json'})
with urllib.request.urlopen(r,timeout=20) as response:return json.load(response)
def get(source,path,params=None):return grafana('/api/datasources/proxy/uid/'+source+path,params)
def query(expr):return get('prometheus','/api/v1/query',{'query':expr})['data']['result']
def scalar(expr,default=None):
x=query(expr)
v=float(x[0]['value'][1]) if x else default
return v if v is not None and math.isfinite(v) else default
def collect():
names=['messenger_http_requests_total','messenger_jobs_total','messenger_job_duration_seconds_count','messenger_job_ready_duration_seconds_count','messenger_db_pool_used','messenger_worker_busy','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total','messenger_observation_timestamp','ALERTS','otelcol_exporter_send_failed_spans_total','otelcol_exporter_sent_spans_total','otelcol_receiver_accepted_spans_total']
return {'timestamp':time.time(),'queries':{n:query(n) for n in names}}
if __name__=='__main__':
import argparse
p=argparse.ArgumentParser();p.add_argument('--output',default='evidence/signals.json');a=p.parse_args()
result=collect();Path(a.output).write_text(json.dumps(result,indent=2)+'\n')
print('Saved signal snapshot:',a.output)
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Reproduce ordinary and retried uploads and save real trace/log relationships."""
import json,subprocess,sys,time,uuid,os
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client,check
from observations import get
C=[str(ROOT/'scripts/compose.sh')]
def compose(*args):subprocess.run(C+list(args),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=120)
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True).strip()
def wait(fn,seconds=60):
end=time.monotonic()+seconds
while time.monotonic()<end:
try:
x=fn()
if x:return x
except Exception:pass
time.sleep(1)
raise RuntimeError('Timed out waiting for correlation evidence')
def trace_summary(d):
spans=[]
for b in d.get('batches',d.get('resourceSpans',[])):
attrs={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])}
for scope in b.get('scopeSpans',b.get('instrumentationLibrarySpans',[])):
for s in scope.get('spans',[]):
spans.append({k:s.get(k) for k in ['name','kind','traceId','spanId','parentSpanId','startTimeUnixNano','endTimeUnixNano','status'] }|{'service':attrs.get('service.name')})
return spans
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
results=[];faults=ROOT/'.lab6-faults';faults.mkdir(exist_ok=True);faults.chmod(0o777)
for retry in [False,True]:
traceid=uuid.uuid4().hex
marker=None
try:
if retry:compose('stop','worker')
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=a,raw=data,headers={'Content-Type':'image/png','traceparent':'00-'+traceid+'-'+uuid.uuid4().hex[:16]+'-01'},schema='Attachment')['json']
jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%str(uuid.UUID(att['id'])))
if retry:
marker=faults/('before_objects.'+jid);marker.write_text('temporary');marker.chmod(0o644)
compose('-f','compose.yaml','-f','compose.faults.yaml','up','-d','--no-deps','worker')
wait(lambda:int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid))>=2)
marker.unlink()
ready=c.wait_ready(a,att)
def complete():
spans=trace_summary(get('tempo','/api/traces/'+traceid))
return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None
spans=wait(complete)
def logs():
d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'})
rows=d['data']['result']
return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None
rows=wait(logs)
events=[]
for r in rows:
stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values'])
check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing')
check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing')
producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]}
consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]]
check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer')
check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost')
results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events})
print('PASS: correlated upload, retry='+str(retry),flush=True)
finally:
if marker:marker.unlink(missing_ok=True)
if retry:compose('up','-d','--no-deps','worker')
(ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n')
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
"""Confirm that Prometheus sees the first 5xx spike after zero-series initialization."""
import concurrent.futures,json,subprocess,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client,check
from observations import query,scalar
C=[str(ROOT/'scripts/compose.sh')]
def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip()
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
before=query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}')
proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',"SET application_name='lab6_counter_probe'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(5); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
def request():return c.call('GET','/api/v1/chats/'+chat['id']+'/messages',expected=[200,500,503],actor=a)['status']
try:
end=time.time()+5
while time.time()<end:
if sql("SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING(pid) WHERE a.application_name='lab6_counter_probe' AND l.mode='AccessExclusiveLock' AND l.granted")!='0':break
time.sleep(.05)
futures=[]
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
start=time.monotonic()
for i in range(50):
time.sleep(max(0,start+i*.1-time.monotonic()));futures.append(pool.submit(request))
statuses=[f.result() for f in futures]
finally:
sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_counter_probe'");proc.wait(timeout=10)
time.sleep(15)
increase=scalar('sum(increase(messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}[1m]))')
check(increase is not None and increase>0,'first spike missing from rate/increase')
out={'at':time.time(),'before':before,'after':query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'),'increase_1m':increase,'statuses':{str(s):statuses.count(s) for s in set(statuses)},'passed':True}
(ROOT/'evidence/first-spike.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: first 5xx spike appears in Prometheus increase:',round(increase,2))
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env python3
"""Attach container/volume evidence to the existing public persistence scenario."""
import json,subprocess,time
from pathlib import Path
root=Path(__file__).resolve().parents[1]
compose=[str(root/'scripts/compose.sh')]
def snapshot():
ids=subprocess.check_output(compose+['ps','-q'],text=True).split()
rows=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
return {r['Name'].lstrip('/'):{'service':r['Config']['Labels']['com.docker.compose.service'],'started_at':r['State']['StartedAt'],'volumes':sorted(m['Name'] for m in r['Mounts'] if m['Type']=='volume')} for r in rows}
before=snapshot();start=time.time()
p=subprocess.run(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],cwd=root,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
(root/'evidence/persistence.txt').write_text(p.stdout)
after=snapshot();stores=['postgres','redis','s3','broker','prometheus','loki','tempo','grafana']
checks={}
for service in stores:
old=next(v for v in before.values() if v['service']==service);new=next(v for v in after.values() if v['service']==service)
checks[service]={'restarted':old['started_at']!=new['started_at'],'same_volumes':old['volumes']==new['volumes']}
out={'started':start,'finished':time.time(),'command':['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],'returncode':p.returncode,'before':before,'after':after,'checks':checks}
(root/'evidence/persistence-processes.json').write_text(json.dumps(out,indent=2)+'\n')
if p.returncode or not all(v['restarted'] and v['same_volumes'] for v in checks.values()):raise SystemExit('FAIL: persistence or actual store restart evidence')
print('PASS: persistence, real restart of all stores, unchanged volume identities')
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Inspect real stored signals, label cardinality and synthetic privacy canaries."""
import json,sys,time,uuid
from pathlib import Path
root=Path(__file__).resolve().parents[1];sys.path[:0]=[str(root/'tests'),str(root/'scripts')]
from client import Client,check
from observations import get,grafana,query,collect
c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap()
marker='private_canary_'+uuid.uuid4().hex
c.message(a,chat,marker)
c.call('POST','/api/v1/chats',400,actor=a,body={marker:marker})
att,_=c.attachment(a,chat);c.wait_ready(a,att)
time.sleep(12)
series=query('{__name__=~"messenger_.*"}')
for row in series:
labels=row['metric']
check(not set(labels).intersection({'user_id','chat_id','message_id','attachment_id','request_id','filename','url'}),'high-cardinality metric label')
check(not __import__('re').search(r'[0-9a-f]{8}-[0-9a-f]{4}-',labels.get('route','')),'raw route contains UUID')
required=['messenger_http_requests_total','messenger_http_duration_seconds_bucket','messenger_jobs_total','messenger_job_duration_seconds_bucket','messenger_job_ready_duration_seconds_bucket','messenger_db_pool_used','messenger_db_pool_waiting','messenger_worker_busy','messenger_worker_capacity','messenger_process_memory','messenger_process_cpu','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total']
found={r['metric']['__name__'] for r in series}
check(set(required)<=found,'missing required metrics: '+str(set(required)-found))
logs=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} |= "'+marker+'"','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))})
check(not logs['data']['result'],'canary leaked into logs')
# Search trace attributes; HTTP bodies and headers are never recorded.
traces=get('tempo','/api/search',{'q':'{ resource.service.name = "messenger-api" }','limit':10})
trace_rows=[]
for tr in traces.get('traces',[]):
d=get('tempo','/api/traces/'+tr['traceID']);check(marker not in json.dumps(d),'canary leaked into traces');trace_rows.append(tr['traceID'])
check(bool(trace_rows),'trace search returned no observations')
# Verify that the imported dashboard references valid expressions.
dashboard=grafana('/api/dashboards/uid/messenger-lab6')['dashboard']
check(len(dashboard['panels'])>=18,'dashboard not provisioned')
provisioned=grafana('/api/datasources');check({'prometheus','loki','tempo'}<=set(d['uid'] for d in provisioned),'datasources not provisioned')
for panel in dashboard['panels']:
for target in panel.get('targets',[]):query(target['expr'])
out=collect();out.update({'checks':{'required_metrics_present':True,'forbidden_labels_absent':True,'raw_uuid_routes_absent':True,'synthetic_canary_absent_in_logs':True,'synthetic_canary_absent_in_traces':True,'dashboard_expressions_valid':True},'series_count':len(series),'trace_ids_checked':trace_rows})
(root/'evidence/signals.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: stored metrics, safe labels, privacy canaries, trace search and dashboard queries')
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
./scripts/compose.sh stop api lb worker dispatcher
./scripts/compose.sh restart postgres redis s3 broker prometheus loki tempo otel-collector grafana
./scripts/compose.sh run --rm -T s3-init
./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Drain and replace API processes individually, retaining other container IDs."""
import json,os,subprocess,time
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def run(args,input=None):
p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100)
if p.returncode:raise RuntimeError(p.stderr.strip()[:300])
return p.stdout.strip()
def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n')
def ids():return run(['./scripts/compose.sh','ps','-q','api']).split()
def states(lb):
lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split()
return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()]
def main():
initial=ids()
if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas')
lb=run(['./scripts/compose.sh','ps','-q','lb'])
for old in initial:
info=json.loads(run(['docker','inspect',old]))[0]
ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress']
backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api')
runtime(lb,f'set server api/{backend} state drain')
try:
deadline=time.monotonic()+10
while True:
# Existing transactions get a chance to finish before SIGTERM.
import csv,io
raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv'])
row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend)
if int(row['scur'])==0:break
if time.monotonic()>deadline:raise RuntimeError('backend did not drain')
time.sleep(.1)
before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old])
run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api'])
after=set(ids())
if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly')
new=(after-before).pop();deadline=time.monotonic()+60
while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy':
if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness')
time.sleep(.5)
print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True)
finally:runtime(lb,f'set server api/{backend} state ready')
time.sleep(3) # allow DNS refresh and a successful active readiness check
if __name__=='__main__':main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/rolling-update.py
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?missing target alias}"
for id in $(./scripts/compose.sh ps -q api); do
current=$(docker exec "$id" wget -T 2 -qO- http://127.0.0.1:8080/internal/metrics | python3 -c 'import json,sys; print(json.load(sys.stdin)["instance_id"])')
if [ "$current" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 8 "$id" >/dev/null
python3 - "$id" "$TARGET_INSTANCE_ID" <<'PYLOG'
import json,sys,time
from pathlib import Path
with Path('evidence/failover-action.jsonl').open('a') as f:
f.write(json.dumps({'at':time.time(),'container':sys.argv[1],'instance_id':sys.argv[2],'action':'docker stop','project':'mtusi2026lab6'})+'\n')
PYLOG
exit 0
fi
done
printf '%s\n' 'Target does not match a running lab6 API process' >&2
exit 1
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Derive incident timing from recorded observations, without inventing missing data."""
import json
from pathlib import Path
root=Path(__file__).resolve().parents[1]
x=json.loads((root/'evidence/slo-experiment.json').read_text())
if not x.get('completed'):raise SystemExit('Experiment has not completed successfully')
rows=x['observations'];events=x['events'];results=[]
for i,event in enumerate(events):
name=event['phase']
if not any(name.startswith(s) for s in ['worker_short_','db_short_','worker_long_','db_long_']):continue
if 'recovery' in name:continue
prefix=name.rsplit('_',1)[0];start=event['at']
phase=next(p for p in x['phases'] if p['name']==name);recovered=phase['end']
nextfault=next((e['at'] for e in events[i+1:] if ('_short_' in e['phase'] or '_long_' in e['phase']) and 'recovery' not in e['phase']),x.get('finished',float('inf')))
observed=[r for r in rows if start<=r['at']<nextfault]
expected=['QueueLag','ReadyBurnRateFast','ReadyBurnRateSlow'] if name.startswith('worker') else ['ApiBurnRateFast','ApiBurnRateSlow']
alerts={}
for alert in expected:
def firing(r):return any(a['labels']['alertname']==alert and a['state']=='firing' for a in r.get('alerts',[]))
first=next((r['at'] for r in observed if firing(r)),None)
resolved=next((r['at'] for r in observed if first is not None and r['at']>max(first,recovered) and not firing(r)),None)
alerts[alert]={'detection_seconds':None if first is None else first-start,'resolved_after_restore_seconds':None if resolved is None else resolved-recovered,'fired':first is not None}
results.append({'phase':name,'duration':recovered-start,'alerts':alerts,'successful_rps':phase['successful_rps'],'error_ratio':phase['error_ratio'],'latency_success_ms':phase['latency_success_ms']})
summary={'completed':x.get('completed',False),'duration_seconds':x.get('finished',0)-x['started'],'parameters':x['parameters'],'incidents':results,'traffic':x.get('traffic'),'attachment_states':x.get('attachment_states'),'image_id':x.get('image_id')}
(root/'evidence/slo-summary.json').write_text(json.dumps(summary,indent=2)+'\n')
print(json.dumps(summary,indent=2))
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p bin
(cd service && GOOS=linux GOARCH=$(docker info --format '{{.Architecture}}' | sed 's/aarch64/arm64/;s/x86_64/amd64/') CGO_ENABLED=0 go test -c -o ../bin/storage.test ./internal/adapter/repository/storage)
database="lab5_atomic_$(date +%s)_$$"
./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" >/dev/null
trap './scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "DROP DATABASE IF EXISTS $database WITH (FORCE)" >/dev/null' EXIT
id=$(./scripts/compose.sh ps -q api | head -n 1)
docker cp bin/storage.test "$id:/tmp/storage.test" >/dev/null
docker exec -e "TEST_DATABASE_NAME=$database" "$id" /tmp/storage.test -test.v
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env python3
"""Wait for the business endpoint and each telemetry backend, not only running processes."""
import subprocess,time,urllib.request
from pathlib import Path
root=Path(__file__).resolve().parents[1]
checks=['http://otel-collector:13133/','http://prometheus:9090/-/ready','http://loki:3100/ready','http://tempo:3200/ready','http://grafana:3000/api/health']
end=time.monotonic()+120
while time.monotonic()<end:
try:
with urllib.request.urlopen('http://127.0.0.1:18086/health/ready',timeout=2) as r:
if r.status!=200:raise RuntimeError()
for url in checks:
subprocess.run([str(root/'scripts/compose.sh'),'exec','-T','--index','1','api','wget','-q','-O','/dev/null',url],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=5)
print('PASS: business, Collector, Prometheus, Loki, Tempo and Grafana ready');break
except Exception:time.sleep(2)
else:raise SystemExit('FAIL: stand readiness timeout')