forked from mtusi2026/mtusi2026lab6
Implement lab6 observability with OpenTelemetry and Grafana
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion.
|
||||
|
||||
Admission metrics measure upload/enqueue requests, not worker throughput.
|
||||
Completion is checked separately so an accepted-but-stuck job cannot pass.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from client import Client, check, run
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--output', default='evidence/attachments-concurrent.json')
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--wait-seconds', type=int, default=180)
|
||||
args = parser.parse_args()
|
||||
check(args.concurrency > 0, 'concurrency must be positive')
|
||||
check(args.wait_seconds > 0, 'wait-seconds must be positive')
|
||||
|
||||
client = Client(args.base_url)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png'
|
||||
|
||||
def upload(_):
|
||||
cli = Client(args.base_url)
|
||||
response = cli.call('POST', path, 202, actor=alice, raw=source,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
att = response['json']
|
||||
check(att['status'] == 'queued', '202 must return a queued attachment')
|
||||
check(att['size_bytes'] == len(source), 'size mismatch')
|
||||
check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'],
|
||||
'incorrect attachment ownership')
|
||||
return att['id'], response['elapsed']
|
||||
|
||||
started = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(upload, range(args.concurrency)))
|
||||
accepted_seconds = time.monotonic() - started
|
||||
ids = [att_id for att_id, _ in results]
|
||||
timings = [duration for _, duration in results]
|
||||
check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs')
|
||||
|
||||
# A 202 response alone only proves acceptance, not successful processing.
|
||||
pending = set(ids)
|
||||
deadline = time.monotonic() + args.wait_seconds
|
||||
while pending and time.monotonic() < deadline:
|
||||
for att_id in list(pending):
|
||||
att = client.call('GET', '/api/v1/attachments/' + att_id,
|
||||
actor=alice, schema='Attachment')['json']
|
||||
check(att['status'] != 'failed', 'attachment reached failed: ' + att_id)
|
||||
if att['status'] == 'ready':
|
||||
variants = [v['name'] for v in att['variants']]
|
||||
check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'},
|
||||
'missing/duplicate image variants: ' + att_id)
|
||||
pending.remove(att_id)
|
||||
if pending:
|
||||
time.sleep(0.5)
|
||||
check(not pending, 'attachments not ready before deadline: ' + str(len(pending)))
|
||||
complete_seconds = time.monotonic() - started
|
||||
|
||||
data = {
|
||||
'concurrency': args.concurrency,
|
||||
'accepted_uploads': len(results),
|
||||
'ready_attachments': args.concurrency,
|
||||
'failed_requests': 0,
|
||||
'acceptance_elapsed_seconds': round(accepted_seconds, 3),
|
||||
'acceptance_rps': round(len(results) / accepted_seconds, 3),
|
||||
'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1),
|
||||
'upload_latency_max_ms': round(max(timings) * 1000, 1),
|
||||
'all_ready_elapsed_seconds': round(complete_seconds, 3),
|
||||
'bytes_per_file': len(source),
|
||||
'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.',
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(exist_ok=True, parents=True)
|
||||
output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency)
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab5 negative API tests with isolated synthetic users (requires live stack)."""
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
from client import Client, arguments, run, check
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Lab5 negative file/ACL/security tests')
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
base = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
|
||||
data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
|
||||
c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'})
|
||||
c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'})
|
||||
uploaded = c.attachment(alice,chat)[0]
|
||||
ready = c.wait_ready(alice, uploaded)
|
||||
unknown = str(uuid.uuid4())
|
||||
c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice)
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve)
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve)
|
||||
# A ready Lab5 attachment has a downloadable thumbnail for chat members.
|
||||
# A non-member still gets 404, and an unknown variant must not resolve.
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve)
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice)
|
||||
check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant')
|
||||
thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice)
|
||||
check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME')
|
||||
other_chat = c.call('POST','/api/v1/chats',201,actor=alice,
|
||||
body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
|
||||
path = '/api/v1/chats/'+other_chat['id']+'/messages'
|
||||
c.call('POST',path,404,actor=alice,
|
||||
body={'text':'cross-chat','attachment_ids':[uploaded['id']]},
|
||||
headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice,
|
||||
body={'text':'unknown','attachment_ids':[unknown]},
|
||||
headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json']
|
||||
check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked')
|
||||
print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks')
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas."""
|
||||
import concurrent.futures
|
||||
import sys
|
||||
import uuid
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Check atomic idempotency across API replicas")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "Concurrent replay " + key}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def worker(_):
|
||||
# Independent request client; one shared bearer token and idempotency key.
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
return client.call("POST", path, expected=201, actor=alice, body=body,
|
||||
headers={"Idempotency-Key": key}, schema="Message")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
|
||||
results = list(executor.map(worker, range(30)))
|
||||
ids = {result["json"]["id"] for result in results}
|
||||
instances = {result["headers"]["X-Instance-Id"] for result in results}
|
||||
check(len(ids) == 1, "parallel replay created different messages")
|
||||
check(len(instances) >= 2, "LB did not serve requests via both API replicas")
|
||||
check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result")
|
||||
c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key})
|
||||
c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message")
|
||||
history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
|
||||
check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history")
|
||||
print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prevent a mismatch between SQL placeholders and parameter tuple in worker result persistence."""
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
source = (Path(__file__).resolve().parents[1] / "app" / "jobs.py").read_text(encoding="utf-8")
|
||||
tree = ast.parse(source)
|
||||
fn = next(x for x in tree.body if isinstance(x, ast.FunctionDef) and x.name == "publish_result")
|
||||
matches = []
|
||||
for node in ast.walk(fn):
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
|
||||
continue
|
||||
if node.func.attr != "execute" or len(node.args) < 2:
|
||||
continue
|
||||
if not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str):
|
||||
continue
|
||||
sql = node.args[0].value
|
||||
if "INSERT INTO attachment_variants" not in sql:
|
||||
continue
|
||||
assert isinstance(node.args[1], ast.Tuple), "Expected literal parameters tuple"
|
||||
placeholders = sql.count("%s")
|
||||
params = len(node.args[1].elts)
|
||||
columns_match = re.search(r"INSERT INTO attachment_variants\s*\(([^)]*)\)", sql, re.I | re.S)
|
||||
assert columns_match, "Columns missing in INSERT"
|
||||
columns = len(columns_match.group(1).split(","))
|
||||
assert placeholders == columns == params, f"SQL mismatch: {columns} columns, {placeholders} placeholders, {params} parameters"
|
||||
matches.append((columns, placeholders, params))
|
||||
assert len(matches) == 1, "Expected exactly one attachment_variants INSERT"
|
||||
print("PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments")
|
||||
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab6 live evidence. Only synthetic test stack; no volume removal.
|
||||
Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import math
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, check, run
|
||||
from pipeline_extended import compose, pg, scalar, wait_for
|
||||
from pipeline_resilience import verify_ready
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
PROM='http://127.0.0.1:9090'
|
||||
LOKI='http://127.0.0.1:3100'
|
||||
TEMPO='http://127.0.0.1:3200'
|
||||
GRAFANA='http://127.0.0.1:3000'
|
||||
ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)'
|
||||
LAG='max(lab_queue_oldest_age_seconds)'
|
||||
DEPTH='max(lab_queue_depth)'
|
||||
FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
|
||||
# Explicit Prometheus readers can append unit suffixes to Collector self-metrics.
|
||||
# Missing gauges are not zero: absent telemetry must fail the outage test.
|
||||
QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})'
|
||||
QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})'
|
||||
RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
||||
|
||||
|
||||
def get(url, params=None, *, as_json=True):
|
||||
try:
|
||||
with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r:
|
||||
raw=r.read(32*1024*1024)
|
||||
return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace')
|
||||
except (urllib.error.URLError,TimeoutError,ValueError) as exc:
|
||||
raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None
|
||||
|
||||
|
||||
def prom(expr):
|
||||
data=get(PROM+'/api/v1/query',{'query':expr})
|
||||
check(data.get('status')=='success','PromQL query failed')
|
||||
return data['data']['result']
|
||||
|
||||
|
||||
def val(expr):
|
||||
rows=prom(expr)
|
||||
return float(rows[0]['value'][1]) if rows else None
|
||||
|
||||
|
||||
def alert_state(name):
|
||||
rules=get(PROM+'/api/v1/rules')['data']['groups']
|
||||
for group in rules:
|
||||
for rule in group['rules']:
|
||||
if rule['name']==name:
|
||||
check(rule.get('health')=='ok','alert rule is not healthy: '+name)
|
||||
return rule['state']
|
||||
raise Failure('alert rule missing: '+name)
|
||||
|
||||
|
||||
def wait_alert(name,state,seconds=100):
|
||||
return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2)
|
||||
|
||||
|
||||
def ready():
|
||||
for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'):
|
||||
wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2)
|
||||
wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100)
|
||||
wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40)
|
||||
wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40)
|
||||
|
||||
|
||||
def trace_spans(tid):
|
||||
raw=get(TEMPO+'/api/traces/'+tid)
|
||||
spans=[]
|
||||
def walk(obj):
|
||||
if isinstance(obj,dict):
|
||||
if {'spanId','traceId','name'}<=set(obj): spans.append(obj)
|
||||
else:
|
||||
for v in obj.values():walk(v)
|
||||
elif isinstance(obj,list):
|
||||
for v in obj:walk(v)
|
||||
walk(raw)
|
||||
return spans
|
||||
|
||||
|
||||
def logs(tid,service,request_id=None):
|
||||
expr='{service_name="%s"} | trace_id="%s"' % (service,tid)
|
||||
if request_id: expr+=' | request_id="%s"'%request_id
|
||||
data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100})
|
||||
return data['data']['result']
|
||||
|
||||
|
||||
def upload(c,alice,chat):
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
tid=r['headers'].get('X-Trace-Id','')
|
||||
check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement')
|
||||
return r['json'],raw,tid,r['headers']['X-Request-Id']
|
||||
|
||||
|
||||
def span_attrs(s):
|
||||
attrs={}
|
||||
for a in s.get('attributes',[]):
|
||||
value=a.get('value',{})
|
||||
attrs[a['key']]=next(iter(value.values()),None)
|
||||
return attrs
|
||||
|
||||
|
||||
def chain_ok(spans):
|
||||
names={s['name'] for s in spans}
|
||||
required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'}
|
||||
if not required<=names:return False
|
||||
producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'}
|
||||
consumers=[s for s in spans if s['name']=='rabbitmq.consume']
|
||||
outcomes={span_attrs(s).get('job.outcome') for s in consumers}
|
||||
return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers)
|
||||
|
||||
|
||||
def trace_retry(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
compose('stop','worker')
|
||||
try:
|
||||
att,raw,tid,rid=upload(c,alice,chat)
|
||||
# Override only worker, never a public endpoint; reset in finally.
|
||||
env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'}
|
||||
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180)
|
||||
verify_ready(c,bob,att,raw,seconds=180)
|
||||
spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3)
|
||||
wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
|
||||
def worker_events(streams):
|
||||
return {v[1] for row in streams for v in row.get('values',[])}
|
||||
wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90)
|
||||
# Keep only allowlisted names/IDs, not arbitrary span values or log payloads.
|
||||
summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans]
|
||||
result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True}
|
||||
print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid)
|
||||
return result
|
||||
finally:
|
||||
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180)
|
||||
|
||||
|
||||
def queue_lag(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
wait_alert('LabQueueLag','inactive')
|
||||
compose('stop','worker')
|
||||
try:
|
||||
att,raw,tid,rid=upload(c,alice,chat)
|
||||
t=time.monotonic()
|
||||
wait_alert('LabQueueLag','firing')
|
||||
lag=val(LAG);depth=val(DEPTH)
|
||||
check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics')
|
||||
detected=time.monotonic()-t
|
||||
print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True)
|
||||
compose('up','-d','worker')
|
||||
verify_ready(c,bob,att,raw,seconds=180)
|
||||
wait_alert('LabQueueLag','inactive',seconds=120)
|
||||
wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2)
|
||||
print('PASS queue-lag: firing -> ready -> resolved')
|
||||
return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)}
|
||||
finally:compose('up','-d','worker')
|
||||
|
||||
|
||||
def db_error(c):
|
||||
alice,_,_,_=c.bootstrap()
|
||||
wait_alert('LabAPIHighErrorRate','inactive')
|
||||
user=compose('exec','-T','postgres','printenv','POSTGRES_USER')
|
||||
db=compose('exec','-T','postgres','printenv','POSTGRES_DB')
|
||||
# Bounded, isolated table lock. Health SELECT 1 works, authenticated business
|
||||
# read times out at statement_timeout. Never stop DB (LB would hide API 5xx).
|
||||
cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;']
|
||||
lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15)
|
||||
start=time.monotonic(); traces=[]; codes=[]
|
||||
while time.monotonic()-start < 43:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
||||
def get_user(_):
|
||||
r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User')
|
||||
return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id')
|
||||
for code,tid,rid in pool.map(get_user,range(3)):
|
||||
codes.append(code)
|
||||
if code==503 and tid:traces.append((tid,rid))
|
||||
if traces and alert_state('LabAPIHighErrorRate')=='firing':break
|
||||
check(bool(traces),'no real HTTP 503 from DB lock')
|
||||
wait_alert('LabAPIHighErrorRate','firing',seconds=20)
|
||||
ratio=val(ERR)
|
||||
check(ratio is not None and ratio>0.05,'error ratio below threshold')
|
||||
tid,rid=traces[0]
|
||||
spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2)
|
||||
wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60)
|
||||
print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True)
|
||||
finally:
|
||||
# SQL transaction has server-side 60s bound; wait for normal COMMIT.
|
||||
try:lock.wait(timeout=75)
|
||||
except subprocess.TimeoutExpired:
|
||||
lock.terminate();lock.wait(timeout=10)
|
||||
# Keep successful business traffic in the evaluation window to prove recovery.
|
||||
t=time.monotonic()
|
||||
while time.monotonic()-t<65:
|
||||
c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
|
||||
if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break
|
||||
time.sleep(1)
|
||||
wait_alert('LabAPIHighErrorRate','inactive',seconds=40)
|
||||
print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved')
|
||||
return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]}
|
||||
|
||||
|
||||
def collector_sample():
|
||||
check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down')
|
||||
rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY)
|
||||
check(rss is not None and math.isfinite(rss) and rss>0,
|
||||
'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus')
|
||||
check(queue is not None and math.isfinite(queue) and queue>=0,
|
||||
'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics')
|
||||
check(capacity is not None and math.isfinite(capacity) and capacity==64,
|
||||
'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity))
|
||||
return rss,queue,capacity
|
||||
|
||||
|
||||
def backend_down(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
baseline=val(FAILED) or 0
|
||||
cid=compose('ps','-q','otel-collector')
|
||||
before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
|
||||
memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip())
|
||||
check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit))
|
||||
rss,queue,capacity=collector_sample()
|
||||
print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True)
|
||||
compose('stop','tempo')
|
||||
try:
|
||||
start=time.monotonic();n=0;memory=[];queues=[];failures=[]
|
||||
att,raw,tid,_=upload(c,alice,chat)
|
||||
while time.monotonic()-start<40:
|
||||
c.message(alice,chat,'telemetry-backend-offline')
|
||||
n+=1
|
||||
rss,queue,capacity=collector_sample()
|
||||
memory.append(rss);queues.append(queue)
|
||||
x=val(FAILED)
|
||||
if x is not None:failures.append(x)
|
||||
time.sleep(.3)
|
||||
verify_ready(c,bob,att,raw,seconds=180)
|
||||
check(n>0,'no successful writes during backend outage')
|
||||
check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics')
|
||||
rss,queue,capacity=collector_sample()
|
||||
memory.append(rss);queues.append(queue)
|
||||
print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True)
|
||||
after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
|
||||
check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after)
|
||||
check(max(memory)<memory_limit,f'Collector RSS {max(memory):g} bytes exceeds/equal to container limit {memory_limit} bytes')
|
||||
check(max(queues)<=capacity,'collector queue exceeds configured cap')
|
||||
result={'successful_messages_during_tempo_outage':n,'trace_id':tid,'max_sampled_collector_rss_bytes':max(memory),'collector_memory_limit_bytes':memory_limit,'rss_sample_count':len(memory),'max_export_queue':max(queues),'export_queue_capacity':capacity,'failed_span_counter_delta':max(failures)-baseline,'restart_state':after}
|
||||
finally:compose('up','-d','tempo')
|
||||
ready()
|
||||
# Confirm a fresh post-recovery trace can be retrieved.
|
||||
r=c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
|
||||
tid2=r['headers']['X-Trace-Id']
|
||||
wait_for('post-recovery trace',lambda:trace_spans(tid2),bool,seconds=90)
|
||||
result['recovered_trace_id']=tid2
|
||||
print('PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered')
|
||||
return result
|
||||
|
||||
|
||||
def metrics_snapshot(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
att,raw,_,_=upload(c,alice,chat)
|
||||
verify_ready(c,bob,att,raw,seconds=150)
|
||||
until=time.monotonic()+12
|
||||
while time.monotonic()<until:
|
||||
c.message(alice,chat,'synthetic-metric-check')
|
||||
time.sleep(5)
|
||||
# Read the real provisioned Grafana dashboard/datasource list, without
|
||||
# printing credentials or copying the response into the evidence.
|
||||
secret=compose('exec','-T','grafana','printenv','GF_SECURITY_ADMIN_PASSWORD')
|
||||
auth=base64.b64encode(('admin:'+secret).encode()).decode()
|
||||
def grafana_json(path):
|
||||
r=urllib.request.Request(GRAFANA+path,headers={'Authorization':'Basic '+auth})
|
||||
try:
|
||||
with urllib.request.urlopen(r,timeout=8) as f:return json.load(f)
|
||||
except (urllib.error.URLError,TimeoutError,ValueError):
|
||||
raise Failure('Grafana authenticated provisioning check failed; verify admin password/volume') from None
|
||||
dashboard=grafana_json('/api/dashboards/uid/lab6-messenger')['dashboard']
|
||||
check(len(dashboard['panels'])>=15,'provisioned dashboard incomplete')
|
||||
check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing')
|
||||
queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR,
|
||||
'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
||||
'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
||||
'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
||||
'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count',
|
||||
'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting',
|
||||
'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'}
|
||||
output={}
|
||||
for key,expr in queries.items():
|
||||
rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2)
|
||||
if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load')
|
||||
output[key]={'query':expr,'result':rows}
|
||||
forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'}
|
||||
for item in output.values():
|
||||
check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels')
|
||||
print('PASS metrics snapshot: all expected series present, no forbidden ID labels')
|
||||
return output
|
||||
|
||||
|
||||
def history(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
att,raw,tid,rid=upload(c,alice,chat)
|
||||
verify_ready(c,bob,att,raw,seconds=150)
|
||||
wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90)
|
||||
wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
|
||||
time.sleep(6)
|
||||
historic_time=time.time()
|
||||
expr='sum(lab_http_requests_total{route=~"/api/.*"})'
|
||||
def historic():
|
||||
rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result']
|
||||
return float(rows[0]['value'][1]) if rows else None
|
||||
before=historic();check(before is not None and before>0,'no historic metric')
|
||||
compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180)
|
||||
ready()
|
||||
wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100)
|
||||
wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100)
|
||||
after=historic();check(after==before,'historic metric changed or disappeared')
|
||||
print('PASS history: saved metrics, logs and traces survive backend restart')
|
||||
return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after}
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args()
|
||||
check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack')
|
||||
ready();c=Client(timeout=12)
|
||||
fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario]
|
||||
result=fun(c)
|
||||
file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
|
||||
print('Saved',file.relative_to(ROOT))
|
||||
|
||||
if __name__=='__main__':run(main)
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fast non-Docker checks for input contracts and cursor scoping."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from app.core import make_cursor, read_cursor, parse_limit, APIError
|
||||
from app.models import CreateUser, CreateChat, CreateMessage
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
def reject(fn):
|
||||
try:
|
||||
fn()
|
||||
except (APIError, ValidationError, ValueError):
|
||||
return
|
||||
raise AssertionError("Expected invalid input")
|
||||
|
||||
|
||||
def main():
|
||||
token = str(uuid.uuid4())
|
||||
dt = datetime.now(timezone.utc)
|
||||
cursor = make_cursor("messages", token, dt, str(uuid.uuid4()))
|
||||
assert read_cursor(cursor, "messages", token)[0] == dt
|
||||
reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4())))
|
||||
reject(lambda: read_cursor(cursor, "chats", token))
|
||||
assert parse_limit(None) == 50
|
||||
reject(lambda: parse_limit("0"))
|
||||
reject(lambda: parse_limit("101"))
|
||||
reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret"))
|
||||
reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3))
|
||||
assert CreateMessage(text="A\u0000B").text == "A\u0000B"
|
||||
reject(lambda: CreateMessage(text=" \t"))
|
||||
reject(lambda: CreateChat(title="title", member_ids=[token] * 2))
|
||||
reject(lambda: CreateChat(title="title", member_ids=[token] * 101))
|
||||
print("PASS: models, cursor scoping, limits, Unicode U+0000")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
|
||||
|
||||
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
|
||||
broker restart without a running dispatcher or worker. This differs from the
|
||||
outbox-restoration tests where dispatcher publishes *after* broker recovery.
|
||||
|
||||
failed-replay: safely inject a deterministic checksum mismatch into one fresh
|
||||
synthetic attachment's DB metadata, verify permanent failure, restore the
|
||||
original hash, replay via the operator-only CLI, and verify success plus audit.
|
||||
|
||||
staging-cleanup: verify the staged S3 object is absent after ready (with a
|
||||
matching cleared staging_key in the DB).
|
||||
|
||||
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
|
||||
run only against a disposable LAB5 environment and never concurrently.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, Failure, check, run
|
||||
from pipeline_resilience import verify_ready, wait_api
|
||||
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, expected=(0,), timeout=240):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
|
||||
)
|
||||
if proc.returncode not in expected:
|
||||
# Don't print env vars, tokens or command stdout/stderr by default.
|
||||
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
|
||||
(' '.join(args[:2]), proc.returncode))
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def pg(sql):
|
||||
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
|
||||
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
|
||||
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
|
||||
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
|
||||
|
||||
|
||||
def scalar(sql):
|
||||
return pg(sql).splitlines()[-1].strip()
|
||||
|
||||
|
||||
def wait_for(label, action, predicate, seconds=30, interval=0.5):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
last = action()
|
||||
if predicate(last):
|
||||
return last
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(interval)
|
||||
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
|
||||
|
||||
|
||||
def job_query(att_id, fields):
|
||||
uuid.UUID(att_id)
|
||||
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
|
||||
(fields, att_id))
|
||||
|
||||
|
||||
def queue_ready():
|
||||
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
|
||||
'name', 'messages_ready', 'messages_unacknowledged')
|
||||
for line in out.splitlines():
|
||||
parts = re.split(r'\s+', line.strip())
|
||||
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
|
||||
return int(parts[1]), int(parts[2])
|
||||
raise Failure('RabbitMQ durable queue missing')
|
||||
|
||||
|
||||
def make_upload(c):
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att, original = c.attachment(alice, chat)
|
||||
check(att['status'] == 'queued', 'upload must return 202 queued')
|
||||
uuid.UUID(att['id'])
|
||||
return alice, bob, att, original
|
||||
|
||||
|
||||
def wait_staging_cleared(att_id, seconds=20):
|
||||
return wait_for('staging_key cleared after ready',
|
||||
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
|
||||
lambda value: value == '<NULL>', seconds=seconds)
|
||||
|
||||
|
||||
def verify_staging_absent(att_id):
|
||||
# This executes *inside* the worker container with its normal S3 credentials.
|
||||
# Never expose S3 secrets to the host or write them to evidence files.
|
||||
code = '''import sys
|
||||
from botocore.exceptions import ClientError
|
||||
from app.attachments import s3_client, BUCKET
|
||||
key='staging/'+sys.argv[1]+'/original'
|
||||
try:
|
||||
s3_client().head_object(Bucket=BUCKET,Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
|
||||
print('ABSENT')
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
print('STILL_PRESENT')'''
|
||||
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
|
||||
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
|
||||
|
||||
|
||||
def broker_persistence(c):
|
||||
compose('stop', 'worker')
|
||||
compose('up', '-d', 'dispatcher')
|
||||
baseline = queue_ready()[0]
|
||||
alice, _bob, att, original = make_upload(c)
|
||||
print('202 queued:', att['id'], flush=True)
|
||||
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
|
||||
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
|
||||
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
|
||||
lambda n: n >= baseline + 1, seconds=15)
|
||||
compose('stop', 'dispatcher')
|
||||
before = queue_ready()[0]
|
||||
check(before >= baseline + 1, 'published message disappeared before restart')
|
||||
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
|
||||
compose('restart', 'broker')
|
||||
compose('up', '-d', '--wait', 'broker')
|
||||
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
|
||||
lambda n: n >= before, seconds=40, interval=1)
|
||||
check(scalar(sql) == 't', 'outbox publish confirmation lost')
|
||||
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
|
||||
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
|
||||
# can deliver this job to the worker.
|
||||
compose('up', '-d', 'worker')
|
||||
verify_ready(c, alice, att, original, seconds=180)
|
||||
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
|
||||
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
|
||||
|
||||
|
||||
def failed_replay(c):
|
||||
compose('stop', 'worker', 'dispatcher')
|
||||
alice, _bob, att, original = make_upload(c)
|
||||
ident = att['id']
|
||||
original_sha = hashlib.sha256(original).hexdigest()
|
||||
check(original_sha == att['sha256'], 'fixture hash mismatch')
|
||||
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
|
||||
'DB hash unexpected before controlled fault')
|
||||
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
|
||||
changed = False
|
||||
try:
|
||||
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
|
||||
(fault_sha, ident))
|
||||
check(ident in updated, 'failed to inject synthetic checksum mismatch')
|
||||
changed = True
|
||||
compose('up', '-d', 'dispatcher', 'worker')
|
||||
|
||||
def get_state():
|
||||
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
|
||||
failed = wait_for('permanent failed status', get_state,
|
||||
lambda a: a['status'] == 'failed', seconds=90)
|
||||
check(failed['error_code'] == 'invalid_image',
|
||||
'expected permanent invalid_image, got ' + str(failed['error_code']))
|
||||
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
|
||||
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
|
||||
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
|
||||
'failed job created variants')
|
||||
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
|
||||
# Repair only the generated synthetic row before explicit operator replay.
|
||||
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
|
||||
(original_sha, ident))
|
||||
check(ident in restored, 'synthetic metadata could not be restored')
|
||||
changed = False
|
||||
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
|
||||
verify_ready(c, alice, att, original, seconds=180)
|
||||
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
|
||||
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
|
||||
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
|
||||
attempt = job_query(ident, 'attempt')
|
||||
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
|
||||
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
|
||||
'replay', '--id', ident, expected=(1,))
|
||||
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
|
||||
finally:
|
||||
if changed:
|
||||
# Never leave the synthetic attachment with altered metadata.
|
||||
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
|
||||
|
||||
|
||||
def staging_cleanup(c):
|
||||
compose('up', '-d', 'dispatcher', 'worker')
|
||||
alice, _bob, att, original = make_upload(c)
|
||||
verify_ready(c, alice, att, original, seconds=180)
|
||||
wait_staging_cleared(att['id'], seconds=30)
|
||||
verify_staging_absent(att['id'])
|
||||
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--scenario', required=True,
|
||||
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
|
||||
args = parser.parse_args()
|
||||
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
|
||||
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
|
||||
c = Client('http://localhost:8080')
|
||||
try:
|
||||
wait_api()
|
||||
if args.scenario == 'broker-persistence':
|
||||
broker_persistence(c)
|
||||
elif args.scenario == 'failed-replay':
|
||||
failed_replay(c)
|
||||
else:
|
||||
staging_cleanup(c)
|
||||
finally:
|
||||
# Restore normal services even when an assertion fails. NO volume deletion.
|
||||
try:
|
||||
compose('up', '-d', '--wait', 'broker')
|
||||
compose('up', '-d', 'dispatcher', 'worker')
|
||||
except Failure:
|
||||
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator evidence for Lab5 queue durability and failure windows.
|
||||
|
||||
Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test
|
||||
preserves user/session secrets strictly in process memory and prints no tokens.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from client import Client, Failure, check, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, expected=(0,)):
|
||||
p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240)
|
||||
if p.returncode not in expected:
|
||||
raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs'
|
||||
% (' '.join(args[:2]), p.returncode))
|
||||
return p
|
||||
|
||||
|
||||
def wait_api(timeout=60):
|
||||
import urllib.request
|
||||
deadline=time.monotonic()+timeout
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as r:
|
||||
if r.status==200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise Failure('API did not become ready')
|
||||
|
||||
|
||||
def make_queued(client):
|
||||
alice,bob,eve,chat=client.bootstrap()
|
||||
att,original=client.attachment(alice,chat)
|
||||
check(att['status']=='queued','202 must return queued snapshot')
|
||||
c=client.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
|
||||
attached=client.message(alice,chat,'',attachments=[att['id']])['json']
|
||||
check(attached['attachment_ids']==[att['id']],'queued attachment cannot be linked')
|
||||
return alice,bob,att,original
|
||||
|
||||
|
||||
def verify_ready(client,actor,att,original,seconds=180):
|
||||
r=client.wait_ready(actor,att,wait_seconds=seconds)
|
||||
check(len(r['variants']) == 3, 'expected exactly three variants, no duplicate entries')
|
||||
variants={v['name']:v for v in r['variants']}
|
||||
check(set(variants)=={'thumbnail','crop','optimized'},'missing/extra variants')
|
||||
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==(256,160), 'thumbnail dimensions')
|
||||
check((variants['crop']['width'],variants['crop']['height'])==(128,128), 'crop dimensions')
|
||||
raw=client.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
|
||||
check(raw['bytes']==original,'original changed across asynchronous pipeline')
|
||||
for name in ('thumbnail','crop','optimized'):
|
||||
v=variants[name]
|
||||
response=client.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
|
||||
check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],
|
||||
'variant checksum mismatch: '+name)
|
||||
return r
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--scenario',choices=['paused','outbox-crash','worker-crash'],required=True)
|
||||
args=parser.parse_args()
|
||||
if os.getenv('LAB5_ALLOW_FAILURE_TEST')!='yes':
|
||||
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes in PowerShell to run destructive failure test')
|
||||
c=Client('http://localhost:8080')
|
||||
# All tests make a fresh attachment while dispatcher/worker are stopped.
|
||||
compose('stop','worker','dispatcher')
|
||||
try:
|
||||
alice,bob,att,original=make_queued(c)
|
||||
print('Created durable queued attachment',att['id'],flush=True)
|
||||
if args.scenario=='paused':
|
||||
compose('restart','api1','api2','broker')
|
||||
compose('up','-d','--wait','broker')
|
||||
wait_api()
|
||||
elif args.scenario=='outbox-crash':
|
||||
# Transaction committed, dispatcher crashes before publish.
|
||||
compose('run','--rm','-e','LAB5_FAULT_BEFORE_PUBLISH=once',
|
||||
'dispatcher',expected=(78,))
|
||||
print('Injected dispatcher exit 78 after committed outbox, before publish')
|
||||
else:
|
||||
compose('up','-d','dispatcher')
|
||||
# Fault after all S3 writes and before PostgreSQL result COMMIT and ack.
|
||||
compose('run','--rm','-e','LAB5_FAULT_AFTER_S3=once',
|
||||
'worker',expected=(77,))
|
||||
print('Injected worker exit 77 after S3 effects, before result/ack')
|
||||
compose('up','-d','dispatcher','worker')
|
||||
wait_api()
|
||||
ready=verify_ready(c,bob,att,original)
|
||||
check(ready['status']=='ready','attachment not ready')
|
||||
print('PASS:',args.scenario,'durable queued->ready; original/3 variants verified; no duplicates')
|
||||
finally:
|
||||
# Restore the normal topology even after an assertion failure.
|
||||
compose('up','-d','dispatcher','worker')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real Python OTLP SDK wire-format test against a local in-process receiver.
|
||||
Requires requirements.txt (does not require Docker or external services).
|
||||
This is not a Collector/backend integration test.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
|
||||
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
|
||||
from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
|
||||
from app import telemetry as ot
|
||||
|
||||
received={k:[] for k in ('/v1/traces','/v1/logs','/v1/metrics')}
|
||||
class Receiver(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
data=self.rfile.read(int(self.headers['Content-Length']))
|
||||
received[self.path].append(data)
|
||||
self.send_response(200);self.send_header('Content-Type','application/x-protobuf');self.send_header('Content-Length','0');self.end_headers()
|
||||
def log_message(self,*args): pass
|
||||
|
||||
def main():
|
||||
server=ThreadingHTTPServer(('127.0.0.1',0),Receiver)
|
||||
threading.Thread(target=server.serve_forever,daemon=True).start()
|
||||
os.environ['OTEL_ENABLED']='yes'
|
||||
os.environ['OTEL_EXPORTER_OTLP_ENDPOINT']='http://127.0.0.1:'+str(server.server_port)
|
||||
ot.init('worker')
|
||||
with ot.span('POST /api/v1/chats/{chat_id}/attachments',kind=SpanKind.SERVER) as root:
|
||||
stored=ot.inject(); trace_id=ot.trace_id()
|
||||
with ot.span('rabbitmq.publish',kind=SpanKind.PRODUCER,context=ot.extract(stored)) as producer:
|
||||
sent=ot.inject()
|
||||
with ot.span('rabbitmq.consume',kind=SpanKind.CONSUMER,context=ot.extract(sent)) as consumer:
|
||||
ot.event('job.ready',job_id='synthetic-id',attempt=2)
|
||||
ot.count('lab_jobs',{'job_type':'attachment.process.v1','outcome':'ready'})
|
||||
ot.observe('lab_job_duration',2.5,{'job_type':'attachment.process.v1','outcome':'ready'})
|
||||
ot.observe('lab_ready_duration',5.0,{'job_type':'attachment.process.v1'})
|
||||
class FakeS3:
|
||||
def put_object(self,**kw): return True
|
||||
ot.S3Tracing(FakeS3()).put_object(Key='SECRET_FILENAME_SENTINEL',Body=b'SECRET_CONTENT_SENTINEL')
|
||||
ot.flush()
|
||||
for provider in reversed(ot.providers): provider.shutdown()
|
||||
server.shutdown()
|
||||
spans=[s for raw in received['/v1/traces'] for r in ExportTraceServiceRequest.FromString(raw).resource_spans for scope in r.scope_spans for s in scope.spans]
|
||||
byname={s.name:s for s in spans}
|
||||
assert byname['rabbitmq.publish'].parent_span_id==root.get_span_context().span_id.to_bytes(8,'big')
|
||||
assert byname['rabbitmq.consume'].parent_span_id==producer.get_span_context().span_id.to_bytes(8,'big')
|
||||
assert all(s.trace_id.hex()==trace_id for s in spans)
|
||||
assert 's3.put_object' in byname
|
||||
logs=[l for raw in received['/v1/logs'] for r in ExportLogsServiceRequest.FromString(raw).resource_logs for scope in r.scope_logs for l in scope.log_records]
|
||||
assert any(l.trace_id.hex()==trace_id and l.span_id==consumer.get_span_context().span_id.to_bytes(8,'big') and l.body.string_value=='job.ready' for l in logs)
|
||||
points=[m for raw in received['/v1/metrics'] for r in ExportMetricsServiceRequest.FromString(raw).resource_metrics for scope in r.scope_metrics for m in scope.metrics]
|
||||
assert any(m.name=='lab_jobs' and m.sum.data_points[0].as_int==1 for m in points)
|
||||
assert any(m.name=='lab_job_duration' and list(m.histogram.data_points[0].explicit_bounds)==list(ot.BUCKETS) for m in points)
|
||||
for raw in sum(received.values(),[]): assert b'SECRET_FILENAME_SENTINEL' not in raw and b'SECRET_CONTENT_SENTINEL' not in raw
|
||||
print('PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes')
|
||||
|
||||
if __name__=='__main__': main()
|
||||
Reference in New Issue
Block a user