"""Short real-backend check; optional bounded incidents for the grade-4 evidence.""" import json import os import subprocess import time import urllib.parse import urllib.request import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path from client import Client, Failure, arguments, check, run ROOT=Path(__file__).resolve().parents[1] EVIDENCE=ROOT/'evidence' def get(url): request=urllib.request.Request(url,headers={'Accept':'application/json'}) with urllib.request.urlopen(request,timeout=5) as response: return json.load(response) def wait(fn, label, seconds=35): deadline=time.monotonic()+seconds error=None while time.monotonic()0,'transient retry',25) compose('unpause','minio');minio_paused=False wait(lambda:c.call('GET','/health/ready')['status']==200,'API restored') c.wait_ready(alice,queued) def retry_chain(): d=trace_data(retry_trace) return d if sum(span['name']=='queue.consume' for span in spans(d))>=2 else None retry_data=wait(retry_chain,'retry chain') (EVIDENCE/'lab6-retry-trace.json').write_text(json.dumps(retry_data,indent=2)) record('retry_keeps_upload_trace',trace_id=retry_trace,consumer_spans=sum(span['name']=='queue.consume' for span in spans(retry_data))) wait(lambda:value('max(queue_depth)')==0 and value('max(ALERTS{alertname="QueueLagGrowing",alertstate="firing"})')==0,'queue alert resolved') record('incident_worker_resolved') # Block INSERT while SELECT readiness remains available. This isolates # DB latency from application CPU/HTTP handler latency. api=compose('ps','-q','api').split()[0] code='''import asyncio,asyncpg,sys from src.core.settings import settings async def main(): c=await asyncpg.connect(settings.database_url.get_secret_value()) await c.execute('BEGIN; LOCK TABLE messages IN SHARE MODE') print('LOCKED',flush=True) await asyncio.to_thread(sys.stdin.readline) await c.execute('ROLLBACK');await c.close() asyncio.run(main())''' locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',code],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True) check(locker.stdout.readline().strip()=='LOCKED','DB fault not active') def failing(_): return Client(c.base,timeout=12).call('POST','/api/v1/chats/'+chat['id']+'/messages',503, actor=alice,body={'text':'bounded DB incident'},headers={'Idempotency-Key':uuid.uuid4().hex}) with ThreadPoolExecutor(max_workers=4) as pool:errors=list(pool.map(failing,range(12))) wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==1,'error alert firing',20) db_trace=wait(lambda: trace_data(errors[0]['headers']['X-Trace-Id']),'DB trace') failures=[span for span in spans(db_trace) if span['name'].startswith('db.') and span.get('status',{}).get('code') in (2,'STATUS_CODE_ERROR')] check(failures,'DB error span missing') db_logs=wait(lambda:logs(event='db.error'),'DB error logs') (EVIDENCE/'lab6-db-incident-trace.json').write_text(json.dumps(db_trace,indent=2)) (EVIDENCE/'lab6-db-incident-logs.json').write_text(json.dumps(db_logs,indent=2)) record('incident_DB_firing',errors=len(errors),db_error_spans=len(failures), diagnosis='db.INSERT timeout, matching db.error event; API readiness remains available') locker.stdin.write('release\n');locker.stdin.flush();locker.wait(timeout=10);locker=None c.message(alice,chat,'DB recovered') wait(lambda:value('max(ALERTS{alertname="ApiHighErrorRate",alertstate="firing"})')==0,'error alert resolved',40) record('incident_DB_resolved') compose('stop','-t','2','tempo');tempo_stopped=True started=time.monotonic() for _ in range(15):c.message(alice,chat,'telemetry outage') other=c.attachment(alice,chat)[0];c.wait_ready(alice,other) # Bounded retry is 10s; wait only for an observed failure, not a benchmark. wait(lambda:value('sum(otelcol_exporter_send_failed_spans_total)')>0 or value('sum(otelcol_exporter_send_failed_spans)')>0,'export failure counter',20) usage=json.loads(compose('exec','-T','api','python','-c','from pathlib import Path;import json;print(json.dumps({"bytes":int(Path("/sys/fs/cgroup/memory.current").read_text()),"limit":int(Path("/sys/fs/cgroup/memory.max").read_text())}))').splitlines()[0]) check(usage['bytes']