forked from mtusi2026/mtusi2026lab6
291 lines
14 KiB
Python
291 lines
14 KiB
Python
"""Lab 6 acceptance: real signals, two incidents, retry propagation and bounded outage.
|
|
|
|
Uses only the standard library. Stops services only in this Compose project and
|
|
always starts them again; never deletes volumes. Run separately from other tests.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
PROM = 'http://127.0.0.1:9090'
|
|
LOKI = 'http://127.0.0.1:3100'
|
|
TEMPO = 'http://127.0.0.1:3200'
|
|
|
|
|
|
def stamp():
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def get(url, **params):
|
|
if params:
|
|
url += '?' + urllib.parse.urlencode(params)
|
|
with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=10) as r:
|
|
return json.load(r)
|
|
|
|
|
|
def query(expr):
|
|
data = get(PROM + '/api/v1/query', query=expr)
|
|
check(data['status'] == 'success', 'PromQL failed')
|
|
return data['data']['result']
|
|
|
|
|
|
def number(expr):
|
|
values = query(expr)
|
|
return float(values[0]['value'][1]) if values else 0.0
|
|
|
|
|
|
def wait(label, fn, timeout=90):
|
|
end = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < end:
|
|
try:
|
|
value = fn()
|
|
if value:
|
|
return value
|
|
except (urllib.error.URLError, TimeoutError, AssertionError) as exc:
|
|
last = type(exc).__name__
|
|
time.sleep(2)
|
|
raise AssertionError(f'Timeout: {label} ({last})')
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90)
|
|
|
|
|
|
def trace_data(trace_id):
|
|
data = get(TEMPO + '/api/traces/' + trace_id)
|
|
spans = []
|
|
for batch in data.get('batches', data.get('resourceSpans', [])):
|
|
resource = {a['key']: a['value'].get('stringValue') for a in batch.get('resource', {}).get('attributes', [])}
|
|
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
|
|
for span in scope.get('spans', []):
|
|
spans.append({**span, 'service': resource.get('service.name')})
|
|
return data, spans
|
|
|
|
|
|
def logs(trace_id):
|
|
return get(LOKI + '/loki/api/v1/query_range',
|
|
query='{service_name=~"messenger-.*"} | trace_id="' + trace_id + '"',
|
|
start=str(time.time_ns() - 3_600_000_000_000), limit=1000)['data']['result']
|
|
|
|
|
|
def attrs(span):
|
|
return {a['key']: next(iter(a['value'].values())) for a in span.get('attributes', [])}
|
|
|
|
|
|
def pipeline(trace_id, *, retry=False):
|
|
data, spans = trace_data(trace_id)
|
|
names = {s['name'] for s in spans}
|
|
required = {'POST /api/v1/chats/{chat_id}/attachments', 'attachment-jobs publish',
|
|
'attachment-jobs process', 'image.process', 's3.GetObject', 's3.PutObject', 'db.UPDATE'}
|
|
if not required <= names:
|
|
return None
|
|
by_id = {s['spanId']: s for s in spans}
|
|
for name, parent_name in [('attachment-jobs publish', 'POST /api/v1/chats/{chat_id}/attachments'),
|
|
('attachment-jobs process', 'attachment-jobs publish'),
|
|
('image.process', 'attachment-jobs process')]:
|
|
for s in (s for s in spans if s['name'] == name):
|
|
check(by_id.get(s.get('parentSpanId'), {}).get('name') == parent_name,
|
|
'Broken trace parent for ' + name)
|
|
check(s.get('traceState') == 'lab=check', 'W3C tracestate lost at ' + name)
|
|
attempts = sorted({int(attrs(s)['job.attempt']) for s in spans
|
|
if s['name'] == 'attachment-jobs process'})
|
|
if retry and len(attempts) < 2:
|
|
return None
|
|
records = logs(trace_id)
|
|
services = {r['stream'].get('service_name') for r in records}
|
|
if not {'messenger-api', 'messenger-worker', 'messenger-dispatcher'} <= services:
|
|
return None
|
|
return {'trace_id': trace_id, 'span_count': len(spans), 'span_names': sorted(names),
|
|
'log_services': sorted(services), 'attempts': attempts, 'parent_chain_verified': True,
|
|
'tracestate_preserved': True,
|
|
'_raw': json.dumps([data, records])}
|
|
|
|
|
|
class Scenario:
|
|
def __init__(self, base_url):
|
|
self.client = Client(base_url)
|
|
self.alice, _, _, self.chat = self.client.bootstrap()
|
|
self.private_markers = [self.alice['token'], self.alice['registration']['password']]
|
|
|
|
def upload(self):
|
|
trace_id, request_id = uuid.uuid4().hex, str(uuid.uuid4())
|
|
filename = 'private_filename_' + uuid.uuid4().hex + '.png'
|
|
self.private_markers.append(filename)
|
|
result = self.client.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=' + filename,
|
|
202, actor=self.alice, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
|
|
headers={'Content-Type': 'image/png', 'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01',
|
|
'tracestate': 'lab=check', 'X-Request-Id': request_id}, schema='Attachment')
|
|
check(result['headers']['X-Request-Id'] == request_id, 'Request ID changed')
|
|
return result['json'], trace_id, request_id
|
|
|
|
def check_pipeline(self, trace_id, request_id, retry=False):
|
|
result = wait('pipeline spans/logs', lambda: pipeline(trace_id, retry=retry))
|
|
raw = result.pop('_raw')
|
|
check(all(value not in raw for value in self.private_markers), 'Private input leaked into telemetry')
|
|
check(request_id in raw, 'Request ID is missing from trace/log correlation')
|
|
result['request_id'] = request_id
|
|
result['private_inputs_absent'] = True
|
|
return result
|
|
|
|
def business(self, expected=200, trace_id=None):
|
|
headers = {'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01'} if trace_id else {}
|
|
return self.client.call('GET', '/api/v1/chats/' + self.chat['id'], expected,
|
|
actor=self.alice, headers=headers, schema='Chat' if expected == 200 else None)
|
|
|
|
def smoke(self):
|
|
att, trace_id, request_id = self.upload()
|
|
self.client.wait_ready(self.alice, att)
|
|
result = self.check_pipeline(trace_id, request_id)
|
|
names = ['http_requests_total', 'http_duration_seconds_bucket', 'jobs_processed_total',
|
|
'job_duration_seconds_bucket', 'attachment_time_to_ready_seconds_bucket',
|
|
'queue_depth', 'queue_oldest_age_seconds', 'db_pool_used', 'db_pool_waiting',
|
|
'worker_busy', 'worker_capacity', 'container_cpu_seconds_total', 'container_memory_bytes']
|
|
for name in names:
|
|
wait(name, lambda name=name: query(name))
|
|
series = get(PROM + '/api/v1/series', **{'match[]': '{job="messenger"}', 'start': str(time.time() - 300)})['data']
|
|
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url'}
|
|
check(all(not forbidden.intersection(row) for row in series), 'High-cardinality metric label')
|
|
check(all(self.chat['id'] not in json.dumps(row) for row in series), 'Raw route ID in metrics')
|
|
result['metrics_present'] = names
|
|
return result
|
|
|
|
def queue(self):
|
|
result = {'started_at': stamp()}
|
|
compose('stop', 'worker')
|
|
try:
|
|
att, trace_id, request_id = self.upload()
|
|
self.client.message(self.alice, self.chat, 'queued attachment', attachments=[att['id']])
|
|
wait('QueueLagHigh firing', lambda: number('ALERTS{alertname="QueueLagHigh",alertstate="firing"}') == 1)
|
|
result.update(firing_at=stamp(), depth=number('max(queue_depth)'),
|
|
oldest_age_seconds=number('max(queue_oldest_age_seconds)'))
|
|
finally:
|
|
compose('start', 'worker')
|
|
ready = self.client.wait_ready(self.alice, att)
|
|
wait('queue alert resolved', lambda: not query('ALERTS{alertname="QueueLagHigh"}'))
|
|
result.update(resolved_at=stamp(), ready=ready['status'], depth_after=number('max(queue_depth)'),
|
|
pipeline=self.check_pipeline(trace_id, request_id))
|
|
return result
|
|
|
|
def database(self):
|
|
result = {'started_at': stamp()}
|
|
trace_id = uuid.uuid4().hex
|
|
compose('stop', 'postgres')
|
|
try:
|
|
errors = 0
|
|
def load_until_firing():
|
|
nonlocal errors
|
|
self.business(expected=503, trace_id=trace_id if errors == 0 else None)
|
|
errors += 1
|
|
return number('ALERTS{alertname="ApiHighErrorRate",alertstate="firing"}') == 1
|
|
wait('ApiHighErrorRate firing under sustained traffic', load_until_firing, timeout=120)
|
|
result['failed_requests'] = errors
|
|
result.update(firing_at=stamp(), error_ratio=number('messenger:http_error_ratio:2m'))
|
|
def failed_trace():
|
|
_, spans = trace_data(trace_id)
|
|
failed = [s for s in spans if s['name'].startswith('db.')
|
|
and s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')]
|
|
return [{'name': s['name'], 'error_type': attrs(s).get('error.type'),
|
|
'duration_seconds': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9}
|
|
for s in failed]
|
|
result['db_error_spans'] = wait('failed DB span', failed_trace)
|
|
result['trace_id'] = trace_id
|
|
result['correlated_log_streams'] = len(wait('DB error logs', lambda: logs(trace_id)))
|
|
finally:
|
|
compose('start', 'postgres')
|
|
wait('API recovered', self.business)
|
|
for _ in range(10):
|
|
self.business()
|
|
wait('API alert resolved', lambda: not query('ALERTS{alertname="ApiHighErrorRate"}'), timeout=160)
|
|
result['resolved_at'] = stamp()
|
|
return result
|
|
|
|
def retry(self):
|
|
failures = number('sum(jobs_processed_total{outcome="failed"})')
|
|
compose('stop', 'worker')
|
|
try:
|
|
att, trace_id, request_id = self.upload()
|
|
compose('stop', 's3')
|
|
# `compose start worker` can also start its dependency graph, including S3.
|
|
compose('up', '-d', '--no-deps', '--no-recreate', 'worker')
|
|
wait('failed processing attempt', lambda: number('sum(jobs_processed_total{outcome="failed"})') > failures, timeout=40)
|
|
finally:
|
|
compose('start', 's3', 'worker')
|
|
wait('S3/API ready', lambda: self.client.call('GET', '/health/ready', schema='Health'))
|
|
self.client.wait_ready(self.alice, att)
|
|
return self.check_pipeline(trace_id, request_id, retry=True)
|
|
|
|
def backend(self):
|
|
accepted = 'sum(otelcol_receiver_accepted_spans_total)'
|
|
sent = 'sum(otelcol_exporter_sent_spans_total{exporter="otlphttp/tempo"})'
|
|
queue = 'sum(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
|
|
# A quiet interval on both ends makes accepted-minus-sent a loss count,
|
|
# not a count of failed export attempts or currently buffered spans.
|
|
time.sleep(8)
|
|
wait('trace queue drained', lambda: number(queue) == 0)
|
|
before = {'accepted': number(accepted), 'sent': number(sent)}
|
|
result = {'started_at': stamp(), 'before': before}
|
|
samples, writes = [], 0
|
|
compose('stop', 'tempo')
|
|
try:
|
|
end = time.monotonic() + 60
|
|
att, _, _ = self.upload()
|
|
self.client.wait_ready(self.alice, att)
|
|
while time.monotonic() < end:
|
|
body = 'private_message_' + uuid.uuid4().hex
|
|
self.private_markers.append(body)
|
|
self.client.message(self.alice, self.chat, body)
|
|
writes += 1
|
|
samples.append({'at': stamp(), 'collector_rss_bytes': number('otelcol_process_memory_rss_bytes'),
|
|
'queue_batches': number(queue)})
|
|
time.sleep(0.3)
|
|
check(writes > 10, 'Not enough writes during telemetry outage')
|
|
check(max(s['queue_batches'] for s in samples) <= 32, 'Collector queue is unbounded')
|
|
check(0 < max(s['collector_rss_bytes'] for s in samples) < 256 * 1024 * 1024,
|
|
'Collector exceeded its memory budget')
|
|
finally:
|
|
compose('start', 'tempo')
|
|
wait('trace queue drained', lambda: number(queue) == 0)
|
|
time.sleep(10) # Include final Prometheus scrape after drain.
|
|
after = {'accepted': number(accepted), 'sent': number(sent)}
|
|
lost = (after['accepted'] - before['accepted']) - (after['sent'] - before['sent'])
|
|
check(lost >= 0, 'Loss window was not quiescent; rerun backend separately')
|
|
result.update(after=after, lost_accepted_spans=lost, successful_writes=writes,
|
|
attachment_ready_during_outage=True, samples=samples, recovered_at=stamp())
|
|
att, trace_id, request_id = self.upload()
|
|
self.client.wait_ready(self.alice, att)
|
|
result['recovered_pipeline'] = self.check_pipeline(trace_id, request_id)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--scenario', choices=['smoke', 'queue', 'database', 'retry', 'backend', 'all'], default='smoke')
|
|
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
|
args = parser.parse_args()
|
|
scenario = Scenario(args.base_url)
|
|
path = ROOT / 'evidence/observability.json'
|
|
evidence = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {}
|
|
for name in (['smoke', 'queue', 'database', 'retry', 'backend'] if args.scenario == 'all' else [args.scenario]):
|
|
print('Running ' + name, flush=True)
|
|
evidence[name] = {'checked_at': stamp(), **getattr(scenario, name)()}
|
|
path.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
|
|
print('PASS: ' + name, flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|