Files
2026-11-16 16:08:15 +03:00

291 lines
14 KiB
Python

"""Lab 6 acceptance: real signals, two incidents, retry propagation and bounded outage.
Uses only the standard library. Stops services only in this Compose project and
always starts them again; never deletes volumes. Run separately from other tests.
"""
import argparse
import json
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
PROM = 'http://127.0.0.1:9090'
LOKI = 'http://127.0.0.1:3100'
TEMPO = 'http://127.0.0.1:3200'
def stamp():
return datetime.now(timezone.utc).isoformat()
def get(url, **params):
if params:
url += '?' + urllib.parse.urlencode(params)
with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=10) as r:
return json.load(r)
def query(expr):
data = get(PROM + '/api/v1/query', query=expr)
check(data['status'] == 'success', 'PromQL failed')
return data['data']['result']
def number(expr):
values = query(expr)
return float(values[0]['value'][1]) if values else 0.0
def wait(label, fn, timeout=90):
end = time.monotonic() + timeout
last = None
while time.monotonic() < end:
try:
value = fn()
if value:
return value
except (urllib.error.URLError, TimeoutError, AssertionError) as exc:
last = type(exc).__name__
time.sleep(2)
raise AssertionError(f'Timeout: {label} ({last})')
def compose(*args):
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90)
def trace_data(trace_id):
data = get(TEMPO + '/api/traces/' + trace_id)
spans = []
for batch in data.get('batches', data.get('resourceSpans', [])):
resource = {a['key']: a['value'].get('stringValue') for a in batch.get('resource', {}).get('attributes', [])}
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
for span in scope.get('spans', []):
spans.append({**span, 'service': resource.get('service.name')})
return data, spans
def logs(trace_id):
return get(LOKI + '/loki/api/v1/query_range',
query='{service_name=~"messenger-.*"} | trace_id="' + trace_id + '"',
start=str(time.time_ns() - 3_600_000_000_000), limit=1000)['data']['result']
def attrs(span):
return {a['key']: next(iter(a['value'].values())) for a in span.get('attributes', [])}
def pipeline(trace_id, *, retry=False):
data, spans = trace_data(trace_id)
names = {s['name'] for s in spans}
required = {'POST /api/v1/chats/{chat_id}/attachments', 'attachment-jobs publish',
'attachment-jobs process', 'image.process', 's3.GetObject', 's3.PutObject', 'db.UPDATE'}
if not required <= names:
return None
by_id = {s['spanId']: s for s in spans}
for name, parent_name in [('attachment-jobs publish', 'POST /api/v1/chats/{chat_id}/attachments'),
('attachment-jobs process', 'attachment-jobs publish'),
('image.process', 'attachment-jobs process')]:
for s in (s for s in spans if s['name'] == name):
check(by_id.get(s.get('parentSpanId'), {}).get('name') == parent_name,
'Broken trace parent for ' + name)
check(s.get('traceState') == 'lab=check', 'W3C tracestate lost at ' + name)
attempts = sorted({int(attrs(s)['job.attempt']) for s in spans
if s['name'] == 'attachment-jobs process'})
if retry and len(attempts) < 2:
return None
records = logs(trace_id)
services = {r['stream'].get('service_name') for r in records}
if not {'messenger-api', 'messenger-worker', 'messenger-dispatcher'} <= services:
return None
return {'trace_id': trace_id, 'span_count': len(spans), 'span_names': sorted(names),
'log_services': sorted(services), 'attempts': attempts, 'parent_chain_verified': True,
'tracestate_preserved': True,
'_raw': json.dumps([data, records])}
class Scenario:
def __init__(self, base_url):
self.client = Client(base_url)
self.alice, _, _, self.chat = self.client.bootstrap()
self.private_markers = [self.alice['token'], self.alice['registration']['password']]
def upload(self):
trace_id, request_id = uuid.uuid4().hex, str(uuid.uuid4())
filename = 'private_filename_' + uuid.uuid4().hex + '.png'
self.private_markers.append(filename)
result = self.client.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=' + filename,
202, actor=self.alice, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
headers={'Content-Type': 'image/png', 'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01',
'tracestate': 'lab=check', 'X-Request-Id': request_id}, schema='Attachment')
check(result['headers']['X-Request-Id'] == request_id, 'Request ID changed')
return result['json'], trace_id, request_id
def check_pipeline(self, trace_id, request_id, retry=False):
result = wait('pipeline spans/logs', lambda: pipeline(trace_id, retry=retry))
raw = result.pop('_raw')
check(all(value not in raw for value in self.private_markers), 'Private input leaked into telemetry')
check(request_id in raw, 'Request ID is missing from trace/log correlation')
result['request_id'] = request_id
result['private_inputs_absent'] = True
return result
def business(self, expected=200, trace_id=None):
headers = {'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01'} if trace_id else {}
return self.client.call('GET', '/api/v1/chats/' + self.chat['id'], expected,
actor=self.alice, headers=headers, schema='Chat' if expected == 200 else None)
def smoke(self):
att, trace_id, request_id = self.upload()
self.client.wait_ready(self.alice, att)
result = self.check_pipeline(trace_id, request_id)
names = ['http_requests_total', 'http_duration_seconds_bucket', 'jobs_processed_total',
'job_duration_seconds_bucket', 'attachment_time_to_ready_seconds_bucket',
'queue_depth', 'queue_oldest_age_seconds', 'db_pool_used', 'db_pool_waiting',
'worker_busy', 'worker_capacity', 'container_cpu_seconds_total', 'container_memory_bytes']
for name in names:
wait(name, lambda name=name: query(name))
series = get(PROM + '/api/v1/series', **{'match[]': '{job="messenger"}', 'start': str(time.time() - 300)})['data']
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url'}
check(all(not forbidden.intersection(row) for row in series), 'High-cardinality metric label')
check(all(self.chat['id'] not in json.dumps(row) for row in series), 'Raw route ID in metrics')
result['metrics_present'] = names
return result
def queue(self):
result = {'started_at': stamp()}
compose('stop', 'worker')
try:
att, trace_id, request_id = self.upload()
self.client.message(self.alice, self.chat, 'queued attachment', attachments=[att['id']])
wait('QueueLagHigh firing', lambda: number('ALERTS{alertname="QueueLagHigh",alertstate="firing"}') == 1)
result.update(firing_at=stamp(), depth=number('max(queue_depth)'),
oldest_age_seconds=number('max(queue_oldest_age_seconds)'))
finally:
compose('start', 'worker')
ready = self.client.wait_ready(self.alice, att)
wait('queue alert resolved', lambda: not query('ALERTS{alertname="QueueLagHigh"}'))
result.update(resolved_at=stamp(), ready=ready['status'], depth_after=number('max(queue_depth)'),
pipeline=self.check_pipeline(trace_id, request_id))
return result
def database(self):
result = {'started_at': stamp()}
trace_id = uuid.uuid4().hex
compose('stop', 'postgres')
try:
errors = 0
def load_until_firing():
nonlocal errors
self.business(expected=503, trace_id=trace_id if errors == 0 else None)
errors += 1
return number('ALERTS{alertname="ApiHighErrorRate",alertstate="firing"}') == 1
wait('ApiHighErrorRate firing under sustained traffic', load_until_firing, timeout=120)
result['failed_requests'] = errors
result.update(firing_at=stamp(), error_ratio=number('messenger:http_error_ratio:2m'))
def failed_trace():
_, spans = trace_data(trace_id)
failed = [s for s in spans if s['name'].startswith('db.')
and s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')]
return [{'name': s['name'], 'error_type': attrs(s).get('error.type'),
'duration_seconds': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9}
for s in failed]
result['db_error_spans'] = wait('failed DB span', failed_trace)
result['trace_id'] = trace_id
result['correlated_log_streams'] = len(wait('DB error logs', lambda: logs(trace_id)))
finally:
compose('start', 'postgres')
wait('API recovered', self.business)
for _ in range(10):
self.business()
wait('API alert resolved', lambda: not query('ALERTS{alertname="ApiHighErrorRate"}'), timeout=160)
result['resolved_at'] = stamp()
return result
def retry(self):
failures = number('sum(jobs_processed_total{outcome="failed"})')
compose('stop', 'worker')
try:
att, trace_id, request_id = self.upload()
compose('stop', 's3')
# `compose start worker` can also start its dependency graph, including S3.
compose('up', '-d', '--no-deps', '--no-recreate', 'worker')
wait('failed processing attempt', lambda: number('sum(jobs_processed_total{outcome="failed"})') > failures, timeout=40)
finally:
compose('start', 's3', 'worker')
wait('S3/API ready', lambda: self.client.call('GET', '/health/ready', schema='Health'))
self.client.wait_ready(self.alice, att)
return self.check_pipeline(trace_id, request_id, retry=True)
def backend(self):
accepted = 'sum(otelcol_receiver_accepted_spans_total)'
sent = 'sum(otelcol_exporter_sent_spans_total{exporter="otlphttp/tempo"})'
queue = 'sum(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
# A quiet interval on both ends makes accepted-minus-sent a loss count,
# not a count of failed export attempts or currently buffered spans.
time.sleep(8)
wait('trace queue drained', lambda: number(queue) == 0)
before = {'accepted': number(accepted), 'sent': number(sent)}
result = {'started_at': stamp(), 'before': before}
samples, writes = [], 0
compose('stop', 'tempo')
try:
end = time.monotonic() + 60
att, _, _ = self.upload()
self.client.wait_ready(self.alice, att)
while time.monotonic() < end:
body = 'private_message_' + uuid.uuid4().hex
self.private_markers.append(body)
self.client.message(self.alice, self.chat, body)
writes += 1
samples.append({'at': stamp(), 'collector_rss_bytes': number('otelcol_process_memory_rss_bytes'),
'queue_batches': number(queue)})
time.sleep(0.3)
check(writes > 10, 'Not enough writes during telemetry outage')
check(max(s['queue_batches'] for s in samples) <= 32, 'Collector queue is unbounded')
check(0 < max(s['collector_rss_bytes'] for s in samples) < 256 * 1024 * 1024,
'Collector exceeded its memory budget')
finally:
compose('start', 'tempo')
wait('trace queue drained', lambda: number(queue) == 0)
time.sleep(10) # Include final Prometheus scrape after drain.
after = {'accepted': number(accepted), 'sent': number(sent)}
lost = (after['accepted'] - before['accepted']) - (after['sent'] - before['sent'])
check(lost >= 0, 'Loss window was not quiescent; rerun backend separately')
result.update(after=after, lost_accepted_spans=lost, successful_writes=writes,
attachment_ready_during_outage=True, samples=samples, recovered_at=stamp())
att, trace_id, request_id = self.upload()
self.client.wait_ready(self.alice, att)
result['recovered_pipeline'] = self.check_pipeline(trace_id, request_id)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--scenario', choices=['smoke', 'queue', 'database', 'retry', 'backend', 'all'], default='smoke')
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
args = parser.parse_args()
scenario = Scenario(args.base_url)
path = ROOT / 'evidence/observability.json'
evidence = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {}
for name in (['smoke', 'queue', 'database', 'retry', 'backend'] if args.scenario == 'all' else [args.scenario]):
print('Running ' + name, flush=True)
evidence[name] = {'checked_at': stamp(), **getattr(scenario, name)()}
path.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
print('PASS: ' + name, flush=True)
if __name__ == '__main__':
main()