Files
mtusi2026lab6-fork/tests/observability.py

129 lines
7.2 KiB
Python

#!/usr/bin/env python3
import json
import re
import time
import uuid
from client import Client, arguments, check, run
from faults import ops, wait_s3
from obs import loki, prom, series, wait_loki, wait_trace_spans
from stack import compose, restore_worker, upload, wait_status
SIGNALS = [
'http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'jobs_processed_total',
'job_processing_duration_seconds_bucket', 'job_time_to_ready_seconds_bucket', 'queue_depth',
'queue_oldest_job_age_seconds', 'db_pool_connections', 'db_pool_max_connections', 'worker_busy',
'worker_capacity', 'process_cpu_time_seconds_total', 'process_memory_usage_bytes',
]
FORBIDDEN = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url', 'http_url', 'path', 'text'}
UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-')
def signals_and_labels():
for name in SIGNALS:
check(prom('count(%s)' % name), 'metric %s has no samples' % name)
seen = set()
for name in SIGNALS:
for s in series(name):
seen |= set(s) - {'__name__'}
check(not UUID.search(s.get('http_route', '')), 'raw identifier in route label')
check(not (seen & FORBIDDEN), 'high-cardinality labels present: %s' % sorted(seen & FORBIDDEN))
routes = {s['http_route'] for s in series('http_server_requests_total')}
check('/api/v1/chats/{chat_id}/messages' in routes, 'route template missing')
print('metrics: %d signals present, labels=%s, routes are templates (%d)' % (len(SIGNALS), sorted(seen), len(routes)))
def chain(c, alice, bob, chat):
resp = upload(c, alice, chat)
request_id = resp['headers']['X-Request-Id']
att = resp['json']
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
check(ready['status'] == 'ready', 'attachment did not become ready')
rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id)
trace_id = rows[-1][1].get('trace_id')
check(trace_id and re.fullmatch(r'[0-9a-f]{32}', trace_id), 'request log has no trace_id')
check(rows[-1][1].get('span_id'), 'request log has no span_id')
spans = wait_trace_spans(trace_id, lambda s: any(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s)
and any(x['name'] == 'db COMMIT' and x['service'] == 'messenger-worker' for x in s)
and any(x['name'] == 's3 remove' for x in s))
by_id = {s['id']: s for s in spans}
server = [s for s in spans if s['kind'] == 'SPAN_KIND_SERVER'][0]
producer = [s for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER'][0]
consumer = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'][0]
check(server['service'] == 'messenger-api' and producer['service'] == 'messenger-api', 'producer must belong to api')
check(consumer['service'] == 'messenger-worker', 'consumer must belong to worker')
check(producer['parent'] == server['id'], 'producer span is not a child of the request span')
check(consumer['parent'] == producer['id'], 'consumer span is not a child of the producer span')
check(consumer['attributes'].get('attachment.id') == att['id'], 'consumer span misses attachment.id')
names = {(s['service'], s['name']) for s in spans}
for want in [('messenger-api', 's3 put'), ('messenger-api', 'db INSERT'), ('messenger-worker', 's3 get'),
('messenger-worker', 'image render'), ('messenger-worker', 's3 put'), ('messenger-worker', 'db COMMIT'),
('messenger-worker', 's3 remove')]:
check(want in names, 'span %s missing' % (want,))
check(all(s['parent'] in by_id or not s['parent'] for s in spans), 'orphan spans in trace')
worker_logs = wait_loki('{service_name="messenger-worker"} | event="job ready" | trace_id="%s"' % trace_id)
check(worker_logs[-1][1].get('span_id'), 'worker log has no span_id')
print('chain: request_id=%s trace_id=%s spans=%d api->producer->consumer->s3/db linked, api and worker logs carry trace_id'
% (request_id, trace_id, len(spans)))
return att, trace_id
def secrets(c, alice, chat, trace_id):
marker = 'secret-body-' + uuid.uuid4().hex
resp = c.message(alice, chat, marker)
request_id = resp['headers']['X-Request-Id']
time.sleep(8)
rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id)
message_trace = rows[-1][1].get('trace_id')
raw = json.dumps(wait_trace_spans(message_trace, lambda s: len(s) > 1)) + json.dumps(wait_trace_spans(trace_id, lambda s: True))
for needle, what in ((marker, 'message body'), (alice['token'], 'session token')):
check(needle not in raw, '%s leaked into traces' % what)
check(not loki('{service_name=~"messenger-.*"} |= "%s"' % needle), '%s leaked into logs' % what)
check(not loki('{service_name=~"messenger-.*"} |~ "(?i)authorization|bearer "'), 'authorization material in logs')
print('secrets: message body and session token absent from Loki and Tempo')
def retry_trace(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
compose('stop', 's3')
try:
compose('up', '-d', '--no-deps', 'worker')
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
check(failed['status'] == 'failed', 'expected retry_exhausted while S3 is down')
finally:
compose('start', 's3')
wait_s3()
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
check(wait_status(c, bob, att['id'], ('ready', 'failed'), 120)['status'] == 'ready', 'replay did not finish')
rows = wait_loki('{service_name="messenger-worker"} | event="job ready" | attachment_id="%s"' % att['id'])
trace_id = rows[-1][1]['trace_id']
spans = wait_trace_spans(trace_id, lambda s: sum(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s) >= 6
and any(x['name'] == 'db COMMIT' for x in s))
consumers = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER']
attempts = sorted(int(s['attributes']['job.attempt']) for s in consumers)
check(attempts == list(range(1, len(attempts) + 1)), 'attempts are not sequential: %s' % attempts)
errors = [s for s in consumers if s['status'] == 'STATUS_CODE_ERROR']
check(len(errors) == 5, 'expected 5 failed attempts in the trace, got %d' % len(errors))
check(sum(1 for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER') == len(consumers), 'each attempt needs its publish span')
print('retry: one trace %s holds %d attempts (%d failed with S3 down, attempt %d ready after replay)'
% (trace_id, len(consumers), len(errors), attempts[-1]))
def main():
args = arguments('Observability signals, trace propagation, correlation and redaction')
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = c.bootstrap()
try:
att, trace_id = chain(c, alice, bob, chat)
signals_and_labels()
secrets(c, alice, chat, trace_id)
retry_trace(c, alice, bob, chat)
finally:
compose('start', 's3', check_rc=False)
restore_worker()
print('PASS: signals, producer/consumer trace chain, log correlation, retry in trace, no secrets in telemetry')
if __name__ == '__main__':
run(main)