forked from mtusi2026/mtusi2026lab6
129 lines
7.2 KiB
Python
129 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import re
|
|
import time
|
|
import uuid
|
|
from client import Client, arguments, check, run
|
|
from faults import ops, wait_s3
|
|
from obs import loki, prom, series, wait_loki, wait_trace_spans
|
|
from stack import compose, restore_worker, upload, wait_status
|
|
|
|
SIGNALS = [
|
|
'http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'jobs_processed_total',
|
|
'job_processing_duration_seconds_bucket', 'job_time_to_ready_seconds_bucket', 'queue_depth',
|
|
'queue_oldest_job_age_seconds', 'db_pool_connections', 'db_pool_max_connections', 'worker_busy',
|
|
'worker_capacity', 'process_cpu_time_seconds_total', 'process_memory_usage_bytes',
|
|
]
|
|
FORBIDDEN = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url', 'http_url', 'path', 'text'}
|
|
UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-')
|
|
|
|
|
|
def signals_and_labels():
|
|
for name in SIGNALS:
|
|
check(prom('count(%s)' % name), 'metric %s has no samples' % name)
|
|
seen = set()
|
|
for name in SIGNALS:
|
|
for s in series(name):
|
|
seen |= set(s) - {'__name__'}
|
|
check(not UUID.search(s.get('http_route', '')), 'raw identifier in route label')
|
|
check(not (seen & FORBIDDEN), 'high-cardinality labels present: %s' % sorted(seen & FORBIDDEN))
|
|
routes = {s['http_route'] for s in series('http_server_requests_total')}
|
|
check('/api/v1/chats/{chat_id}/messages' in routes, 'route template missing')
|
|
print('metrics: %d signals present, labels=%s, routes are templates (%d)' % (len(SIGNALS), sorted(seen), len(routes)))
|
|
|
|
|
|
def chain(c, alice, bob, chat):
|
|
resp = upload(c, alice, chat)
|
|
request_id = resp['headers']['X-Request-Id']
|
|
att = resp['json']
|
|
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
|
check(ready['status'] == 'ready', 'attachment did not become ready')
|
|
rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id)
|
|
trace_id = rows[-1][1].get('trace_id')
|
|
check(trace_id and re.fullmatch(r'[0-9a-f]{32}', trace_id), 'request log has no trace_id')
|
|
check(rows[-1][1].get('span_id'), 'request log has no span_id')
|
|
spans = wait_trace_spans(trace_id, lambda s: any(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s)
|
|
and any(x['name'] == 'db COMMIT' and x['service'] == 'messenger-worker' for x in s)
|
|
and any(x['name'] == 's3 remove' for x in s))
|
|
by_id = {s['id']: s for s in spans}
|
|
server = [s for s in spans if s['kind'] == 'SPAN_KIND_SERVER'][0]
|
|
producer = [s for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER'][0]
|
|
consumer = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER'][0]
|
|
check(server['service'] == 'messenger-api' and producer['service'] == 'messenger-api', 'producer must belong to api')
|
|
check(consumer['service'] == 'messenger-worker', 'consumer must belong to worker')
|
|
check(producer['parent'] == server['id'], 'producer span is not a child of the request span')
|
|
check(consumer['parent'] == producer['id'], 'consumer span is not a child of the producer span')
|
|
check(consumer['attributes'].get('attachment.id') == att['id'], 'consumer span misses attachment.id')
|
|
names = {(s['service'], s['name']) for s in spans}
|
|
for want in [('messenger-api', 's3 put'), ('messenger-api', 'db INSERT'), ('messenger-worker', 's3 get'),
|
|
('messenger-worker', 'image render'), ('messenger-worker', 's3 put'), ('messenger-worker', 'db COMMIT'),
|
|
('messenger-worker', 's3 remove')]:
|
|
check(want in names, 'span %s missing' % (want,))
|
|
check(all(s['parent'] in by_id or not s['parent'] for s in spans), 'orphan spans in trace')
|
|
worker_logs = wait_loki('{service_name="messenger-worker"} | event="job ready" | trace_id="%s"' % trace_id)
|
|
check(worker_logs[-1][1].get('span_id'), 'worker log has no span_id')
|
|
print('chain: request_id=%s trace_id=%s spans=%d api->producer->consumer->s3/db linked, api and worker logs carry trace_id'
|
|
% (request_id, trace_id, len(spans)))
|
|
return att, trace_id
|
|
|
|
|
|
def secrets(c, alice, chat, trace_id):
|
|
marker = 'secret-body-' + uuid.uuid4().hex
|
|
resp = c.message(alice, chat, marker)
|
|
request_id = resp['headers']['X-Request-Id']
|
|
time.sleep(8)
|
|
rows = wait_loki('{service_name="messenger-api"} | event="request" | request_id="%s"' % request_id)
|
|
message_trace = rows[-1][1].get('trace_id')
|
|
raw = json.dumps(wait_trace_spans(message_trace, lambda s: len(s) > 1)) + json.dumps(wait_trace_spans(trace_id, lambda s: True))
|
|
for needle, what in ((marker, 'message body'), (alice['token'], 'session token')):
|
|
check(needle not in raw, '%s leaked into traces' % what)
|
|
check(not loki('{service_name=~"messenger-.*"} |= "%s"' % needle), '%s leaked into logs' % what)
|
|
check(not loki('{service_name=~"messenger-.*"} |~ "(?i)authorization|bearer "'), 'authorization material in logs')
|
|
print('secrets: message body and session token absent from Loki and Tempo')
|
|
|
|
|
|
def retry_trace(c, alice, bob, chat):
|
|
compose('stop', 'worker')
|
|
att = upload(c, alice, chat)['json']
|
|
compose('stop', 's3')
|
|
try:
|
|
compose('up', '-d', '--no-deps', 'worker')
|
|
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
|
|
check(failed['status'] == 'failed', 'expected retry_exhausted while S3 is down')
|
|
finally:
|
|
compose('start', 's3')
|
|
wait_s3()
|
|
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
|
|
check(wait_status(c, bob, att['id'], ('ready', 'failed'), 120)['status'] == 'ready', 'replay did not finish')
|
|
rows = wait_loki('{service_name="messenger-worker"} | event="job ready" | attachment_id="%s"' % att['id'])
|
|
trace_id = rows[-1][1]['trace_id']
|
|
spans = wait_trace_spans(trace_id, lambda s: sum(x['kind'] == 'SPAN_KIND_CONSUMER' for x in s) >= 6
|
|
and any(x['name'] == 'db COMMIT' for x in s))
|
|
consumers = [s for s in spans if s['kind'] == 'SPAN_KIND_CONSUMER']
|
|
attempts = sorted(int(s['attributes']['job.attempt']) for s in consumers)
|
|
check(attempts == list(range(1, len(attempts) + 1)), 'attempts are not sequential: %s' % attempts)
|
|
errors = [s for s in consumers if s['status'] == 'STATUS_CODE_ERROR']
|
|
check(len(errors) == 5, 'expected 5 failed attempts in the trace, got %d' % len(errors))
|
|
check(sum(1 for s in spans if s['kind'] == 'SPAN_KIND_PRODUCER') == len(consumers), 'each attempt needs its publish span')
|
|
print('retry: one trace %s holds %d attempts (%d failed with S3 down, attempt %d ready after replay)'
|
|
% (trace_id, len(consumers), len(errors), attempts[-1]))
|
|
|
|
|
|
def main():
|
|
args = arguments('Observability signals, trace propagation, correlation and redaction')
|
|
c = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
try:
|
|
att, trace_id = chain(c, alice, bob, chat)
|
|
signals_and_labels()
|
|
secrets(c, alice, chat, trace_id)
|
|
retry_trace(c, alice, bob, chat)
|
|
finally:
|
|
compose('start', 's3', check_rc=False)
|
|
restore_worker()
|
|
print('PASS: signals, producer/consumer trace chain, log correlation, retry in trace, no secrets in telemetry')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|