Files
mtusi2026lab6-fork/student-tests/observability.py
T
2026-11-19 19:24:45 +03:00

356 lines
17 KiB
Python

"""Black-box checks on the dedicated, local Lab6 Compose project."""
import argparse
import base64
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
from urllib.error import HTTPError
from urllib.parse import urlencode, urlsplit
from urllib.request import urlopen
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def local_port(name, default):
return int(os.environ.get(name, default))
BASE = os.environ.get('BASE_URL', f'http://127.0.0.1:{local_port("APP_PORT", 8080)}')
PROM = f'http://127.0.0.1:{local_port("PROMETHEUS_PORT", 9090)}'
LOKI = f'http://127.0.0.1:{local_port("LOKI_PORT", 3100)}'
TEMPO = f'http://127.0.0.1:{local_port("TEMPO_PORT", 3200)}'
GRAFANA = f'http://127.0.0.1:{local_port("GRAFANA_PORT", 3000)}'
COMPOSE = ['docker', 'compose', '--project-directory', str(ROOT)]
FIXTURE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
def compose(*args):
return subprocess.run([*COMPOSE, *args], cwd=ROOT, check=True, capture_output=True,
text=True, encoding='utf-8', errors='replace', timeout=60).stdout.strip()
def backend(base, path, **params):
query = '?' + urlencode(params) if params else ''
with urlopen(base + path + query, timeout=5) as response:
return json.load(response)
def tempo_ready():
with urlopen(TEMPO + '/ready', timeout=5) as response:
return response.status == 200
def prometheus(expr, at=None):
params = {'query': expr}
if at is not None:
params['time'] = at
vector = backend(PROM, '/api/v1/query', **params)['data']['result']
return [{**item['metric'], 'value': float(item['value'][1])} for item in vector]
def wait_for(probe, message, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
result = probe()
if result:
return result
except (HTTPError, OSError):
pass
time.sleep(1)
raise AssertionError(message)
def save(output, name, result):
output.mkdir(parents=True, exist_ok=True)
(output / name).write_text(json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + '\n',
encoding='utf-8')
def lab_only():
check(urlsplit(BASE).hostname in {'localhost', '127.0.0.1', '::1'}, 'use the dedicated local lab')
project = json.loads(compose('config', '--format', 'json'))['name']
check(project.startswith('mtusi-lab6'), 'Compose project name must begin with mtusi-lab6')
for service in ('worker', 'dispatcher', 'db', 'tempo', 'otel-collector'):
check(bool(compose('ps', '-q', '--status', 'running', service)), f'{service} must be running')
def actor_and_chat(client, label):
actor = client.user(label)
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Синтетическая Lab6', 'member_ids': [actor['id']]}, schema='Chat')['json']
return actor, chat
def upload(client, actor, chat, filename='synthetic.png'):
return client.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename={filename}', 202,
actor=actor, raw=FIXTURE, headers={'Content-Type': 'image/png'}, schema='Attachment')
def loki_records(expression, lookback=300):
now = time.time_ns()
data = backend(LOKI, '/loki/api/v1/query_range', query=expression, start=now - lookback * 10**9,
end=now, limit=100)['data']['result']
result = []
for stream in data:
for timestamp, line, *extra in stream['values']:
metadata = extra[0] if extra else {}
result.append({**stream['stream'], **metadata, 'timestamp_ns': timestamp, 'body': line})
return result
def api_log(request_id, lookback=300):
logs = loki_records(f'{{service_name="messenger-api"}} | request_id="{request_id}"', lookback)
return logs[0] if logs else None
def otlp_value(item):
return next(iter(item.values()))
def span_id(value):
if not value or re.fullmatch('[0-9a-f]{16}', value):
return value
return base64.b64decode(value, validate=True).hex()
def trace_by_id(trace_id):
data = backend(TEMPO, '/api/traces/' + trace_id)
spans = []
for batch in data.get('batches', data.get('resourceSpans', [])):
resource = {a['key']: otlp_value(a['value']) for a in batch['resource']['attributes']}
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
for span in scope['spans']:
spans.append({
'name': span['name'], 'service': resource['service.name'],
'service_version': resource['service.version'],
'environment': resource['deployment.environment.name'],
'id': span_id(span['spanId']), 'parent': span_id(span.get('parentSpanId')),
'kind': span['kind'], 'status': span.get('status', {}),
'duration_ms': round((int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6, 3),
'attributes': {a['key']: otlp_value(a['value']) for a in span.get('attributes', [])},
})
return spans
def assert_privacy(records, secrets):
serialized = json.dumps(records, ensure_ascii=False)
check(not any(secret in serialized for secret in secrets), 'unexpected synthetic sensitive value in telemetry')
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename',
'url', 'url_full', 'url_path', 'text', 'trace_id', 'span_id'}
sample = prometheus('count by (__name__) ({__name__=~"messenger_.+"})')
check(bool(sample), 'no application metric series')
series = backend(PROM, '/api/v1/series', **{'match[]': '{__name__=~"messenger_.+"}'})['data']
check(not any(forbidden & entry.keys() for entry in series), 'forbidden metric label')
return len(series)
def correlation(output):
client = Client(BASE)
actor, chat = actor_and_chat(client, 'trace')
canary = 'private-' + uuid4().hex
message = client.message(actor, chat, canary)
compose('stop', 'worker')
extra_worker = 'mtusi-lab6-trace-' + uuid4().hex[:8]
try:
received = upload(client, actor, chat, canary + '.png')
attachment = received['json']['id']
compose('run', '-d', '--no-deps', '--name', extra_worker, '-e', 'LAB5_FAULT=temporary_once',
'-e', 'LAB5_FAULT_ATTACHMENT=' + attachment, 'worker')
wait_for(lambda: client.call('GET', '/api/v1/attachments/' + attachment, actor=actor,
schema='Attachment')['json']['status'] == 'ready',
'retry did not produce a ready result')
request_id = received['headers']['X-Request-Id']
log = wait_for(lambda: api_log(request_id), 'no upload log in Loki')
trace_id = log['trace_id']
check(bool(re.fullmatch('[0-9a-f]{32}', trace_id)), 'invalid Loki trace ID')
logs = wait_for(lambda: enough_attempts(trace_id), 'no log for both worker attempts')
spans = wait_for(lambda: enough_spans(trace_id), 'missing worker trace or S3 spans')
server = next(span for span in spans if span['name'].startswith('POST /api/v1/chats/'))
check(log['span_id'] == server['id'], 'API log has the wrong span')
producers = {int(s['attributes']['job.attempt']): s for s in spans if s['name'].endswith(' publish')}
consumers = {int(s['attributes']['job.attempt']): s for s in spans if s['name'].endswith(' process')}
check(set(producers) == set(consumers) == {1, 2}, 'expected one failed attempt and one retry')
for attempt in (1, 2):
check(producers[attempt]['parent'] == server['id'], 'producer parent is not the upload')
check(consumers[attempt]['parent'] == producers[attempt]['id'], 'consumer parent is not the producer')
check(any(s['span_id'] == consumers[attempt]['id'] for s in logs), 'worker log span mismatch')
check({log['outcome'] for log in logs} == {'queued', 'ready'}, 'missing retry/ready outcomes')
check(any(s['name'] == 'postgresql query' and s['parent'] == consumers[2]['id'] for s in spans),
'missing SQL span for the successful worker attempt')
message_log = wait_for(lambda: api_log(message['headers']['X-Request-Id']), 'missing message log')
message_spans = wait_for(lambda: trace_by_id(message_log['trace_id']), 'missing message trace')
series = assert_privacy([log, logs, spans, message_log, message_spans],
[canary, actor['registration']['password'], actor['token']])
dashboard = backend(GRAFANA, '/api/dashboards/uid/messenger-lab6')['dashboard']
sources = {ds['uid'] for ds in backend(GRAFANA, '/api/datasources')}
check({'lab-prom', 'lab-loki', 'lab-tempo'} <= sources, 'missing provisioned datasource')
save(output, 'correlation.json', {
'trace_id': trace_id, 'request_id': request_id, 'logs': [log, *logs], 'spans': spans,
'privacy': {'synthetic_canaries_absent': True, 'metric_series_checked': series},
'grafana': {'dashboard_uid': dashboard['uid'], 'panels': len(dashboard['panels']),
'datasources': sorted(sources)},
})
finally:
subprocess.run(['docker', 'stop', '-t', '5', extra_worker], capture_output=True, timeout=20)
subprocess.run(['docker', 'rm', extra_worker], capture_output=True, timeout=20)
compose('start', 'worker')
def enough_attempts(trace_id):
logs = loki_records(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="image.attempt"')
return logs if len(logs) >= 2 else None
def enough_spans(trace_id):
spans = trace_by_id(trace_id)
return spans if sum(span['name'] == 's3 put result' for span in spans) == 4 else None
def queue_restart(output):
def stream_entry(attachment):
raw = compose('exec', '-T', 'broker', 'sh', '-c',
'REDISCLI_AUTH="$(cat /run/secrets/broker_password)" '
'exec valkey-cli --json XRANGE attachment-jobs - +')
for identifier, fields in json.loads(raw):
values = dict(zip(fields[::2], fields[1::2]))
if json.loads(values.get('job', '{}')).get('attachment_id') == attachment:
return identifier
return None
def started_at():
containers = compose('ps', '-q', 'api', 'broker').splitlines()
check(len(containers) >= 3, 'expected two API replicas and a broker running')
raw = subprocess.check_output(
['docker', 'inspect', '--format', '{{.Id}} {{.State.StartedAt}}', *containers], text=True, timeout=15)
return dict(line.split() for line in raw.splitlines())
def api_ready():
with urlopen(BASE + '/health/ready', timeout=5) as response:
return response.status == 200
client = Client(BASE)
actor, chat = actor_and_chat(client, 'restart')
compose('stop', 'worker')
try:
accepted = upload(client, actor, chat)['json']
identifier = accepted['id']
path = '/api/v1/attachments/' + identifier
message = client.message(actor, chat, 'synthetic queued attachment', attachments=[identifier])['json']
entry = wait_for(lambda: stream_entry(identifier), 'outbox did not publish the queued job')
before = started_at()
compose('restart', 'api', 'broker')
wait_for(api_ready, 'API did not recover after restart')
after = started_at()
check(before.keys() == after.keys() and all(before[key] != after[key] for key in before),
'restart did not affect every API replica and the broker')
check(not compose('ps', '-q', '--status', 'running', 'worker'), 'worker processed the job during restart')
check(wait_for(lambda: stream_entry(identifier), 'Stream lost the published job') == entry,
'original Stream entry was lost')
check(client.call('GET', path, actor=actor, schema='Attachment')['json']['status'] == 'queued',
'attachment did not stay queued')
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor)['bytes']
check(message['id'] in history.decode('utf-8'), 'message with queued attachment disappeared')
finally:
compose('start', 'worker')
ready = client.wait_ready(actor, accepted)
original = client.call('GET', path + '/content', actor=actor)['bytes']
thumbnail = client.call('GET', path + '/content?variant=thumbnail', actor=actor)['bytes']
check(original == FIXTURE and thumbnail.startswith(b'\xff\xd8'), 'published file or thumbnail is invalid')
save(output, 'queue-restart.json', {
'api_and_broker_started_at_before': before, 'api_and_broker_started_at_after': after,
'stream_entry_before_and_after': entry, 'worker_stopped_during_restart': True,
'attachment_status_before': 'queued', 'attachment_status_after': ready['status'],
'message_preserved': True, 'original_equal': True, 'thumbnail_jpeg': True,
})
def outage(output):
loss = 'sum({__name__=~"otelcol_exporter_send_failed_spans(_total)?",exporter="otlphttp/tempo"})'
queue = 'max(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?",exporter="otlphttp/tempo"})'
queries = {'send_failed': loss, 'enqueue_failed': refused, 'queue': queue,
'queue_capacity': 'max(otelcol_exporter_queue_capacity{exporter="otlphttp/tempo"})',
'collector_rss_bytes': 'max({__name__=~"otelcol_process_memory_rss(_bytes)?"})'}
def sample():
return {name: next(iter(prometheus(expr)), {}).get('value', 0) for name, expr in queries.items()}
client = Client(BASE)
actor, chat = actor_and_chat(client, 'backend')
baseline = sample()
collector = compose('ps', '-q', 'otel-collector')
inspect = ['docker', 'inspect', '--format',
'{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.HostConfig.Memory}}', collector]
identity = subprocess.check_output(inspect, text=True).strip()
timeline = []
delivered = 0
compose('stop', 'tempo')
started = time.monotonic()
try:
snapshot = started
while time.monotonic() - started < 35:
client.message(actor, chat, 'synthetic during Tempo outage')
delivered += 1
if time.monotonic() >= snapshot:
timeline.append({'elapsed_s': round(time.monotonic() - started, 3), **sample()})
snapshot += 2
time.sleep(.1)
attachment = upload(client, actor, chat)['json']['id']
wait_for(lambda: client.call('GET', '/api/v1/attachments/' + attachment, actor=actor,
schema='Attachment')['json']['status'] == 'ready', 'worker blocked on Tempo')
timeline.append({'elapsed_s': round(time.monotonic() - started, 3), **sample()})
check(delivered >= 50, 'not enough successful writes during outage')
check(max(s['send_failed'] for s in timeline) > baseline['send_failed'], 'no accounted span loss')
check(subprocess.check_output(inspect, text=True).strip() == identity, 'collector restarted or OOMed')
finally:
recovery_started = time.monotonic()
compose('start', 'tempo')
wait_for(tempo_ready, 'Tempo did not become ready after restart')
probes = []
def resumed_trace():
resumed = client.message(actor, chat, 'synthetic after Tempo restart')
probes.append(resumed['headers']['X-Request-Id'])
for request_id in probes[-6:]:
log = api_log(request_id)
if log:
try:
if trace_by_id(log['trace_id']):
return log['trace_id']
except HTTPError as error:
if error.code != 404:
raise
return None
recovered = wait_for(resumed_trace, 'traces did not recover')
save(output, 'backend-outage.json', {
'fault': 'tempo stopped', 'baseline': baseline, 'samples': timeline,
'successful_message_writes': delivered, 'upload_ready_while_tempo_stopped': True,
'collector_started_at_restarts_oom_limit_bytes': identity,
'recovery_trace_id': recovered,
'fault_duration_s': round(recovery_started - started, 3),
'new_trace_after_restart_s': round(time.monotonic() - recovery_started, 3),
})
def main():
args = argparse.ArgumentParser(description=__doc__)
args.add_argument('scenario', choices=['correlation', 'outage', 'queue-restart'])
args.add_argument('--output', type=Path, default=ROOT / 'evidence')
options = args.parse_args()
lab_only()
{'correlation': correlation, 'outage': outage, 'queue-restart': queue_restart}[options.scenario](options.output)
print(options.scenario + ' passed')
if __name__ == '__main__':
main()