forked from mtusi2026/mtusi2026lab6
356 lines
17 KiB
Python
356 lines
17 KiB
Python
"""Black-box checks on the dedicated, local Lab6 Compose project."""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import urlencode, urlsplit
|
|
from urllib.request import urlopen
|
|
from uuid import uuid4
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def local_port(name, default):
|
|
return int(os.environ.get(name, default))
|
|
|
|
|
|
BASE = os.environ.get('BASE_URL', f'http://127.0.0.1:{local_port("APP_PORT", 8080)}')
|
|
PROM = f'http://127.0.0.1:{local_port("PROMETHEUS_PORT", 9090)}'
|
|
LOKI = f'http://127.0.0.1:{local_port("LOKI_PORT", 3100)}'
|
|
TEMPO = f'http://127.0.0.1:{local_port("TEMPO_PORT", 3200)}'
|
|
GRAFANA = f'http://127.0.0.1:{local_port("GRAFANA_PORT", 3000)}'
|
|
COMPOSE = ['docker', 'compose', '--project-directory', str(ROOT)]
|
|
FIXTURE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
|
|
|
|
def compose(*args):
|
|
return subprocess.run([*COMPOSE, *args], cwd=ROOT, check=True, capture_output=True,
|
|
text=True, encoding='utf-8', errors='replace', timeout=60).stdout.strip()
|
|
|
|
|
|
def backend(base, path, **params):
|
|
query = '?' + urlencode(params) if params else ''
|
|
with urlopen(base + path + query, timeout=5) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def tempo_ready():
|
|
with urlopen(TEMPO + '/ready', timeout=5) as response:
|
|
return response.status == 200
|
|
|
|
|
|
def prometheus(expr, at=None):
|
|
params = {'query': expr}
|
|
if at is not None:
|
|
params['time'] = at
|
|
vector = backend(PROM, '/api/v1/query', **params)['data']['result']
|
|
return [{**item['metric'], 'value': float(item['value'][1])} for item in vector]
|
|
|
|
|
|
def wait_for(probe, message, timeout=45):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
result = probe()
|
|
if result:
|
|
return result
|
|
except (HTTPError, OSError):
|
|
pass
|
|
time.sleep(1)
|
|
raise AssertionError(message)
|
|
|
|
|
|
def save(output, name, result):
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
(output / name).write_text(json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + '\n',
|
|
encoding='utf-8')
|
|
|
|
|
|
def lab_only():
|
|
check(urlsplit(BASE).hostname in {'localhost', '127.0.0.1', '::1'}, 'use the dedicated local lab')
|
|
project = json.loads(compose('config', '--format', 'json'))['name']
|
|
check(project.startswith('mtusi-lab6'), 'Compose project name must begin with mtusi-lab6')
|
|
for service in ('worker', 'dispatcher', 'db', 'tempo', 'otel-collector'):
|
|
check(bool(compose('ps', '-q', '--status', 'running', service)), f'{service} must be running')
|
|
|
|
|
|
def actor_and_chat(client, label):
|
|
actor = client.user(label)
|
|
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Синтетическая Lab6', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
return actor, chat
|
|
|
|
|
|
def upload(client, actor, chat, filename='synthetic.png'):
|
|
return client.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename={filename}', 202,
|
|
actor=actor, raw=FIXTURE, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
|
|
|
|
def loki_records(expression, lookback=300):
|
|
now = time.time_ns()
|
|
data = backend(LOKI, '/loki/api/v1/query_range', query=expression, start=now - lookback * 10**9,
|
|
end=now, limit=100)['data']['result']
|
|
result = []
|
|
for stream in data:
|
|
for timestamp, line, *extra in stream['values']:
|
|
metadata = extra[0] if extra else {}
|
|
result.append({**stream['stream'], **metadata, 'timestamp_ns': timestamp, 'body': line})
|
|
return result
|
|
|
|
|
|
def api_log(request_id, lookback=300):
|
|
logs = loki_records(f'{{service_name="messenger-api"}} | request_id="{request_id}"', lookback)
|
|
return logs[0] if logs else None
|
|
|
|
|
|
def otlp_value(item):
|
|
return next(iter(item.values()))
|
|
|
|
|
|
def span_id(value):
|
|
if not value or re.fullmatch('[0-9a-f]{16}', value):
|
|
return value
|
|
return base64.b64decode(value, validate=True).hex()
|
|
|
|
|
|
def trace_by_id(trace_id):
|
|
data = backend(TEMPO, '/api/traces/' + trace_id)
|
|
spans = []
|
|
for batch in data.get('batches', data.get('resourceSpans', [])):
|
|
resource = {a['key']: otlp_value(a['value']) for a in batch['resource']['attributes']}
|
|
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
|
|
for span in scope['spans']:
|
|
spans.append({
|
|
'name': span['name'], 'service': resource['service.name'],
|
|
'service_version': resource['service.version'],
|
|
'environment': resource['deployment.environment.name'],
|
|
'id': span_id(span['spanId']), 'parent': span_id(span.get('parentSpanId')),
|
|
'kind': span['kind'], 'status': span.get('status', {}),
|
|
'duration_ms': round((int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6, 3),
|
|
'attributes': {a['key']: otlp_value(a['value']) for a in span.get('attributes', [])},
|
|
})
|
|
return spans
|
|
|
|
|
|
def assert_privacy(records, secrets):
|
|
serialized = json.dumps(records, ensure_ascii=False)
|
|
check(not any(secret in serialized for secret in secrets), 'unexpected synthetic sensitive value in telemetry')
|
|
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename',
|
|
'url', 'url_full', 'url_path', 'text', 'trace_id', 'span_id'}
|
|
sample = prometheus('count by (__name__) ({__name__=~"messenger_.+"})')
|
|
check(bool(sample), 'no application metric series')
|
|
series = backend(PROM, '/api/v1/series', **{'match[]': '{__name__=~"messenger_.+"}'})['data']
|
|
check(not any(forbidden & entry.keys() for entry in series), 'forbidden metric label')
|
|
return len(series)
|
|
|
|
|
|
def correlation(output):
|
|
client = Client(BASE)
|
|
actor, chat = actor_and_chat(client, 'trace')
|
|
canary = 'private-' + uuid4().hex
|
|
message = client.message(actor, chat, canary)
|
|
compose('stop', 'worker')
|
|
extra_worker = 'mtusi-lab6-trace-' + uuid4().hex[:8]
|
|
try:
|
|
received = upload(client, actor, chat, canary + '.png')
|
|
attachment = received['json']['id']
|
|
compose('run', '-d', '--no-deps', '--name', extra_worker, '-e', 'LAB5_FAULT=temporary_once',
|
|
'-e', 'LAB5_FAULT_ATTACHMENT=' + attachment, 'worker')
|
|
wait_for(lambda: client.call('GET', '/api/v1/attachments/' + attachment, actor=actor,
|
|
schema='Attachment')['json']['status'] == 'ready',
|
|
'retry did not produce a ready result')
|
|
request_id = received['headers']['X-Request-Id']
|
|
log = wait_for(lambda: api_log(request_id), 'no upload log in Loki')
|
|
trace_id = log['trace_id']
|
|
check(bool(re.fullmatch('[0-9a-f]{32}', trace_id)), 'invalid Loki trace ID')
|
|
logs = wait_for(lambda: enough_attempts(trace_id), 'no log for both worker attempts')
|
|
spans = wait_for(lambda: enough_spans(trace_id), 'missing worker trace or S3 spans')
|
|
server = next(span for span in spans if span['name'].startswith('POST /api/v1/chats/'))
|
|
check(log['span_id'] == server['id'], 'API log has the wrong span')
|
|
producers = {int(s['attributes']['job.attempt']): s for s in spans if s['name'].endswith(' publish')}
|
|
consumers = {int(s['attributes']['job.attempt']): s for s in spans if s['name'].endswith(' process')}
|
|
check(set(producers) == set(consumers) == {1, 2}, 'expected one failed attempt and one retry')
|
|
for attempt in (1, 2):
|
|
check(producers[attempt]['parent'] == server['id'], 'producer parent is not the upload')
|
|
check(consumers[attempt]['parent'] == producers[attempt]['id'], 'consumer parent is not the producer')
|
|
check(any(s['span_id'] == consumers[attempt]['id'] for s in logs), 'worker log span mismatch')
|
|
check({log['outcome'] for log in logs} == {'queued', 'ready'}, 'missing retry/ready outcomes')
|
|
check(any(s['name'] == 'postgresql query' and s['parent'] == consumers[2]['id'] for s in spans),
|
|
'missing SQL span for the successful worker attempt')
|
|
message_log = wait_for(lambda: api_log(message['headers']['X-Request-Id']), 'missing message log')
|
|
message_spans = wait_for(lambda: trace_by_id(message_log['trace_id']), 'missing message trace')
|
|
series = assert_privacy([log, logs, spans, message_log, message_spans],
|
|
[canary, actor['registration']['password'], actor['token']])
|
|
dashboard = backend(GRAFANA, '/api/dashboards/uid/messenger-lab6')['dashboard']
|
|
sources = {ds['uid'] for ds in backend(GRAFANA, '/api/datasources')}
|
|
check({'lab-prom', 'lab-loki', 'lab-tempo'} <= sources, 'missing provisioned datasource')
|
|
save(output, 'correlation.json', {
|
|
'trace_id': trace_id, 'request_id': request_id, 'logs': [log, *logs], 'spans': spans,
|
|
'privacy': {'synthetic_canaries_absent': True, 'metric_series_checked': series},
|
|
'grafana': {'dashboard_uid': dashboard['uid'], 'panels': len(dashboard['panels']),
|
|
'datasources': sorted(sources)},
|
|
})
|
|
finally:
|
|
subprocess.run(['docker', 'stop', '-t', '5', extra_worker], capture_output=True, timeout=20)
|
|
subprocess.run(['docker', 'rm', extra_worker], capture_output=True, timeout=20)
|
|
compose('start', 'worker')
|
|
|
|
|
|
def enough_attempts(trace_id):
|
|
logs = loki_records(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="image.attempt"')
|
|
return logs if len(logs) >= 2 else None
|
|
|
|
|
|
def enough_spans(trace_id):
|
|
spans = trace_by_id(trace_id)
|
|
return spans if sum(span['name'] == 's3 put result' for span in spans) == 4 else None
|
|
|
|
|
|
def queue_restart(output):
|
|
def stream_entry(attachment):
|
|
raw = compose('exec', '-T', 'broker', 'sh', '-c',
|
|
'REDISCLI_AUTH="$(cat /run/secrets/broker_password)" '
|
|
'exec valkey-cli --json XRANGE attachment-jobs - +')
|
|
for identifier, fields in json.loads(raw):
|
|
values = dict(zip(fields[::2], fields[1::2]))
|
|
if json.loads(values.get('job', '{}')).get('attachment_id') == attachment:
|
|
return identifier
|
|
return None
|
|
|
|
def started_at():
|
|
containers = compose('ps', '-q', 'api', 'broker').splitlines()
|
|
check(len(containers) >= 3, 'expected two API replicas and a broker running')
|
|
raw = subprocess.check_output(
|
|
['docker', 'inspect', '--format', '{{.Id}} {{.State.StartedAt}}', *containers], text=True, timeout=15)
|
|
return dict(line.split() for line in raw.splitlines())
|
|
|
|
def api_ready():
|
|
with urlopen(BASE + '/health/ready', timeout=5) as response:
|
|
return response.status == 200
|
|
|
|
client = Client(BASE)
|
|
actor, chat = actor_and_chat(client, 'restart')
|
|
compose('stop', 'worker')
|
|
try:
|
|
accepted = upload(client, actor, chat)['json']
|
|
identifier = accepted['id']
|
|
path = '/api/v1/attachments/' + identifier
|
|
message = client.message(actor, chat, 'synthetic queued attachment', attachments=[identifier])['json']
|
|
entry = wait_for(lambda: stream_entry(identifier), 'outbox did not publish the queued job')
|
|
before = started_at()
|
|
compose('restart', 'api', 'broker')
|
|
wait_for(api_ready, 'API did not recover after restart')
|
|
after = started_at()
|
|
check(before.keys() == after.keys() and all(before[key] != after[key] for key in before),
|
|
'restart did not affect every API replica and the broker')
|
|
check(not compose('ps', '-q', '--status', 'running', 'worker'), 'worker processed the job during restart')
|
|
check(wait_for(lambda: stream_entry(identifier), 'Stream lost the published job') == entry,
|
|
'original Stream entry was lost')
|
|
check(client.call('GET', path, actor=actor, schema='Attachment')['json']['status'] == 'queued',
|
|
'attachment did not stay queued')
|
|
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor)['bytes']
|
|
check(message['id'] in history.decode('utf-8'), 'message with queued attachment disappeared')
|
|
finally:
|
|
compose('start', 'worker')
|
|
ready = client.wait_ready(actor, accepted)
|
|
original = client.call('GET', path + '/content', actor=actor)['bytes']
|
|
thumbnail = client.call('GET', path + '/content?variant=thumbnail', actor=actor)['bytes']
|
|
check(original == FIXTURE and thumbnail.startswith(b'\xff\xd8'), 'published file or thumbnail is invalid')
|
|
save(output, 'queue-restart.json', {
|
|
'api_and_broker_started_at_before': before, 'api_and_broker_started_at_after': after,
|
|
'stream_entry_before_and_after': entry, 'worker_stopped_during_restart': True,
|
|
'attachment_status_before': 'queued', 'attachment_status_after': ready['status'],
|
|
'message_preserved': True, 'original_equal': True, 'thumbnail_jpeg': True,
|
|
})
|
|
|
|
|
|
def outage(output):
|
|
loss = 'sum({__name__=~"otelcol_exporter_send_failed_spans(_total)?",exporter="otlphttp/tempo"})'
|
|
queue = 'max(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
|
|
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?",exporter="otlphttp/tempo"})'
|
|
queries = {'send_failed': loss, 'enqueue_failed': refused, 'queue': queue,
|
|
'queue_capacity': 'max(otelcol_exporter_queue_capacity{exporter="otlphttp/tempo"})',
|
|
'collector_rss_bytes': 'max({__name__=~"otelcol_process_memory_rss(_bytes)?"})'}
|
|
|
|
def sample():
|
|
return {name: next(iter(prometheus(expr)), {}).get('value', 0) for name, expr in queries.items()}
|
|
|
|
client = Client(BASE)
|
|
actor, chat = actor_and_chat(client, 'backend')
|
|
baseline = sample()
|
|
collector = compose('ps', '-q', 'otel-collector')
|
|
inspect = ['docker', 'inspect', '--format',
|
|
'{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.HostConfig.Memory}}', collector]
|
|
identity = subprocess.check_output(inspect, text=True).strip()
|
|
timeline = []
|
|
delivered = 0
|
|
compose('stop', 'tempo')
|
|
started = time.monotonic()
|
|
try:
|
|
snapshot = started
|
|
while time.monotonic() - started < 35:
|
|
client.message(actor, chat, 'synthetic during Tempo outage')
|
|
delivered += 1
|
|
if time.monotonic() >= snapshot:
|
|
timeline.append({'elapsed_s': round(time.monotonic() - started, 3), **sample()})
|
|
snapshot += 2
|
|
time.sleep(.1)
|
|
attachment = upload(client, actor, chat)['json']['id']
|
|
wait_for(lambda: client.call('GET', '/api/v1/attachments/' + attachment, actor=actor,
|
|
schema='Attachment')['json']['status'] == 'ready', 'worker blocked on Tempo')
|
|
timeline.append({'elapsed_s': round(time.monotonic() - started, 3), **sample()})
|
|
check(delivered >= 50, 'not enough successful writes during outage')
|
|
check(max(s['send_failed'] for s in timeline) > baseline['send_failed'], 'no accounted span loss')
|
|
check(subprocess.check_output(inspect, text=True).strip() == identity, 'collector restarted or OOMed')
|
|
finally:
|
|
recovery_started = time.monotonic()
|
|
compose('start', 'tempo')
|
|
wait_for(tempo_ready, 'Tempo did not become ready after restart')
|
|
probes = []
|
|
|
|
def resumed_trace():
|
|
resumed = client.message(actor, chat, 'synthetic after Tempo restart')
|
|
probes.append(resumed['headers']['X-Request-Id'])
|
|
for request_id in probes[-6:]:
|
|
log = api_log(request_id)
|
|
if log:
|
|
try:
|
|
if trace_by_id(log['trace_id']):
|
|
return log['trace_id']
|
|
except HTTPError as error:
|
|
if error.code != 404:
|
|
raise
|
|
return None
|
|
|
|
recovered = wait_for(resumed_trace, 'traces did not recover')
|
|
save(output, 'backend-outage.json', {
|
|
'fault': 'tempo stopped', 'baseline': baseline, 'samples': timeline,
|
|
'successful_message_writes': delivered, 'upload_ready_while_tempo_stopped': True,
|
|
'collector_started_at_restarts_oom_limit_bytes': identity,
|
|
'recovery_trace_id': recovered,
|
|
'fault_duration_s': round(recovery_started - started, 3),
|
|
'new_trace_after_restart_s': round(time.monotonic() - recovery_started, 3),
|
|
})
|
|
|
|
|
|
def main():
|
|
args = argparse.ArgumentParser(description=__doc__)
|
|
args.add_argument('scenario', choices=['correlation', 'outage', 'queue-restart'])
|
|
args.add_argument('--output', type=Path, default=ROOT / 'evidence')
|
|
options = args.parse_args()
|
|
lab_only()
|
|
{'correlation': correlation, 'outage': outage, 'queue-restart': queue_restart}[options.scenario](options.output)
|
|
print(options.scenario + ' passed')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|