#!/usr/bin/env python3 """Проверки наблюдаемости lab6 на Docker-стенде (только по сохранённым наблюдениям). signals уровень 3–4: метрики контракта есть и с правильными метками; запрос → трасса (Tempo) → лог с тем же trace_id (Loki); upload → API → relay (producer) → worker (consumer) → S3/DB в одной трассе; нет утечек пароля, токена, текста сообщения; dashboard из provisioning, его запросы возвращают данные retry-trace S3 недоступен во время обработки: попытки с retry в той же трассе upload incident-worker инцидент 1: worker остановлен → возраст задач растёт → alert firing → worker запущен → backlog уходит → resolved; диагностика по метрикам/логам/трассам incident-db инцидент 2: блокировка таблицы messages (БД «зависла») под нагрузкой → 5xx → MessengerHighErrorRate firing → resolved; трасса показывает, что время ушло в запрос к БД, а не в обработчик (сравнение с медленным upload) backend-down Tempo и Loki остановлены под нагрузкой: API без ошибок, память Collector ограничена, потери считаются; затем остановлен сам Collector Печатает JSON-итог и PASS/FAIL. Токены и пароли не печатаются. """ import json import re import statistics import subprocess import threading import time import uuid from pathlib import Path import labutil # noqa: F401 (добавляет tests/ в sys.path) from client import Client, Failure, arguments, check, run from labutil import ROOT, compose, psql, sh, wait_status from loadgen import Load import obsutil as o SAMPLE = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() FORBIDDEN_LABELS = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'url', 'url_full', 'http_target', 'filename', 'text', 'username'} UUID_RE = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}') def upload(c, actor, chat, rid=None, data=SAMPLE, name='p.png'): h = {'Content-Type': 'image/png'} if rid: h['X-Request-Id'] = rid return c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), 202, actor=actor, raw=data, headers=h, schema='Attachment')['json'] def tree(spans): """Краткое дерево трассы: сервис, имя, родитель (по имени), длительность.""" by_id = {s['span_id']: s for s in spans} out = [] for s in spans: p = by_id.get(s['parent']) out.append({'service': s['service'], 'name': s['name'], 'kind': s['kind'], 'ms': s['ms'], 'parent': (p['service'] + ': ' + p['name']) if p else None, 'outcome': s['attrs'].get('messenger.job.outcome'), 'attempt': s['attrs'].get('messenger.job.attempt'), 'status': s['status']}) return out def chain_ok(spans, child_name, parent_name): by_id = {s['span_id']: s for s in spans} for s in spans: if s['name'] == child_name and by_id.get(s['parent'], {}).get('name') == parent_name: return True return False def wait_alert(name, state, timeout): t0 = time.time() got = o.wait_until(lambda: (o.alerts().get(name) == state) if state else (name not in o.alerts()), timeout, 2) return round(time.time() - t0, 1) if got else None # ---------------- signals ---------------- METRICS = { 'http requests (counter = _count)': 'http_server_request_duration_seconds_count{job="messenger-api"}', 'http duration histogram (s)': 'http_server_request_duration_seconds_bucket{job="messenger-api"}', 'jobs processed by outcome': 'messenger_jobs_processed_total', 'job duration histogram (s)': 'messenger_job_duration_seconds_bucket', 'time to ready histogram (s)': 'messenger_attachment_time_to_ready_seconds_bucket', 'queue backlog by state': 'messenger_jobs_backlog', 'oldest job age (s)': 'messenger_jobs_oldest_age_seconds', 'outbox pending': 'messenger_outbox_pending', 'rabbitmq queue messages': '{__name__=~"rabbitmq_detailed_queue_messages_ready|rabbitmq_queue_messages_ready"}', 'worker busy': 'messenger_worker_busy', 'worker capacity': 'messenger_worker_capacity', 'db pool acquired': '{__name__=~"pgxpool_acquired_connections.*"}', 'db pool waits': '{__name__=~"pgxpool_empty_acquire.*"}', 'process cpu': '{__name__=~"process_cpu_time.*"}', 'go memory': '{__name__=~"go_memory_used.*"}', } def scenario_signals(c): o.wait_grafana() alice, bob, _, chat = c.bootstrap() t0 = time.time() - 5 secret_text = 'секрет-текст-' + uuid.uuid4().hex rid_msg = 'obs-msg-' + uuid.uuid4().hex[:16] c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': secret_text}, headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': rid_msg}) rid_up = 'obs-upload-' + uuid.uuid4().hex[:16] att = upload(c, alice, chat, rid_up) meta = wait_status(c, bob, att['id'], ('ready',)) c.call('GET', '/api/v1/chats/%s/messages' % chat['id'], actor=bob) res = {} # 1. Контролируемый запрос → трасса → лог API с тем же trace_id tid = o.find_trace('{span.request_id="%s"}' % rid_msg, t0) check(tid, 'trace for controlled request not found in Tempo') spans, _ = o.tempo_trace(tid) server = [s for s in spans if s['kind'] in ('SPAN_KIND_SERVER', 2, '2') or s['name'].startswith('POST /api')] check(server, 'no server span') logs = o.wait_until(lambda: o.loki('{service_name="messenger-api"} | trace_id="%s"' % tid, t0), 120) check(logs, 'no API log with trace_id ' + tid) log_line = logs[0][1] res['request'] = {'request_id': rid_msg, 'trace_id': tid, 'spans': tree(spans)[:12], 'log': {'line': log_line[:300], 'labels': {k: v for k, v in logs[0][2].items() if k in ('service_name', 'event', 'route', 'status', 'trace_id', 'span_id', 'detected_level', 'severity_text', 'request_id')}}} # 2. Upload → relay → worker → S3/DB в одной трассе (W3C через outbox и AMQP) tid2 = o.find_trace('{span.request_id="%s"}' % rid_up, t0) check(tid2, 'upload trace not found') def full(): sp, _ = o.tempo_trace(tid2) return sp if any(s['service'] == 'messenger-worker' for s in sp) else None spans2 = o.wait_until(full, 60) check(spans2, 'worker spans not in the upload trace') services = sorted({s['service'] for s in spans2}) links = { 'api server → relay producer': chain_ok(spans2, 'attachment.process publish', 'POST /api/v1/chats/{chat_id}/attachments'), 'relay producer → worker consumer': chain_ok(spans2, 'attachment.process process', 'attachment.process publish'), 'worker consumer → S3 PUT': any(s['name'] == 'S3 PUT' for s in spans2), 'worker consumer → image.decode': chain_ok(spans2, 'image.decode', 'attachment.process process'), } db_in_worker = sorted({s['name'] for s in spans2 if s['service'] == 'messenger-worker' and s['name'] not in ('attachment.process process', 'image.decode', 'image.variants') and not s['name'].startswith('S3')}) check(all(links.values()), 'broken causal chain: %s' % links) wlogs = o.wait_until(lambda: o.loki('{service_name="messenger-worker"} | trace_id="%s"' % tid2, t0), 120) check(wlogs, 'no worker log with the upload trace_id') res['upload'] = {'request_id': rid_up, 'trace_id': tid2, 'services': services, 'links': links, 'span_count': len(spans2), 'worker_db_spans': db_in_worker[:10], 'tree': tree(spans2)[:40], 'worker_log_events': sorted({l[2].get('event', '') for l in wlogs})} # 3. Метрики контракта и метки time.sleep(12) # период экспорта метрик 5 s + scrape 5 s found = {} for title, sel in METRICS.items(): r = o.prom(sel) names = sorted({x['metric'].get('__name__') for x in r}) found[title] = {'series': len(r), 'names': names[:4]} missing = [k for k, v in found.items() if v['series'] == 0] check(not missing, 'metrics missing: %s' % missing) labels = set(o.prom_labels()) bad = sorted(labels & FORBIDDEN_LABELS) check(not bad, 'forbidden metric labels: %s' % bad) routes = o.prom_label_values('http_route') check(not [r for r in routes if UUID_RE.search(r)], 'raw ids in http_route: %s' % routes) sample_series = o.prom('http_server_request_duration_seconds_count{job="messenger-api"}')[0]['metric'] res['metrics'] = {'found': found, 'labels_total': len(labels), 'forbidden_labels': bad, 'http_route_values': routes, 'example_series_labels': sorted(sample_series)} # 4. Утечки: пароль, токен, текст сообщения, Authorization — ни в логах (Loki, stdout), ни в трассах needles = {'password': alice['registration']['password'], 'session_token': alice['token'], 'message_text': secret_text, 'bearer_header': 'Bearer ' + alice['token'][:12]} leaks = {} for k, v in needles.items(): hits = o.loki('{service_name=~"messenger-.+"} |= `%s`' % v.replace('`', ''), t0, limit=5) leaks['loki:' + k] = len(hits) raw_traces = json.dumps(o.tempo_trace(tid)[1]) + json.dumps(o.tempo_trace(tid2)[1]) for k, v in needles.items(): leaks['tempo:' + k] = raw_traces.count(v) api_stdout = compose('logs', '--no-color', '--since', '15m', 'api', 'worker', 'relay').stdout for k, v in needles.items(): leaks['stdout:' + k] = api_stdout.count(v) check(sum(leaks.values()) == 0, 'telemetry leak: %s' % {k: v for k, v in leaks.items() if v}) res['leak_check'] = leaks # 5. Provisioning: datasources и dashboard из Git; каждый Prometheus-запрос панели возвращает данные ds = sorted(d['type'] + ':' + d['uid'] for d in o.grafana('/api/datasources')) found_d = o.grafana('/api/search', {'query': 'Messenger'}) check(any(d.get('uid') == 'messenger-overview' for d in found_d), 'dashboard not provisioned') dash = o.grafana('/api/dashboards/uid/messenger-overview')['dashboard'] panel_check = [] for p in dash['panels']: for t in p.get('targets', []): if t.get('datasource', {}).get('uid') != 'prometheus': continue expr = t['expr'].replace('$__rate_interval', '1m') try: n = len(o.prom(expr)) except Exception as e: n = 'error: %s' % type(e).__name__ panel_check.append({'panel': p['title'], 'legend': t.get('legendFormat', ''), 'series': n}) empty = [x for x in panel_check if x['series'] == 0] res['grafana'] = {'datasources': ds, 'dashboard': dash['title'], 'panels': len(dash['panels']), 'prometheus_targets': len(panel_check), 'targets_without_data_now': empty, 'targets': panel_check} errors = [x for x in panel_check if isinstance(x['series'], str)] check(not errors, 'dashboard queries failed: %s' % errors) res['attachment'] = {'id': att['id'], 'status': meta['status']} return res # ---------------- retry в трассе ---------------- def scenario_retry_trace(c): o.wait_grafana() alice, bob, _, chat = c.bootstrap() t0 = time.time() - 5 worker = compose('ps', '-q', 'worker').stdout.split()[0] s3 = compose('ps', '-q', 's3').stdout.split()[0] compose('stop', 'worker') rid = 'obs-retry-' + uuid.uuid4().hex[:16] att = upload(c, alice, chat, rid) time.sleep(2) sh('docker', 'stop', s3) sh('docker', 'start', worker) time.sleep(6) # 1–2 попытки упадут на недоступном S3 sh('docker', 'start', s3) meta = wait_status(c, bob, att['id'], ('ready',), timeout=150) tid = o.find_trace('{span.request_id="%s"}' % rid, t0) check(tid, 'trace not found') def done(): sp, _ = o.tempo_trace(tid) return sp if any(s['attrs'].get('messenger.job.outcome') == 'done' for s in sp) else None spans = o.wait_until(done, 90) check(spans, 'final attempt not in trace') attempts = [(s['attrs'].get('messenger.job.attempt'), s['attrs'].get('messenger.job.outcome'), s['status']) for s in spans if s['name'] == 'attachment.process process'] publishes = sum(1 for s in spans if s['name'] == 'attachment.process publish') check(any(a[1] == 'retry' for a in attempts), 'no retry attempt in trace: %s' % attempts) ev = psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE job_id = " "(SELECT job_id FROM jobs WHERE attachment_id = '%s')" % att['id']) return {'request_id': rid, 'trace_id': tid, 'attempts_in_trace': attempts, 'publish_spans': publishes, 'job_events': ev, 'final_status': meta['status'], 'tree': [x for x in tree(spans) if x['name'].startswith('attachment.') or x['name'].startswith('POST')]} # ---------------- инцидент 1: worker ---------------- def scenario_incident_worker(c): o.wait_grafana() alice, bob, _, chat = c.bootstrap() t_stop = time.time() compose('stop', 'worker') ids = [upload(c, alice, chat, 'obs-inc1-%d-%s' % (i, uuid.uuid4().hex[:8]))['id'] for i in range(3)] t_firing = wait_alert('MessengerQueueLagGrowing', 'firing', 240) check(t_firing, 'MessengerQueueLagGrowing did not fire') detect = round(time.time() - t_stop, 1) al = o.alerts() # Диагностика «как оператор»: только по сохранённым наблюдениям diag = { 'alerts_firing': sorted(k for k, v in al.items() if v == 'firing'), 'backlog_queued': o.prom_value('sum(messenger_jobs_backlog{state="queued"})'), 'oldest_age_s': o.prom_value('max(messenger_jobs_oldest_age_seconds)'), 'jobs_processed_rate_1m': o.prom_value('sum(rate(messenger_jobs_processed_total[1m]))', default=0.0), 'worker_capacity_series': len(o.prom('messenger_worker_capacity')), 'rabbitmq_ready': o.prom_value('sum(rabbitmq_detailed_queue_messages_ready{queue="attachment.process"})'), 'api_5xx_ratio': o.prom_value('messenger:http_error_ratio:rate1m', default=0.0), 'producer_spans_since_stop': len(o.tempo_search('{name="attachment.process publish"}', t_stop)), 'consumer_spans_since_stop': len(o.tempo_search('{name="attachment.process process"}', t_stop + 2)), 'last_worker_log_events': sorted({l[2].get('event', '') for l in o.loki('{service_name="messenger-worker"}', t_stop - 30, limit=50)}), } t_start = time.time() compose('start', 'worker') for i in ids: wait_status(c, bob, i, ('ready',), timeout=120) t_ready = round(time.time() - t_start, 1) t_resolved = wait_alert('MessengerQueueLagGrowing', None, 180) check(t_resolved is not None, 'alert did not resolve') series = o.prom_range('max(messenger_jobs_oldest_age_seconds)', t_stop - 30, time.time(), 10) return {'detection_seconds_from_worker_stop': detect, 'diagnosis': diag, 'backlog_ready_seconds_after_start': t_ready, 'resolved_seconds_after_worker_start': round(t_ready + t_resolved, 1), 'oldest_age_series': [[round(float(t) - t_stop), round(float(v), 1)] for t, v in (series[0]['values'] if series else [])]} # ---------------- инцидент 2: БД ---------------- def is_db(s): return (s['attrs'].get('db.system') == 'postgresql' or s['attrs'].get('db.system.name') == 'postgresql' or s['name'].split(' ')[0] in ('query', 'acquire', 'batch', 'prepare', 'connect', 'copy_from')) def breakdown(spans): """Доля времени server span, проведённая в дочерних spans БД (pgx) и S3.""" by_parent = {} for s in spans: by_parent.setdefault(s['parent'], []).append(s) srv = next((s for s in spans if s['parent'] == '' or s['name'].startswith(('POST /api', 'GET /api'))), None) if not srv: return None db = sum(s['ms'] for s in spans if s['service'] == srv['service'] and is_db(s)) s3 = sum(s['ms'] for s in spans if s['name'].startswith('S3')) return {'server_span': srv['name'], 'server_ms': srv['ms'], 'status': srv['attrs'].get('http.response.status_code'), 'db_ms': round(db, 1), 's3_ms': round(s3, 1), 'db_share': round(db / srv['ms'], 3) if srv['ms'] else None, 'db_spans': [(s['name'], s['ms']) for s in spans if is_db(s)][:6]} def scenario_incident_db(c): o.wait_grafana() alice, bob, _, chat = c.bootstrap() load = Load(c.base, None, alice, chat, concurrency=4, timeout=10, prefix='obs-inc2').start() time.sleep(20) t_lock = time.time() lock = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c', 'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(110); COMMIT;'], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) t_firing = wait_alert('MessengerHighErrorRate', 'firing', 150) detect = round(time.time() - t_lock, 1) if t_firing else None snap = { 'error_ratio_1m': o.prom_value('messenger:http_error_ratio:rate1m'), 'p99_s': o.prom_value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'), 'p99_by_route': {r['metric'].get('http_route'): round(float(r['value'][1]), 3) for r in o.prom( 'histogram_quantile(0.99, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))')}, 'db_pool_acquired': o.prom_value('sum({__name__=~"pgxpool_acquired_connections.*", job="messenger-api"})'), 'health_ready_5xx_rate': o.prom_value('sum(rate(http_server_request_duration_seconds_count{http_route="/health/ready", http_response_status_class="5xx"}[1m]))', default=0.0), } lock.wait(timeout=200) t_unlock = time.time() t_resolved = wait_alert('MessengerHighErrorRate', None, 180) load.stop() # Трассы ошибок: где время? (по сохранённым трассам Tempo) err_traces = o.tempo_search('{resource.service.name="messenger-api" && span.http.response.status_code >= 500}', t_lock - 1, limit=5) analysis = [] for t in err_traces[:3]: sp, _ = o.tempo_trace(t['traceID']) analysis.append(breakdown(sp)) # Сравнение: медленный upload большого файла — время не в БД rid = 'obs-slowhandler-' + uuid.uuid4().hex[:8] from labutil import noise_png t_up = time.time() - 2 upload(c, alice, chat, rid, data=noise_png(4 * 1024 * 1024), name='big.png') tid = o.find_trace('{span.request_id="%s"}' % rid, t_up) slow_handler = breakdown(o.tempo_trace(tid)[0]) if tid else None err_logs = o.loki('{service_name="messenger-api"} | detected_level=~"error|ERROR"', t_lock, t_unlock + 5, limit=200) check(detect is not None, 'MessengerHighErrorRate did not fire') check(t_resolved is not None, 'MessengerHighErrorRate did not resolve') check(analysis and all(a and a['db_share'] and a['db_share'] > 0.8 for a in analysis), 'error traces do not point to DB: %s' % analysis) return {'fault': 'LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 110 s under load (4 clients)', 'detection_seconds': detect, 'resolved_seconds_after_unlock': t_resolved, 'metrics_at_firing': snap, 'error_trace_breakdown': analysis, 'slow_handler_comparison': slow_handler, 'error_logs': {'count': len(err_logs), 'events': sorted({l[2].get('event', '') for l in err_logs}), 'routes': sorted({l[2].get('route', '') for l in err_logs})}, 'load': load.summary(exclude=[(t_lock, t_unlock)])} # ---------------- отказ backend телеметрии ---------------- def mem_of(service): cid = compose('ps', '-q', service).stdout.split() return round(o.mem_bytes(cid[0]) / 2**20, 1) if cid and o.mem_bytes(cid[0]) else None def scenario_backend_down(c): o.wait_grafana() alice, bob, _, chat = c.bootstrap() load = Load(c.base, None, alice, chat, concurrency=4, timeout=10, prefix='obs-bd').start() samples = [] stop = threading.Event() col = compose('ps', '-q', 'otel-collector').stdout.split()[0] def sampler(): while not stop.is_set(): samples.append({'t': round(time.time() - t0, 1), 'collector_mib': round((o.mem_bytes(col) or 0) / 2**20, 1), 'api_mib': [round((o.mem_bytes(x) or 0) / 2**20, 1) for x in compose('ps', '-q', 'api').stdout.split()]}) time.sleep(5) t0 = time.time() threading.Thread(target=sampler, daemon=True).start() time.sleep(15) compose('stop', 'tempo', 'loki') t_down = time.time() phase = {} fired = wait_alert('MessengerTelemetryExportFailing', 'firing', 150) phase['export_failing_alert_seconds'] = round(time.time() - t_down, 1) if fired else None time.sleep(max(0, 130 - (time.time() - t_down))) q = lambda e: o.prom_value(e, default=0.0) phase['during'] = { 'exporter_queue_size_max': q('max_over_time(sum(otelcol_exporter_queue_size)[2m:5s])'), 'exporter_queue_capacity': q('sum(otelcol_exporter_queue_capacity)'), 'send_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_send_failed_spans(_total)?"}[3m]))'), 'send_failed_log_records': q('sum(increase({__name__=~"otelcol_exporter_send_failed_log_records(_total)?"}[3m]))'), 'enqueue_failed_spans': q('sum(increase({__name__=~"otelcol_exporter_enqueue_failed_spans(_total)?"}[3m]))'), 'refused_spans_memory_limiter': q('sum(increase({__name__=~"otelcol_receiver_refused_spans(_total)?"}[3m]))'), 'metrics_still_scraped': q('sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m]))'), } compose('start', 'tempo', 'loki') compose('up', '-d', '--wait', '--no-deps', 'tempo', 'loki', timeout=180) t_up = time.time() rid = 'obs-after-' + uuid.uuid4().hex[:8] sent0 = q('sum({__name__=~"otelcol_exporter_sent_spans(_total)?", exporter="otlp/tempo"})') # Новые запросы после возврата Tempo; трасса ищется по ID из заголовка traceresponse # (минуя поиск) — сначала каждые 5 s новый запрос, пока трасса не появится. got, tids = None, [] end = time.time() + 240 while time.time() < end and not got: r = c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': 'after'}, headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': rid}) tid = (r['headers'].get('traceresponse') or '--').split('-')[1] tids.append(tid) for t in tids: try: if o.tempo_trace(t)[0]: got = t break except Exception: pass if not got: time.sleep(5) phase['tempo_spans_exported_after_up'] = q('sum({__name__=~"otelcol_exporter_sent_spans(_total)?", exporter="otlp/tempo"})') - sent0 phase['refused_spans_total'] = q('sum({__name__=~"otelcol_receiver_refused_spans(_total)?"})') phase['new_trace_after_recovery'] = bool(got) phase['traces_visible_seconds_after_tempo_up'] = round(time.time() - t_up, 1) if got else None phase['probe_requests_after_up'] = len(tids) # Фаза 2: остановлен сам Collector — SDK отбрасывает по ограниченной очереди compose('stop', 'otel-collector') t_cd = time.time() time.sleep(40) compose('start', 'otel-collector') time.sleep(10) load.stop() stop.set() summary = load.summary() col_max = max(s['collector_mib'] for s in samples) api_max = max(max(s['api_mib'] or [0]) for s in samples) check(summary['errors'] == 0, 'API errors while telemetry backend was down: %s' % summary) check(col_max < 256, 'collector memory not bounded') check(phase['new_trace_after_recovery'], 'no traces after backend recovery') return {'load': summary, 'collector_limit_mib': 256, 'collector_max_mib': col_max, 'api_max_mib': api_max, 'api_limit_mib': 256, 'phases': phase, 'timeline': {'tempo_loki_down_s': round(t_down - t0), 'tempo_loki_up_s': round(t_up - t0), 'collector_down_s': round(t_cd - t0), 'collector_down_for_s': 40}, 'memory_samples': samples} SCENARIOS = {'signals': scenario_signals, 'retry-trace': scenario_retry_trace, 'incident-worker': scenario_incident_worker, 'incident-db': scenario_incident_db, 'backend-down': scenario_backend_down} def main(): def configure(p): p.add_argument('scenario', choices=sorted(SCENARIOS)) p.add_argument('--output', default=None) args = arguments('Lab6 observability scenarios on the Docker stand', configure) c = Client(args.base_url, args.ca_file, timeout=30) try: result = SCENARIOS[args.scenario](c) finally: compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 's3', 'api', 'otel-collector', 'tempo', 'loki', timeout=300, check=False) text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(text + '\n', encoding='utf-8') print(text) print('PASS: ' + args.scenario) if __name__ == '__main__': run(main)