"""Shared lab6 helpers: Prometheus/Loki/Tempo/Grafana client, controlled app environment, SQL table lock, batch drain. Python 3.9+, standard library only.""" import base64 import binascii import contextlib import json import os import select import subprocess import sys import time import urllib.error import urllib.parse import urllib.request import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from scripts.lab5 import stand # noqa: E402 Failure = stand.Failure HTTP_TIMEOUT = 10 SCRAPE_JOBS = ('prometheus', 'otel-collector', 'haproxy', 'rabbitmq', 'loki', 'tempo') RULE_ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing') DASHBOARD_UIDS = ('messenger-overview', 'messenger-telemetry') DATASOURCE_UIDS = ('prometheus', 'loki', 'tempo') LOCKABLE_TABLES = ('messages', 'attachments') # Keys the controller may change/restore (identical for api and worker). MUTABLE_KEYS = ( 'OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG', 'OTEL_METRIC_EXPORT_INTERVAL', 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_EXPORTER_OTLP_TIMEOUT', 'OTEL_RESOURCE_ATTRIBUTES', 'OTEL_BSP_MAX_QUEUE_SIZE', 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE', 'OTEL_BSP_SCHEDULE_DELAY', 'OTEL_BLRP_MAX_QUEUE_SIZE', 'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', 'OTEL_BLRP_SCHEDULE_DELAY', 'LOG_LEVEL', 'CACHE_MODE', 'FAULT_INJECTION_ENABLED', ) # Recorded as diagnostics, never settable. DIAGNOSTIC_KEYS = ('OTEL_SERVICE_NAME', 'SERVICE_VERSION') # Baseline of the SDK knobs (compose defaults) used by track1 and incidents. SDK_DEFAULTS = { 'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1', 'OTEL_METRIC_EXPORT_INTERVAL': '10000', 'OTEL_EXPORTER_OTLP_TIMEOUT': '5000', 'OTEL_BSP_MAX_QUEUE_SIZE': '2048', 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE': '512', 'OTEL_BSP_SCHEDULE_DELAY': '1000', 'OTEL_BLRP_MAX_QUEUE_SIZE': '2048', 'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE': '512', 'OTEL_BLRP_SCHEDULE_DELAY': '1000', 'LOG_LEVEL': 'INFO', } def utc_now(): return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) class _NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): return None _OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()) def http_get(url, headers=None, timeout=HTTP_TIMEOUT): """GET returning (status, bytes); never prints the URL query/headers.""" request = urllib.request.Request(url, headers=headers or {}) try: with _OPENER.open(request, timeout=timeout) as response: return response.status, response.read() except urllib.error.HTTPError as error: with error: return error.code, error.read() except (urllib.error.URLError, TimeoutError, OSError) as exc: raise Failure('GET ' + url.split('?')[0] + ': connection failure (' + type(exc).__name__ + ')') from None def http_json(url, headers=None, timeout=HTTP_TIMEOUT, allow=(200,)): status, data = http_get(url, headers, timeout) if status not in allow: raise Failure('GET ' + url.split('?')[0] + ': status ' + str(status)) try: return status, json.loads(data) if data else None except (ValueError, UnicodeError): raise Failure('GET ' + url.split('?')[0] + ': invalid JSON') from None # ---------------------------------------------------------------- trace JSON def normalize_id(value, nbytes): """Hex or base64 ID -> lowercase hex of exactly nbytes; else Failure.""" if not isinstance(value, str): raise Failure('trace id is not a string') if value == '': return '' if len(value) == nbytes * 2: try: int(value, 16) return value.lower() except ValueError: pass try: raw = base64.b64decode(value, validate=True) except (binascii.Error, ValueError): raise Failure('unrecognised trace/span id encoding') from None if len(raw) != nbytes: raise Failure('trace/span id has wrong length') return raw.hex() def _any_value(node): if not isinstance(node, dict): return node for key in ('stringValue', 'boolValue', 'doubleValue'): if key in node: return node[key] if 'intValue' in node: return int(node['intValue']) if 'arrayValue' in node: return [_any_value(v) for v in node['arrayValue'].get('values', [])] return None def _attributes(items): out = {} for item in items or []: out[item['key']] = _any_value(item.get('value')) return out _KIND = {0: 'UNSPECIFIED', 1: 'INTERNAL', 2: 'SERVER', 3: 'CLIENT', 4: 'PRODUCER', 5: 'CONSUMER'} _STATUS = {0: 'UNSET', 1: 'OK', 2: 'ERROR'} def normalize_trace(document): """Tempo JSON (batches or resourceSpans) -> list of span dicts.""" if not isinstance(document, dict): raise Failure('unknown Tempo response format') batches = document.get('batches', document.get('resourceSpans')) if batches is None: raise Failure('unknown Tempo response format (no batches/resourceSpans)') spans = [] for batch in batches: resource = _attributes(batch.get('resource', {}).get('attributes')) scopes = batch.get('scopeSpans', batch.get('instrumentationLibrarySpans')) if scopes is None: raise Failure('unknown Tempo batch format (no scopeSpans)') for scope in scopes: for span in scope.get('spans', []): kind = span.get('kind', 0) if isinstance(kind, str): kind = kind.replace('SPAN_KIND_', '') else: kind = _KIND.get(kind, str(kind)) status = span.get('status', {}).get('code', 0) if isinstance(status, str): status = status.replace('STATUS_CODE_', '') else: status = _STATUS.get(status, str(status)) spans.append({ 'trace_id': normalize_id(span.get('traceId', ''), 16), 'span_id': normalize_id(span.get('spanId', ''), 8), 'parent_span_id': normalize_id( span.get('parentSpanId', ''), 8), 'name': span.get('name', ''), 'kind': kind, 'start_ns': int(span.get('startTimeUnixNano', 0)), 'end_ns': int(span.get('endTimeUnixNano', 0)), 'attributes': _attributes(span.get('attributes')), 'status': status, 'links': [{ 'trace_id': normalize_id(link.get('traceId', ''), 16), 'span_id': normalize_id(link.get('spanId', ''), 8), } for link in span.get('links', [])], 'service': resource.get('service.name'), 'instance': resource.get('service.instance.id'), }) return spans # ------------------------------------------------------------------- client class ObsClient: def __init__(self, prometheus_url, grafana_url, password_file=None, base_url=None, ca_file=None): self.prometheus = prometheus_url.rstrip('/') self.grafana = grafana_url.rstrip('/') self.base_url = base_url self.ca_file = ca_file path = Path(password_file) if password_file else ( ROOT / 'secrets/grafana_admin_password') self._auth = None if path.exists(): token = base64.b64encode( b'admin:' + path.read_bytes().strip()).decode() self._auth = {'Authorization': 'Basic ' + token} # -- Prometheus def prom_query(self, expr, at=None): params = {'query': expr} if at is not None: params['time'] = str(at) _, body = http_json(self.prometheus + '/api/v1/query?' + urllib.parse.urlencode(params)) if not isinstance(body, dict) or body.get('status') != 'success': raise Failure('Prometheus query failed') return body['data']['result'] def prom_scalar(self, expr, default=None): result = self.prom_query(expr) if not result: return default return float(result[0]['value'][1]) def prom_range(self, expr, start, end, step): params = {'query': expr, 'start': str(start), 'end': str(end), 'step': str(step)} _, body = http_json(self.prometheus + '/api/v1/query_range?' + urllib.parse.urlencode(params)) if not isinstance(body, dict) or body.get('status') != 'success': raise Failure('Prometheus range query failed') return body['data']['result'] def prom_names(self): _, body = http_json(self.prometheus + '/api/v1/label/__name__/values') return sorted(body['data']) # -- Grafana proxied backends def _grafana_get(self, path, allow=(200,)): if self._auth is None: raise Failure('Grafana password file is missing') return http_get(self.grafana + path, self._auth) def grafana_json(self, path, allow=(200,)): status, data = self._grafana_get(path) if status not in allow: raise Failure('Grafana ' + path.split('?')[0] + ': status ' + str(status)) try: return json.loads(data) except (ValueError, UnicodeError): raise Failure('Grafana ' + path.split('?')[0] + ': invalid JSON') from None def loki_rows(self, query, start_ns, end_ns, deadline_s=60): """All log rows in [start, end]: [{'ts', 'line', 'labels'}].""" limit = 5000 deadline = time.monotonic() + deadline_s cursor = int(start_ns) seen = set() rows = [] while True: if time.monotonic() > deadline: raise Failure('Loki query deadline exceeded; incomplete window') params = urllib.parse.urlencode({ 'query': query, 'start': str(cursor), 'end': str(int(end_ns)), 'limit': str(limit), 'direction': 'forward'}) body = self.grafana_json( '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params) if body.get('status') != 'success': raise Failure('Loki query failed') fresh = 0 page = 0 last_ts = cursor for stream in body['data']['result']: labels = stream.get('stream', {}) key = json.dumps(labels, sort_keys=True) for value in stream.get('values', []): page += 1 ts, line = int(value[0]), value[1] last_ts = max(last_ts, ts) ident = (key, ts, line) if ident in seen: continue seen.add(ident) fresh += 1 rows.append({'ts': ts, 'line': line, 'labels': labels}) if page < limit: return rows if fresh == 0 or last_ts == cursor: raise Failure('Loki pagination made no progress; ' 'incomplete window') cursor = last_ts # overlap on the last timestamp, deduped above def get_trace(self, trace_id, wait=0): """Normalised spans, or None on 404; waits up to `wait` seconds.""" deadline = time.monotonic() + min(wait, 60) path = '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id while True: if self._auth is None: raise Failure('Grafana password file is missing') status, data = http_get( self.grafana + path, {**self._auth, 'Accept': 'application/json'}) if status == 200: try: return normalize_trace(json.loads(data)) except (ValueError, UnicodeError): raise Failure('Tempo returned invalid JSON') from None if status != 404: raise Failure('Tempo trace lookup: status ' + str(status)) if time.monotonic() >= deadline: return None time.sleep(1) # -- closed backends from inside the compose network def internal_get(self, service, port, path): script = ( "const [h,p,u]=process.argv.slice(1);" "fetch('http://'+h+':'+p+u,{signal:AbortSignal.timeout(5000)})" ".then(async r=>{const b=await r.text();" "console.log(JSON.stringify({status:r.status,body:b.slice(0,1048576)}))})" ".catch(e=>{console.log(JSON.stringify({status:0,body:String(e.name)}))})") result = stand.compose('exec', '-T', 'api', 'node', '-e', script, service, str(port), path, check_result=False) try: out = json.loads(result.stdout.strip().splitlines()[-1]) except (ValueError, IndexError): raise Failure('internal_get ' + service + ': no result') from None return out['status'], out['body'] # -- readiness def _check(self, problems, name, ok): if not ok: problems.append(name) def readiness_problems(self, app_telemetry=True, provisioning=True): problems = [] if self.base_url: try: status, _ = http_get(self.base_url.rstrip('/') + '/health/ready') self._check(problems, 'api-ready', status == 200) except Failure: problems.append('api-ready') probes = (('otel-collector', 13133, '/', 'collector-health'), ('loki', 3100, '/ready', 'loki-ready'), ('tempo', 3200, '/ready', 'tempo-ready')) for service, port, path, name in probes: try: status, _ = self.internal_get(service, port, path) self._check(problems, name, status == 200) except Failure: problems.append(name) try: status, _ = http_get(self.prometheus + '/-/ready') self._check(problems, 'prometheus-ready', status == 200) except Failure: problems.append('prometheus-ready') try: status, _ = http_get(self.grafana + '/api/health') self._check(problems, 'grafana-health', status == 200) except Failure: problems.append('grafana-health') try: up = {r['metric']['job']: r['value'][1] for r in self.prom_query('up')} for job in SCRAPE_JOBS: self._check(problems, 'target-up:' + job, up.get(job) == '1') if app_telemetry: ages = self.prom_query( 'time() - timestamp(target_info{job=~"messenger-.*"})') fresh = {} for r in ages: if float(r['value'][1]) < 30: fresh.setdefault(r['metric']['job'], set()).add( r['metric'].get('instance')) self._check(problems, 'api-instances>=2', len(fresh.get('messenger-api', ())) >= 2) self._check(problems, 'worker-instance', len(fresh.get('messenger-worker', ())) >= 1) except Failure: problems.append('prometheus-queries') if provisioning: try: for uid in DATASOURCE_UIDS: if uid == 'tempo': # The Tempo plugin has no health resource; its API echo # through the datasource proxy proves the connection. status, _ = self._grafana_get( '/api/datasources/proxy/uid/tempo/api/echo') self._check(problems, 'datasource:tempo', status == 200) continue body = self.grafana_json( '/api/datasources/uid/' + uid + '/health', allow=(200, 400, 404, 500)) self._check(problems, 'datasource:' + uid, isinstance(body, dict) and body.get('status', '').upper() == 'OK') _, rules = http_json(self.prometheus + '/api/v1/rules') names = {r['name'] for g in rules['data']['groups'] for r in g['rules']} for alert in RULE_ALERTS: self._check(problems, 'rule:' + alert, alert in names) for uid in DASHBOARD_UIDS: status, _ = self._grafana_get('/api/dashboards/uid/' + uid) self._check(problems, 'dashboard:' + uid, status == 200) except Failure: problems.append('provisioning-queries') return problems def wait_ready(self, timeout=120, app_telemetry=True, provisioning=True): deadline = time.monotonic() + timeout problems = ['not-checked'] while time.monotonic() < deadline: problems = self.readiness_problems(app_telemetry, provisioning) if not problems: return time.sleep(2) raise Failure('stack not ready: ' + ', '.join(problems)) def wait_telemetry_flushed(self, timeout=120): """Two new Prometheus scrapes of the Collector and empty queues.""" deadline = time.monotonic() + timeout stamps = [] while time.monotonic() < deadline: result = self.prom_query('timestamp(up{job="otel-collector"})') if result: stamp = float(result[0]['value'][1]) if not stamps or stamp > stamps[-1]: stamps.append(stamp) queue = self.prom_query('sum(otelcol_exporter_queue_size)') empty = bool(queue) and float(queue[0]['value'][1]) == 0 if len(stamps) >= 3 and empty: return time.sleep(2) raise Failure('telemetry did not flush before deadline') # -- controlled application environment def container_ids(self, service): out = stand.compose('ps', '-q', service).stdout.split() if not out: raise Failure('no running container for ' + service) return out def inspect_env(self, service): """{container_id: {'env': {...}, 'image': id, 'restarts': n}}""" result = {} for cid in self.container_ids(service): data = json.loads(stand.docker('inspect', cid).stdout)[0] env = dict(item.split('=', 1) for item in data['Config']['Env'] if '=' in item) result[cid[:12]] = { 'env': {k: env.get(k) for k in MUTABLE_KEYS + DIAGNOSTIC_KEYS}, 'image': data['Image'], 'restarts': data['RestartCount'], 'oom_killed': data['State'].get('OOMKilled', False), } return result def app_state(self): return {'api': self.inspect_env('api'), 'worker': self.inspect_env('worker')} @contextlib.contextmanager def app_environment(self, initial_overrides=None): saved_state = self.app_state() baseline = {} for service in ('api', 'worker'): for info in saved_state[service].values(): for key in MUTABLE_KEYS: baseline.setdefault((service, key), info['env'][key]) saved_environ = {k: os.environ.get(k) for k in MUTABLE_KEYS} touched = set() dirty = [False] def effective(service): return {cid: info for cid, info in self.inspect_env(service).items()} def set_overrides(values, force=False): unknown = set(values) - set(MUTABLE_KEYS) if unknown: raise Failure('not a controllable key: ' + ', '.join(sorted(unknown))) for key in values: api = {i['env'][key] for i in effective('api').values()} worker = {i['env'][key] for i in effective('worker').values()} if len(api | worker) > 1 and api != worker: raise Failure('api/worker disagree on ' + key + ' before change') if not force and all( info['env'][key] == str(value) for service in ('api', 'worker') for info in effective(service).values() for key, value in values.items()): return self.app_state() for key, value in values.items(): os.environ[key] = str(value) touched.add(key) dirty[0] = True stand.compose('up', '-d', '--no-deps', '--force-recreate', '--wait', 'api', 'worker') for service in ('api', 'worker'): state = effective(service) for cid, info in state.items(): for key, value in values.items(): if info['env'][key] != str(value): raise Failure('effective ' + key + ' differs in ' + service + ' ' + cid) sdk_on = os.environ.get('OTEL_SDK_DISABLED', 'false') != 'true' self.wait_ready(app_telemetry=sdk_on, provisioning=True) return self.app_state() try: if initial_overrides: set_overrides(initial_overrides) yield set_overrides finally: if dirty[0]: # Restore the original effective values as controller env so the # recreated containers match what was running before. for key in MUTABLE_KEYS: original = baseline.get(('api', key)) if original is not None and key in touched: os.environ[key] = original stand.compose('up', '-d', '--no-deps', '--force-recreate', '--wait', 'api', 'worker') for key in touched: if saved_environ[key] is None: os.environ.pop(key, None) else: os.environ[key] = saved_environ[key] now = self.app_state() for service in ('api', 'worker'): for cid, info in now[service].items(): for key in touched: if info['env'][key] != baseline.get((service, key)): print('WARNING: ' + key + ' not restored on ' + service, file=sys.stderr) with contextlib.suppress(Failure): self.wait_ready(app_telemetry=True, provisioning=True, timeout=120) # -- foreign faults def foreign_faults(self): result = stand.compose('exec', '-T', 'redis', 'redis-cli', '--scan', '--pattern', 'fault:*') return sorted(line for line in result.stdout.split() if line) def require_no_foreign_faults(self): keys = self.foreign_faults() if keys: raise Failure('foreign fault keys present (finish the other ' 'experiment first): ' + ', '.join(keys)) # --------------------------------------------------------------- PostgreSQL def psql(sql, timeout=30): script = 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 ' \ '-At -F , -c "$SQL"' result = subprocess.run( ['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T', '-e', 'SQL=' + sql, 'postgres', 'sh', '-c', script], cwd=ROOT, text=True, capture_output=True, timeout=timeout) if result.returncode != 0: raise Failure('psql failed: ' + ((result.stderr or '').strip() .splitlines() or ['?'])[-1][:200]) return result.stdout.strip() @contextlib.contextmanager def hold_table_lock(table): """ACCESS EXCLUSIVE lock on messages/attachments held until exit.""" if table not in LOCKABLE_TABLES: raise Failure('table is not lockable: ' + table) name = 'lab6-lock-' + uuid.uuid4().hex[:12] process = subprocess.Popen( ['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T', 'postgres', 'sh', '-c', 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -At'], cwd=ROOT, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1) pid = None try: process.stdin.write( "SET application_name='" + name + "';\n" "BEGIN;\n" "SET LOCAL statement_timeout='120s';\n" "SET LOCAL idle_in_transaction_session_timeout='120s';\n" "LOCK TABLE " + table + " IN ACCESS EXCLUSIVE MODE;\n" "SELECT 'LOCKED:'||pg_backend_pid();\n") process.stdin.flush() deadline = time.monotonic() + 30 while pid is None: if time.monotonic() > deadline: raise Failure('table lock marker not received') ready, _, _ = select.select([process.stdout], [], [], 1) if not ready: if process.poll() is not None: raise Failure('psql exited before locking') continue line = process.stdout.readline() if line.startswith('LOCKED:'): pid = int(line.split(':')[1]) elif line == '' and process.poll() is not None: raise Failure('psql exited before locking') yield pid finally: try: if process.poll() is None: process.stdin.write('ROLLBACK;\n\\q\n') process.stdin.flush() process.stdin.close() except (OSError, ValueError): pass try: process.wait(timeout=10) except subprocess.TimeoutExpired: process.kill() if pid is not None: with contextlib.suppress(Failure, subprocess.TimeoutExpired): psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity " "WHERE pid=" + str(int(pid)) + " AND application_name='" + name + "'") for stream in (process.stdout, process.stderr): with contextlib.suppress(Exception): stream.close() def wait_drained(attachment_ids, obs=None, timeout=120, flush_telemetry=True): """Batch-check jobs/outbox/attachments in PostgreSQL (no HTTP polling). Returns {'drained': bool, 'states': {id: status}, 'queued': n, 'outbox': n, 'seconds': t, 'timestamps': ...}. A vanished known ID is a Failure; non-ready at the deadline is reported, not raised. """ ids = list(attachment_ids) for item in ids: uuid.UUID(item) array = "ARRAY[" + ",".join("'" + i + "'" for i in ids) + "]::uuid[]" \ if ids else "ARRAY[]::uuid[]" sql = ( "SELECT (SELECT count(*) FROM attachment_jobs WHERE status IN " "('queued','processing'))||','||" "(SELECT count(*) FROM job_outbox WHERE published_at IS NULL)||','||" "COALESCE((SELECT string_agg(id::text||':'||status,';') FROM attachments " "WHERE id = ANY(" + array + ")),'')") started = time.monotonic() start_ts = utc_now() states = {} queued = outbox = -1 drained = False while time.monotonic() - started < timeout: out = psql(sql) queued_s, outbox_s, rest = (out.split(',', 2) + ['', ''])[:3] queued, outbox = int(queued_s), int(outbox_s) states = dict(part.split(':', 1) for part in rest.split(';') if part) missing = set(ids) - set(states) if missing: raise Failure('known attachment vanished: ' + sorted(missing)[0]) terminal = all(s in ('ready', 'failed') for s in states.values()) if queued == 0 and outbox == 0 and terminal: drained = True break time.sleep(1) result = {'drained': drained, 'states': states, 'queued_jobs': queued, 'unpublished_outbox': outbox, 'seconds': round(time.monotonic() - started, 3), 'started': start_ts, 'finished': utc_now()} if flush_telemetry and obs is not None: obs.wait_telemetry_flushed(timeout=timeout) result['telemetry_flushed'] = utc_now() return result