Files

676 lines
28 KiB
Python

"""Shared lab6 helpers: Prometheus/Loki/Tempo/Grafana client, controlled app
environment, SQL table lock, batch drain. Python 3.9+, standard library only."""
import base64
import binascii
import contextlib
import json
import os
import select
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.lab5 import stand # noqa: E402
Failure = stand.Failure
HTTP_TIMEOUT = 10
SCRAPE_JOBS = ('prometheus', 'otel-collector', 'haproxy', 'rabbitmq', 'loki',
'tempo')
RULE_ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
DASHBOARD_UIDS = ('messenger-overview', 'messenger-telemetry')
DATASOURCE_UIDS = ('prometheus', 'loki', 'tempo')
LOCKABLE_TABLES = ('messages', 'attachments')
# Keys the controller may change/restore (identical for api and worker).
MUTABLE_KEYS = (
'OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG', 'OTEL_METRIC_EXPORT_INTERVAL',
'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL',
'OTEL_EXPORTER_OTLP_TIMEOUT', 'OTEL_RESOURCE_ATTRIBUTES',
'OTEL_BSP_MAX_QUEUE_SIZE', 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE',
'OTEL_BSP_SCHEDULE_DELAY', 'OTEL_BLRP_MAX_QUEUE_SIZE',
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', 'OTEL_BLRP_SCHEDULE_DELAY',
'LOG_LEVEL', 'CACHE_MODE', 'FAULT_INJECTION_ENABLED',
)
# Recorded as diagnostics, never settable.
DIAGNOSTIC_KEYS = ('OTEL_SERVICE_NAME', 'SERVICE_VERSION')
# Baseline of the SDK knobs (compose defaults) used by track1 and incidents.
SDK_DEFAULTS = {
'OTEL_SDK_DISABLED': 'false',
'OTEL_TRACES_SAMPLER_ARG': '1',
'OTEL_METRIC_EXPORT_INTERVAL': '10000',
'OTEL_EXPORTER_OTLP_TIMEOUT': '5000',
'OTEL_BSP_MAX_QUEUE_SIZE': '2048',
'OTEL_BSP_MAX_EXPORT_BATCH_SIZE': '512',
'OTEL_BSP_SCHEDULE_DELAY': '1000',
'OTEL_BLRP_MAX_QUEUE_SIZE': '2048',
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE': '512',
'OTEL_BLRP_SCHEDULE_DELAY': '1000',
'LOG_LEVEL': 'INFO',
}
def utc_now():
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}),
_NoRedirect())
def http_get(url, headers=None, timeout=HTTP_TIMEOUT):
"""GET returning (status, bytes); never prints the URL query/headers."""
request = urllib.request.Request(url, headers=headers or {})
try:
with _OPENER.open(request, timeout=timeout) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
with error:
return error.code, error.read()
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise Failure('GET ' + url.split('?')[0] + ': connection failure (' +
type(exc).__name__ + ')') from None
def http_json(url, headers=None, timeout=HTTP_TIMEOUT, allow=(200,)):
status, data = http_get(url, headers, timeout)
if status not in allow:
raise Failure('GET ' + url.split('?')[0] + ': status ' + str(status))
try:
return status, json.loads(data) if data else None
except (ValueError, UnicodeError):
raise Failure('GET ' + url.split('?')[0] + ': invalid JSON') from None
# ---------------------------------------------------------------- trace JSON
def normalize_id(value, nbytes):
"""Hex or base64 ID -> lowercase hex of exactly nbytes; else Failure."""
if not isinstance(value, str):
raise Failure('trace id is not a string')
if value == '':
return ''
if len(value) == nbytes * 2:
try:
int(value, 16)
return value.lower()
except ValueError:
pass
try:
raw = base64.b64decode(value, validate=True)
except (binascii.Error, ValueError):
raise Failure('unrecognised trace/span id encoding') from None
if len(raw) != nbytes:
raise Failure('trace/span id has wrong length')
return raw.hex()
def _any_value(node):
if not isinstance(node, dict):
return node
for key in ('stringValue', 'boolValue', 'doubleValue'):
if key in node:
return node[key]
if 'intValue' in node:
return int(node['intValue'])
if 'arrayValue' in node:
return [_any_value(v) for v in node['arrayValue'].get('values', [])]
return None
def _attributes(items):
out = {}
for item in items or []:
out[item['key']] = _any_value(item.get('value'))
return out
_KIND = {0: 'UNSPECIFIED', 1: 'INTERNAL', 2: 'SERVER', 3: 'CLIENT',
4: 'PRODUCER', 5: 'CONSUMER'}
_STATUS = {0: 'UNSET', 1: 'OK', 2: 'ERROR'}
def normalize_trace(document):
"""Tempo JSON (batches or resourceSpans) -> list of span dicts."""
if not isinstance(document, dict):
raise Failure('unknown Tempo response format')
batches = document.get('batches', document.get('resourceSpans'))
if batches is None:
raise Failure('unknown Tempo response format (no batches/resourceSpans)')
spans = []
for batch in batches:
resource = _attributes(batch.get('resource', {}).get('attributes'))
scopes = batch.get('scopeSpans', batch.get('instrumentationLibrarySpans'))
if scopes is None:
raise Failure('unknown Tempo batch format (no scopeSpans)')
for scope in scopes:
for span in scope.get('spans', []):
kind = span.get('kind', 0)
if isinstance(kind, str):
kind = kind.replace('SPAN_KIND_', '')
else:
kind = _KIND.get(kind, str(kind))
status = span.get('status', {}).get('code', 0)
if isinstance(status, str):
status = status.replace('STATUS_CODE_', '')
else:
status = _STATUS.get(status, str(status))
spans.append({
'trace_id': normalize_id(span.get('traceId', ''), 16),
'span_id': normalize_id(span.get('spanId', ''), 8),
'parent_span_id': normalize_id(
span.get('parentSpanId', ''), 8),
'name': span.get('name', ''),
'kind': kind,
'start_ns': int(span.get('startTimeUnixNano', 0)),
'end_ns': int(span.get('endTimeUnixNano', 0)),
'attributes': _attributes(span.get('attributes')),
'status': status,
'links': [{
'trace_id': normalize_id(link.get('traceId', ''), 16),
'span_id': normalize_id(link.get('spanId', ''), 8),
} for link in span.get('links', [])],
'service': resource.get('service.name'),
'instance': resource.get('service.instance.id'),
})
return spans
# ------------------------------------------------------------------- client
class ObsClient:
def __init__(self, prometheus_url, grafana_url, password_file=None,
base_url=None, ca_file=None):
self.prometheus = prometheus_url.rstrip('/')
self.grafana = grafana_url.rstrip('/')
self.base_url = base_url
self.ca_file = ca_file
path = Path(password_file) if password_file else (
ROOT / 'secrets/grafana_admin_password')
self._auth = None
if path.exists():
token = base64.b64encode(
b'admin:' + path.read_bytes().strip()).decode()
self._auth = {'Authorization': 'Basic ' + token}
# -- Prometheus
def prom_query(self, expr, at=None):
params = {'query': expr}
if at is not None:
params['time'] = str(at)
_, body = http_json(self.prometheus + '/api/v1/query?' +
urllib.parse.urlencode(params))
if not isinstance(body, dict) or body.get('status') != 'success':
raise Failure('Prometheus query failed')
return body['data']['result']
def prom_scalar(self, expr, default=None):
result = self.prom_query(expr)
if not result:
return default
return float(result[0]['value'][1])
def prom_range(self, expr, start, end, step):
params = {'query': expr, 'start': str(start), 'end': str(end),
'step': str(step)}
_, body = http_json(self.prometheus + '/api/v1/query_range?' +
urllib.parse.urlencode(params))
if not isinstance(body, dict) or body.get('status') != 'success':
raise Failure('Prometheus range query failed')
return body['data']['result']
def prom_names(self):
_, body = http_json(self.prometheus + '/api/v1/label/__name__/values')
return sorted(body['data'])
# -- Grafana proxied backends
def _grafana_get(self, path, allow=(200,)):
if self._auth is None:
raise Failure('Grafana password file is missing')
return http_get(self.grafana + path, self._auth)
def grafana_json(self, path, allow=(200,)):
status, data = self._grafana_get(path)
if status not in allow:
raise Failure('Grafana ' + path.split('?')[0] + ': status ' +
str(status))
try:
return json.loads(data)
except (ValueError, UnicodeError):
raise Failure('Grafana ' + path.split('?')[0] +
': invalid JSON') from None
def loki_rows(self, query, start_ns, end_ns, deadline_s=60):
"""All log rows in [start, end]: [{'ts', 'line', 'labels'}]."""
limit = 5000
deadline = time.monotonic() + deadline_s
cursor = int(start_ns)
seen = set()
rows = []
while True:
if time.monotonic() > deadline:
raise Failure('Loki query deadline exceeded; incomplete window')
params = urllib.parse.urlencode({
'query': query, 'start': str(cursor), 'end': str(int(end_ns)),
'limit': str(limit), 'direction': 'forward'})
body = self.grafana_json(
'/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' +
params)
if body.get('status') != 'success':
raise Failure('Loki query failed')
fresh = 0
page = 0
last_ts = cursor
for stream in body['data']['result']:
labels = stream.get('stream', {})
key = json.dumps(labels, sort_keys=True)
for value in stream.get('values', []):
page += 1
ts, line = int(value[0]), value[1]
last_ts = max(last_ts, ts)
ident = (key, ts, line)
if ident in seen:
continue
seen.add(ident)
fresh += 1
rows.append({'ts': ts, 'line': line, 'labels': labels})
if page < limit:
return rows
if fresh == 0 or last_ts == cursor:
raise Failure('Loki pagination made no progress; '
'incomplete window')
cursor = last_ts # overlap on the last timestamp, deduped above
def get_trace(self, trace_id, wait=0):
"""Normalised spans, or None on 404; waits up to `wait` seconds."""
deadline = time.monotonic() + min(wait, 60)
path = '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id
while True:
if self._auth is None:
raise Failure('Grafana password file is missing')
status, data = http_get(
self.grafana + path,
{**self._auth, 'Accept': 'application/json'})
if status == 200:
try:
return normalize_trace(json.loads(data))
except (ValueError, UnicodeError):
raise Failure('Tempo returned invalid JSON') from None
if status != 404:
raise Failure('Tempo trace lookup: status ' + str(status))
if time.monotonic() >= deadline:
return None
time.sleep(1)
# -- closed backends from inside the compose network
def internal_get(self, service, port, path):
script = (
"const [h,p,u]=process.argv.slice(1);"
"fetch('http://'+h+':'+p+u,{signal:AbortSignal.timeout(5000)})"
".then(async r=>{const b=await r.text();"
"console.log(JSON.stringify({status:r.status,body:b.slice(0,1048576)}))})"
".catch(e=>{console.log(JSON.stringify({status:0,body:String(e.name)}))})")
result = stand.compose('exec', '-T', 'api', 'node', '-e', script,
service, str(port), path, check_result=False)
try:
out = json.loads(result.stdout.strip().splitlines()[-1])
except (ValueError, IndexError):
raise Failure('internal_get ' + service + ': no result') from None
return out['status'], out['body']
# -- readiness
def _check(self, problems, name, ok):
if not ok:
problems.append(name)
def readiness_problems(self, app_telemetry=True, provisioning=True):
problems = []
if self.base_url:
try:
status, _ = http_get(self.base_url.rstrip('/') + '/health/ready')
self._check(problems, 'api-ready', status == 200)
except Failure:
problems.append('api-ready')
probes = (('otel-collector', 13133, '/', 'collector-health'),
('loki', 3100, '/ready', 'loki-ready'),
('tempo', 3200, '/ready', 'tempo-ready'))
for service, port, path, name in probes:
try:
status, _ = self.internal_get(service, port, path)
self._check(problems, name, status == 200)
except Failure:
problems.append(name)
try:
status, _ = http_get(self.prometheus + '/-/ready')
self._check(problems, 'prometheus-ready', status == 200)
except Failure:
problems.append('prometheus-ready')
try:
status, _ = http_get(self.grafana + '/api/health')
self._check(problems, 'grafana-health', status == 200)
except Failure:
problems.append('grafana-health')
try:
up = {r['metric']['job']: r['value'][1]
for r in self.prom_query('up')}
for job in SCRAPE_JOBS:
self._check(problems, 'target-up:' + job, up.get(job) == '1')
if app_telemetry:
ages = self.prom_query(
'time() - timestamp(target_info{job=~"messenger-.*"})')
fresh = {}
for r in ages:
if float(r['value'][1]) < 30:
fresh.setdefault(r['metric']['job'], set()).add(
r['metric'].get('instance'))
self._check(problems, 'api-instances>=2',
len(fresh.get('messenger-api', ())) >= 2)
self._check(problems, 'worker-instance',
len(fresh.get('messenger-worker', ())) >= 1)
except Failure:
problems.append('prometheus-queries')
if provisioning:
try:
for uid in DATASOURCE_UIDS:
if uid == 'tempo':
# The Tempo plugin has no health resource; its API echo
# through the datasource proxy proves the connection.
status, _ = self._grafana_get(
'/api/datasources/proxy/uid/tempo/api/echo')
self._check(problems, 'datasource:tempo', status == 200)
continue
body = self.grafana_json(
'/api/datasources/uid/' + uid + '/health',
allow=(200, 400, 404, 500))
self._check(problems, 'datasource:' + uid,
isinstance(body, dict) and
body.get('status', '').upper() == 'OK')
_, rules = http_json(self.prometheus + '/api/v1/rules')
names = {r['name'] for g in rules['data']['groups']
for r in g['rules']}
for alert in RULE_ALERTS:
self._check(problems, 'rule:' + alert, alert in names)
for uid in DASHBOARD_UIDS:
status, _ = self._grafana_get('/api/dashboards/uid/' + uid)
self._check(problems, 'dashboard:' + uid, status == 200)
except Failure:
problems.append('provisioning-queries')
return problems
def wait_ready(self, timeout=120, app_telemetry=True, provisioning=True):
deadline = time.monotonic() + timeout
problems = ['not-checked']
while time.monotonic() < deadline:
problems = self.readiness_problems(app_telemetry, provisioning)
if not problems:
return
time.sleep(2)
raise Failure('stack not ready: ' + ', '.join(problems))
def wait_telemetry_flushed(self, timeout=120):
"""Two new Prometheus scrapes of the Collector and empty queues."""
deadline = time.monotonic() + timeout
stamps = []
while time.monotonic() < deadline:
result = self.prom_query('timestamp(up{job="otel-collector"})')
if result:
stamp = float(result[0]['value'][1])
if not stamps or stamp > stamps[-1]:
stamps.append(stamp)
queue = self.prom_query('sum(otelcol_exporter_queue_size)')
empty = bool(queue) and float(queue[0]['value'][1]) == 0
if len(stamps) >= 3 and empty:
return
time.sleep(2)
raise Failure('telemetry did not flush before deadline')
# -- controlled application environment
def container_ids(self, service):
out = stand.compose('ps', '-q', service).stdout.split()
if not out:
raise Failure('no running container for ' + service)
return out
def inspect_env(self, service):
"""{container_id: {'env': {...}, 'image': id, 'restarts': n}}"""
result = {}
for cid in self.container_ids(service):
data = json.loads(stand.docker('inspect', cid).stdout)[0]
env = dict(item.split('=', 1) for item in data['Config']['Env']
if '=' in item)
result[cid[:12]] = {
'env': {k: env.get(k) for k in MUTABLE_KEYS + DIAGNOSTIC_KEYS},
'image': data['Image'],
'restarts': data['RestartCount'],
'oom_killed': data['State'].get('OOMKilled', False),
}
return result
def app_state(self):
return {'api': self.inspect_env('api'),
'worker': self.inspect_env('worker')}
@contextlib.contextmanager
def app_environment(self, initial_overrides=None):
saved_state = self.app_state()
baseline = {}
for service in ('api', 'worker'):
for info in saved_state[service].values():
for key in MUTABLE_KEYS:
baseline.setdefault((service, key), info['env'][key])
saved_environ = {k: os.environ.get(k) for k in MUTABLE_KEYS}
touched = set()
dirty = [False]
def effective(service):
return {cid: info for cid, info in self.inspect_env(service).items()}
def set_overrides(values, force=False):
unknown = set(values) - set(MUTABLE_KEYS)
if unknown:
raise Failure('not a controllable key: ' + ', '.join(sorted(unknown)))
for key in values:
api = {i['env'][key] for i in effective('api').values()}
worker = {i['env'][key] for i in effective('worker').values()}
if len(api | worker) > 1 and api != worker:
raise Failure('api/worker disagree on ' + key + ' before change')
if not force and all(
info['env'][key] == str(value)
for service in ('api', 'worker')
for info in effective(service).values()
for key, value in values.items()):
return self.app_state()
for key, value in values.items():
os.environ[key] = str(value)
touched.add(key)
dirty[0] = True
stand.compose('up', '-d', '--no-deps', '--force-recreate',
'--wait', 'api', 'worker')
for service in ('api', 'worker'):
state = effective(service)
for cid, info in state.items():
for key, value in values.items():
if info['env'][key] != str(value):
raise Failure('effective ' + key + ' differs in ' +
service + ' ' + cid)
sdk_on = os.environ.get('OTEL_SDK_DISABLED', 'false') != 'true'
self.wait_ready(app_telemetry=sdk_on, provisioning=True)
return self.app_state()
try:
if initial_overrides:
set_overrides(initial_overrides)
yield set_overrides
finally:
if dirty[0]:
# Restore the original effective values as controller env so the
# recreated containers match what was running before.
for key in MUTABLE_KEYS:
original = baseline.get(('api', key))
if original is not None and key in touched:
os.environ[key] = original
stand.compose('up', '-d', '--no-deps', '--force-recreate',
'--wait', 'api', 'worker')
for key in touched:
if saved_environ[key] is None:
os.environ.pop(key, None)
else:
os.environ[key] = saved_environ[key]
now = self.app_state()
for service in ('api', 'worker'):
for cid, info in now[service].items():
for key in touched:
if info['env'][key] != baseline.get((service, key)):
print('WARNING: ' + key + ' not restored on ' +
service, file=sys.stderr)
with contextlib.suppress(Failure):
self.wait_ready(app_telemetry=True, provisioning=True,
timeout=120)
# -- foreign faults
def foreign_faults(self):
result = stand.compose('exec', '-T', 'redis', 'redis-cli', '--scan',
'--pattern', 'fault:*')
return sorted(line for line in result.stdout.split() if line)
def require_no_foreign_faults(self):
keys = self.foreign_faults()
if keys:
raise Failure('foreign fault keys present (finish the other '
'experiment first): ' + ', '.join(keys))
# --------------------------------------------------------------- PostgreSQL
def psql(sql, timeout=30):
script = 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 ' \
'-At -F , -c "$SQL"'
result = subprocess.run(
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
'-e', 'SQL=' + sql, 'postgres', 'sh', '-c', script],
cwd=ROOT, text=True, capture_output=True, timeout=timeout)
if result.returncode != 0:
raise Failure('psql failed: ' + ((result.stderr or '').strip()
.splitlines() or ['?'])[-1][:200])
return result.stdout.strip()
@contextlib.contextmanager
def hold_table_lock(table):
"""ACCESS EXCLUSIVE lock on messages/attachments held until exit."""
if table not in LOCKABLE_TABLES:
raise Failure('table is not lockable: ' + table)
name = 'lab6-lock-' + uuid.uuid4().hex[:12]
process = subprocess.Popen(
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
'postgres', 'sh', '-c',
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -At'],
cwd=ROOT, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
pid = None
try:
process.stdin.write(
"SET application_name='" + name + "';\n"
"BEGIN;\n"
"SET LOCAL statement_timeout='120s';\n"
"SET LOCAL idle_in_transaction_session_timeout='120s';\n"
"LOCK TABLE " + table + " IN ACCESS EXCLUSIVE MODE;\n"
"SELECT 'LOCKED:'||pg_backend_pid();\n")
process.stdin.flush()
deadline = time.monotonic() + 30
while pid is None:
if time.monotonic() > deadline:
raise Failure('table lock marker not received')
ready, _, _ = select.select([process.stdout], [], [], 1)
if not ready:
if process.poll() is not None:
raise Failure('psql exited before locking')
continue
line = process.stdout.readline()
if line.startswith('LOCKED:'):
pid = int(line.split(':')[1])
elif line == '' and process.poll() is not None:
raise Failure('psql exited before locking')
yield pid
finally:
try:
if process.poll() is None:
process.stdin.write('ROLLBACK;\n\\q\n')
process.stdin.flush()
process.stdin.close()
except (OSError, ValueError):
pass
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
if pid is not None:
with contextlib.suppress(Failure, subprocess.TimeoutExpired):
psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE pid=" + str(int(pid)) +
" AND application_name='" + name + "'")
for stream in (process.stdout, process.stderr):
with contextlib.suppress(Exception):
stream.close()
def wait_drained(attachment_ids, obs=None, timeout=120, flush_telemetry=True):
"""Batch-check jobs/outbox/attachments in PostgreSQL (no HTTP polling).
Returns {'drained': bool, 'states': {id: status}, 'queued': n,
'outbox': n, 'seconds': t, 'timestamps': ...}. A vanished known ID is a
Failure; non-ready at the deadline is reported, not raised.
"""
ids = list(attachment_ids)
for item in ids:
uuid.UUID(item)
array = "ARRAY[" + ",".join("'" + i + "'" for i in ids) + "]::uuid[]" \
if ids else "ARRAY[]::uuid[]"
sql = (
"SELECT (SELECT count(*) FROM attachment_jobs WHERE status IN "
"('queued','processing'))||','||"
"(SELECT count(*) FROM job_outbox WHERE published_at IS NULL)||','||"
"COALESCE((SELECT string_agg(id::text||':'||status,';') FROM attachments "
"WHERE id = ANY(" + array + ")),'')")
started = time.monotonic()
start_ts = utc_now()
states = {}
queued = outbox = -1
drained = False
while time.monotonic() - started < timeout:
out = psql(sql)
queued_s, outbox_s, rest = (out.split(',', 2) + ['', ''])[:3]
queued, outbox = int(queued_s), int(outbox_s)
states = dict(part.split(':', 1) for part in rest.split(';') if part)
missing = set(ids) - set(states)
if missing:
raise Failure('known attachment vanished: ' + sorted(missing)[0])
terminal = all(s in ('ready', 'failed') for s in states.values())
if queued == 0 and outbox == 0 and terminal:
drained = True
break
time.sleep(1)
result = {'drained': drained, 'states': states, 'queued_jobs': queued,
'unpublished_outbox': outbox,
'seconds': round(time.monotonic() - started, 3),
'started': start_ts, 'finished': utc_now()}
if flush_telemetry and obs is not None:
obs.wait_telemetry_flushed(timeout=timeout)
result['telemetry_flushed'] = utc_now()
return result