676 lines
28 KiB
Python
676 lines
28 KiB
Python
"""Shared lab6 helpers: Prometheus/Loki/Tempo/Grafana client, controlled app
|
|
environment, SQL table lock, batch drain. Python 3.9+, standard library only."""
|
|
import base64
|
|
import binascii
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import select
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from scripts.lab5 import stand # noqa: E402
|
|
|
|
Failure = stand.Failure
|
|
|
|
HTTP_TIMEOUT = 10
|
|
SCRAPE_JOBS = ('prometheus', 'otel-collector', 'haproxy', 'rabbitmq', 'loki',
|
|
'tempo')
|
|
RULE_ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
|
|
DASHBOARD_UIDS = ('messenger-overview', 'messenger-telemetry')
|
|
DATASOURCE_UIDS = ('prometheus', 'loki', 'tempo')
|
|
LOCKABLE_TABLES = ('messages', 'attachments')
|
|
|
|
# Keys the controller may change/restore (identical for api and worker).
|
|
MUTABLE_KEYS = (
|
|
'OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG', 'OTEL_METRIC_EXPORT_INTERVAL',
|
|
'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL',
|
|
'OTEL_EXPORTER_OTLP_TIMEOUT', 'OTEL_RESOURCE_ATTRIBUTES',
|
|
'OTEL_BSP_MAX_QUEUE_SIZE', 'OTEL_BSP_MAX_EXPORT_BATCH_SIZE',
|
|
'OTEL_BSP_SCHEDULE_DELAY', 'OTEL_BLRP_MAX_QUEUE_SIZE',
|
|
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', 'OTEL_BLRP_SCHEDULE_DELAY',
|
|
'LOG_LEVEL', 'CACHE_MODE', 'FAULT_INJECTION_ENABLED',
|
|
)
|
|
# Recorded as diagnostics, never settable.
|
|
DIAGNOSTIC_KEYS = ('OTEL_SERVICE_NAME', 'SERVICE_VERSION')
|
|
|
|
# Baseline of the SDK knobs (compose defaults) used by track1 and incidents.
|
|
SDK_DEFAULTS = {
|
|
'OTEL_SDK_DISABLED': 'false',
|
|
'OTEL_TRACES_SAMPLER_ARG': '1',
|
|
'OTEL_METRIC_EXPORT_INTERVAL': '10000',
|
|
'OTEL_EXPORTER_OTLP_TIMEOUT': '5000',
|
|
'OTEL_BSP_MAX_QUEUE_SIZE': '2048',
|
|
'OTEL_BSP_MAX_EXPORT_BATCH_SIZE': '512',
|
|
'OTEL_BSP_SCHEDULE_DELAY': '1000',
|
|
'OTEL_BLRP_MAX_QUEUE_SIZE': '2048',
|
|
'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE': '512',
|
|
'OTEL_BLRP_SCHEDULE_DELAY': '1000',
|
|
'LOG_LEVEL': 'INFO',
|
|
}
|
|
|
|
|
|
def utc_now():
|
|
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
|
|
|
|
|
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *args, **kwargs):
|
|
return None
|
|
|
|
|
|
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}),
|
|
_NoRedirect())
|
|
|
|
|
|
def http_get(url, headers=None, timeout=HTTP_TIMEOUT):
|
|
"""GET returning (status, bytes); never prints the URL query/headers."""
|
|
request = urllib.request.Request(url, headers=headers or {})
|
|
try:
|
|
with _OPENER.open(request, timeout=timeout) as response:
|
|
return response.status, response.read()
|
|
except urllib.error.HTTPError as error:
|
|
with error:
|
|
return error.code, error.read()
|
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
raise Failure('GET ' + url.split('?')[0] + ': connection failure (' +
|
|
type(exc).__name__ + ')') from None
|
|
|
|
|
|
def http_json(url, headers=None, timeout=HTTP_TIMEOUT, allow=(200,)):
|
|
status, data = http_get(url, headers, timeout)
|
|
if status not in allow:
|
|
raise Failure('GET ' + url.split('?')[0] + ': status ' + str(status))
|
|
try:
|
|
return status, json.loads(data) if data else None
|
|
except (ValueError, UnicodeError):
|
|
raise Failure('GET ' + url.split('?')[0] + ': invalid JSON') from None
|
|
|
|
|
|
# ---------------------------------------------------------------- trace JSON
|
|
|
|
def normalize_id(value, nbytes):
|
|
"""Hex or base64 ID -> lowercase hex of exactly nbytes; else Failure."""
|
|
if not isinstance(value, str):
|
|
raise Failure('trace id is not a string')
|
|
if value == '':
|
|
return ''
|
|
if len(value) == nbytes * 2:
|
|
try:
|
|
int(value, 16)
|
|
return value.lower()
|
|
except ValueError:
|
|
pass
|
|
try:
|
|
raw = base64.b64decode(value, validate=True)
|
|
except (binascii.Error, ValueError):
|
|
raise Failure('unrecognised trace/span id encoding') from None
|
|
if len(raw) != nbytes:
|
|
raise Failure('trace/span id has wrong length')
|
|
return raw.hex()
|
|
|
|
|
|
def _any_value(node):
|
|
if not isinstance(node, dict):
|
|
return node
|
|
for key in ('stringValue', 'boolValue', 'doubleValue'):
|
|
if key in node:
|
|
return node[key]
|
|
if 'intValue' in node:
|
|
return int(node['intValue'])
|
|
if 'arrayValue' in node:
|
|
return [_any_value(v) for v in node['arrayValue'].get('values', [])]
|
|
return None
|
|
|
|
|
|
def _attributes(items):
|
|
out = {}
|
|
for item in items or []:
|
|
out[item['key']] = _any_value(item.get('value'))
|
|
return out
|
|
|
|
|
|
_KIND = {0: 'UNSPECIFIED', 1: 'INTERNAL', 2: 'SERVER', 3: 'CLIENT',
|
|
4: 'PRODUCER', 5: 'CONSUMER'}
|
|
_STATUS = {0: 'UNSET', 1: 'OK', 2: 'ERROR'}
|
|
|
|
|
|
def normalize_trace(document):
|
|
"""Tempo JSON (batches or resourceSpans) -> list of span dicts."""
|
|
if not isinstance(document, dict):
|
|
raise Failure('unknown Tempo response format')
|
|
batches = document.get('batches', document.get('resourceSpans'))
|
|
if batches is None:
|
|
raise Failure('unknown Tempo response format (no batches/resourceSpans)')
|
|
spans = []
|
|
for batch in batches:
|
|
resource = _attributes(batch.get('resource', {}).get('attributes'))
|
|
scopes = batch.get('scopeSpans', batch.get('instrumentationLibrarySpans'))
|
|
if scopes is None:
|
|
raise Failure('unknown Tempo batch format (no scopeSpans)')
|
|
for scope in scopes:
|
|
for span in scope.get('spans', []):
|
|
kind = span.get('kind', 0)
|
|
if isinstance(kind, str):
|
|
kind = kind.replace('SPAN_KIND_', '')
|
|
else:
|
|
kind = _KIND.get(kind, str(kind))
|
|
status = span.get('status', {}).get('code', 0)
|
|
if isinstance(status, str):
|
|
status = status.replace('STATUS_CODE_', '')
|
|
else:
|
|
status = _STATUS.get(status, str(status))
|
|
spans.append({
|
|
'trace_id': normalize_id(span.get('traceId', ''), 16),
|
|
'span_id': normalize_id(span.get('spanId', ''), 8),
|
|
'parent_span_id': normalize_id(
|
|
span.get('parentSpanId', ''), 8),
|
|
'name': span.get('name', ''),
|
|
'kind': kind,
|
|
'start_ns': int(span.get('startTimeUnixNano', 0)),
|
|
'end_ns': int(span.get('endTimeUnixNano', 0)),
|
|
'attributes': _attributes(span.get('attributes')),
|
|
'status': status,
|
|
'links': [{
|
|
'trace_id': normalize_id(link.get('traceId', ''), 16),
|
|
'span_id': normalize_id(link.get('spanId', ''), 8),
|
|
} for link in span.get('links', [])],
|
|
'service': resource.get('service.name'),
|
|
'instance': resource.get('service.instance.id'),
|
|
})
|
|
return spans
|
|
|
|
|
|
# ------------------------------------------------------------------- client
|
|
|
|
class ObsClient:
|
|
def __init__(self, prometheus_url, grafana_url, password_file=None,
|
|
base_url=None, ca_file=None):
|
|
self.prometheus = prometheus_url.rstrip('/')
|
|
self.grafana = grafana_url.rstrip('/')
|
|
self.base_url = base_url
|
|
self.ca_file = ca_file
|
|
path = Path(password_file) if password_file else (
|
|
ROOT / 'secrets/grafana_admin_password')
|
|
self._auth = None
|
|
if path.exists():
|
|
token = base64.b64encode(
|
|
b'admin:' + path.read_bytes().strip()).decode()
|
|
self._auth = {'Authorization': 'Basic ' + token}
|
|
|
|
# -- Prometheus
|
|
def prom_query(self, expr, at=None):
|
|
params = {'query': expr}
|
|
if at is not None:
|
|
params['time'] = str(at)
|
|
_, body = http_json(self.prometheus + '/api/v1/query?' +
|
|
urllib.parse.urlencode(params))
|
|
if not isinstance(body, dict) or body.get('status') != 'success':
|
|
raise Failure('Prometheus query failed')
|
|
return body['data']['result']
|
|
|
|
def prom_scalar(self, expr, default=None):
|
|
result = self.prom_query(expr)
|
|
if not result:
|
|
return default
|
|
return float(result[0]['value'][1])
|
|
|
|
def prom_range(self, expr, start, end, step):
|
|
params = {'query': expr, 'start': str(start), 'end': str(end),
|
|
'step': str(step)}
|
|
_, body = http_json(self.prometheus + '/api/v1/query_range?' +
|
|
urllib.parse.urlencode(params))
|
|
if not isinstance(body, dict) or body.get('status') != 'success':
|
|
raise Failure('Prometheus range query failed')
|
|
return body['data']['result']
|
|
|
|
def prom_names(self):
|
|
_, body = http_json(self.prometheus + '/api/v1/label/__name__/values')
|
|
return sorted(body['data'])
|
|
|
|
# -- Grafana proxied backends
|
|
def _grafana_get(self, path, allow=(200,)):
|
|
if self._auth is None:
|
|
raise Failure('Grafana password file is missing')
|
|
return http_get(self.grafana + path, self._auth)
|
|
|
|
def grafana_json(self, path, allow=(200,)):
|
|
status, data = self._grafana_get(path)
|
|
if status not in allow:
|
|
raise Failure('Grafana ' + path.split('?')[0] + ': status ' +
|
|
str(status))
|
|
try:
|
|
return json.loads(data)
|
|
except (ValueError, UnicodeError):
|
|
raise Failure('Grafana ' + path.split('?')[0] +
|
|
': invalid JSON') from None
|
|
|
|
def loki_rows(self, query, start_ns, end_ns, deadline_s=60):
|
|
"""All log rows in [start, end]: [{'ts', 'line', 'labels'}]."""
|
|
limit = 5000
|
|
deadline = time.monotonic() + deadline_s
|
|
cursor = int(start_ns)
|
|
seen = set()
|
|
rows = []
|
|
while True:
|
|
if time.monotonic() > deadline:
|
|
raise Failure('Loki query deadline exceeded; incomplete window')
|
|
params = urllib.parse.urlencode({
|
|
'query': query, 'start': str(cursor), 'end': str(int(end_ns)),
|
|
'limit': str(limit), 'direction': 'forward'})
|
|
body = self.grafana_json(
|
|
'/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' +
|
|
params)
|
|
if body.get('status') != 'success':
|
|
raise Failure('Loki query failed')
|
|
fresh = 0
|
|
page = 0
|
|
last_ts = cursor
|
|
for stream in body['data']['result']:
|
|
labels = stream.get('stream', {})
|
|
key = json.dumps(labels, sort_keys=True)
|
|
for value in stream.get('values', []):
|
|
page += 1
|
|
ts, line = int(value[0]), value[1]
|
|
last_ts = max(last_ts, ts)
|
|
ident = (key, ts, line)
|
|
if ident in seen:
|
|
continue
|
|
seen.add(ident)
|
|
fresh += 1
|
|
rows.append({'ts': ts, 'line': line, 'labels': labels})
|
|
if page < limit:
|
|
return rows
|
|
if fresh == 0 or last_ts == cursor:
|
|
raise Failure('Loki pagination made no progress; '
|
|
'incomplete window')
|
|
cursor = last_ts # overlap on the last timestamp, deduped above
|
|
|
|
def get_trace(self, trace_id, wait=0):
|
|
"""Normalised spans, or None on 404; waits up to `wait` seconds."""
|
|
deadline = time.monotonic() + min(wait, 60)
|
|
path = '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id
|
|
while True:
|
|
if self._auth is None:
|
|
raise Failure('Grafana password file is missing')
|
|
status, data = http_get(
|
|
self.grafana + path,
|
|
{**self._auth, 'Accept': 'application/json'})
|
|
if status == 200:
|
|
try:
|
|
return normalize_trace(json.loads(data))
|
|
except (ValueError, UnicodeError):
|
|
raise Failure('Tempo returned invalid JSON') from None
|
|
if status != 404:
|
|
raise Failure('Tempo trace lookup: status ' + str(status))
|
|
if time.monotonic() >= deadline:
|
|
return None
|
|
time.sleep(1)
|
|
|
|
# -- closed backends from inside the compose network
|
|
def internal_get(self, service, port, path):
|
|
script = (
|
|
"const [h,p,u]=process.argv.slice(1);"
|
|
"fetch('http://'+h+':'+p+u,{signal:AbortSignal.timeout(5000)})"
|
|
".then(async r=>{const b=await r.text();"
|
|
"console.log(JSON.stringify({status:r.status,body:b.slice(0,1048576)}))})"
|
|
".catch(e=>{console.log(JSON.stringify({status:0,body:String(e.name)}))})")
|
|
result = stand.compose('exec', '-T', 'api', 'node', '-e', script,
|
|
service, str(port), path, check_result=False)
|
|
try:
|
|
out = json.loads(result.stdout.strip().splitlines()[-1])
|
|
except (ValueError, IndexError):
|
|
raise Failure('internal_get ' + service + ': no result') from None
|
|
return out['status'], out['body']
|
|
|
|
# -- readiness
|
|
def _check(self, problems, name, ok):
|
|
if not ok:
|
|
problems.append(name)
|
|
|
|
def readiness_problems(self, app_telemetry=True, provisioning=True):
|
|
problems = []
|
|
if self.base_url:
|
|
try:
|
|
status, _ = http_get(self.base_url.rstrip('/') + '/health/ready')
|
|
self._check(problems, 'api-ready', status == 200)
|
|
except Failure:
|
|
problems.append('api-ready')
|
|
probes = (('otel-collector', 13133, '/', 'collector-health'),
|
|
('loki', 3100, '/ready', 'loki-ready'),
|
|
('tempo', 3200, '/ready', 'tempo-ready'))
|
|
for service, port, path, name in probes:
|
|
try:
|
|
status, _ = self.internal_get(service, port, path)
|
|
self._check(problems, name, status == 200)
|
|
except Failure:
|
|
problems.append(name)
|
|
try:
|
|
status, _ = http_get(self.prometheus + '/-/ready')
|
|
self._check(problems, 'prometheus-ready', status == 200)
|
|
except Failure:
|
|
problems.append('prometheus-ready')
|
|
try:
|
|
status, _ = http_get(self.grafana + '/api/health')
|
|
self._check(problems, 'grafana-health', status == 200)
|
|
except Failure:
|
|
problems.append('grafana-health')
|
|
try:
|
|
up = {r['metric']['job']: r['value'][1]
|
|
for r in self.prom_query('up')}
|
|
for job in SCRAPE_JOBS:
|
|
self._check(problems, 'target-up:' + job, up.get(job) == '1')
|
|
if app_telemetry:
|
|
ages = self.prom_query(
|
|
'time() - timestamp(target_info{job=~"messenger-.*"})')
|
|
fresh = {}
|
|
for r in ages:
|
|
if float(r['value'][1]) < 30:
|
|
fresh.setdefault(r['metric']['job'], set()).add(
|
|
r['metric'].get('instance'))
|
|
self._check(problems, 'api-instances>=2',
|
|
len(fresh.get('messenger-api', ())) >= 2)
|
|
self._check(problems, 'worker-instance',
|
|
len(fresh.get('messenger-worker', ())) >= 1)
|
|
except Failure:
|
|
problems.append('prometheus-queries')
|
|
if provisioning:
|
|
try:
|
|
for uid in DATASOURCE_UIDS:
|
|
if uid == 'tempo':
|
|
# The Tempo plugin has no health resource; its API echo
|
|
# through the datasource proxy proves the connection.
|
|
status, _ = self._grafana_get(
|
|
'/api/datasources/proxy/uid/tempo/api/echo')
|
|
self._check(problems, 'datasource:tempo', status == 200)
|
|
continue
|
|
body = self.grafana_json(
|
|
'/api/datasources/uid/' + uid + '/health',
|
|
allow=(200, 400, 404, 500))
|
|
self._check(problems, 'datasource:' + uid,
|
|
isinstance(body, dict) and
|
|
body.get('status', '').upper() == 'OK')
|
|
_, rules = http_json(self.prometheus + '/api/v1/rules')
|
|
names = {r['name'] for g in rules['data']['groups']
|
|
for r in g['rules']}
|
|
for alert in RULE_ALERTS:
|
|
self._check(problems, 'rule:' + alert, alert in names)
|
|
for uid in DASHBOARD_UIDS:
|
|
status, _ = self._grafana_get('/api/dashboards/uid/' + uid)
|
|
self._check(problems, 'dashboard:' + uid, status == 200)
|
|
except Failure:
|
|
problems.append('provisioning-queries')
|
|
return problems
|
|
|
|
def wait_ready(self, timeout=120, app_telemetry=True, provisioning=True):
|
|
deadline = time.monotonic() + timeout
|
|
problems = ['not-checked']
|
|
while time.monotonic() < deadline:
|
|
problems = self.readiness_problems(app_telemetry, provisioning)
|
|
if not problems:
|
|
return
|
|
time.sleep(2)
|
|
raise Failure('stack not ready: ' + ', '.join(problems))
|
|
|
|
def wait_telemetry_flushed(self, timeout=120):
|
|
"""Two new Prometheus scrapes of the Collector and empty queues."""
|
|
deadline = time.monotonic() + timeout
|
|
stamps = []
|
|
while time.monotonic() < deadline:
|
|
result = self.prom_query('timestamp(up{job="otel-collector"})')
|
|
if result:
|
|
stamp = float(result[0]['value'][1])
|
|
if not stamps or stamp > stamps[-1]:
|
|
stamps.append(stamp)
|
|
queue = self.prom_query('sum(otelcol_exporter_queue_size)')
|
|
empty = bool(queue) and float(queue[0]['value'][1]) == 0
|
|
if len(stamps) >= 3 and empty:
|
|
return
|
|
time.sleep(2)
|
|
raise Failure('telemetry did not flush before deadline')
|
|
|
|
# -- controlled application environment
|
|
def container_ids(self, service):
|
|
out = stand.compose('ps', '-q', service).stdout.split()
|
|
if not out:
|
|
raise Failure('no running container for ' + service)
|
|
return out
|
|
|
|
def inspect_env(self, service):
|
|
"""{container_id: {'env': {...}, 'image': id, 'restarts': n}}"""
|
|
result = {}
|
|
for cid in self.container_ids(service):
|
|
data = json.loads(stand.docker('inspect', cid).stdout)[0]
|
|
env = dict(item.split('=', 1) for item in data['Config']['Env']
|
|
if '=' in item)
|
|
result[cid[:12]] = {
|
|
'env': {k: env.get(k) for k in MUTABLE_KEYS + DIAGNOSTIC_KEYS},
|
|
'image': data['Image'],
|
|
'restarts': data['RestartCount'],
|
|
'oom_killed': data['State'].get('OOMKilled', False),
|
|
}
|
|
return result
|
|
|
|
def app_state(self):
|
|
return {'api': self.inspect_env('api'),
|
|
'worker': self.inspect_env('worker')}
|
|
|
|
@contextlib.contextmanager
|
|
def app_environment(self, initial_overrides=None):
|
|
saved_state = self.app_state()
|
|
baseline = {}
|
|
for service in ('api', 'worker'):
|
|
for info in saved_state[service].values():
|
|
for key in MUTABLE_KEYS:
|
|
baseline.setdefault((service, key), info['env'][key])
|
|
saved_environ = {k: os.environ.get(k) for k in MUTABLE_KEYS}
|
|
touched = set()
|
|
dirty = [False]
|
|
|
|
def effective(service):
|
|
return {cid: info for cid, info in self.inspect_env(service).items()}
|
|
|
|
def set_overrides(values, force=False):
|
|
unknown = set(values) - set(MUTABLE_KEYS)
|
|
if unknown:
|
|
raise Failure('not a controllable key: ' + ', '.join(sorted(unknown)))
|
|
for key in values:
|
|
api = {i['env'][key] for i in effective('api').values()}
|
|
worker = {i['env'][key] for i in effective('worker').values()}
|
|
if len(api | worker) > 1 and api != worker:
|
|
raise Failure('api/worker disagree on ' + key + ' before change')
|
|
if not force and all(
|
|
info['env'][key] == str(value)
|
|
for service in ('api', 'worker')
|
|
for info in effective(service).values()
|
|
for key, value in values.items()):
|
|
return self.app_state()
|
|
for key, value in values.items():
|
|
os.environ[key] = str(value)
|
|
touched.add(key)
|
|
dirty[0] = True
|
|
stand.compose('up', '-d', '--no-deps', '--force-recreate',
|
|
'--wait', 'api', 'worker')
|
|
for service in ('api', 'worker'):
|
|
state = effective(service)
|
|
for cid, info in state.items():
|
|
for key, value in values.items():
|
|
if info['env'][key] != str(value):
|
|
raise Failure('effective ' + key + ' differs in ' +
|
|
service + ' ' + cid)
|
|
sdk_on = os.environ.get('OTEL_SDK_DISABLED', 'false') != 'true'
|
|
self.wait_ready(app_telemetry=sdk_on, provisioning=True)
|
|
return self.app_state()
|
|
|
|
try:
|
|
if initial_overrides:
|
|
set_overrides(initial_overrides)
|
|
yield set_overrides
|
|
finally:
|
|
if dirty[0]:
|
|
# Restore the original effective values as controller env so the
|
|
# recreated containers match what was running before.
|
|
for key in MUTABLE_KEYS:
|
|
original = baseline.get(('api', key))
|
|
if original is not None and key in touched:
|
|
os.environ[key] = original
|
|
stand.compose('up', '-d', '--no-deps', '--force-recreate',
|
|
'--wait', 'api', 'worker')
|
|
for key in touched:
|
|
if saved_environ[key] is None:
|
|
os.environ.pop(key, None)
|
|
else:
|
|
os.environ[key] = saved_environ[key]
|
|
now = self.app_state()
|
|
for service in ('api', 'worker'):
|
|
for cid, info in now[service].items():
|
|
for key in touched:
|
|
if info['env'][key] != baseline.get((service, key)):
|
|
print('WARNING: ' + key + ' not restored on ' +
|
|
service, file=sys.stderr)
|
|
with contextlib.suppress(Failure):
|
|
self.wait_ready(app_telemetry=True, provisioning=True,
|
|
timeout=120)
|
|
|
|
# -- foreign faults
|
|
def foreign_faults(self):
|
|
result = stand.compose('exec', '-T', 'redis', 'redis-cli', '--scan',
|
|
'--pattern', 'fault:*')
|
|
return sorted(line for line in result.stdout.split() if line)
|
|
|
|
def require_no_foreign_faults(self):
|
|
keys = self.foreign_faults()
|
|
if keys:
|
|
raise Failure('foreign fault keys present (finish the other '
|
|
'experiment first): ' + ', '.join(keys))
|
|
|
|
|
|
# --------------------------------------------------------------- PostgreSQL
|
|
|
|
def psql(sql, timeout=30):
|
|
script = 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 ' \
|
|
'-At -F , -c "$SQL"'
|
|
result = subprocess.run(
|
|
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
|
|
'-e', 'SQL=' + sql, 'postgres', 'sh', '-c', script],
|
|
cwd=ROOT, text=True, capture_output=True, timeout=timeout)
|
|
if result.returncode != 0:
|
|
raise Failure('psql failed: ' + ((result.stderr or '').strip()
|
|
.splitlines() or ['?'])[-1][:200])
|
|
return result.stdout.strip()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def hold_table_lock(table):
|
|
"""ACCESS EXCLUSIVE lock on messages/attachments held until exit."""
|
|
if table not in LOCKABLE_TABLES:
|
|
raise Failure('table is not lockable: ' + table)
|
|
name = 'lab6-lock-' + uuid.uuid4().hex[:12]
|
|
process = subprocess.Popen(
|
|
['docker', 'compose', '--env-file', stand.ENV_FILE, 'exec', '-T',
|
|
'postgres', 'sh', '-c',
|
|
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -At'],
|
|
cwd=ROOT, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, bufsize=1)
|
|
pid = None
|
|
try:
|
|
process.stdin.write(
|
|
"SET application_name='" + name + "';\n"
|
|
"BEGIN;\n"
|
|
"SET LOCAL statement_timeout='120s';\n"
|
|
"SET LOCAL idle_in_transaction_session_timeout='120s';\n"
|
|
"LOCK TABLE " + table + " IN ACCESS EXCLUSIVE MODE;\n"
|
|
"SELECT 'LOCKED:'||pg_backend_pid();\n")
|
|
process.stdin.flush()
|
|
deadline = time.monotonic() + 30
|
|
while pid is None:
|
|
if time.monotonic() > deadline:
|
|
raise Failure('table lock marker not received')
|
|
ready, _, _ = select.select([process.stdout], [], [], 1)
|
|
if not ready:
|
|
if process.poll() is not None:
|
|
raise Failure('psql exited before locking')
|
|
continue
|
|
line = process.stdout.readline()
|
|
if line.startswith('LOCKED:'):
|
|
pid = int(line.split(':')[1])
|
|
elif line == '' and process.poll() is not None:
|
|
raise Failure('psql exited before locking')
|
|
yield pid
|
|
finally:
|
|
try:
|
|
if process.poll() is None:
|
|
process.stdin.write('ROLLBACK;\n\\q\n')
|
|
process.stdin.flush()
|
|
process.stdin.close()
|
|
except (OSError, ValueError):
|
|
pass
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
if pid is not None:
|
|
with contextlib.suppress(Failure, subprocess.TimeoutExpired):
|
|
psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
|
"WHERE pid=" + str(int(pid)) +
|
|
" AND application_name='" + name + "'")
|
|
for stream in (process.stdout, process.stderr):
|
|
with contextlib.suppress(Exception):
|
|
stream.close()
|
|
|
|
|
|
def wait_drained(attachment_ids, obs=None, timeout=120, flush_telemetry=True):
|
|
"""Batch-check jobs/outbox/attachments in PostgreSQL (no HTTP polling).
|
|
|
|
Returns {'drained': bool, 'states': {id: status}, 'queued': n,
|
|
'outbox': n, 'seconds': t, 'timestamps': ...}. A vanished known ID is a
|
|
Failure; non-ready at the deadline is reported, not raised.
|
|
"""
|
|
ids = list(attachment_ids)
|
|
for item in ids:
|
|
uuid.UUID(item)
|
|
array = "ARRAY[" + ",".join("'" + i + "'" for i in ids) + "]::uuid[]" \
|
|
if ids else "ARRAY[]::uuid[]"
|
|
sql = (
|
|
"SELECT (SELECT count(*) FROM attachment_jobs WHERE status IN "
|
|
"('queued','processing'))||','||"
|
|
"(SELECT count(*) FROM job_outbox WHERE published_at IS NULL)||','||"
|
|
"COALESCE((SELECT string_agg(id::text||':'||status,';') FROM attachments "
|
|
"WHERE id = ANY(" + array + ")),'')")
|
|
started = time.monotonic()
|
|
start_ts = utc_now()
|
|
states = {}
|
|
queued = outbox = -1
|
|
drained = False
|
|
while time.monotonic() - started < timeout:
|
|
out = psql(sql)
|
|
queued_s, outbox_s, rest = (out.split(',', 2) + ['', ''])[:3]
|
|
queued, outbox = int(queued_s), int(outbox_s)
|
|
states = dict(part.split(':', 1) for part in rest.split(';') if part)
|
|
missing = set(ids) - set(states)
|
|
if missing:
|
|
raise Failure('known attachment vanished: ' + sorted(missing)[0])
|
|
terminal = all(s in ('ready', 'failed') for s in states.values())
|
|
if queued == 0 and outbox == 0 and terminal:
|
|
drained = True
|
|
break
|
|
time.sleep(1)
|
|
result = {'drained': drained, 'states': states, 'queued_jobs': queued,
|
|
'unpublished_outbox': outbox,
|
|
'seconds': round(time.monotonic() - started, 3),
|
|
'started': start_ts, 'finished': utc_now()}
|
|
if flush_telemetry and obs is not None:
|
|
obs.wait_telemetry_flushed(timeout=timeout)
|
|
result['telemetry_flushed'] = utc_now()
|
|
return result
|