182 lines
7.0 KiB
Python
182 lines
7.0 KiB
Python
"""Container resource sampling for lab6: docker stats (container CPU/memory) and
|
|
docker top (process RSS) every few seconds. Stdlib only.
|
|
|
|
MemUsage is *container* memory usage (cgroup, includes page cache); RSS is the
|
|
sum of process RSS from `docker top` (KiB -> bytes). They are never mixed."""
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import sys # noqa: E402
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import measurement # noqa: E402
|
|
import obs # noqa: E402
|
|
|
|
UNITS = {'B': 1, 'kB': 1000, 'MB': 1000 ** 2, 'GB': 1000 ** 3, 'TB': 1000 ** 4,
|
|
'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3, 'TiB': 1024 ** 4}
|
|
SIZE = re.compile(r'^\s*([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)\s*$')
|
|
SERVICES = ('api', 'worker', 'otel-collector', 'prometheus', 'loki', 'tempo',
|
|
'grafana')
|
|
|
|
|
|
def parse_size(text):
|
|
"""'147.5MiB' -> bytes; SI and IEC units differ; unknown unit is an error."""
|
|
match = SIZE.match(text)
|
|
if not match or match.group(2) not in UNITS:
|
|
raise ValueError('unparseable size: ' + repr(text))
|
|
return int(float(match.group(1)) * UNITS[match.group(2)])
|
|
|
|
|
|
def parse_stats_row(row):
|
|
usage, limit = row['MemUsage'].split('/')
|
|
cpu = row['CPUPerc'].strip()
|
|
if not cpu.endswith('%'):
|
|
raise ValueError('unparseable CPU: ' + repr(cpu))
|
|
return {'cpu_pct': float(cpu[:-1]), 'mem_usage_bytes': parse_size(usage),
|
|
'mem_limit_bytes': parse_size(limit)}
|
|
|
|
|
|
def parse_top_rss(text):
|
|
"""`docker top <id> -eo pid,rss` output -> bytes (KiB x 1024)."""
|
|
lines = [line.split() for line in text.strip().splitlines()]
|
|
if not lines or [h.upper() for h in lines[0]] != ['PID', 'RSS']:
|
|
raise ValueError('unexpected docker top header')
|
|
return sum(int(row[1]) for row in lines[1:]) * 1024
|
|
|
|
|
|
def container_map():
|
|
"""{short_id: service} for all observed services."""
|
|
result = {}
|
|
for service in SERVICES:
|
|
ids = obs.stand.compose('ps', '-q', service, check_result=False)
|
|
for cid in ids.stdout.split():
|
|
result[cid[:12]] = service
|
|
return result
|
|
|
|
|
|
class ResourceSampler(threading.Thread):
|
|
def __init__(self, containers, interval=5.0, path=None):
|
|
super().__init__(daemon=True)
|
|
self.containers = dict(containers)
|
|
self.interval = interval
|
|
self.path = Path(path) if path else None
|
|
self.stop_event = threading.Event()
|
|
self.samples = []
|
|
self.errors = []
|
|
|
|
def one_tick(self, tick):
|
|
mono = time.monotonic()
|
|
stamp = obs.utc_now()
|
|
ids = list(self.containers)
|
|
stats = subprocess.run(
|
|
['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + ids,
|
|
capture_output=True, text=True, timeout=60)
|
|
rows = {}
|
|
for line in stats.stdout.splitlines():
|
|
if line.strip():
|
|
row = json.loads(line)
|
|
rows[row['ID'][:12]] = row
|
|
for cid in ids:
|
|
sample = {'tick': tick, 'utc': stamp, 'mono': mono, 'id': cid,
|
|
'service': self.containers[cid]}
|
|
raw = rows.get(cid)
|
|
if raw is None:
|
|
sample['error'] = 'no_stats_row'
|
|
else:
|
|
sample['raw'] = {k: raw[k] for k in ('CPUPerc', 'MemUsage')}
|
|
try:
|
|
sample.update(parse_stats_row(raw))
|
|
except ValueError as exc:
|
|
sample['error'] = str(exc)
|
|
top = subprocess.run(
|
|
['docker', 'top', cid, '-eo', 'pid,rss'],
|
|
capture_output=True, text=True, timeout=30)
|
|
if top.returncode == 0:
|
|
try:
|
|
sample['rss_bytes'] = parse_top_rss(top.stdout)
|
|
except ValueError as exc:
|
|
sample['rss_error'] = str(exc)
|
|
else:
|
|
sample['rss_bytes'] = None
|
|
sample['rss_reason'] = 'not_running'
|
|
self.samples.append(sample)
|
|
if self.path is not None:
|
|
with self.path.open('a') as handle:
|
|
handle.write(json.dumps(sample) + '\n')
|
|
if 'error' in sample:
|
|
self.errors.append(sample['error'])
|
|
|
|
def run(self):
|
|
tick = 0
|
|
while not self.stop_event.is_set():
|
|
begun = time.monotonic()
|
|
try:
|
|
self.one_tick(tick)
|
|
except (subprocess.SubprocessError, ValueError, KeyError) as exc:
|
|
self.errors.append(type(exc).__name__)
|
|
tick += 1
|
|
self.stop_event.wait(max(0.0, self.interval - (time.monotonic() - begun)))
|
|
|
|
def stop(self):
|
|
self.stop_event.set()
|
|
self.join(timeout=90)
|
|
|
|
|
|
def _summarise(values):
|
|
values = [v for v in values if v is not None]
|
|
return {'median': measurement.median(values),
|
|
'p95': measurement.nearest_rank(values, 0.95),
|
|
'max': max(values) if values else None, 'n': len(values)}
|
|
|
|
|
|
def summarize(samples, container_ids=None):
|
|
"""Per-service statistics. For API the replicas are summed on a common
|
|
tick first; individual values are kept in `per_container`."""
|
|
ticks = {}
|
|
for sample in samples:
|
|
if 'error' in sample:
|
|
continue
|
|
ticks.setdefault(sample['tick'], {}).setdefault(
|
|
sample['service'], []).append(sample)
|
|
order = sorted(ticks)
|
|
result = {}
|
|
services = {s['service'] for s in samples}
|
|
for service in sorted(services):
|
|
cpu, mem, rss, times = [], [], [], []
|
|
for tick in order:
|
|
group = ticks[tick].get(service)
|
|
if not group:
|
|
continue
|
|
times.append(min(s['mono'] for s in group))
|
|
cpu.append(sum(s['cpu_pct'] for s in group))
|
|
mem.append(sum(s['mem_usage_bytes'] for s in group))
|
|
values = [s.get('rss_bytes') for s in group]
|
|
rss.append(None if any(v is None for v in values) else sum(values))
|
|
weights = [times[i + 1] - times[i] for i in range(len(times) - 1)]
|
|
if times:
|
|
weights.append(sum(weights) / len(weights) if weights else 1.0)
|
|
mean_cpu = (sum(c * w for c, w in zip(cpu, weights)) / sum(weights)
|
|
if weights and sum(weights) > 0 else None)
|
|
per_container = {}
|
|
for s in samples:
|
|
if s['service'] == service and 'error' not in s:
|
|
per_container.setdefault(s['id'], []).append(s['cpu_pct'])
|
|
result[service] = {
|
|
'cpu_pct_time_weighted_mean': mean_cpu,
|
|
'container_memory_bytes': _summarise(mem),
|
|
'process_rss_bytes': _summarise(rss),
|
|
'mem_limit_bytes': max((s['mem_limit_bytes'] for s in samples
|
|
if s['service'] == service and
|
|
'mem_limit_bytes' in s), default=None),
|
|
'ticks': len(times),
|
|
'per_container_cpu_mean': {k: sum(v) / len(v)
|
|
for k, v in per_container.items()},
|
|
}
|
|
result['sample_errors'] = len(
|
|
[s for s in samples if 'error' in s])
|
|
return result
|