Files
mtusi2026lab6/scripts/lab6/resources.py
T

182 lines
7.0 KiB
Python

"""Container resource sampling for lab6: docker stats (container CPU/memory) and
docker top (process RSS) every few seconds. Stdlib only.
MemUsage is *container* memory usage (cgroup, includes page cache); RSS is the
sum of process RSS from `docker top` (KiB -> bytes). They are never mixed."""
import json
import re
import subprocess
import threading
import time
from pathlib import Path
import sys # noqa: E402
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
UNITS = {'B': 1, 'kB': 1000, 'MB': 1000 ** 2, 'GB': 1000 ** 3, 'TB': 1000 ** 4,
'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3, 'TiB': 1024 ** 4}
SIZE = re.compile(r'^\s*([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)\s*$')
SERVICES = ('api', 'worker', 'otel-collector', 'prometheus', 'loki', 'tempo',
'grafana')
def parse_size(text):
"""'147.5MiB' -> bytes; SI and IEC units differ; unknown unit is an error."""
match = SIZE.match(text)
if not match or match.group(2) not in UNITS:
raise ValueError('unparseable size: ' + repr(text))
return int(float(match.group(1)) * UNITS[match.group(2)])
def parse_stats_row(row):
usage, limit = row['MemUsage'].split('/')
cpu = row['CPUPerc'].strip()
if not cpu.endswith('%'):
raise ValueError('unparseable CPU: ' + repr(cpu))
return {'cpu_pct': float(cpu[:-1]), 'mem_usage_bytes': parse_size(usage),
'mem_limit_bytes': parse_size(limit)}
def parse_top_rss(text):
"""`docker top <id> -eo pid,rss` output -> bytes (KiB x 1024)."""
lines = [line.split() for line in text.strip().splitlines()]
if not lines or [h.upper() for h in lines[0]] != ['PID', 'RSS']:
raise ValueError('unexpected docker top header')
return sum(int(row[1]) for row in lines[1:]) * 1024
def container_map():
"""{short_id: service} for all observed services."""
result = {}
for service in SERVICES:
ids = obs.stand.compose('ps', '-q', service, check_result=False)
for cid in ids.stdout.split():
result[cid[:12]] = service
return result
class ResourceSampler(threading.Thread):
def __init__(self, containers, interval=5.0, path=None):
super().__init__(daemon=True)
self.containers = dict(containers)
self.interval = interval
self.path = Path(path) if path else None
self.stop_event = threading.Event()
self.samples = []
self.errors = []
def one_tick(self, tick):
mono = time.monotonic()
stamp = obs.utc_now()
ids = list(self.containers)
stats = subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + ids,
capture_output=True, text=True, timeout=60)
rows = {}
for line in stats.stdout.splitlines():
if line.strip():
row = json.loads(line)
rows[row['ID'][:12]] = row
for cid in ids:
sample = {'tick': tick, 'utc': stamp, 'mono': mono, 'id': cid,
'service': self.containers[cid]}
raw = rows.get(cid)
if raw is None:
sample['error'] = 'no_stats_row'
else:
sample['raw'] = {k: raw[k] for k in ('CPUPerc', 'MemUsage')}
try:
sample.update(parse_stats_row(raw))
except ValueError as exc:
sample['error'] = str(exc)
top = subprocess.run(
['docker', 'top', cid, '-eo', 'pid,rss'],
capture_output=True, text=True, timeout=30)
if top.returncode == 0:
try:
sample['rss_bytes'] = parse_top_rss(top.stdout)
except ValueError as exc:
sample['rss_error'] = str(exc)
else:
sample['rss_bytes'] = None
sample['rss_reason'] = 'not_running'
self.samples.append(sample)
if self.path is not None:
with self.path.open('a') as handle:
handle.write(json.dumps(sample) + '\n')
if 'error' in sample:
self.errors.append(sample['error'])
def run(self):
tick = 0
while not self.stop_event.is_set():
begun = time.monotonic()
try:
self.one_tick(tick)
except (subprocess.SubprocessError, ValueError, KeyError) as exc:
self.errors.append(type(exc).__name__)
tick += 1
self.stop_event.wait(max(0.0, self.interval - (time.monotonic() - begun)))
def stop(self):
self.stop_event.set()
self.join(timeout=90)
def _summarise(values):
values = [v for v in values if v is not None]
return {'median': measurement.median(values),
'p95': measurement.nearest_rank(values, 0.95),
'max': max(values) if values else None, 'n': len(values)}
def summarize(samples, container_ids=None):
"""Per-service statistics. For API the replicas are summed on a common
tick first; individual values are kept in `per_container`."""
ticks = {}
for sample in samples:
if 'error' in sample:
continue
ticks.setdefault(sample['tick'], {}).setdefault(
sample['service'], []).append(sample)
order = sorted(ticks)
result = {}
services = {s['service'] for s in samples}
for service in sorted(services):
cpu, mem, rss, times = [], [], [], []
for tick in order:
group = ticks[tick].get(service)
if not group:
continue
times.append(min(s['mono'] for s in group))
cpu.append(sum(s['cpu_pct'] for s in group))
mem.append(sum(s['mem_usage_bytes'] for s in group))
values = [s.get('rss_bytes') for s in group]
rss.append(None if any(v is None for v in values) else sum(values))
weights = [times[i + 1] - times[i] for i in range(len(times) - 1)]
if times:
weights.append(sum(weights) / len(weights) if weights else 1.0)
mean_cpu = (sum(c * w for c, w in zip(cpu, weights)) / sum(weights)
if weights and sum(weights) > 0 else None)
per_container = {}
for s in samples:
if s['service'] == service and 'error' not in s:
per_container.setdefault(s['id'], []).append(s['cpu_pct'])
result[service] = {
'cpu_pct_time_weighted_mean': mean_cpu,
'container_memory_bytes': _summarise(mem),
'process_rss_bytes': _summarise(rss),
'mem_limit_bytes': max((s['mem_limit_bytes'] for s in samples
if s['service'] == service and
'mem_limit_bytes' in s), default=None),
'ticks': len(times),
'per_container_cpu_mean': {k: sum(v) / len(v)
for k, v in per_container.items()},
}
result['sample_errors'] = len(
[s for s in samples if 'error' in s])
return result