"""Container resource sampling for lab6: docker stats (container CPU/memory) and docker top (process RSS) every few seconds. Stdlib only. MemUsage is *container* memory usage (cgroup, includes page cache); RSS is the sum of process RSS from `docker top` (KiB -> bytes). They are never mixed.""" import json import re import subprocess import threading import time from pathlib import Path import sys # noqa: E402 sys.path.insert(0, str(Path(__file__).resolve().parent)) import measurement # noqa: E402 import obs # noqa: E402 UNITS = {'B': 1, 'kB': 1000, 'MB': 1000 ** 2, 'GB': 1000 ** 3, 'TB': 1000 ** 4, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3, 'TiB': 1024 ** 4} SIZE = re.compile(r'^\s*([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)\s*$') SERVICES = ('api', 'worker', 'otel-collector', 'prometheus', 'loki', 'tempo', 'grafana') def parse_size(text): """'147.5MiB' -> bytes; SI and IEC units differ; unknown unit is an error.""" match = SIZE.match(text) if not match or match.group(2) not in UNITS: raise ValueError('unparseable size: ' + repr(text)) return int(float(match.group(1)) * UNITS[match.group(2)]) def parse_stats_row(row): usage, limit = row['MemUsage'].split('/') cpu = row['CPUPerc'].strip() if not cpu.endswith('%'): raise ValueError('unparseable CPU: ' + repr(cpu)) return {'cpu_pct': float(cpu[:-1]), 'mem_usage_bytes': parse_size(usage), 'mem_limit_bytes': parse_size(limit)} def parse_top_rss(text): """`docker top -eo pid,rss` output -> bytes (KiB x 1024).""" lines = [line.split() for line in text.strip().splitlines()] if not lines or [h.upper() for h in lines[0]] != ['PID', 'RSS']: raise ValueError('unexpected docker top header') return sum(int(row[1]) for row in lines[1:]) * 1024 def container_map(): """{short_id: service} for all observed services.""" result = {} for service in SERVICES: ids = obs.stand.compose('ps', '-q', service, check_result=False) for cid in ids.stdout.split(): result[cid[:12]] = service return result class ResourceSampler(threading.Thread): def __init__(self, containers, interval=5.0, path=None): super().__init__(daemon=True) self.containers = dict(containers) self.interval = interval self.path = Path(path) if path else None self.stop_event = threading.Event() self.samples = [] self.errors = [] def one_tick(self, tick): mono = time.monotonic() stamp = obs.utc_now() ids = list(self.containers) stats = subprocess.run( ['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + ids, capture_output=True, text=True, timeout=60) rows = {} for line in stats.stdout.splitlines(): if line.strip(): row = json.loads(line) rows[row['ID'][:12]] = row for cid in ids: sample = {'tick': tick, 'utc': stamp, 'mono': mono, 'id': cid, 'service': self.containers[cid]} raw = rows.get(cid) if raw is None: sample['error'] = 'no_stats_row' else: sample['raw'] = {k: raw[k] for k in ('CPUPerc', 'MemUsage')} try: sample.update(parse_stats_row(raw)) except ValueError as exc: sample['error'] = str(exc) top = subprocess.run( ['docker', 'top', cid, '-eo', 'pid,rss'], capture_output=True, text=True, timeout=30) if top.returncode == 0: try: sample['rss_bytes'] = parse_top_rss(top.stdout) except ValueError as exc: sample['rss_error'] = str(exc) else: sample['rss_bytes'] = None sample['rss_reason'] = 'not_running' self.samples.append(sample) if self.path is not None: with self.path.open('a') as handle: handle.write(json.dumps(sample) + '\n') if 'error' in sample: self.errors.append(sample['error']) def run(self): tick = 0 while not self.stop_event.is_set(): begun = time.monotonic() try: self.one_tick(tick) except (subprocess.SubprocessError, ValueError, KeyError) as exc: self.errors.append(type(exc).__name__) tick += 1 self.stop_event.wait(max(0.0, self.interval - (time.monotonic() - begun))) def stop(self): self.stop_event.set() self.join(timeout=90) def _summarise(values): values = [v for v in values if v is not None] return {'median': measurement.median(values), 'p95': measurement.nearest_rank(values, 0.95), 'max': max(values) if values else None, 'n': len(values)} def summarize(samples, container_ids=None): """Per-service statistics. For API the replicas are summed on a common tick first; individual values are kept in `per_container`.""" ticks = {} for sample in samples: if 'error' in sample: continue ticks.setdefault(sample['tick'], {}).setdefault( sample['service'], []).append(sample) order = sorted(ticks) result = {} services = {s['service'] for s in samples} for service in sorted(services): cpu, mem, rss, times = [], [], [], [] for tick in order: group = ticks[tick].get(service) if not group: continue times.append(min(s['mono'] for s in group)) cpu.append(sum(s['cpu_pct'] for s in group)) mem.append(sum(s['mem_usage_bytes'] for s in group)) values = [s.get('rss_bytes') for s in group] rss.append(None if any(v is None for v in values) else sum(values)) weights = [times[i + 1] - times[i] for i in range(len(times) - 1)] if times: weights.append(sum(weights) / len(weights) if weights else 1.0) mean_cpu = (sum(c * w for c, w in zip(cpu, weights)) / sum(weights) if weights and sum(weights) > 0 else None) per_container = {} for s in samples: if s['service'] == service and 'error' not in s: per_container.setdefault(s['id'], []).append(s['cpu_pct']) result[service] = { 'cpu_pct_time_weighted_mean': mean_cpu, 'container_memory_bytes': _summarise(mem), 'process_rss_bytes': _summarise(rss), 'mem_limit_bytes': max((s['mem_limit_bytes'] for s in samples if s['service'] == service and 'mem_limit_bytes' in s), default=None), 'ticks': len(times), 'per_container_cpu_mean': {k: sum(v) / len(v) for k, v in per_container.items()}, } result['sample_errors'] = len( [s for s in samples if 'error' in s]) return result