183 lines
7.4 KiB
Python
183 lines
7.4 KiB
Python
"""Сэмплер стенда: раз в секунду — ресурсы контейнеров, LB и PostgreSQL.
|
||
|
||
Работает только внутри контейнера `vlad-bench-sampler` (образ
|
||
приложения: в нём есть Python и asyncpg) с `--cgroupns host`: тогда в
|
||
`/sys/fs/cgroup/docker/<id>/` видны cgroup всех контейнеров ВМ Docker.
|
||
Его запускает `bench/run.py` (`Stand.start_sampler`), на хосте он не
|
||
работает. Пишет строки JSON в файл `--out`; время — часы ВМ (`time.time()`),
|
||
те же, что у генератора.
|
||
|
||
Строка тика:
|
||
{"ts": ..., "cg": {"<id12>": [usage_usec, nr_periods, nr_throttled,
|
||
throttled_usec, memory_current]},
|
||
"vm": [занято_jiffies, всего_jiffies],
|
||
"lb": {"<слот>": {...}, "FRONTEND": {...}}, # режим list
|
||
"pg": {"conn": [[адрес, состояние, ожидание, n], ...],
|
||
"db": {xact_commit, xact_rollback, sessions, numbackends}}}
|
||
|
||
Режимы:
|
||
list — заданные контейнеры (`--container id` …), опрос статистики
|
||
HAProxy (`/stats;csv` на :8404 внутри сети) и одно постоянное
|
||
соединение с PostgreSQL стенда (`pg_stat_activity`,
|
||
`pg_stat_database`); его слот занимает одно из
|
||
`max_connections`. Серии трека 1;
|
||
scan — все каталоги контейнеров в cgroup на каждом тике (новые
|
||
реплики видны с первого тика), без сети, LB и PostgreSQL:
|
||
наблюдатель не обращается к стенду. Профили failover и rollout.
|
||
|
||
Пароль PostgreSQL — только из окружения контейнера (`PGPASSWORD`), в
|
||
вывод не попадает.
|
||
"""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import contextlib
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
CGROUP = Path("/sys/fs/cgroup/docker")
|
||
HEX64 = re.compile(r"[0-9a-f]{64}")
|
||
LB_STATS = "http://haproxy:8404/stats;csv"
|
||
LB_FIELDS = ("scur", "stot", "econ", "eresp", "wretr", "wredis", "qcur", "rate")
|
||
# соединения приложения: application_name из src/constants.py (копия:
|
||
# сэмплер код приложения не импортирует)
|
||
APP_NAME = "mtusi-messenger"
|
||
PG_CONN_SQL = """
|
||
SELECT coalesce(host(client_addr), '-'), coalesce(state, '-'),
|
||
coalesce(wait_event_type, 'CPU'), count(*)
|
||
FROM pg_stat_activity
|
||
WHERE backend_type = 'client backend' AND application_name = $1
|
||
GROUP BY 1, 2, 3
|
||
"""
|
||
PG_DB_SQL = """
|
||
SELECT xact_commit, xact_rollback, sessions, numbackends
|
||
FROM pg_stat_database WHERE datname = current_database()
|
||
"""
|
||
|
||
|
||
def cgroup_line(container: str) -> list[int] | None:
|
||
"""usage_usec, nr_periods, nr_throttled, throttled_usec, memory.current."""
|
||
folder = CGROUP / container
|
||
try:
|
||
stat = (folder / "cpu.stat").read_text()
|
||
memory = int((folder / "memory.current").read_text())
|
||
except (OSError, ValueError):
|
||
return None
|
||
values = dict(line.split() for line in stat.splitlines() if line)
|
||
return [
|
||
int(values.get("usage_usec", 0)),
|
||
int(values.get("nr_periods", 0)),
|
||
int(values.get("nr_throttled", 0)),
|
||
int(values.get("throttled_usec", 0)),
|
||
memory,
|
||
]
|
||
|
||
|
||
def vm_line() -> list[int]:
|
||
fields = Path("/proc/stat").read_text().splitlines()[0].split()[1:9]
|
||
user, nice, system, idle, iowait, irq, softirq, steal = (int(f) for f in fields)
|
||
total = user + nice + system + idle + iowait + irq + softirq + steal
|
||
return [total - idle - iowait, total]
|
||
|
||
|
||
def lb_stats() -> dict | None:
|
||
try:
|
||
with urllib.request.urlopen(LB_STATS, timeout=0.5) as response: # noqa: S310
|
||
text = response.read().decode()
|
||
except OSError:
|
||
return None
|
||
lines = text.splitlines()
|
||
header = lines[0].lstrip("# ").split(",")
|
||
out = {}
|
||
for line in lines[1:]:
|
||
row = dict(zip(header, line.split(","), strict=False))
|
||
if row.get("pxname") != "api" and row.get("pxname") != "entry":
|
||
continue
|
||
name = "FRONTEND" if row["pxname"] == "entry" else row["svname"]
|
||
if name == "FRONTEND" and row["svname"] != "FRONTEND":
|
||
continue
|
||
out[name] = {
|
||
"status": row.get("status", ""),
|
||
"addr": row.get("addr", ""),
|
||
**{field: int(row[field]) for field in LB_FIELDS if row.get(field)},
|
||
}
|
||
return out
|
||
|
||
|
||
async def main() -> None:
|
||
args = parser().parse_args()
|
||
interval = args.interval
|
||
connection = None
|
||
if args.mode == "list":
|
||
import asyncpg # noqa: PLC0415 - нужен только режиму list
|
||
|
||
for _ in range(20):
|
||
try:
|
||
connection = await asyncpg.connect(
|
||
host="postgres",
|
||
user=os.environ["PGUSER"],
|
||
database=os.environ["PGDATABASE"],
|
||
password=os.environ["PGPASSWORD"],
|
||
server_settings={"application_name": "bench-sampler"},
|
||
timeout=3,
|
||
)
|
||
break
|
||
except (OSError, asyncpg.PostgresError):
|
||
await asyncio.sleep(0.5)
|
||
out = open(args.out, "a", encoding="ascii") # noqa: ASYNC230, SIM115
|
||
next_tick = time.monotonic()
|
||
while True:
|
||
ts = time.time()
|
||
if args.mode == "scan":
|
||
# файлы cgroup — псевдофайловая система ядра: чтение без ожидания
|
||
try:
|
||
ids = [
|
||
p.name
|
||
for p in CGROUP.iterdir() # noqa: ASYNC240
|
||
if HEX64.fullmatch(p.name)
|
||
]
|
||
except OSError:
|
||
ids = []
|
||
else:
|
||
ids = args.container
|
||
tick: dict = {"ts": round(ts, 3), "cg": {}, "vm": vm_line()}
|
||
for container in ids:
|
||
line = cgroup_line(container)
|
||
if line is not None:
|
||
tick["cg"][container[:12]] = line
|
||
if args.mode == "list":
|
||
tick["lb"] = await asyncio.to_thread(lb_stats)
|
||
if connection is not None:
|
||
with contextlib.suppress(Exception):
|
||
rows = await connection.fetch(PG_CONN_SQL, APP_NAME)
|
||
db = await connection.fetchrow(PG_DB_SQL)
|
||
# соединения приложения: адрес реплики, состояние, ожидание
|
||
tick["pg"] = {
|
||
"conn": [list(row) for row in rows],
|
||
"db": dict(db),
|
||
}
|
||
out.write(json.dumps(tick, separators=(",", ":")) + "\n")
|
||
out.flush()
|
||
next_tick += interval
|
||
await asyncio.sleep(max(0.0, next_tick - time.monotonic()))
|
||
|
||
|
||
def parser() -> argparse.ArgumentParser:
|
||
root = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
root.add_argument("--mode", choices=("list", "scan"), required=True)
|
||
root.add_argument("--interval", type=float, default=1.0)
|
||
root.add_argument("--out", required=True)
|
||
root.add_argument("--container", action="append", default=[])
|
||
return root
|
||
|
||
|
||
if __name__ == "__main__":
|
||
with contextlib.suppress(KeyboardInterrupt):
|
||
asyncio.run(main())
|
||
sys.exit(0)
|