115 lines
5.1 KiB
Python
115 lines
5.1 KiB
Python
"""Low-cardinality saturation/backlog signals and safe dependency spans (lab6/4)."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from datetime import datetime, timezone
|
|
import os
|
|
import resource
|
|
import time
|
|
|
|
from opentelemetry import metrics
|
|
from opentelemetry.metrics import Observation
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.telemetry import span
|
|
|
|
meter = metrics.get_meter("messenger")
|
|
waiting = 0
|
|
busy = 0
|
|
snapshot: dict[str, float] = {}
|
|
ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
|
|
retries = meter.create_counter("messenger.jobs.retries", unit="{retry}")
|
|
|
|
|
|
class ObservedSession(AsyncSession):
|
|
async def execute(self, statement, params=None, **kwargs):
|
|
global waiting
|
|
# Number of acquisition requests in progress (includes new connects).
|
|
# Acquisition is separate from query execution to diagnose pool waits.
|
|
waiting += 1
|
|
try:
|
|
with span("db.acquire", {"db.system": "postgresql"}):
|
|
await self.connection()
|
|
finally:
|
|
waiting -= 1
|
|
operation = str(statement).lstrip().split(None, 1)[0].upper()
|
|
if operation not in {"SELECT", "INSERT", "UPDATE", "DELETE", "SET"}:
|
|
operation = "OTHER"
|
|
with span("db.query", {"db.system": "postgresql", "db.operation": operation}):
|
|
return await super().execute(statement, params, **kwargs)
|
|
|
|
async def commit(self):
|
|
with span("db.commit", {"db.system": "postgresql"}):
|
|
return await super().commit()
|
|
|
|
|
|
def install_gauges(engine) -> None:
|
|
pool = engine.sync_engine.pool
|
|
meter.create_observable_gauge("messenger.db.pool.used",
|
|
callbacks=[lambda _: [Observation(pool.checkedout())]])
|
|
meter.create_observable_gauge("messenger.db.pool.waiting",
|
|
callbacks=[lambda _: [Observation(waiting)]])
|
|
meter.create_observable_gauge("messenger.db.pool.capacity",
|
|
callbacks=[lambda _: [Observation(pool.size() + pool._max_overflow)]])
|
|
meter.create_observable_gauge("messenger.worker.busy",
|
|
callbacks=[lambda _: [Observation(busy)]])
|
|
capacity = 1 if os.getenv("OTEL_SERVICE_NAME") == "messenger-worker" else 0
|
|
meter.create_observable_gauge("messenger.worker.capacity",
|
|
callbacks=[lambda _: [Observation(capacity)]])
|
|
if capacity:
|
|
meter.create_observable_gauge("messenger.worker.heartbeat", unit="s",
|
|
callbacks=[lambda _: [Observation(time.time())]])
|
|
meter.create_observable_counter("messenger.process.cpu", unit="s",
|
|
callbacks=[lambda _: [Observation(time.process_time())]])
|
|
|
|
def rss(_):
|
|
# Linux /proc resident pages, current RSS (not ru_maxrss high watermark).
|
|
try:
|
|
with open("/proc/self/statm", encoding="ascii") as statm:
|
|
value = int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE")
|
|
except OSError:
|
|
value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
|
|
return [Observation(value)]
|
|
|
|
meter.create_observable_gauge("messenger.process.memory", unit="By", callbacks=[rss])
|
|
if os.getenv("OTEL_SERVICE_NAME") == "messenger-publisher":
|
|
for name, unit, key in (("depth", "", "depth"),
|
|
("oldest_age", "s", "oldest_age"),
|
|
("sample_age", "s", "sample_age")):
|
|
def observe(_, key=key):
|
|
if "sampled_at" not in snapshot:
|
|
return []
|
|
value = (time.monotonic() - snapshot["sampled_at"]
|
|
if key == "sample_age" else snapshot[key])
|
|
return [Observation(value, {"queue": "attachments"})]
|
|
meter.create_observable_gauge("messenger.queue." + name, unit=unit,
|
|
callbacks=[observe])
|
|
|
|
|
|
async def backlog_loop(session_factory) -> None:
|
|
# Authoritative logical queue = queued + processing, including durable
|
|
# outbox and delayed retries. Remains observable when worker is stopped.
|
|
while True:
|
|
try:
|
|
async with session_factory() as db:
|
|
result = await db.execute(text("""
|
|
SELECT count(*) AS depth,
|
|
COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS oldest_age
|
|
FROM attachments WHERE status IN ('queued','processing')
|
|
"""))
|
|
row = result.mappings().one()
|
|
snapshot.update(depth=float(row["depth"]), oldest_age=float(row["oldest_age"]),
|
|
sampled_at=time.monotonic())
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
# Retain last sample; sample_age makes stale observations explicit.
|
|
pass
|
|
await asyncio.sleep(2)
|
|
|
|
|
|
def observe_ready(row) -> None:
|
|
elapsed = (datetime.now(timezone.utc) - row["created_at"]).total_seconds()
|
|
ready_duration.record(max(0, elapsed), {"job_type": "attachment.process.v1"})
|