Files
mtusi2026lab6/app/advanced.py
T

115 lines
5.1 KiB
Python

"""Low-cardinality saturation/backlog signals and safe dependency spans (lab6/4)."""
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
import os
import resource
import time
from opentelemetry import metrics
from opentelemetry.metrics import Observation
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.telemetry import span
meter = metrics.get_meter("messenger")
waiting = 0
busy = 0
snapshot: dict[str, float] = {}
ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
retries = meter.create_counter("messenger.jobs.retries", unit="{retry}")
class ObservedSession(AsyncSession):
async def execute(self, statement, params=None, **kwargs):
global waiting
# Number of acquisition requests in progress (includes new connects).
# Acquisition is separate from query execution to diagnose pool waits.
waiting += 1
try:
with span("db.acquire", {"db.system": "postgresql"}):
await self.connection()
finally:
waiting -= 1
operation = str(statement).lstrip().split(None, 1)[0].upper()
if operation not in {"SELECT", "INSERT", "UPDATE", "DELETE", "SET"}:
operation = "OTHER"
with span("db.query", {"db.system": "postgresql", "db.operation": operation}):
return await super().execute(statement, params, **kwargs)
async def commit(self):
with span("db.commit", {"db.system": "postgresql"}):
return await super().commit()
def install_gauges(engine) -> None:
pool = engine.sync_engine.pool
meter.create_observable_gauge("messenger.db.pool.used",
callbacks=[lambda _: [Observation(pool.checkedout())]])
meter.create_observable_gauge("messenger.db.pool.waiting",
callbacks=[lambda _: [Observation(waiting)]])
meter.create_observable_gauge("messenger.db.pool.capacity",
callbacks=[lambda _: [Observation(pool.size() + pool._max_overflow)]])
meter.create_observable_gauge("messenger.worker.busy",
callbacks=[lambda _: [Observation(busy)]])
capacity = 1 if os.getenv("OTEL_SERVICE_NAME") == "messenger-worker" else 0
meter.create_observable_gauge("messenger.worker.capacity",
callbacks=[lambda _: [Observation(capacity)]])
if capacity:
meter.create_observable_gauge("messenger.worker.heartbeat", unit="s",
callbacks=[lambda _: [Observation(time.time())]])
meter.create_observable_counter("messenger.process.cpu", unit="s",
callbacks=[lambda _: [Observation(time.process_time())]])
def rss(_):
# Linux /proc resident pages, current RSS (not ru_maxrss high watermark).
try:
with open("/proc/self/statm", encoding="ascii") as statm:
value = int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE")
except OSError:
value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
return [Observation(value)]
meter.create_observable_gauge("messenger.process.memory", unit="By", callbacks=[rss])
if os.getenv("OTEL_SERVICE_NAME") == "messenger-publisher":
for name, unit, key in (("depth", "", "depth"),
("oldest_age", "s", "oldest_age"),
("sample_age", "s", "sample_age")):
def observe(_, key=key):
if "sampled_at" not in snapshot:
return []
value = (time.monotonic() - snapshot["sampled_at"]
if key == "sample_age" else snapshot[key])
return [Observation(value, {"queue": "attachments"})]
meter.create_observable_gauge("messenger.queue." + name, unit=unit,
callbacks=[observe])
async def backlog_loop(session_factory) -> None:
# Authoritative logical queue = queued + processing, including durable
# outbox and delayed retries. Remains observable when worker is stopped.
while True:
try:
async with session_factory() as db:
result = await db.execute(text("""
SELECT count(*) AS depth,
COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS oldest_age
FROM attachments WHERE status IN ('queued','processing')
"""))
row = result.mappings().one()
snapshot.update(depth=float(row["depth"]), oldest_age=float(row["oldest_age"]),
sampled_at=time.monotonic())
except asyncio.CancelledError:
raise
except Exception:
# Retain last sample; sample_age makes stale observations explicit.
pass
await asyncio.sleep(2)
def observe_ready(row) -> None:
elapsed = (datetime.now(timezone.utc) - row["created_at"]).total_seconds()
ready_duration.record(max(0, elapsed), {"job_type": "attachment.process.v1"})