Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.

This commit is contained in:
2026-11-07 01:36:19 +03:00
parent 470d9d6f1e
commit 7d4e1237a3
56 changed files with 4419 additions and 53 deletions
+73
View File
@@ -0,0 +1,73 @@
from pathlib import Path
from app.settings import settings
def _text(name):
path = Path(settings.fault_dir) / name
try:
return path.read_text(encoding="utf-8").strip()
except (FileNotFoundError, OSError):
return ""
def db_slow_seconds():
raw = _text("db_slow_ms")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 10000.0) / 1000.0
def db_fail():
return _text("db_fail") == "1"
def http_error_ratio():
raw = _text("http_error_ratio")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 1.0)
def handler_slow_seconds():
raw = _text("handler_slow_ms")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 10000.0) / 1000.0
def consume_process_failure():
path = Path(settings.fault_dir) / "process_fail_remaining"
try:
raw = path.read_text(encoding="utf-8").strip()
except (FileNotFoundError, OSError):
return False
try:
remaining = int(raw or "0")
except ValueError:
return False
if remaining <= 0:
return False
try:
path.write_text(str(remaining - 1), encoding="utf-8")
except OSError:
return False
return True