Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.

This commit is contained in:
2026-11-07 01:36:19 +03:00
parent 470d9d6f1e
commit 7d4e1237a3
56 changed files with 4419 additions and 53 deletions
+31
View File
@@ -0,0 +1,31 @@
"""Find the newest worker job.retry / job.ready lines and summarize that trace."""
import json
import urllib.parse
import urllib.request
def lines(event):
query = '{service_name="messenger-worker"} |= "' + event + '"'
payload = json.load(
urllib.request.urlopen(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(query)
+ "&limit=20",
timeout=10,
)
)
found = []
for stream in payload["data"]["result"]:
for ts, line in stream["values"]:
try:
body = json.loads(line)
except ValueError:
continue
if body.get("event") == event:
found.append((int(ts), body.get("attempt"), body.get("trace_id")))
found.sort()
return found[-5:]
print("retry", lines("job.retry"))
print("ready", lines("job.ready"))