Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
"""Find the newest worker job.retry / job.ready lines and summarize that trace."""
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
def lines(event):
|
||||
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
||||
payload = json.load(
|
||||
urllib.request.urlopen(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(query)
|
||||
+ "&limit=20",
|
||||
timeout=10,
|
||||
)
|
||||
)
|
||||
found = []
|
||||
for stream in payload["data"]["result"]:
|
||||
for ts, line in stream["values"]:
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if body.get("event") == event:
|
||||
found.append((int(ts), body.get("attempt"), body.get("trace_id")))
|
||||
found.sort()
|
||||
return found[-5:]
|
||||
|
||||
|
||||
print("retry", lines("job.retry"))
|
||||
print("ready", lines("job.ready"))
|
||||
Reference in New Issue
Block a user