Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.

This commit is contained in:
2026-11-07 01:36:19 +03:00
parent 470d9d6f1e
commit 7d4e1237a3
56 changed files with 4419 additions and 53 deletions
+109
View File
@@ -0,0 +1,109 @@
"""Re-check worker-stop alerts and a Tempo outage after the rule fix."""
import json
import subprocess
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
OUT = ROOT / "evidence" / "incidents.json"
def prom(expr):
url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)["data"]["result"]
def alerts():
return {
item["metric"].get("alertname"): item["metric"].get("alertstate")
for item in prom("ALERTS")
}
def reload_rules():
request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST")
with urllib.request.urlopen(request, timeout=10) as response:
return response.status
def main():
print("reload", reload_rules(), flush=True)
client = Client("http://127.0.0.1:8080", timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
report = {"alerts_before": alerts()}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
client.attachment(alice, chat)
started = time.monotonic()
seen = {}
samples = []
while time.monotonic() - started < 130:
state = alerts()
age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})')
consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})')
sample = {
"t": round(time.monotonic() - started, 1),
"alerts": state,
"age": age[0]["value"][1] if age else None,
"api_consumers": consumers[0]["value"][1] if consumers else None,
}
samples.append(sample)
for name in ("QueueLagHigh", "WorkerConsumersZero"):
if name not in seen and state.get(name) == "firing":
seen[name] = sample["t"]
if len(seen) == 2:
break
time.sleep(5)
report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
cleared = {}
restore_started = time.monotonic()
while time.monotonic() - restore_started < 90:
state = alerts()
for name in ("QueueLagHigh", "WorkerConsumersZero"):
if name not in cleared and name not in state:
cleared[name] = round(time.monotonic() - restore_started, 1)
if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared:
break
time.sleep(5)
report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()}
before_failed = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
counts = {}
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
time.sleep(0.5)
time.sleep(5)
memory = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
text=True,
).strip()
queue = prom("otelcol_exporter_queue_size")
failed = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
report["tempo_down_40s"] = {
"api_results": counts,
"failed_before": [item["value"][1] for item in before_failed],
"failed_after": [item["value"][1] for item in failed],
"queue_size": [item["value"][1] for item in queue],
"collector_memory": memory,
}
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print("wrote", OUT, flush=True)
if __name__ == "__main__":
main()
+13
View File
@@ -0,0 +1,13 @@
import json
import urllib.parse
import urllib.request
query = '{service_name="messenger-api"} |= "http.request.completed" |= "content"'
url = "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + urllib.parse.quote(query) + "&limit=15"
data = json.load(urllib.request.urlopen(url, timeout=10))
values = []
for stream in data["data"]["result"]:
values.extend(stream["values"])
print("n", len(values))
for _, line in values[-8:]:
print(line[:500])
+31
View File
@@ -0,0 +1,31 @@
"""Find the newest worker job.retry / job.ready lines and summarize that trace."""
import json
import urllib.parse
import urllib.request
def lines(event):
query = '{service_name="messenger-worker"} |= "' + event + '"'
payload = json.load(
urllib.request.urlopen(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(query)
+ "&limit=20",
timeout=10,
)
)
found = []
for stream in payload["data"]["result"]:
for ts, line in stream["values"]:
try:
body = json.loads(line)
except ValueError:
continue
if body.get("event") == event:
found.append((int(ts), body.get("attempt"), body.get("trace_id")))
found.sort()
return found[-5:]
print("retry", lines("job.retry"))
print("ready", lines("job.ready"))
+47
View File
@@ -0,0 +1,47 @@
"""Print a short, secret-free snapshot of metrics, one log, and recent traces."""
import json
import urllib.parse
import urllib.request
def get(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)
def query(expr):
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
names = get("http://127.0.0.1:9090/api/v1/label/__name__/values")["data"]
app = [name for name in names if name.startswith("http_") or name.startswith("messenger_")]
print("NAMES")
print("\n".join(app))
for expr in (
"messenger_jobs_total",
"messenger_queue_depth",
"messenger_db_up",
"messenger_worker_busy",
"messenger_worker_consumers",
):
print(expr, query(expr))
log_query = '{service_name="messenger-worker"}'
logs = get(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(log_query)
+ "&limit=2"
)
streams = logs["data"]["result"]
print("LOG_STREAMS", len(streams))
if streams:
print(streams[0]["values"][-1][1][:600])
tempo_query = '{resource.service.name="messenger-api"}'
traces = get(
"http://127.0.0.1:3200/api/search?q=" + urllib.parse.quote(tempo_query) + "&limit=5"
)
print("TRACES")
for item in traces.get("traces", []):
print(item.get("rootServiceName"), item.get("rootTraceName"), item.get("traceID"), item.get("durationMs"))
+31
View File
@@ -0,0 +1,31 @@
"""Summarize one Tempo trace by id. Prints span names and parent links only."""
import json
import sys
import urllib.request
trace_id = sys.argv[1]
payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:3200/api/traces/{trace_id}", timeout=10))
batches = payload.get("batches") or payload.get("resourceSpans") or []
# Tempo may return OTLP JSON under batches or the classic trace format.
if "trace" in payload:
spans = payload["trace"].get("spans") or []
for span in spans:
print(span.get("name") or span.get("operationName"), span.get("spanID"), span.get("parentSpanID"))
raise SystemExit
def walk(node):
if isinstance(node, dict):
if "name" in node and ("spanId" in node or "span_id" in node):
print(
node.get("name"),
"id=" + str(node.get("spanId") or node.get("span_id")),
"parent=" + str(node.get("parentSpanId") or node.get("parent_span_id") or "-"),
)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
+32
View File
@@ -0,0 +1,32 @@
"""Write the local SeaweedFS identity file from .env. The file is gitignored."""
import json
from pathlib import Path
root = Path(__file__).resolve().parents[1]
env = {}
for line in (root / ".env").read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
env[key] = value
out = root / "infra" / "seaweed" / "s3.json"
out.parent.mkdir(parents=True, exist_ok=True)
document = {
"identities": [
{
"name": "app",
"credentials": [
{
"accessKey": env["S3_ACCESS_KEY"],
"secretKey": env["S3_SECRET_KEY"],
}
],
"actions": ["Admin", "Read", "List", "Write", "Tagging"],
}
]
}
out.write_text(json.dumps(document), encoding="utf-8")
print("wrote infra/seaweed/s3.json")
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
wait_healthy() {
i=0
while [ "$i" -lt 40 ]; do
ok=1
for svc in "$@"; do
health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}')
if [ "$health" != "healthy" ]; then
ok=0
fi
done
if [ "$ok" -eq 1 ]; then
return 0
fi
i=$((i + 1))
sleep 1
done
echo "timed out waiting for healthy: $*" >&2
return 1
}
docker compose --env-file .env restart -t 5 postgres s3 rabbitmq
wait_healthy postgres s3 rabbitmq
docker compose --env-file .env restart -t 5 api-1 api-2 worker
wait_healthy api-1 api-2 worker
docker compose --env-file .env restart -t 2 nginx
+25
View File
@@ -0,0 +1,25 @@
"""Run the public resilience test on Windows by executing the shell hook with Git Bash.
The public test launches the hook directly. That works on Linux. This runner does not
change tests/resilience.py; it only prefixes the same script with bash.
"""
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
import resilience # noqa: E402
BASH = r"C:\Program Files\Git\bin\bash.exe"
_popen = resilience.subprocess.Popen
def popen(args, **kwargs):
if args and str(args[0]).endswith(".sh"):
args = [BASH, str(args[0])]
return _popen(args, **kwargs)
resilience.subprocess.Popen = popen
resilience.run(resilience.main)
+270
View File
@@ -0,0 +1,270 @@
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
import json
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
FAULTS = ROOT / "faults"
OUT = ROOT / "evidence" / "slo-track2.json"
def get(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)
def prom(expr):
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
def alert_states():
found = {}
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
name = item["metric"].get("alertname")
found[name] = item["metric"].get("alertstate")
return found
def write_fault(name, value):
path = FAULTS / name
if value is None:
path.unlink(missing_ok=True)
else:
path.write_text(str(value), encoding="utf-8")
def clear_faults():
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
write_fault(name, None)
def drive(client, actor, seconds, rps):
interval = 1.0 / rps
deadline = time.monotonic() + seconds
counts = {}
while time.monotonic() < deadline:
started = time.monotonic()
try:
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
status = result["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
sleep = interval - (time.monotonic() - started)
if sleep > 0:
time.sleep(sleep)
return counts
def wait_state(name, wanted, timeout):
started = time.monotonic()
last = {}
while time.monotonic() - started < timeout:
last = alert_states()
if last.get(name) == wanted or (wanted is None and name not in last):
return round(time.monotonic() - started, 1), last
time.sleep(5)
return None, last
def span_summary(trace_id):
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
spans = []
def walk(node):
if isinstance(node, dict):
if "name" in node and "spanId" in node:
start = node.get("startTimeUnixNano")
end = node.get("endTimeUnixNano")
duration = None
try:
duration = round((int(end) - int(start)) / 1e6, 1)
except (TypeError, ValueError):
duration = None
spans.append(
{
"name": node.get("name"),
"span": node.get("spanId"),
"parent": node.get("parentSpanId") or "",
"ms": duration,
}
)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return spans
def latest_worker_trace(event):
query = '{service_name="messenger-worker"} |= "' + event + '"'
payload = get(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(query)
+ "&limit=5"
)
lines = []
for stream in payload["data"]["result"]:
for _, line in stream["values"]:
try:
body = json.loads(line)
except ValueError:
continue
if body.get("event") == event:
lines.append(body)
if not lines:
return None
body = lines[-1]
return {
"event": body.get("event"),
"attempt": body.get("attempt"),
"trace_id": body.get("trace_id"),
"span_id": body.get("span_id"),
"service": body.get("service.name"),
}
def main():
clear_faults()
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
client = Client("http://127.0.0.1:8080", timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
print("bootstrap ok", flush=True)
write_fault("process_fail_remaining", "1")
attachment, _content = client.attachment(alice, chat)
ready = client.wait_ready(alice, attachment, wait_seconds=40)
time.sleep(3)
log = latest_worker_trace("job.ready")
spans = []
if log and log.get("trace_id"):
try:
spans = span_summary(log["trace_id"])
except Exception as exc:
report["retry_trace_error"] = type(exc).__name__
report["retry_upload"] = {
"final_status": ready["status"],
"log": log,
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
"span_names": [span["name"] for span in spans],
"parents_linked": any(span["parent"] for span in spans),
}
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
print("warmup", flush=True)
report["warmup"] = drive(client, alice, 90, 2)
report["alerts_after_warmup"] = alert_states()
write_fault("http_error_ratio", "1")
print("short spike", flush=True)
report["short_spike"] = drive(client, alice, 10, 2)
time.sleep(35)
report["alerts_after_short_spike"] = alert_states()
clear_faults()
report["recovery_after_spike"] = drive(client, alice, 20, 2)
write_fault("http_error_ratio", "1")
print("sustained", flush=True)
sustained_started = time.monotonic()
seen = None
counts = {}
deadline = time.monotonic() + 100
interval = 0.5
while time.monotonic() < deadline:
started = time.monotonic()
try:
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
states = alert_states()
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
seen = round(time.monotonic() - sustained_started, 1)
report["fast_burn_detected_after_seconds"] = seen
report["alerts_when_fast_burn_fired"] = states
break
sleep = interval - (time.monotonic() - started)
if sleep > 0:
time.sleep(sleep)
report["sustained_counts_until_detection_or_timeout"] = counts
if seen is None:
report["fast_burn_detected_after_seconds"] = None
report["alerts_after_sustained"] = alert_states()
clear_faults()
print("resolve", flush=True)
resolve_started = time.monotonic()
resolved = None
resolve_deadline = time.monotonic() + 180
while time.monotonic() < resolve_deadline:
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
if "ApiErrorBudgetFastBurn" not in alert_states():
resolved = round(time.monotonic() - resolve_started, 1)
break
time.sleep(5)
report["fast_burn_resolved_after_seconds"] = resolved
report["alerts_after_resolve"] = alert_states()
print("worker stop", flush=True)
import subprocess
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
client.attachment(alice, chat)
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
report["worker_stopped"] = {
"consumers_firing_after_seconds": consumer_wait,
"queue_lag_firing_after_seconds": queue_wait,
"alerts": queue_state or consumer_state,
}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
print("latency split", flush=True)
write_fault("db_slow_ms", "800")
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
time.sleep(3)
write_fault("db_slow_ms", None)
write_fault("handler_slow_ms", "800")
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
time.sleep(3)
write_fault("handler_slow_ms", None)
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
print("tempo down", flush=True)
before = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
tempo_counts = drive(client, alice, 8, 2)
time.sleep(6)
after = prom("otelcol_exporter_send_failed_spans")
stats = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
text=True,
).strip()
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
report["tempo_stopped"] = {
"api_results": tempo_counts,
"failed_spans_before": [item["value"][1] for item in before],
"failed_spans_after": [item["value"][1] for item in after],
"collector_memory": stats,
}
clear_faults()
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print("wrote", OUT, flush=True)
if __name__ == "__main__":
try:
main()
finally:
clear_faults()
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
api-1|api-2) ;;
*) echo "unexpected instance" >&2; exit 1 ;;
esac
docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"