Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
"""Re-check worker-stop alerts and a Tempo outage after the rule fix."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
OUT = ROOT / "evidence" / "incidents.json"
|
||||
|
||||
|
||||
def prom(expr):
|
||||
url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)["data"]["result"]
|
||||
|
||||
|
||||
def alerts():
|
||||
return {
|
||||
item["metric"].get("alertname"): item["metric"].get("alertstate")
|
||||
for item in prom("ALERTS")
|
||||
}
|
||||
|
||||
|
||||
def reload_rules():
|
||||
request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST")
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return response.status
|
||||
|
||||
|
||||
def main():
|
||||
print("reload", reload_rules(), flush=True)
|
||||
client = Client("http://127.0.0.1:8080", timeout=8)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
report = {"alerts_before": alerts()}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
||||
client.attachment(alice, chat)
|
||||
started = time.monotonic()
|
||||
seen = {}
|
||||
samples = []
|
||||
while time.monotonic() - started < 130:
|
||||
state = alerts()
|
||||
age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})')
|
||||
consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})')
|
||||
sample = {
|
||||
"t": round(time.monotonic() - started, 1),
|
||||
"alerts": state,
|
||||
"age": age[0]["value"][1] if age else None,
|
||||
"api_consumers": consumers[0]["value"][1] if consumers else None,
|
||||
}
|
||||
samples.append(sample)
|
||||
for name in ("QueueLagHigh", "WorkerConsumersZero"):
|
||||
if name not in seen and state.get(name) == "firing":
|
||||
seen[name] = sample["t"]
|
||||
if len(seen) == 2:
|
||||
break
|
||||
time.sleep(5)
|
||||
report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
||||
cleared = {}
|
||||
restore_started = time.monotonic()
|
||||
while time.monotonic() - restore_started < 90:
|
||||
state = alerts()
|
||||
for name in ("QueueLagHigh", "WorkerConsumersZero"):
|
||||
if name not in cleared and name not in state:
|
||||
cleared[name] = round(time.monotonic() - restore_started, 1)
|
||||
if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared:
|
||||
break
|
||||
time.sleep(5)
|
||||
report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()}
|
||||
|
||||
before_failed = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
||||
counts = {}
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
time.sleep(0.5)
|
||||
time.sleep(5)
|
||||
memory = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
||||
text=True,
|
||||
).strip()
|
||||
queue = prom("otelcol_exporter_queue_size")
|
||||
failed = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
||||
report["tempo_down_40s"] = {
|
||||
"api_results": counts,
|
||||
"failed_before": [item["value"][1] for item in before_failed],
|
||||
"failed_after": [item["value"][1] for item in failed],
|
||||
"queue_size": [item["value"][1] for item in queue],
|
||||
"collector_memory": memory,
|
||||
}
|
||||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print("wrote", OUT, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
query = '{service_name="messenger-api"} |= "http.request.completed" |= "content"'
|
||||
url = "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + urllib.parse.quote(query) + "&limit=15"
|
||||
data = json.load(urllib.request.urlopen(url, timeout=10))
|
||||
values = []
|
||||
for stream in data["data"]["result"]:
|
||||
values.extend(stream["values"])
|
||||
print("n", len(values))
|
||||
for _, line in values[-8:]:
|
||||
print(line[:500])
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Find the newest worker job.retry / job.ready lines and summarize that trace."""
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
def lines(event):
|
||||
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
||||
payload = json.load(
|
||||
urllib.request.urlopen(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(query)
|
||||
+ "&limit=20",
|
||||
timeout=10,
|
||||
)
|
||||
)
|
||||
found = []
|
||||
for stream in payload["data"]["result"]:
|
||||
for ts, line in stream["values"]:
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if body.get("event") == event:
|
||||
found.append((int(ts), body.get("attempt"), body.get("trace_id")))
|
||||
found.sort()
|
||||
return found[-5:]
|
||||
|
||||
|
||||
print("retry", lines("job.retry"))
|
||||
print("ready", lines("job.ready"))
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Print a short, secret-free snapshot of metrics, one log, and recent traces."""
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def query(expr):
|
||||
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
|
||||
|
||||
|
||||
names = get("http://127.0.0.1:9090/api/v1/label/__name__/values")["data"]
|
||||
app = [name for name in names if name.startswith("http_") or name.startswith("messenger_")]
|
||||
print("NAMES")
|
||||
print("\n".join(app))
|
||||
for expr in (
|
||||
"messenger_jobs_total",
|
||||
"messenger_queue_depth",
|
||||
"messenger_db_up",
|
||||
"messenger_worker_busy",
|
||||
"messenger_worker_consumers",
|
||||
):
|
||||
print(expr, query(expr))
|
||||
|
||||
log_query = '{service_name="messenger-worker"}'
|
||||
logs = get(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(log_query)
|
||||
+ "&limit=2"
|
||||
)
|
||||
streams = logs["data"]["result"]
|
||||
print("LOG_STREAMS", len(streams))
|
||||
if streams:
|
||||
print(streams[0]["values"][-1][1][:600])
|
||||
|
||||
tempo_query = '{resource.service.name="messenger-api"}'
|
||||
traces = get(
|
||||
"http://127.0.0.1:3200/api/search?q=" + urllib.parse.quote(tempo_query) + "&limit=5"
|
||||
)
|
||||
print("TRACES")
|
||||
for item in traces.get("traces", []):
|
||||
print(item.get("rootServiceName"), item.get("rootTraceName"), item.get("traceID"), item.get("durationMs"))
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Summarize one Tempo trace by id. Prints span names and parent links only."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
trace_id = sys.argv[1]
|
||||
payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:3200/api/traces/{trace_id}", timeout=10))
|
||||
batches = payload.get("batches") or payload.get("resourceSpans") or []
|
||||
# Tempo may return OTLP JSON under batches or the classic trace format.
|
||||
if "trace" in payload:
|
||||
spans = payload["trace"].get("spans") or []
|
||||
for span in spans:
|
||||
print(span.get("name") or span.get("operationName"), span.get("spanID"), span.get("parentSpanID"))
|
||||
raise SystemExit
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if "name" in node and ("spanId" in node or "span_id" in node):
|
||||
print(
|
||||
node.get("name"),
|
||||
"id=" + str(node.get("spanId") or node.get("span_id")),
|
||||
"parent=" + str(node.get("parentSpanId") or node.get("parent_span_id") or "-"),
|
||||
)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Write the local SeaweedFS identity file from .env. The file is gitignored."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
env = {}
|
||||
for line in (root / ".env").read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
env[key] = value
|
||||
|
||||
out = root / "infra" / "seaweed" / "s3.json"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
document = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "app",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": env["S3_ACCESS_KEY"],
|
||||
"secretKey": env["S3_SECRET_KEY"],
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Write", "Tagging"],
|
||||
}
|
||||
]
|
||||
}
|
||||
out.write_text(json.dumps(document), encoding="utf-8")
|
||||
print("wrote infra/seaweed/s3.json")
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
wait_healthy() {
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
ok=1
|
||||
for svc in "$@"; do
|
||||
health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}')
|
||||
if [ "$health" != "healthy" ]; then
|
||||
ok=0
|
||||
fi
|
||||
done
|
||||
if [ "$ok" -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "timed out waiting for healthy: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
docker compose --env-file .env restart -t 5 postgres s3 rabbitmq
|
||||
wait_healthy postgres s3 rabbitmq
|
||||
docker compose --env-file .env restart -t 5 api-1 api-2 worker
|
||||
wait_healthy api-1 api-2 worker
|
||||
docker compose --env-file .env restart -t 2 nginx
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Run the public resilience test on Windows by executing the shell hook with Git Bash.
|
||||
|
||||
The public test launches the hook directly. That works on Linux. This runner does not
|
||||
change tests/resilience.py; it only prefixes the same script with bash.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
import resilience # noqa: E402
|
||||
|
||||
BASH = r"C:\Program Files\Git\bin\bash.exe"
|
||||
_popen = resilience.subprocess.Popen
|
||||
|
||||
|
||||
def popen(args, **kwargs):
|
||||
if args and str(args[0]).endswith(".sh"):
|
||||
args = [BASH, str(args[0])]
|
||||
return _popen(args, **kwargs)
|
||||
|
||||
|
||||
resilience.subprocess.Popen = popen
|
||||
resilience.run(resilience.main)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
FAULTS = ROOT / "faults"
|
||||
OUT = ROOT / "evidence" / "slo-track2.json"
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def prom(expr):
|
||||
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
|
||||
|
||||
|
||||
def alert_states():
|
||||
found = {}
|
||||
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
|
||||
name = item["metric"].get("alertname")
|
||||
found[name] = item["metric"].get("alertstate")
|
||||
return found
|
||||
|
||||
|
||||
def write_fault(name, value):
|
||||
path = FAULTS / name
|
||||
if value is None:
|
||||
path.unlink(missing_ok=True)
|
||||
else:
|
||||
path.write_text(str(value), encoding="utf-8")
|
||||
|
||||
|
||||
def clear_faults():
|
||||
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
|
||||
write_fault(name, None)
|
||||
|
||||
|
||||
def drive(client, actor, seconds, rps):
|
||||
interval = 1.0 / rps
|
||||
deadline = time.monotonic() + seconds
|
||||
counts = {}
|
||||
while time.monotonic() < deadline:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
|
||||
status = result["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
sleep = interval - (time.monotonic() - started)
|
||||
if sleep > 0:
|
||||
time.sleep(sleep)
|
||||
return counts
|
||||
|
||||
|
||||
def wait_state(name, wanted, timeout):
|
||||
started = time.monotonic()
|
||||
last = {}
|
||||
while time.monotonic() - started < timeout:
|
||||
last = alert_states()
|
||||
if last.get(name) == wanted or (wanted is None and name not in last):
|
||||
return round(time.monotonic() - started, 1), last
|
||||
time.sleep(5)
|
||||
return None, last
|
||||
|
||||
|
||||
def span_summary(trace_id):
|
||||
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
|
||||
spans = []
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if "name" in node and "spanId" in node:
|
||||
start = node.get("startTimeUnixNano")
|
||||
end = node.get("endTimeUnixNano")
|
||||
duration = None
|
||||
try:
|
||||
duration = round((int(end) - int(start)) / 1e6, 1)
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
spans.append(
|
||||
{
|
||||
"name": node.get("name"),
|
||||
"span": node.get("spanId"),
|
||||
"parent": node.get("parentSpanId") or "",
|
||||
"ms": duration,
|
||||
}
|
||||
)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return spans
|
||||
|
||||
|
||||
def latest_worker_trace(event):
|
||||
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
||||
payload = get(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(query)
|
||||
+ "&limit=5"
|
||||
)
|
||||
lines = []
|
||||
for stream in payload["data"]["result"]:
|
||||
for _, line in stream["values"]:
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if body.get("event") == event:
|
||||
lines.append(body)
|
||||
if not lines:
|
||||
return None
|
||||
body = lines[-1]
|
||||
return {
|
||||
"event": body.get("event"),
|
||||
"attempt": body.get("attempt"),
|
||||
"trace_id": body.get("trace_id"),
|
||||
"span_id": body.get("span_id"),
|
||||
"service": body.get("service.name"),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
clear_faults()
|
||||
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
||||
client = Client("http://127.0.0.1:8080", timeout=8)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
print("bootstrap ok", flush=True)
|
||||
|
||||
write_fault("process_fail_remaining", "1")
|
||||
attachment, _content = client.attachment(alice, chat)
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=40)
|
||||
time.sleep(3)
|
||||
log = latest_worker_trace("job.ready")
|
||||
spans = []
|
||||
if log and log.get("trace_id"):
|
||||
try:
|
||||
spans = span_summary(log["trace_id"])
|
||||
except Exception as exc:
|
||||
report["retry_trace_error"] = type(exc).__name__
|
||||
report["retry_upload"] = {
|
||||
"final_status": ready["status"],
|
||||
"log": log,
|
||||
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
|
||||
"span_names": [span["name"] for span in spans],
|
||||
"parents_linked": any(span["parent"] for span in spans),
|
||||
}
|
||||
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
|
||||
|
||||
print("warmup", flush=True)
|
||||
report["warmup"] = drive(client, alice, 90, 2)
|
||||
report["alerts_after_warmup"] = alert_states()
|
||||
write_fault("http_error_ratio", "1")
|
||||
print("short spike", flush=True)
|
||||
report["short_spike"] = drive(client, alice, 10, 2)
|
||||
time.sleep(35)
|
||||
report["alerts_after_short_spike"] = alert_states()
|
||||
clear_faults()
|
||||
report["recovery_after_spike"] = drive(client, alice, 20, 2)
|
||||
|
||||
write_fault("http_error_ratio", "1")
|
||||
print("sustained", flush=True)
|
||||
sustained_started = time.monotonic()
|
||||
seen = None
|
||||
counts = {}
|
||||
deadline = time.monotonic() + 100
|
||||
interval = 0.5
|
||||
while time.monotonic() < deadline:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
states = alert_states()
|
||||
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
|
||||
seen = round(time.monotonic() - sustained_started, 1)
|
||||
report["fast_burn_detected_after_seconds"] = seen
|
||||
report["alerts_when_fast_burn_fired"] = states
|
||||
break
|
||||
sleep = interval - (time.monotonic() - started)
|
||||
if sleep > 0:
|
||||
time.sleep(sleep)
|
||||
report["sustained_counts_until_detection_or_timeout"] = counts
|
||||
if seen is None:
|
||||
report["fast_burn_detected_after_seconds"] = None
|
||||
report["alerts_after_sustained"] = alert_states()
|
||||
clear_faults()
|
||||
print("resolve", flush=True)
|
||||
resolve_started = time.monotonic()
|
||||
resolved = None
|
||||
resolve_deadline = time.monotonic() + 180
|
||||
while time.monotonic() < resolve_deadline:
|
||||
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
||||
if "ApiErrorBudgetFastBurn" not in alert_states():
|
||||
resolved = round(time.monotonic() - resolve_started, 1)
|
||||
break
|
||||
time.sleep(5)
|
||||
report["fast_burn_resolved_after_seconds"] = resolved
|
||||
report["alerts_after_resolve"] = alert_states()
|
||||
|
||||
print("worker stop", flush=True)
|
||||
import subprocess
|
||||
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
||||
client.attachment(alice, chat)
|
||||
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
|
||||
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
|
||||
report["worker_stopped"] = {
|
||||
"consumers_firing_after_seconds": consumer_wait,
|
||||
"queue_lag_firing_after_seconds": queue_wait,
|
||||
"alerts": queue_state or consumer_state,
|
||||
}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
||||
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
|
||||
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
|
||||
|
||||
print("latency split", flush=True)
|
||||
write_fault("db_slow_ms", "800")
|
||||
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
|
||||
time.sleep(3)
|
||||
write_fault("db_slow_ms", None)
|
||||
write_fault("handler_slow_ms", "800")
|
||||
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
||||
time.sleep(3)
|
||||
write_fault("handler_slow_ms", None)
|
||||
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
|
||||
|
||||
print("tempo down", flush=True)
|
||||
before = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
||||
tempo_counts = drive(client, alice, 8, 2)
|
||||
time.sleep(6)
|
||||
after = prom("otelcol_exporter_send_failed_spans")
|
||||
stats = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
||||
text=True,
|
||||
).strip()
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
||||
report["tempo_stopped"] = {
|
||||
"api_results": tempo_counts,
|
||||
"failed_spans_before": [item["value"][1] for item in before],
|
||||
"failed_spans_after": [item["value"][1] for item in after],
|
||||
"collector_memory": stats,
|
||||
}
|
||||
clear_faults()
|
||||
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print("wrote", OUT, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
clear_faults()
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1|api-2) ;;
|
||||
*) echo "unexpected instance" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user