Preserve distributed job context and expose grade 4 operational signals
This commit is contained in:
@@ -131,6 +131,7 @@ def tempo_spans(body):
|
||||
duration = (int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"])) / 1e6
|
||||
spans.append({"service": service, "name": span["name"], "kind": span.get("kind"),
|
||||
"status": span.get("status", {}).get("code", "STATUS_CODE_UNSET"),
|
||||
"span_id": span.get("spanId"), "parent_span_id": span.get("parentSpanId"),
|
||||
"duration_ms": round(duration, 3), "links": len(span.get("links", [])),
|
||||
"attributes": values})
|
||||
return spans
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for
|
||||
from obs_recovery import backends, compose
|
||||
|
||||
|
||||
def actor():
|
||||
api = Api(os.environ["BASE_URL"])
|
||||
user, username, password, _ = api.register()
|
||||
auth, _ = api.login(username, password)
|
||||
return api, auth, api.chat(auth, user)
|
||||
|
||||
|
||||
def upload(api, auth, chat):
|
||||
return api.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png",
|
||||
Path("tests/fixtures/sample.png").read_bytes(),
|
||||
dict(auth, **{"Content-Type": "image/png"}),
|
||||
expect=(202,),
|
||||
)
|
||||
|
||||
|
||||
def ready(api, auth, attachment):
|
||||
def check():
|
||||
item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0]
|
||||
if item["status"] == "failed":
|
||||
raise RuntimeError("attachment failed")
|
||||
return item if item["status"] == "ready" else None
|
||||
|
||||
return wait_for("ready", check, 120, 1)
|
||||
|
||||
|
||||
def value(b, expression):
|
||||
result = b.promql(expression)
|
||||
return float(result[0]["value"][1]) if result else None
|
||||
|
||||
|
||||
def alert(b, name, firing):
|
||||
def check():
|
||||
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
|
||||
found = [
|
||||
a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing"
|
||||
]
|
||||
if bool(found) == firing:
|
||||
return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found}
|
||||
return None
|
||||
|
||||
return wait_for(name, check, 150)
|
||||
|
||||
|
||||
def chain(b, retry):
|
||||
api, auth, chat = actor()
|
||||
started = time.time()
|
||||
if retry:
|
||||
compose("stop", "worker")
|
||||
attachment, rid = upload(api, auth, chat)
|
||||
try:
|
||||
if retry:
|
||||
compose("stop", "s3")
|
||||
compose("start", "worker")
|
||||
query = (
|
||||
'{service_name="messenger-worker"} | attachment_id="'
|
||||
+ attachment["id"]
|
||||
+ '" | outcome="retry_scheduled"'
|
||||
)
|
||||
wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60)
|
||||
finally:
|
||||
if retry:
|
||||
compose("up", "-d", "--no-deps", "--wait", "s3", "worker")
|
||||
ready(api, auth, attachment)
|
||||
logs = wait_for(
|
||||
"upload log",
|
||||
lambda: b.logs(
|
||||
'{service_name="messenger-api"} | request_id="' + rid + '"',
|
||||
started - 1,
|
||||
time.time() + 1,
|
||||
),
|
||||
60,
|
||||
)
|
||||
trace_id = logs[0][0]["trace_id"]
|
||||
|
||||
def complete():
|
||||
spans = tempo_spans(b.trace(trace_id)[0])
|
||||
return (
|
||||
spans
|
||||
if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans)
|
||||
else None
|
||||
)
|
||||
|
||||
spans = wait_for("complete trace", complete, 90)
|
||||
roots = [
|
||||
s
|
||||
for s in spans
|
||||
if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid
|
||||
]
|
||||
ids = {s["span_id"] for s in spans}
|
||||
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
|
||||
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
|
||||
clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"]
|
||||
if (
|
||||
not roots
|
||||
or not producers
|
||||
or not consumers
|
||||
or any(s["parent_span_id"] not in ids for s in producers + consumers)
|
||||
):
|
||||
raise Failure("broken distributed parent chain")
|
||||
if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any(
|
||||
s["attributes"].get("db.system") == "postgresql" for s in clients
|
||||
):
|
||||
raise Failure("missing dependency spans")
|
||||
outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers]
|
||||
if retry and "retry_scheduled" not in outcomes:
|
||||
raise Failure("retry left original trace")
|
||||
api.logout(auth)
|
||||
fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms")
|
||||
return {
|
||||
"upload_request_id": rid,
|
||||
"trace_id": trace_id,
|
||||
"retry": retry,
|
||||
"outcomes": outcomes,
|
||||
"spans": [
|
||||
{**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")}
|
||||
for s in spans
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def worker(b):
|
||||
api, auth, chat = actor()
|
||||
compose("stop", "worker")
|
||||
stopped = now()
|
||||
try:
|
||||
items = [upload(api, auth, chat)[0] for _ in range(8)]
|
||||
firing = alert(b, "MessengerQueueLag", True)
|
||||
observed = {
|
||||
"depth": value(b, "max(messenger_queue_depth)"),
|
||||
"age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"),
|
||||
}
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "worker")
|
||||
restored = now()
|
||||
for item in items:
|
||||
ready(api, auth, item)
|
||||
resolved = alert(b, "MessengerQueueLag", False)
|
||||
api.logout(auth)
|
||||
return {
|
||||
"stopped": stopped,
|
||||
"restored": restored,
|
||||
"jobs": len(items),
|
||||
"backlog": observed,
|
||||
"firing": firing,
|
||||
"resolved": resolved,
|
||||
"depth_after": value(b, "max(messenger_queue_depth)"),
|
||||
}
|
||||
|
||||
|
||||
def database(b):
|
||||
api, auth, _ = actor()
|
||||
started = time.time()
|
||||
prefix = "db-incident-" + uuid.uuid4().hex[:12]
|
||||
|
||||
def request(index):
|
||||
rid = prefix + "-" + str(index)
|
||||
status = http(
|
||||
"GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid})
|
||||
)[0]
|
||||
return {"request_id": rid, "status": status}
|
||||
|
||||
for index in range(10):
|
||||
request(index)
|
||||
time.sleep(12)
|
||||
compose("pause", "postgres")
|
||||
paused = now()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
responses = list(pool.map(request, range(10, 34)))
|
||||
firing = alert(b, "MessengerHighErrorRate", True)
|
||||
signals = {
|
||||
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
|
||||
"pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"),
|
||||
}
|
||||
finally:
|
||||
compose("unpause", "postgres")
|
||||
restored = now()
|
||||
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
|
||||
for index in range(34, 74):
|
||||
request(index)
|
||||
time.sleep(0.15)
|
||||
resolved = alert(b, "MessengerHighErrorRate", False)
|
||||
traces = b.traceql(
|
||||
'{ span.messenger.request_id =~ "' + prefix + '.*" }',
|
||||
started - 1,
|
||||
time.time() + 1,
|
||||
limit=100,
|
||||
)
|
||||
dependency = []
|
||||
for trace in traces:
|
||||
spans = tempo_spans(b.trace(trace["traceID"])[0])
|
||||
failed = [
|
||||
s
|
||||
for s in spans
|
||||
if s["attributes"].get("db.system") == "postgresql"
|
||||
and s["status"] == "STATUS_CODE_ERROR"
|
||||
]
|
||||
if failed:
|
||||
dependency.append(
|
||||
{
|
||||
"trace_id": trace["traceID"],
|
||||
"db_errors": [
|
||||
{k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed
|
||||
],
|
||||
}
|
||||
)
|
||||
if not dependency or not any(r["status"] == 503 for r in responses):
|
||||
raise Failure("database incident has no dependency failure trace")
|
||||
api.logout(auth)
|
||||
return {
|
||||
"paused": paused,
|
||||
"restored": restored,
|
||||
"requests": responses,
|
||||
"signals": signals,
|
||||
"firing": firing,
|
||||
"resolved": resolved,
|
||||
"dependency_traces": dependency,
|
||||
}
|
||||
|
||||
|
||||
def collector_metrics():
|
||||
source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())'
|
||||
raw = compose("exec", "-T", "api", "python", "-c", source)
|
||||
return {
|
||||
line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1])
|
||||
for line in raw.splitlines()
|
||||
if line.startswith(
|
||||
(
|
||||
"otelcol_receiver_accepted_",
|
||||
"otelcol_exporter_sent_",
|
||||
"otelcol_exporter_send_failed_",
|
||||
"otelcol_exporter_queue_",
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def memory():
|
||||
ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines()
|
||||
raw = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
|
||||
)
|
||||
return [
|
||||
{k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")}
|
||||
for row in map(json.loads, raw.splitlines())
|
||||
]
|
||||
|
||||
|
||||
def storage(b):
|
||||
api, auth, _ = actor()
|
||||
before = collector_metrics()
|
||||
samples = [{"utc": now(), "containers": memory()}]
|
||||
compose("stop", "prometheus", "loki", "tempo")
|
||||
stopped = now()
|
||||
requests = []
|
||||
try:
|
||||
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True)
|
||||
until = time.monotonic() + 85
|
||||
while time.monotonic() < until:
|
||||
rid = "storage-outage-" + uuid.uuid4().hex
|
||||
status = http(
|
||||
"GET",
|
||||
api.base + "/api/v1/users/me",
|
||||
headers=dict(auth, **{"X-Request-Id": rid}),
|
||||
expect=(200,),
|
||||
)[0]
|
||||
requests.append({"id": rid, "status": status})
|
||||
if len(requests) % 15 == 0:
|
||||
samples.append({"utc": now(), "containers": memory()})
|
||||
time.sleep(1)
|
||||
during = collector_metrics()
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo")
|
||||
restored = now()
|
||||
time.sleep(20)
|
||||
after = collector_metrics()
|
||||
samples.append({"utc": now(), "containers": memory()})
|
||||
found = 0
|
||||
for item in requests:
|
||||
found += bool(
|
||||
b.traceql(
|
||||
'{ span.messenger.request_id = "' + item["id"] + '" }',
|
||||
time.time() - 300,
|
||||
time.time() + 1,
|
||||
)
|
||||
)
|
||||
peak_percent = max(
|
||||
float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"]
|
||||
)
|
||||
if peak_percent >= 100:
|
||||
raise Failure("container memory limit reached")
|
||||
api.logout(auth)
|
||||
return {
|
||||
"stopped": stopped,
|
||||
"restored": restored,
|
||||
"peak_memory_percent_of_limit": peak_percent,
|
||||
"smoke": result.stdout,
|
||||
"successful_probes": len(requests),
|
||||
"stored_probe_traces": found,
|
||||
"lost_probe_traces": len(requests) - found,
|
||||
"memory_samples": samples,
|
||||
"collector_before": before,
|
||||
"collector_during": during,
|
||||
"collector_after": after,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage"))
|
||||
args = parser.parse_args()
|
||||
b = backends()
|
||||
started = now()
|
||||
report = (
|
||||
chain(b, args.scenario == "retry")
|
||||
if args.scenario in ("chain", "retry")
|
||||
else {"worker": worker, "database": database, "storage": storage}[args.scenario](b)
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"started": started,
|
||||
"finished": now(),
|
||||
"result": "PASS",
|
||||
"scenario": args.scenario,
|
||||
"observations": report,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user