Files

388 lines
12 KiB
Python

import argparse
import concurrent.futures
import json
import os
import subprocess
import time
import uuid
from pathlib import Path
from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for
from obs_recovery import backends, compose
def actor():
api = Api(os.environ["BASE_URL"])
user, username, password, _ = api.register()
auth, _ = api.login(username, password)
return api, auth, api.chat(auth, user)
def upload(api, auth, chat):
return api.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png",
Path("tests/fixtures/sample.png").read_bytes(),
dict(auth, **{"Content-Type": "image/png"}),
expect=(202,),
)
def ready(api, auth, attachment):
def check():
item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0]
if item["status"] == "failed":
raise RuntimeError("attachment failed")
return item if item["status"] == "ready" else None
return wait_for("ready", check, 120, 1)
def value(b, expression):
result = b.promql(expression)
return float(result[0]["value"][1]) if result else None
def alert(b, name, firing):
def check():
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
found = [
a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing"
]
if bool(found) == firing:
return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found}
return None
return wait_for(name, check, 150)
def chain(b, retry):
api, auth, chat = actor()
started = time.time()
if retry:
compose("stop", "worker")
attachment, rid = upload(api, auth, chat)
try:
if retry:
compose("stop", "s3")
compose("up", "-d", "--no-deps", "--wait", "worker")
query = (
'{service_name="messenger-worker"} | attachment_id="'
+ attachment["id"]
+ '" | outcome="retry_scheduled"'
)
wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60)
finally:
if retry:
compose("up", "-d", "--no-deps", "--wait", "s3", "worker")
ready(api, auth, attachment)
logs = wait_for(
"upload log",
lambda: b.logs(
'{service_name="messenger-api"} | request_id="' + rid + '"',
started - 1,
time.time() + 1,
),
60,
)
trace_id = logs[0][0]["trace_id"]
def complete():
spans = tempo_spans(b.trace(trace_id)[0])
return (
spans
if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans)
else None
)
spans = wait_for("complete trace", complete, 90)
roots = [
s
for s in spans
if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid
]
ids = {s["span_id"] for s in spans}
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"]
if (
not roots
or not producers
or not consumers
or any(s["parent_span_id"] not in ids for s in producers + consumers)
):
raise Failure("broken distributed parent chain")
if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any(
s["attributes"].get("db.system") == "postgresql" for s in clients
):
raise Failure("missing dependency spans")
outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers]
if retry and "retry_scheduled" not in outcomes:
raise Failure("retry left original trace")
api.logout(auth)
fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms")
return {
"upload_request_id": rid,
"trace_id": trace_id,
"retry": retry,
"outcomes": outcomes,
"spans": [
{**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")}
for s in spans
],
}
def worker(b):
api, auth, chat = actor()
compose("stop", "worker")
stopped = now()
try:
items = [upload(api, auth, chat)[0] for _ in range(8)]
firing = alert(b, "MessengerQueueLag", True)
observed = {
"depth": value(b, "max(messenger_queue_depth)"),
"age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"),
}
finally:
compose("up", "-d", "--no-deps", "--wait", "worker")
restored = now()
for item in items:
ready(api, auth, item)
resolved = alert(b, "MessengerQueueLag", False)
api.logout(auth)
return {
"stopped": stopped,
"restored": restored,
"jobs": len(items),
"backlog": observed,
"firing": firing,
"resolved": resolved,
"depth_after": value(b, "max(messenger_queue_depth)"),
}
def database(b):
api, auth, _ = actor()
started = time.time()
prefix = "db-incident-" + uuid.uuid4().hex[:12]
def request(index):
rid = prefix + "-" + str(index)
status = http(
"GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid})
)[0]
return {"request_id": rid, "status": status}
for index in range(10):
request(index)
time.sleep(12)
psql = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"sh",
"-c",
'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"',
"--",
"-c",
]
lock_sql = (
"SET application_name='lab6-db-lock'; BEGIN; "
"LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(120); ROLLBACK;"
)
process = subprocess.Popen(
[*psql, lock_sql], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
paused = now()
responses = []
try:
time.sleep(1)
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
for batch in range(20):
responses.extend(pool.map(request, range(10 + batch * 12, 22 + batch * 12)))
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
if any(
a["labels"].get("alertname") == "MessengerHighErrorRate"
and a["state"] == "firing"
for a in alerts
):
break
time.sleep(1)
firing = alert(b, "MessengerHighErrorRate", True)
signals = {
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
"pool_waiting": value(b, "max(max_over_time(messenger_db_pool_waiting[1m]))"),
}
finally:
subprocess.run(
[
*psql,
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE application_name='lab6-db-lock'",
],
check=True,
capture_output=True,
)
process.wait(timeout=10)
restored = now()
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
for index in range(34, 74):
request(index)
time.sleep(0.15)
resolved = alert(b, "MessengerHighErrorRate", False)
traces = b.traceql(
'{ span.messenger.request_id =~ "' + prefix + '.*" }',
started - 1,
time.time() + 1,
limit=100,
)
dependency = []
for trace in traces:
spans = tempo_spans(b.trace(trace["traceID"])[0])
failed = [
s
for s in spans
if s["attributes"].get("db.system") == "postgresql"
and s["status"] == "STATUS_CODE_ERROR"
]
if failed:
dependency.append(
{
"trace_id": trace["traceID"],
"db_errors": [
{k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed
],
}
)
if not dependency or not any(r["status"] == 503 for r in responses):
raise Failure("database incident has no dependency failure trace")
api.logout(auth)
return {
"paused": paused,
"restored": restored,
"requests": responses,
"signals": signals,
"firing": firing,
"resolved": resolved,
"dependency_traces": dependency,
}
def collector_metrics():
source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())'
raw = compose("exec", "-T", "api", "python", "-c", source)
return {
line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1])
for line in raw.splitlines()
if line.startswith(
(
"otelcol_receiver_accepted_",
"otelcol_exporter_sent_",
"otelcol_exporter_send_failed_",
"otelcol_exporter_queue_",
)
)
}
def memory():
ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines()
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
)
return [
{k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")}
for row in map(json.loads, raw.splitlines())
]
def storage(b):
api, auth, _ = actor()
before = collector_metrics()
samples = [{"utc": now(), "containers": memory()}]
compose("stop", "prometheus", "loki", "tempo")
stopped = now()
requests = []
try:
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True)
until = time.monotonic() + 85
while time.monotonic() < until:
rid = "storage-outage-" + uuid.uuid4().hex
status = http(
"GET",
api.base + "/api/v1/users/me",
headers=dict(auth, **{"X-Request-Id": rid}),
expect=(200,),
)[0]
requests.append({"id": rid, "status": status})
if len(requests) % 15 == 0:
samples.append({"utc": now(), "containers": memory()})
time.sleep(1)
during = collector_metrics()
finally:
compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo")
restored = now()
time.sleep(20)
after = collector_metrics()
samples.append({"utc": now(), "containers": memory()})
found = 0
for item in requests:
found += bool(
b.traceql(
'{ span.messenger.request_id = "' + item["id"] + '" }',
time.time() - 300,
time.time() + 1,
)
)
peak_percent = max(
float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"]
)
if peak_percent >= 100:
raise Failure("container memory limit reached")
api.logout(auth)
return {
"stopped": stopped,
"restored": restored,
"peak_memory_percent_of_limit": peak_percent,
"smoke": result.stdout,
"successful_probes": len(requests),
"stored_probe_traces": found,
"lost_probe_traces": len(requests) - found,
"memory_samples": samples,
"collector_before": before,
"collector_during": during,
"collector_after": after,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage"))
args = parser.parse_args()
b = backends()
started = now()
report = (
chain(b, args.scenario == "retry")
if args.scenario in ("chain", "retry")
else {"worker": worker, "database": database, "storage": storage}[args.scenario](b)
)
print(
json.dumps(
{
"started": started,
"finished": now(),
"result": "PASS",
"scenario": args.scenario,
"observations": report,
},
indent=2,
)
)
if __name__ == "__main__":
main()