forked from mtusi2026/mtusi2026lab2
388 lines
12 KiB
Python
388 lines
12 KiB
Python
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for
|
|
from obs_recovery import backends, compose
|
|
|
|
|
|
def actor():
|
|
api = Api(os.environ["BASE_URL"])
|
|
user, username, password, _ = api.register()
|
|
auth, _ = api.login(username, password)
|
|
return api, auth, api.chat(auth, user)
|
|
|
|
|
|
def upload(api, auth, chat):
|
|
return api.call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png",
|
|
Path("tests/fixtures/sample.png").read_bytes(),
|
|
dict(auth, **{"Content-Type": "image/png"}),
|
|
expect=(202,),
|
|
)
|
|
|
|
|
|
def ready(api, auth, attachment):
|
|
def check():
|
|
item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0]
|
|
if item["status"] == "failed":
|
|
raise RuntimeError("attachment failed")
|
|
return item if item["status"] == "ready" else None
|
|
|
|
return wait_for("ready", check, 120, 1)
|
|
|
|
|
|
def value(b, expression):
|
|
result = b.promql(expression)
|
|
return float(result[0]["value"][1]) if result else None
|
|
|
|
|
|
def alert(b, name, firing):
|
|
def check():
|
|
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
|
|
found = [
|
|
a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing"
|
|
]
|
|
if bool(found) == firing:
|
|
return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found}
|
|
return None
|
|
|
|
return wait_for(name, check, 150)
|
|
|
|
|
|
def chain(b, retry):
|
|
api, auth, chat = actor()
|
|
started = time.time()
|
|
if retry:
|
|
compose("stop", "worker")
|
|
attachment, rid = upload(api, auth, chat)
|
|
try:
|
|
if retry:
|
|
compose("stop", "s3")
|
|
compose("up", "-d", "--no-deps", "--wait", "worker")
|
|
query = (
|
|
'{service_name="messenger-worker"} | attachment_id="'
|
|
+ attachment["id"]
|
|
+ '" | outcome="retry_scheduled"'
|
|
)
|
|
wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60)
|
|
finally:
|
|
if retry:
|
|
compose("up", "-d", "--no-deps", "--wait", "s3", "worker")
|
|
ready(api, auth, attachment)
|
|
logs = wait_for(
|
|
"upload log",
|
|
lambda: b.logs(
|
|
'{service_name="messenger-api"} | request_id="' + rid + '"',
|
|
started - 1,
|
|
time.time() + 1,
|
|
),
|
|
60,
|
|
)
|
|
trace_id = logs[0][0]["trace_id"]
|
|
|
|
def complete():
|
|
spans = tempo_spans(b.trace(trace_id)[0])
|
|
return (
|
|
spans
|
|
if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans)
|
|
else None
|
|
)
|
|
|
|
spans = wait_for("complete trace", complete, 90)
|
|
roots = [
|
|
s
|
|
for s in spans
|
|
if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid
|
|
]
|
|
ids = {s["span_id"] for s in spans}
|
|
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
|
|
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
|
|
clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"]
|
|
if (
|
|
not roots
|
|
or not producers
|
|
or not consumers
|
|
or any(s["parent_span_id"] not in ids for s in producers + consumers)
|
|
):
|
|
raise Failure("broken distributed parent chain")
|
|
if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any(
|
|
s["attributes"].get("db.system") == "postgresql" for s in clients
|
|
):
|
|
raise Failure("missing dependency spans")
|
|
outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers]
|
|
if retry and "retry_scheduled" not in outcomes:
|
|
raise Failure("retry left original trace")
|
|
api.logout(auth)
|
|
fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms")
|
|
return {
|
|
"upload_request_id": rid,
|
|
"trace_id": trace_id,
|
|
"retry": retry,
|
|
"outcomes": outcomes,
|
|
"spans": [
|
|
{**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")}
|
|
for s in spans
|
|
],
|
|
}
|
|
|
|
|
|
def worker(b):
|
|
api, auth, chat = actor()
|
|
compose("stop", "worker")
|
|
stopped = now()
|
|
try:
|
|
items = [upload(api, auth, chat)[0] for _ in range(8)]
|
|
firing = alert(b, "MessengerQueueLag", True)
|
|
observed = {
|
|
"depth": value(b, "max(messenger_queue_depth)"),
|
|
"age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"),
|
|
}
|
|
finally:
|
|
compose("up", "-d", "--no-deps", "--wait", "worker")
|
|
restored = now()
|
|
for item in items:
|
|
ready(api, auth, item)
|
|
resolved = alert(b, "MessengerQueueLag", False)
|
|
api.logout(auth)
|
|
return {
|
|
"stopped": stopped,
|
|
"restored": restored,
|
|
"jobs": len(items),
|
|
"backlog": observed,
|
|
"firing": firing,
|
|
"resolved": resolved,
|
|
"depth_after": value(b, "max(messenger_queue_depth)"),
|
|
}
|
|
|
|
|
|
def database(b):
|
|
api, auth, _ = actor()
|
|
started = time.time()
|
|
prefix = "db-incident-" + uuid.uuid4().hex[:12]
|
|
|
|
def request(index):
|
|
rid = prefix + "-" + str(index)
|
|
status = http(
|
|
"GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid})
|
|
)[0]
|
|
return {"request_id": rid, "status": status}
|
|
|
|
for index in range(10):
|
|
request(index)
|
|
time.sleep(12)
|
|
psql = [
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"sh",
|
|
"-c",
|
|
'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"',
|
|
"--",
|
|
"-c",
|
|
]
|
|
lock_sql = (
|
|
"SET application_name='lab6-db-lock'; BEGIN; "
|
|
"LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(120); ROLLBACK;"
|
|
)
|
|
process = subprocess.Popen(
|
|
[*psql, lock_sql], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
|
)
|
|
paused = now()
|
|
responses = []
|
|
try:
|
|
time.sleep(1)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
|
for batch in range(20):
|
|
responses.extend(pool.map(request, range(10 + batch * 12, 22 + batch * 12)))
|
|
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
|
|
if any(
|
|
a["labels"].get("alertname") == "MessengerHighErrorRate"
|
|
and a["state"] == "firing"
|
|
for a in alerts
|
|
):
|
|
break
|
|
time.sleep(1)
|
|
firing = alert(b, "MessengerHighErrorRate", True)
|
|
signals = {
|
|
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
|
|
"pool_waiting": value(b, "max(max_over_time(messenger_db_pool_waiting[1m]))"),
|
|
}
|
|
finally:
|
|
subprocess.run(
|
|
[
|
|
*psql,
|
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
|
"WHERE application_name='lab6-db-lock'",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
process.wait(timeout=10)
|
|
restored = now()
|
|
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
|
|
for index in range(34, 74):
|
|
request(index)
|
|
time.sleep(0.15)
|
|
resolved = alert(b, "MessengerHighErrorRate", False)
|
|
traces = b.traceql(
|
|
'{ span.messenger.request_id =~ "' + prefix + '.*" }',
|
|
started - 1,
|
|
time.time() + 1,
|
|
limit=100,
|
|
)
|
|
dependency = []
|
|
for trace in traces:
|
|
spans = tempo_spans(b.trace(trace["traceID"])[0])
|
|
failed = [
|
|
s
|
|
for s in spans
|
|
if s["attributes"].get("db.system") == "postgresql"
|
|
and s["status"] == "STATUS_CODE_ERROR"
|
|
]
|
|
if failed:
|
|
dependency.append(
|
|
{
|
|
"trace_id": trace["traceID"],
|
|
"db_errors": [
|
|
{k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed
|
|
],
|
|
}
|
|
)
|
|
if not dependency or not any(r["status"] == 503 for r in responses):
|
|
raise Failure("database incident has no dependency failure trace")
|
|
api.logout(auth)
|
|
return {
|
|
"paused": paused,
|
|
"restored": restored,
|
|
"requests": responses,
|
|
"signals": signals,
|
|
"firing": firing,
|
|
"resolved": resolved,
|
|
"dependency_traces": dependency,
|
|
}
|
|
|
|
|
|
def collector_metrics():
|
|
source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())'
|
|
raw = compose("exec", "-T", "api", "python", "-c", source)
|
|
return {
|
|
line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1])
|
|
for line in raw.splitlines()
|
|
if line.startswith(
|
|
(
|
|
"otelcol_receiver_accepted_",
|
|
"otelcol_exporter_sent_",
|
|
"otelcol_exporter_send_failed_",
|
|
"otelcol_exporter_queue_",
|
|
)
|
|
)
|
|
}
|
|
|
|
|
|
def memory():
|
|
ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines()
|
|
raw = subprocess.check_output(
|
|
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
|
|
)
|
|
return [
|
|
{k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")}
|
|
for row in map(json.loads, raw.splitlines())
|
|
]
|
|
|
|
|
|
def storage(b):
|
|
api, auth, _ = actor()
|
|
before = collector_metrics()
|
|
samples = [{"utc": now(), "containers": memory()}]
|
|
compose("stop", "prometheus", "loki", "tempo")
|
|
stopped = now()
|
|
requests = []
|
|
try:
|
|
result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True)
|
|
until = time.monotonic() + 85
|
|
while time.monotonic() < until:
|
|
rid = "storage-outage-" + uuid.uuid4().hex
|
|
status = http(
|
|
"GET",
|
|
api.base + "/api/v1/users/me",
|
|
headers=dict(auth, **{"X-Request-Id": rid}),
|
|
expect=(200,),
|
|
)[0]
|
|
requests.append({"id": rid, "status": status})
|
|
if len(requests) % 15 == 0:
|
|
samples.append({"utc": now(), "containers": memory()})
|
|
time.sleep(1)
|
|
during = collector_metrics()
|
|
finally:
|
|
compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo")
|
|
restored = now()
|
|
time.sleep(20)
|
|
after = collector_metrics()
|
|
samples.append({"utc": now(), "containers": memory()})
|
|
found = 0
|
|
for item in requests:
|
|
found += bool(
|
|
b.traceql(
|
|
'{ span.messenger.request_id = "' + item["id"] + '" }',
|
|
time.time() - 300,
|
|
time.time() + 1,
|
|
)
|
|
)
|
|
peak_percent = max(
|
|
float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"]
|
|
)
|
|
if peak_percent >= 100:
|
|
raise Failure("container memory limit reached")
|
|
api.logout(auth)
|
|
return {
|
|
"stopped": stopped,
|
|
"restored": restored,
|
|
"peak_memory_percent_of_limit": peak_percent,
|
|
"smoke": result.stdout,
|
|
"successful_probes": len(requests),
|
|
"stored_probe_traces": found,
|
|
"lost_probe_traces": len(requests) - found,
|
|
"memory_samples": samples,
|
|
"collector_before": before,
|
|
"collector_during": during,
|
|
"collector_after": after,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage"))
|
|
args = parser.parse_args()
|
|
b = backends()
|
|
started = now()
|
|
report = (
|
|
chain(b, args.scenario == "retry")
|
|
if args.scenario in ("chain", "retry")
|
|
else {"worker": worker, "database": database, "storage": storage}[args.scenario](b)
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"started": started,
|
|
"finished": now(),
|
|
"result": "PASS",
|
|
"scenario": args.scenario,
|
|
"observations": report,
|
|
},
|
|
indent=2,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|