import argparse import concurrent.futures import json import os import subprocess import time import uuid from pathlib import Path from obs_check import Api, Failure, get_json, http, now, tempo_spans, wait_for from obs_recovery import backends, compose def actor(): api = Api(os.environ["BASE_URL"]) user, username, password, _ = api.register() auth, _ = api.login(username, password) return api, auth, api.chat(auth, user) def upload(api, auth, chat): return api.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=observation.png", Path("tests/fixtures/sample.png").read_bytes(), dict(auth, **{"Content-Type": "image/png"}), expect=(202,), ) def ready(api, auth, attachment): def check(): item = api.call("GET", "/api/v1/attachments/" + attachment["id"], headers=auth)[0] if item["status"] == "failed": raise RuntimeError("attachment failed") return item if item["status"] == "ready" else None return wait_for("ready", check, 120, 1) def value(b, expression): result = b.promql(expression) return float(result[0]["value"][1]) if result else None def alert(b, name, firing): def check(): alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"] found = [ a for a in alerts if a["labels"].get("alertname") == name and a["state"] == "firing" ] if bool(found) == firing: return {"utc": now(), "state": "firing" if firing else "resolved", "alerts": found} return None return wait_for(name, check, 150) def chain(b, retry): api, auth, chat = actor() started = time.time() if retry: compose("stop", "worker") attachment, rid = upload(api, auth, chat) try: if retry: compose("stop", "s3") compose("up", "-d", "--no-deps", "--wait", "worker") query = ( '{service_name="messenger-worker"} | attachment_id="' + attachment["id"] + '" | outcome="retry_scheduled"' ) wait_for("real S3 retry", lambda: b.logs(query, started, time.time() + 1), 60) finally: if retry: compose("up", "-d", "--no-deps", "--wait", "s3", "worker") ready(api, auth, attachment) logs = wait_for( "upload log", lambda: b.logs( '{service_name="messenger-api"} | request_id="' + rid + '"', started - 1, time.time() + 1, ), 60, ) trace_id = logs[0][0]["trace_id"] def complete(): spans = tempo_spans(b.trace(trace_id)[0]) return ( spans if any(s["attributes"].get("messenger.job.outcome") == "done" for s in spans) else None ) spans = wait_for("complete trace", complete, 90) roots = [ s for s in spans if s["kind"] == "SPAN_KIND_SERVER" and s["attributes"].get("messenger.request_id") == rid ] ids = {s["span_id"] for s in spans} producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"] consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"] clients = [s for s in spans if s["kind"] == "SPAN_KIND_CLIENT"] if ( not roots or not producers or not consumers or any(s["parent_span_id"] not in ids for s in producers + consumers) ): raise Failure("broken distributed parent chain") if not any(s["attributes"].get("rpc.service") == "S3" for s in clients) or not any( s["attributes"].get("db.system") == "postgresql" for s in clients ): raise Failure("missing dependency spans") outcomes = [s["attributes"].get("messenger.job.outcome") for s in consumers] if retry and "retry_scheduled" not in outcomes: raise Failure("retry left original trace") api.logout(auth) fields = ("service", "name", "kind", "status", "span_id", "parent_span_id", "duration_ms") return { "upload_request_id": rid, "trace_id": trace_id, "retry": retry, "outcomes": outcomes, "spans": [ {**{k: s[k] for k in fields}, "attempt": s["attributes"].get("messenger.job.attempt")} for s in spans ], } def worker(b): api, auth, chat = actor() compose("stop", "worker") stopped = now() try: items = [upload(api, auth, chat)[0] for _ in range(8)] firing = alert(b, "MessengerQueueLag", True) observed = { "depth": value(b, "max(messenger_queue_depth)"), "age_seconds": value(b, "max(messenger_queue_oldest_age_seconds)"), } finally: compose("up", "-d", "--no-deps", "--wait", "worker") restored = now() for item in items: ready(api, auth, item) resolved = alert(b, "MessengerQueueLag", False) api.logout(auth) return { "stopped": stopped, "restored": restored, "jobs": len(items), "backlog": observed, "firing": firing, "resolved": resolved, "depth_after": value(b, "max(messenger_queue_depth)"), } def database(b): api, auth, _ = actor() started = time.time() prefix = "db-incident-" + uuid.uuid4().hex[:12] def request(index): rid = prefix + "-" + str(index) status = http( "GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid}) )[0] return {"request_id": rid, "status": status} for index in range(10): request(index) time.sleep(12) psql = [ "docker", "compose", "exec", "-T", "postgres", "sh", "-c", 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"', "--", "-c", ] lock_sql = ( "SET application_name='lab6-db-lock'; BEGIN; " "LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(120); ROLLBACK;" ) process = subprocess.Popen( [*psql, lock_sql], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) paused = now() responses = [] try: time.sleep(1) with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: for batch in range(20): responses.extend(pool.map(request, range(10 + batch * 12, 22 + batch * 12))) alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"] if any( a["labels"].get("alertname") == "MessengerHighErrorRate" and a["state"] == "firing" for a in alerts ): break time.sleep(1) firing = alert(b, "MessengerHighErrorRate", True) signals = { "error_ratio": value(b, "messenger:http_errors:ratio_rate1m"), "pool_waiting": value(b, "max(max_over_time(messenger_db_pool_waiting[1m]))"), } finally: subprocess.run( [ *psql, "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " "WHERE application_name='lab6-db-lock'", ], check=True, capture_output=True, ) process.wait(timeout=10) restored = now() wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90) for index in range(34, 74): request(index) time.sleep(0.15) resolved = alert(b, "MessengerHighErrorRate", False) traces = b.traceql( '{ span.messenger.request_id =~ "' + prefix + '.*" }', started - 1, time.time() + 1, limit=100, ) dependency = [] for trace in traces: spans = tempo_spans(b.trace(trace["traceID"])[0]) failed = [ s for s in spans if s["attributes"].get("db.system") == "postgresql" and s["status"] == "STATUS_CODE_ERROR" ] if failed: dependency.append( { "trace_id": trace["traceID"], "db_errors": [ {k: s[k] for k in ("name", "kind", "status", "duration_ms")} for s in failed ], } ) if not dependency or not any(r["status"] == 503 for r in responses): raise Failure("database incident has no dependency failure trace") api.logout(auth) return { "paused": paused, "restored": restored, "requests": responses, "signals": signals, "firing": firing, "resolved": resolved, "dependency_traces": dependency, } def collector_metrics(): source = 'import urllib.request; print(urllib.request.urlopen("http://otel-collector:8888/metrics").read().decode())' raw = compose("exec", "-T", "api", "python", "-c", source) return { line.rsplit(" ", 1)[0]: float(line.rsplit(" ", 1)[1]) for line in raw.splitlines() if line.startswith( ( "otelcol_receiver_accepted_", "otelcol_exporter_sent_", "otelcol_exporter_send_failed_", "otelcol_exporter_queue_", ) ) } def memory(): ids = compose("ps", "-q", "api", "relay", "worker", "otel-collector").splitlines() raw = subprocess.check_output( ["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True ) return [ {k: row[k] for k in ("Name", "MemUsage", "MemPerc", "CPUPerc")} for row in map(json.loads, raw.splitlines()) ] def storage(b): api, auth, _ = actor() before = collector_metrics() samples = [{"utc": now(), "containers": memory()}] compose("stop", "prometheus", "loki", "tempo") stopped = now() requests = [] try: result = subprocess.run(["make", "test"], capture_output=True, text=True, check=True) until = time.monotonic() + 85 while time.monotonic() < until: rid = "storage-outage-" + uuid.uuid4().hex status = http( "GET", api.base + "/api/v1/users/me", headers=dict(auth, **{"X-Request-Id": rid}), expect=(200,), )[0] requests.append({"id": rid, "status": status}) if len(requests) % 15 == 0: samples.append({"utc": now(), "containers": memory()}) time.sleep(1) during = collector_metrics() finally: compose("up", "-d", "--no-deps", "--wait", "prometheus", "loki", "tempo") restored = now() time.sleep(20) after = collector_metrics() samples.append({"utc": now(), "containers": memory()}) found = 0 for item in requests: found += bool( b.traceql( '{ span.messenger.request_id = "' + item["id"] + '" }', time.time() - 300, time.time() + 1, ) ) peak_percent = max( float(row["MemPerc"].rstrip("%")) for sample in samples for row in sample["containers"] ) if peak_percent >= 100: raise Failure("container memory limit reached") api.logout(auth) return { "stopped": stopped, "restored": restored, "peak_memory_percent_of_limit": peak_percent, "smoke": result.stdout, "successful_probes": len(requests), "stored_probe_traces": found, "lost_probe_traces": len(requests) - found, "memory_samples": samples, "collector_before": before, "collector_during": during, "collector_after": after, } def main(): parser = argparse.ArgumentParser() parser.add_argument("scenario", choices=("chain", "retry", "worker", "database", "storage")) args = parser.parse_args() b = backends() started = now() report = ( chain(b, args.scenario == "retry") if args.scenario in ("chain", "retry") else {"worker": worker, "database": database, "storage": storage}[args.scenario](b) ) print( json.dumps( { "started": started, "finished": now(), "result": "PASS", "scenario": args.scenario, "observations": report, }, indent=2, ) ) if __name__ == "__main__": main()