From cd9d91550aa4eeba31489e07b71ee2767acfc7d8 Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 21:12:24 +0300 Subject: [PATCH] Exercise error alerts with a bounded PostgreSQL lock incident --- scripts/obs_incidents.py | 46 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/scripts/obs_incidents.py b/scripts/obs_incidents.py index c8fede46..2b12455b 100644 --- a/scripts/obs_incidents.py +++ b/scripts/obs_incidents.py @@ -177,18 +177,56 @@ def database(b): for index in range(10): request(index) time.sleep(12) - compose("pause", "postgres") + psql = [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "sh", + "-c", + 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"', + "--", + "-c", + ] + lock_sql = ( + "SET application_name='lab6-db-lock'; BEGIN; " + "LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(120); ROLLBACK;" + ) + process = subprocess.Popen( + [*psql, lock_sql], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) paused = now() + responses = [] try: + time.sleep(1) with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: - responses = list(pool.map(request, range(10, 34))) + for batch in range(20): + responses.extend(pool.map(request, range(10 + batch * 12, 22 + batch * 12))) + alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"] + if any( + a["labels"].get("alertname") == "MessengerHighErrorRate" + and a["state"] == "firing" + for a in alerts + ): + break + time.sleep(1) firing = alert(b, "MessengerHighErrorRate", True) signals = { "error_ratio": value(b, "messenger:http_errors:ratio_rate1m"), - "pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"), + "pool_waiting": value(b, "max(max_over_time(messenger_db_pool_waiting[1m]))"), } finally: - compose("unpause", "postgres") + subprocess.run( + [ + *psql, + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + "WHERE application_name='lab6-db-lock'", + ], + check=True, + capture_output=True, + ) + process.wait(timeout=10) restored = now() wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90) for index in range(34, 74):