Exercise error alerts with a bounded PostgreSQL lock incident

This commit is contained in:
Artyom
2026-11-06 21:12:24 +03:00
parent 447a6507f9
commit cd9d91550a
+42 -4
View File
@@ -177,18 +177,56 @@ def database(b):
for index in range(10):
request(index)
time.sleep(12)
compose("pause", "postgres")
psql = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"sh",
"-c",
'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"',
"--",
"-c",
]
lock_sql = (
"SET application_name='lab6-db-lock'; BEGIN; "
"LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(120); ROLLBACK;"
)
process = subprocess.Popen(
[*psql, lock_sql], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
paused = now()
responses = []
try:
time.sleep(1)
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
responses = list(pool.map(request, range(10, 34)))
for batch in range(20):
responses.extend(pool.map(request, range(10 + batch * 12, 22 + batch * 12)))
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
if any(
a["labels"].get("alertname") == "MessengerHighErrorRate"
and a["state"] == "firing"
for a in alerts
):
break
time.sleep(1)
firing = alert(b, "MessengerHighErrorRate", True)
signals = {
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
"pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"),
"pool_waiting": value(b, "max(max_over_time(messenger_db_pool_waiting[1m]))"),
}
finally:
compose("unpause", "postgres")
subprocess.run(
[
*psql,
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE application_name='lab6-db-lock'",
],
check=True,
capture_output=True,
)
process.wait(timeout=10)
restored = now()
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
for index in range(34, 74):