forked from mtusi2026/mtusi2026lab2
Exercise error alerts with a bounded PostgreSQL lock incident
This commit is contained in:
@@ -177,18 +177,56 @@ def database(b):
|
||||
for index in range(10):
|
||||
request(index)
|
||||
time.sleep(12)
|
||||
compose("pause", "postgres")
|
||||
psql = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"sh",
|
||||
"-c",
|
||||
'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"',
|
||||
"--",
|
||||
"-c",
|
||||
]
|
||||
lock_sql = (
|
||||
"SET application_name='lab6-db-lock'; BEGIN; "
|
||||
"LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(120); ROLLBACK;"
|
||||
)
|
||||
process = subprocess.Popen(
|
||||
[*psql, lock_sql], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
paused = now()
|
||||
responses = []
|
||||
try:
|
||||
time.sleep(1)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
responses = list(pool.map(request, range(10, 34)))
|
||||
for batch in range(20):
|
||||
responses.extend(pool.map(request, range(10 + batch * 12, 22 + batch * 12)))
|
||||
alerts = get_json(b.prometheus + "/api/v1/alerts")["data"]["alerts"]
|
||||
if any(
|
||||
a["labels"].get("alertname") == "MessengerHighErrorRate"
|
||||
and a["state"] == "firing"
|
||||
for a in alerts
|
||||
):
|
||||
break
|
||||
time.sleep(1)
|
||||
firing = alert(b, "MessengerHighErrorRate", True)
|
||||
signals = {
|
||||
"error_ratio": value(b, "messenger:http_errors:ratio_rate1m"),
|
||||
"pool_waiting": value(b, "max_over_time(messenger_db_pool_waiting[1m])"),
|
||||
"pool_waiting": value(b, "max(max_over_time(messenger_db_pool_waiting[1m]))"),
|
||||
}
|
||||
finally:
|
||||
compose("unpause", "postgres")
|
||||
subprocess.run(
|
||||
[
|
||||
*psql,
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE application_name='lab6-db-lock'",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
process.wait(timeout=10)
|
||||
restored = now()
|
||||
wait_for("database recovery", lambda: http("GET", api.base + "/health/ready")[0] == 200, 90)
|
||||
for index in range(34, 74):
|
||||
|
||||
Reference in New Issue
Block a user