Files
mtusi2026lab4/student-tests/postgres_pause.py
T

177 lines
5.4 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import concurrent.futures
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
BASE_URL = "http://127.0.0.1:18082"
MAX_RESPONSE_SECONDS = 6.0
WATCHDOG_SECONDS = 8.0
COMPOSE = [
"docker", "compose",
"-f", "compose.yaml",
"-f", "compose.s3.yaml",
"--env-file", ".env",
]
def docker(*args):
return subprocess.check_output(
["docker", *args],
cwd=ROOT,
text=True,
timeout=20,
stderr=subprocess.PIPE,
).strip()
def main():
pg_id = subprocess.check_output(
[*COMPOSE, "ps", "-q", "postgres"],
cwd=ROOT,
text=True,
timeout=20,
).strip()
assert pg_id, "PostgreSQL container not found"
assert docker(
"inspect", "-f", "{{.State.Paused}}", pg_id
) == "false", "PostgreSQL is already paused"
c = Client(BASE_URL, None)
alice, _, _, chat = c.bootstrap()
message = c.message(alice, chat, "postgres-pause regression")["json"]
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
baseline_instances = set()
for _ in range(6):
response = c.call(
"GET", path, actor=alice, schema="MessagePage"
)
assert any(
item["id"] == message["id"]
for item in response["json"]["items"]
), "Baseline message is missing"
instance = response["headers"].get("X-Instance-Id")
assert instance, "Baseline response has no X-Instance-Id"
baseline_instances.add(instance)
assert len(baseline_instances) >= 2, (
"Baseline did not reach both API replicas"
)
print("Baseline: history is available through both replicas", flush=True)
def probe():
started = time.monotonic()
try:
client = Client(BASE_URL, None)
response = client.call("GET", path, 503, actor=alice)
elapsed = time.monotonic() - started
instance = response["headers"].get("X-Instance-Id")
code = response["json"]["error"]["code"]
return {
"ok": bool(instance) and code == "unavailable",
"status": 503,
"code": code,
"instance": instance,
"elapsed_s": round(elapsed, 4),
}
except Exception as exc:
return {
"ok": False,
"error_type": type(exc).__name__,
"elapsed_s": round(time.monotonic() - started, 4),
}
executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
paused_by_test = False
try:
paused_by_test = True
docker("pause", pg_id)
assert docker(
"inspect", "-f", "{{.State.Paused}}", pg_id
) == "true"
print("PostgreSQL paused; requesting history", flush=True)
futures = [executor.submit(probe) for _ in range(4)]
_, pending = concurrent.futures.wait(
futures, timeout=WATCHDOG_SECONDS
)
assert not pending, "History requests exceeded the test watchdog"
results = [future.result() for future in futures]
print(json.dumps(results, indent=2), flush=True)
assert all(item["ok"] for item in results), (
"Expected API JSON 503 unavailable with X-Instance-Id"
)
assert all(
item["elapsed_s"] < MAX_RESPONSE_SECONDS
for item in results
), "API response exceeded the declared deadline"
failed_instances = {item["instance"] for item in results}
assert baseline_instances.issubset(failed_instances), (
"Paused-DB requests did not exercise both baseline replicas"
)
finally:
try:
if paused_by_test:
state = docker(
"inspect", "-f", "{{.State.Paused}}", pg_id
)
if state == "true":
docker("unpause", pg_id)
print("PostgreSQL unpaused", flush=True)
finally:
executor.shutdown(wait=False, cancel_futures=True)
recovery_started = time.monotonic()
recovery_deadline = recovery_started + 60
while True:
try:
response = c.call(
"GET", path, actor=alice, schema="MessagePage"
)
assert any(
item["id"] == message["id"]
for item in response["json"]["items"]
), "Confirmed message disappeared"
break
except Exception:
if time.monotonic() >= recovery_deadline:
raise AssertionError("History did not recover") from None
time.sleep(1)
report = {
"scenario": "postgres_pause_history",
"passed": True,
"max_response_seconds": MAX_RESPONSE_SECONDS,
"baseline_instances": sorted(baseline_instances),
"paused_requests": results,
"recovery_observed_after_s": round(
time.monotonic() - recovery_started, 4
),
"confirmed_message_preserved": True,
}
target = ROOT / "evidence" / "postgres_pause.json"
target.parent.mkdir(exist_ok=True)
target.write_text(json.dumps(report, indent=2) + "\n")
print("PASS: bounded API 503 on both replicas; history recovered", flush=True)
if __name__ == "__main__":
main()