177 lines
5.4 KiB
Python
177 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client
|
|
|
|
|
|
BASE_URL = "http://127.0.0.1:18082"
|
|
MAX_RESPONSE_SECONDS = 6.0
|
|
WATCHDOG_SECONDS = 8.0
|
|
|
|
COMPOSE = [
|
|
"docker", "compose",
|
|
"-f", "compose.yaml",
|
|
"-f", "compose.s3.yaml",
|
|
"--env-file", ".env",
|
|
]
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(
|
|
["docker", *args],
|
|
cwd=ROOT,
|
|
text=True,
|
|
timeout=20,
|
|
stderr=subprocess.PIPE,
|
|
).strip()
|
|
|
|
|
|
def main():
|
|
pg_id = subprocess.check_output(
|
|
[*COMPOSE, "ps", "-q", "postgres"],
|
|
cwd=ROOT,
|
|
text=True,
|
|
timeout=20,
|
|
).strip()
|
|
assert pg_id, "PostgreSQL container not found"
|
|
assert docker(
|
|
"inspect", "-f", "{{.State.Paused}}", pg_id
|
|
) == "false", "PostgreSQL is already paused"
|
|
|
|
c = Client(BASE_URL, None)
|
|
alice, _, _, chat = c.bootstrap()
|
|
message = c.message(alice, chat, "postgres-pause regression")["json"]
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
|
|
|
baseline_instances = set()
|
|
for _ in range(6):
|
|
response = c.call(
|
|
"GET", path, actor=alice, schema="MessagePage"
|
|
)
|
|
assert any(
|
|
item["id"] == message["id"]
|
|
for item in response["json"]["items"]
|
|
), "Baseline message is missing"
|
|
instance = response["headers"].get("X-Instance-Id")
|
|
assert instance, "Baseline response has no X-Instance-Id"
|
|
baseline_instances.add(instance)
|
|
|
|
assert len(baseline_instances) >= 2, (
|
|
"Baseline did not reach both API replicas"
|
|
)
|
|
print("Baseline: history is available through both replicas", flush=True)
|
|
|
|
def probe():
|
|
started = time.monotonic()
|
|
try:
|
|
client = Client(BASE_URL, None)
|
|
response = client.call("GET", path, 503, actor=alice)
|
|
elapsed = time.monotonic() - started
|
|
instance = response["headers"].get("X-Instance-Id")
|
|
code = response["json"]["error"]["code"]
|
|
return {
|
|
"ok": bool(instance) and code == "unavailable",
|
|
"status": 503,
|
|
"code": code,
|
|
"instance": instance,
|
|
"elapsed_s": round(elapsed, 4),
|
|
}
|
|
except Exception as exc:
|
|
return {
|
|
"ok": False,
|
|
"error_type": type(exc).__name__,
|
|
"elapsed_s": round(time.monotonic() - started, 4),
|
|
}
|
|
|
|
executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
|
|
paused_by_test = False
|
|
try:
|
|
paused_by_test = True
|
|
docker("pause", pg_id)
|
|
assert docker(
|
|
"inspect", "-f", "{{.State.Paused}}", pg_id
|
|
) == "true"
|
|
print("PostgreSQL paused; requesting history", flush=True)
|
|
|
|
futures = [executor.submit(probe) for _ in range(4)]
|
|
_, pending = concurrent.futures.wait(
|
|
futures, timeout=WATCHDOG_SECONDS
|
|
)
|
|
assert not pending, "History requests exceeded the test watchdog"
|
|
|
|
results = [future.result() for future in futures]
|
|
print(json.dumps(results, indent=2), flush=True)
|
|
|
|
assert all(item["ok"] for item in results), (
|
|
"Expected API JSON 503 unavailable with X-Instance-Id"
|
|
)
|
|
assert all(
|
|
item["elapsed_s"] < MAX_RESPONSE_SECONDS
|
|
for item in results
|
|
), "API response exceeded the declared deadline"
|
|
|
|
failed_instances = {item["instance"] for item in results}
|
|
assert baseline_instances.issubset(failed_instances), (
|
|
"Paused-DB requests did not exercise both baseline replicas"
|
|
)
|
|
finally:
|
|
try:
|
|
if paused_by_test:
|
|
state = docker(
|
|
"inspect", "-f", "{{.State.Paused}}", pg_id
|
|
)
|
|
if state == "true":
|
|
docker("unpause", pg_id)
|
|
print("PostgreSQL unpaused", flush=True)
|
|
finally:
|
|
executor.shutdown(wait=False, cancel_futures=True)
|
|
|
|
recovery_started = time.monotonic()
|
|
recovery_deadline = recovery_started + 60
|
|
while True:
|
|
try:
|
|
response = c.call(
|
|
"GET", path, actor=alice, schema="MessagePage"
|
|
)
|
|
assert any(
|
|
item["id"] == message["id"]
|
|
for item in response["json"]["items"]
|
|
), "Confirmed message disappeared"
|
|
break
|
|
except Exception:
|
|
if time.monotonic() >= recovery_deadline:
|
|
raise AssertionError("History did not recover") from None
|
|
time.sleep(1)
|
|
|
|
report = {
|
|
"scenario": "postgres_pause_history",
|
|
"passed": True,
|
|
"max_response_seconds": MAX_RESPONSE_SECONDS,
|
|
"baseline_instances": sorted(baseline_instances),
|
|
"paused_requests": results,
|
|
"recovery_observed_after_s": round(
|
|
time.monotonic() - recovery_started, 4
|
|
),
|
|
"confirmed_message_preserved": True,
|
|
}
|
|
target = ROOT / "evidence" / "postgres_pause.json"
|
|
target.parent.mkdir(exist_ok=True)
|
|
target.write_text(json.dumps(report, indent=2) + "\n")
|
|
|
|
print("PASS: bounded API 503 on both replicas; history recovered", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|