#!/usr/bin/env python3 from __future__ import annotations import concurrent.futures import json from pathlib import Path import subprocess import sys import time ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "tests")) from client import Client BASE_URL = "http://127.0.0.1:18082" MAX_RESPONSE_SECONDS = 6.0 WATCHDOG_SECONDS = 8.0 COMPOSE = [ "docker", "compose", "-f", "compose.yaml", "-f", "compose.s3.yaml", "--env-file", ".env", ] def docker(*args): return subprocess.check_output( ["docker", *args], cwd=ROOT, text=True, timeout=20, stderr=subprocess.PIPE, ).strip() def main(): pg_id = subprocess.check_output( [*COMPOSE, "ps", "-q", "postgres"], cwd=ROOT, text=True, timeout=20, ).strip() assert pg_id, "PostgreSQL container not found" assert docker( "inspect", "-f", "{{.State.Paused}}", pg_id ) == "false", "PostgreSQL is already paused" c = Client(BASE_URL, None) alice, _, _, chat = c.bootstrap() message = c.message(alice, chat, "postgres-pause regression")["json"] path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" baseline_instances = set() for _ in range(6): response = c.call( "GET", path, actor=alice, schema="MessagePage" ) assert any( item["id"] == message["id"] for item in response["json"]["items"] ), "Baseline message is missing" instance = response["headers"].get("X-Instance-Id") assert instance, "Baseline response has no X-Instance-Id" baseline_instances.add(instance) assert len(baseline_instances) >= 2, ( "Baseline did not reach both API replicas" ) print("Baseline: history is available through both replicas", flush=True) def probe(): started = time.monotonic() try: client = Client(BASE_URL, None) response = client.call("GET", path, 503, actor=alice) elapsed = time.monotonic() - started instance = response["headers"].get("X-Instance-Id") code = response["json"]["error"]["code"] return { "ok": bool(instance) and code == "unavailable", "status": 503, "code": code, "instance": instance, "elapsed_s": round(elapsed, 4), } except Exception as exc: return { "ok": False, "error_type": type(exc).__name__, "elapsed_s": round(time.monotonic() - started, 4), } executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) paused_by_test = False try: paused_by_test = True docker("pause", pg_id) assert docker( "inspect", "-f", "{{.State.Paused}}", pg_id ) == "true" print("PostgreSQL paused; requesting history", flush=True) futures = [executor.submit(probe) for _ in range(4)] _, pending = concurrent.futures.wait( futures, timeout=WATCHDOG_SECONDS ) assert not pending, "History requests exceeded the test watchdog" results = [future.result() for future in futures] print(json.dumps(results, indent=2), flush=True) assert all(item["ok"] for item in results), ( "Expected API JSON 503 unavailable with X-Instance-Id" ) assert all( item["elapsed_s"] < MAX_RESPONSE_SECONDS for item in results ), "API response exceeded the declared deadline" failed_instances = {item["instance"] for item in results} assert baseline_instances.issubset(failed_instances), ( "Paused-DB requests did not exercise both baseline replicas" ) finally: try: if paused_by_test: state = docker( "inspect", "-f", "{{.State.Paused}}", pg_id ) if state == "true": docker("unpause", pg_id) print("PostgreSQL unpaused", flush=True) finally: executor.shutdown(wait=False, cancel_futures=True) recovery_started = time.monotonic() recovery_deadline = recovery_started + 60 while True: try: response = c.call( "GET", path, actor=alice, schema="MessagePage" ) assert any( item["id"] == message["id"] for item in response["json"]["items"] ), "Confirmed message disappeared" break except Exception: if time.monotonic() >= recovery_deadline: raise AssertionError("History did not recover") from None time.sleep(1) report = { "scenario": "postgres_pause_history", "passed": True, "max_response_seconds": MAX_RESPONSE_SECONDS, "baseline_instances": sorted(baseline_instances), "paused_requests": results, "recovery_observed_after_s": round( time.monotonic() - recovery_started, 4 ), "confirmed_message_preserved": True, } target = ROOT / "evidence" / "postgres_pause.json" target.parent.mkdir(exist_ok=True) target.write_text(json.dumps(report, indent=2) + "\n") print("PASS: bounded API 503 on both replicas; history recovered", flush=True) if __name__ == "__main__": main()