package main import ( "bytes" "encoding/json" "flag" "fmt" "io" "net/http" "os" "os/exec" "strings" "time" "github.com/google/uuid" ) type apiClient struct { baseURL string client *http.Client token string } func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } fmt.Println("PASS: PostgreSQL and Valkey outage recovery") } func run() error { baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") flag.Parse() client := &apiClient{baseURL: *baseURL, client: &http.Client{Timeout: 10 * time.Second}} if err := client.registerAndLogin(); err != nil { return err } if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil { return err } if err := verifyOutage(client, "valkey"); err != nil { return err } return verifyOutage(client, "postgres") } func (c *apiClient) registerAndLogin() error { username := "dep_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] password := "T3st-" + uuid.NewString() body := map[string]string{"username": username, "display_name": "Dependency check", "password": password} payload, err := json.Marshal(body) if err != nil { return err } request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/users", bytes.NewReader(payload)) if err != nil { return err } request.Header.Set("Content-Type", "application/json") response, err := c.client.Do(request) if err != nil { return fmt.Errorf("register: %w", err) } if err := closeResponse(response, http.StatusCreated); err != nil { return fmt.Errorf("register: %w", err) } request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil) if err != nil { return err } request.SetBasicAuth(username, password) response, err = c.client.Do(request) if err != nil { return fmt.Errorf("login: %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusCreated { return fmt.Errorf("login status = %d", response.StatusCode) } var session struct { Token string `json:"token"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { return fmt.Errorf("decode session: %w", err) } if session.Token == "" { return fmt.Errorf("login returned an empty token") } c.token = session.Token return nil } func verifyOutage(client *apiClient, service string) (returnErr error) { if err := compose("stop", service); err != nil { return err } defer func() { if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); returnErr == nil && err != nil { returnErr = err } }() if err := client.expectEventually("/health/ready", http.StatusServiceUnavailable, false); err != nil { return fmt.Errorf("%s readiness outage: %w", service, err) } if err := client.expectEventually("/api/v1/users/me", http.StatusServiceUnavailable, true); err != nil { return fmt.Errorf("%s business outage: %w", service, err) } if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); err != nil { return err } if err := client.expectEventually("/health/ready", http.StatusOK, false); err != nil { return fmt.Errorf("%s readiness recovery: %w", service, err) } if err := client.expectEventually("/api/v1/users/me", http.StatusOK, true); err != nil { return fmt.Errorf("%s session recovery: %w", service, err) } return nil } func compose(arguments ...string) error { command := exec.Command("docker", append([]string{"compose"}, arguments...)...) if output, err := command.CombinedOutput(); err != nil { return fmt.Errorf("docker compose %v: %w: %s", arguments, err, bytes.TrimSpace(output)) } return nil } func (c *apiClient) expectEventually(path string, status int, authenticated bool) error { deadline := time.Now().Add(20 * time.Second) var lastErr error for time.Now().Before(deadline) { lastErr = c.expect(http.MethodGet, path, status, authenticated) if lastErr == nil { return nil } time.Sleep(200 * time.Millisecond) } return lastErr } func (c *apiClient) expect(method, path string, status int, authenticated bool) error { request, err := http.NewRequest(method, c.baseURL+path, nil) if err != nil { return err } if authenticated { request.Header.Set("Authorization", "Bearer "+c.token) } response, err := c.client.Do(request) if err != nil { return err } return closeResponse(response, status) } func closeResponse(response *http.Response, status int) error { defer func() { _ = response.Body.Close() }() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) if response.StatusCode != status { return fmt.Errorf("status = %d, want %d", response.StatusCode, status) } return nil }