package main import ( "bytes" "crypto/tls" "crypto/x509" "encoding/json" "flag" "fmt" "io" "net/http" "os" "os/exec" "strings" "sync" "time" "github.com/google/uuid" ) type apiClient struct { baseURL string client *http.Client transport *http.Transport token string userID string } func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } fmt.Println("PASS: replica idempotency and dependency recovery") } func run() error { baseURL := flag.String("base-url", "https://localhost:8443", "API base URL") caFile := flag.String("ca-file", ".runtime/security/public-ca/ca.crt", "trusted CA certificate") flag.Parse() caPEM, err := os.ReadFile(*caFile) if err != nil { return fmt.Errorf("read trusted CA: %w", err) } roots := x509.NewCertPool() if !roots.AppendCertsFromPEM(caPEM) { return fmt.Errorf("trusted CA does not contain a certificate") } transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}} defer transport.CloseIdleConnections() client := &apiClient{ baseURL: *baseURL, client: &http.Client{Transport: transport, Timeout: 10 * time.Second}, transport: transport, } if err := client.registerAndLogin(); err != nil { return err } if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil { return err } if err := client.verifyConcurrentIdempotency(); err != nil { return err } if err := verifyOutage(client, "valkey"); err != nil { return err } return verifyOutage(client, "postgres") } func (c *apiClient) registerAndLogin() error { username := "dep_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] password := "T3st-" + uuid.NewString() body := map[string]string{"username": username, "display_name": "Dependency check", "password": password} payload, err := json.Marshal(body) if err != nil { return err } request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/users", bytes.NewReader(payload)) if err != nil { return err } request.Header.Set("Content-Type", "application/json") response, err := c.client.Do(request) if err != nil { return fmt.Errorf("register: %w", err) } registrationBody := response.Body defer func() { _ = registrationBody.Close() }() if response.StatusCode != http.StatusCreated { return fmt.Errorf("register status = %d", response.StatusCode) } var user struct { ID string `json:"id"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil { return fmt.Errorf("decode user: %w", err) } if user.ID == "" { return fmt.Errorf("register returned an empty user id") } c.userID = user.ID request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil) if err != nil { return err } request.SetBasicAuth(username, password) response, err = c.client.Do(request) if err != nil { return fmt.Errorf("login: %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusCreated { return fmt.Errorf("login status = %d", response.StatusCode) } var session struct { Token string `json:"token"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { return fmt.Errorf("decode session: %w", err) } if session.Token == "" { return fmt.Errorf("login returned an empty token") } c.token = session.Token return nil } type messageResult struct { messageID string instanceID string err error } func (c *apiClient) verifyConcurrentIdempotency() error { chatID, err := c.createChat() if err != nil { return err } const workers = 16 var ( key = uuid.NewString() start = make(chan struct{}) results = make(chan messageResult, workers) group sync.WaitGroup instances = make(map[string]struct{}) messageID string ) transport := c.transport.Clone() transport.DisableKeepAlives = true transport.MaxConnsPerHost = workers client := &http.Client{Transport: transport, Timeout: 10 * time.Second} defer transport.CloseIdleConnections() for range workers { group.Add(1) go func() { defer group.Done() <-start id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message") results <- messageResult{messageID: id, instanceID: instance, err: requestErr} }() } close(start) group.Wait() close(results) for result := range results { if result.err != nil { return result.err } if result.instanceID == "" { return fmt.Errorf("message response has an empty X-Instance-Id") } instances[result.instanceID] = struct{}{} if messageID == "" { messageID = result.messageID } if result.messageID != messageID { return fmt.Errorf("idempotent responses returned different message ids") } } if len(instances) < 2 { return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances)) } return c.verifyMessageHistory(chatID, messageID) } func (c *apiClient) createChat() (string, error) { payload, err := json.Marshal(map[string]any{ "title": "Replica check", "member_ids": []string{c.userID}, }) if err != nil { return "", err } request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload)) if err != nil { return "", err } request.Header.Set("Authorization", "Bearer "+c.token) request.Header.Set("Content-Type", "application/json") response, err := c.client.Do(request) if err != nil { return "", fmt.Errorf("create chat: %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusCreated { return "", fmt.Errorf("create chat status = %d", response.StatusCode) } var chat struct { ID string `json:"id"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil { return "", fmt.Errorf("decode chat: %w", err) } return chat.ID, nil } func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) { payload, err := json.Marshal(map[string]string{"text": text}) if err != nil { return "", "", err } request, err := http.NewRequest( http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload), ) if err != nil { return "", "", err } request.Header.Set("Authorization", "Bearer "+c.token) request.Header.Set("Content-Type", "application/json") request.Header.Set("Idempotency-Key", key) response, err := client.Do(request) if err != nil { return "", "", fmt.Errorf("create message: %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusCreated { return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode) } var message struct { ID string `json:"id"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil { return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err) } return message.ID, response.Header.Get("X-Instance-Id"), nil } func (c *apiClient) verifyMessageHistory(chatID, messageID string) error { request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil) if err != nil { return err } request.Header.Set("Authorization", "Bearer "+c.token) response, err := c.client.Do(request) if err != nil { return fmt.Errorf("list messages: %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { return fmt.Errorf("list messages status = %d", response.StatusCode) } var page struct { Items []struct { ID string `json:"id"` } `json:"items"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil { return fmt.Errorf("decode messages: %w", err) } if len(page.Items) != 1 || page.Items[0].ID != messageID { return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items)) } return nil } func verifyOutage(client *apiClient, service string) (returnErr error) { if err := compose("stop", service); err != nil { return err } defer func() { if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); returnErr == nil && err != nil { returnErr = err } }() if err := client.expectEventually("/health/ready", http.StatusServiceUnavailable, false); err != nil { return fmt.Errorf("%s readiness outage: %w", service, err) } if err := client.expectEventually("/api/v1/users/me", http.StatusServiceUnavailable, true); err != nil { return fmt.Errorf("%s business outage: %w", service, err) } if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); err != nil { return err } if err := client.expectEventually("/health/ready", http.StatusOK, false); err != nil { return fmt.Errorf("%s readiness recovery: %w", service, err) } if err := client.expectEventually("/api/v1/users/me", http.StatusOK, true); err != nil { return fmt.Errorf("%s session recovery: %w", service, err) } return nil } func compose(arguments ...string) error { command := exec.Command("docker", append([]string{"compose"}, arguments...)...) if output, err := command.CombinedOutput(); err != nil { return fmt.Errorf("docker compose %v: %w: %s", arguments, err, bytes.TrimSpace(output)) } return nil } func (c *apiClient) expectEventually(path string, status int, authenticated bool) error { deadline := time.Now().Add(20 * time.Second) var lastErr error for time.Now().Before(deadline) { lastErr = c.expect(http.MethodGet, path, status, authenticated) if lastErr == nil { return nil } time.Sleep(200 * time.Millisecond) } return lastErr } func (c *apiClient) expect(method, path string, status int, authenticated bool) error { request, err := http.NewRequest(method, c.baseURL+path, nil) if err != nil { return err } if authenticated { request.Header.Set("Authorization", "Bearer "+c.token) } response, err := c.client.Do(request) if err != nil { return err } return closeResponse(response, status) } func closeResponse(response *http.Response, status int) error { defer func() { _ = response.Body.Close() }() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) if response.StatusCode != status { return fmt.Errorf("status = %d, want %d", response.StatusCode, status) } return nil }