forked from mtusi2026/mtusi2026lab7
331 lines
9.6 KiB
Go
331 lines
9.6 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type apiClient struct {
|
|
baseURL string
|
|
client *http.Client
|
|
token string
|
|
userID string
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(1)
|
|
}
|
|
fmt.Println("PASS: replica idempotency and dependency recovery")
|
|
}
|
|
|
|
func run() error {
|
|
baseURL := flag.String("base-url", "http://localhost:8080", "API base URL")
|
|
flag.Parse()
|
|
client := &apiClient{baseURL: *baseURL, client: &http.Client{Timeout: 10 * time.Second}}
|
|
if err := client.registerAndLogin(); err != nil {
|
|
return err
|
|
}
|
|
if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil {
|
|
return err
|
|
}
|
|
if err := client.verifyConcurrentIdempotency(); err != nil {
|
|
return err
|
|
}
|
|
if err := verifyOutage(client, "valkey"); err != nil {
|
|
return err
|
|
}
|
|
return verifyOutage(client, "postgres")
|
|
}
|
|
|
|
func (c *apiClient) registerAndLogin() error {
|
|
username := "dep_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16]
|
|
password := "T3st-" + uuid.NewString()
|
|
body := map[string]string{"username": username, "display_name": "Dependency check", "password": password}
|
|
payload, err := json.Marshal(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/users", bytes.NewReader(payload))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.Header.Set("Content-Type", "application/json")
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return fmt.Errorf("register: %w", err)
|
|
}
|
|
registrationBody := response.Body
|
|
defer func() { _ = registrationBody.Close() }()
|
|
if response.StatusCode != http.StatusCreated {
|
|
return fmt.Errorf("register status = %d", response.StatusCode)
|
|
}
|
|
var user struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil {
|
|
return fmt.Errorf("decode user: %w", err)
|
|
}
|
|
if user.ID == "" {
|
|
return fmt.Errorf("register returned an empty user id")
|
|
}
|
|
c.userID = user.ID
|
|
|
|
request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.SetBasicAuth(username, password)
|
|
response, err = c.client.Do(request)
|
|
if err != nil {
|
|
return fmt.Errorf("login: %w", err)
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusCreated {
|
|
return fmt.Errorf("login status = %d", response.StatusCode)
|
|
}
|
|
var session struct {
|
|
Token string `json:"token"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
|
|
return fmt.Errorf("decode session: %w", err)
|
|
}
|
|
if session.Token == "" {
|
|
return fmt.Errorf("login returned an empty token")
|
|
}
|
|
c.token = session.Token
|
|
return nil
|
|
}
|
|
|
|
type messageResult struct {
|
|
messageID string
|
|
instanceID string
|
|
err error
|
|
}
|
|
|
|
func (c *apiClient) verifyConcurrentIdempotency() error {
|
|
chatID, err := c.createChat()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
const workers = 16
|
|
var (
|
|
key = uuid.NewString()
|
|
start = make(chan struct{})
|
|
results = make(chan messageResult, workers)
|
|
group sync.WaitGroup
|
|
instances = make(map[string]struct{})
|
|
messageID string
|
|
)
|
|
transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers}
|
|
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
|
|
defer transport.CloseIdleConnections()
|
|
|
|
for range workers {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
<-start
|
|
id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message")
|
|
results <- messageResult{messageID: id, instanceID: instance, err: requestErr}
|
|
}()
|
|
}
|
|
close(start)
|
|
group.Wait()
|
|
close(results)
|
|
|
|
for result := range results {
|
|
if result.err != nil {
|
|
return result.err
|
|
}
|
|
if result.instanceID == "" {
|
|
return fmt.Errorf("message response has an empty X-Instance-Id")
|
|
}
|
|
instances[result.instanceID] = struct{}{}
|
|
if messageID == "" {
|
|
messageID = result.messageID
|
|
}
|
|
if result.messageID != messageID {
|
|
return fmt.Errorf("idempotent responses returned different message ids")
|
|
}
|
|
}
|
|
if len(instances) < 2 {
|
|
return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances))
|
|
}
|
|
return c.verifyMessageHistory(chatID, messageID)
|
|
}
|
|
|
|
func (c *apiClient) createChat() (string, error) {
|
|
payload, err := json.Marshal(map[string]any{
|
|
"title": "Replica check", "member_ids": []string{c.userID},
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
request.Header.Set("Content-Type", "application/json")
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return "", fmt.Errorf("create chat: %w", err)
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusCreated {
|
|
return "", fmt.Errorf("create chat status = %d", response.StatusCode)
|
|
}
|
|
var chat struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil {
|
|
return "", fmt.Errorf("decode chat: %w", err)
|
|
}
|
|
return chat.ID, nil
|
|
}
|
|
|
|
func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) {
|
|
payload, err := json.Marshal(map[string]string{"text": text})
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
request, err := http.NewRequest(
|
|
http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload),
|
|
)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
request.Header.Set("Content-Type", "application/json")
|
|
request.Header.Set("Idempotency-Key", key)
|
|
response, err := client.Do(request)
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("create message: %w", err)
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusCreated {
|
|
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode)
|
|
}
|
|
var message struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil {
|
|
return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err)
|
|
}
|
|
return message.ID, response.Header.Get("X-Instance-Id"), nil
|
|
}
|
|
|
|
func (c *apiClient) verifyMessageHistory(chatID, messageID string) error {
|
|
request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return fmt.Errorf("list messages: %w", err)
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("list messages status = %d", response.StatusCode)
|
|
}
|
|
var page struct {
|
|
Items []struct {
|
|
ID string `json:"id"`
|
|
} `json:"items"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil {
|
|
return fmt.Errorf("decode messages: %w", err)
|
|
}
|
|
if len(page.Items) != 1 || page.Items[0].ID != messageID {
|
|
return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func verifyOutage(client *apiClient, service string) (returnErr error) {
|
|
if err := compose("stop", service); err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); returnErr == nil && err != nil {
|
|
returnErr = err
|
|
}
|
|
}()
|
|
if err := client.expectEventually("/health/ready", http.StatusServiceUnavailable, false); err != nil {
|
|
return fmt.Errorf("%s readiness outage: %w", service, err)
|
|
}
|
|
if err := client.expectEventually("/api/v1/users/me", http.StatusServiceUnavailable, true); err != nil {
|
|
return fmt.Errorf("%s business outage: %w", service, err)
|
|
}
|
|
if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); err != nil {
|
|
return err
|
|
}
|
|
if err := client.expectEventually("/health/ready", http.StatusOK, false); err != nil {
|
|
return fmt.Errorf("%s readiness recovery: %w", service, err)
|
|
}
|
|
if err := client.expectEventually("/api/v1/users/me", http.StatusOK, true); err != nil {
|
|
return fmt.Errorf("%s session recovery: %w", service, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func compose(arguments ...string) error {
|
|
command := exec.Command("docker", append([]string{"compose"}, arguments...)...)
|
|
if output, err := command.CombinedOutput(); err != nil {
|
|
return fmt.Errorf("docker compose %v: %w: %s", arguments, err, bytes.TrimSpace(output))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *apiClient) expectEventually(path string, status int, authenticated bool) error {
|
|
deadline := time.Now().Add(20 * time.Second)
|
|
var lastErr error
|
|
for time.Now().Before(deadline) {
|
|
lastErr = c.expect(http.MethodGet, path, status, authenticated)
|
|
if lastErr == nil {
|
|
return nil
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
return lastErr
|
|
}
|
|
|
|
func (c *apiClient) expect(method, path string, status int, authenticated bool) error {
|
|
request, err := http.NewRequest(method, c.baseURL+path, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if authenticated {
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
}
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return closeResponse(response, status)
|
|
}
|
|
|
|
func closeResponse(response *http.Response, status int) error {
|
|
defer func() { _ = response.Body.Close() }()
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
|
if response.StatusCode != status {
|
|
return fmt.Errorf("status = %d, want %d", response.StatusCode, status)
|
|
}
|
|
return nil
|
|
}
|