Files

523 lines
16 KiB
Go

package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
)
type configuration struct {
baseURL string
prometheusURL string
label string
duration time.Duration
warmup time.Duration
concurrency int
replicas int
apiCPUs string
apiMemory string
dbMaxConns int
output string
}
type apiClient struct {
baseURL string
client *http.Client
token string
chatID string
}
type collector struct {
mu sync.Mutex
latencies []time.Duration
errors map[string]int
instances map[string]struct{}
}
type result struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
GOOS string `json:"goos"`
GOARCH string `json:"goarch"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latencySummary `json:"latency_ms"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type latencySummary struct {
P50 float64 `json:"p50"`
P95 float64 `json:"p95"`
P99 float64 `json:"p99"`
}
type runtimeSnapshot struct {
apiCPUSeconds float64
apiResident float64
healthyReplicas float64
nginxRequests float64
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
settings := parseFlags()
client := &apiClient{
baseURL: strings.TrimRight(settings.baseURL, "/"),
client: &http.Client{Transport: &http.Transport{
MaxIdleConns: settings.concurrency * 2,
MaxIdleConnsPerHost: settings.concurrency,
MaxConnsPerHost: settings.concurrency,
IdleConnTimeout: 30 * time.Second,
}, Timeout: 10 * time.Second},
}
if err := client.waitForReplicas(settings.replicas); err != nil {
return err
}
if err := client.bootstrap(); err != nil {
return err
}
if settings.warmup > 0 {
warmup := newCollector()
if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil {
return fmt.Errorf("warmup: %w", err)
}
}
var before runtimeSnapshot
if settings.prometheusURL != "" {
snapshot, err := collectRuntimeSnapshot()
if err != nil {
return fmt.Errorf("collect metrics before load: %w", err)
}
before = snapshot
}
measurements := newCollector()
started := time.Now()
if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil {
return err
}
elapsed := time.Since(started)
summary := measurements.result(settings, elapsed)
if settings.prometheusURL != "" {
after, err := collectRuntimeSnapshot()
if err != nil {
return fmt.Errorf("collect metrics after load: %w", err)
}
summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed)
if err != nil {
return err
}
}
return writeResult(settings.output, summary)
}
func (c *apiClient) waitForReplicas(expected int) error {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
instances := make(map[string]struct{}, expected)
for ctx.Err() == nil {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody)
if err != nil {
return err
}
response, err := c.client.Do(request)
if err == nil {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
_ = response.Body.Close()
if response.StatusCode == http.StatusOK {
if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" {
instances[instanceID] = struct{}{}
}
if len(instances) == expected {
return nil
}
}
}
time.Sleep(100 * time.Millisecond)
}
return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err())
}
func parseFlags() configuration {
var settings configuration
flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL")
flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL")
flag.StringVar(&settings.label, "label", "manual", "experiment point label")
flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration")
flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration")
flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients")
flag.IntVar(&settings.replicas, "replicas", 2, "API replica count")
flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica")
flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica")
flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica")
flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty")
flag.Parse()
if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 {
fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive")
os.Exit(2)
}
return settings
}
func (c *apiClient) bootstrap() error {
username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
password := "Synthetic-" + uuid.NewString()
var user struct {
ID string `json:"id"`
}
if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{
"username": username, "display_name": "Scale experiment", "password": password,
}, &user, http.StatusCreated); err != nil {
return fmt.Errorf("register: %w", err)
}
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody)
if err != nil {
return err
}
request.SetBasicAuth(username, password)
response, err := c.client.Do(request)
if err != nil {
return fmt.Errorf("login: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return responseError("login", response)
}
var session struct {
Token string `json:"token"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
return fmt.Errorf("decode session: %w", err)
}
c.token = session.Token
var chat struct {
ID string `json:"id"`
}
if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{
"title": "Scale experiment", "member_ids": []string{user.ID},
}, &chat, http.StatusCreated); err != nil {
return fmt.Errorf("create chat: %w", err)
}
c.chatID = chat.ID
return nil
}
func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error {
ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel()
var sequence atomic.Uint64
var group sync.WaitGroup
for range concurrency {
group.Add(1)
go func() {
defer group.Done()
for ctx.Err() == nil {
number := sequence.Add(1)
started := time.Now()
instanceID, status, err := client.createMessage(ctx, number)
if err != nil && ctx.Err() != nil {
return
}
measurements.observe(time.Since(started), instanceID, status, err)
}
}()
}
group.Wait()
if len(measurements.latencies) == 0 {
return errors.New("no successful requests")
}
return nil
}
func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) {
payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)})
if err != nil {
return "", 0, err
}
request, err := http.NewRequestWithContext(
ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload),
)
if err != nil {
return "", 0, err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", uuid.NewString())
response, err := c.client.Do(request)
if err != nil {
return "", 0, err
}
defer func() { _ = response.Body.Close() }()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
if response.StatusCode != http.StatusCreated {
return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status")
}
return response.Header.Get("X-Instance-Id"), response.StatusCode, nil
}
func newCollector() *collector {
return &collector{errors: make(map[string]int), instances: make(map[string]struct{})}
}
func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) {
c.mu.Lock()
defer c.mu.Unlock()
if err != nil {
key := "network"
if status != 0 {
key = strconv.Itoa(status)
}
c.errors[key]++
return
}
c.latencies = append(c.latencies, latency)
if instanceID != "" {
c.instances[instanceID] = struct{}{}
}
}
func (c *collector) result(settings configuration, elapsed time.Duration) result {
sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] })
return result{
Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339),
GOOS: runtime.GOOS, GOARCH: runtime.GOARCH,
DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(),
Concurrency: settings.concurrency, Replicas: settings.replicas,
APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns,
SuccessfulRequests: len(c.latencies),
SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors,
LatencyMS: latencySummary{
P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99),
},
ObservedInstances: len(c.instances),
}
}
func percentile(values []time.Duration, percentile float64) float64 {
if len(values) == 0 {
return 0
}
index := int(float64(len(values)-1) * percentile)
return float64(values[index].Microseconds()) / 1000
}
func collectRuntimeSnapshot() (runtimeSnapshot, error) {
hosts, err := net.LookupHost("api")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err)
}
uniqueHosts := make(map[string]struct{}, len(hosts))
var snapshot runtimeSnapshot
for _, host := range hosts {
if _, duplicate := uniqueHosts[host]; duplicate {
continue
}
uniqueHosts[host] = struct{}{}
metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err)
}
cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total")
if err != nil {
return runtimeSnapshot{}, err
}
resident, err := requiredMetric(metrics, "process_resident_memory_bytes")
if err != nil {
return runtimeSnapshot{}, err
}
snapshot.apiCPUSeconds += cpuSeconds
snapshot.apiResident += resident
snapshot.healthyReplicas++
}
nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err)
}
snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total")
if err != nil {
return runtimeSnapshot{}, err
}
return snapshot, nil
}
func scrapeMetrics(endpoint string) (map[string]float64, error) {
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Get(endpoint)
if err != nil {
return nil, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %d", response.StatusCode)
}
metrics := make(map[string]float64)
scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
name, _, _ := strings.Cut(fields[0], "{")
value, err := strconv.ParseFloat(fields[len(fields)-1], 64)
if err != nil {
return nil, fmt.Errorf("parse metric %s: %w", name, err)
}
metrics[name] += value
}
if err := scanner.Err(); err != nil {
return nil, err
}
return metrics, nil
}
func requiredMetric(metrics map[string]float64, name string) (float64, error) {
value, found := metrics[name]
if !found {
return 0, fmt.Errorf("metric %s not found", name)
}
return value, nil
}
func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) {
postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`)
if err != nil {
return nil, fmt.Errorf("query PostgreSQL connections: %w", err)
}
seconds := elapsed.Seconds()
return map[string]float64{
"api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds,
"api_resident_bytes": after.apiResident,
"healthy_replicas": after.healthyReplicas,
"nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds,
"postgres_connections": postgresConnections,
}, nil
}
func prometheusQuery(baseURL, query string) (float64, error) {
endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query)
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Get(endpoint)
if err != nil {
return 0, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
}
var payload struct {
Data struct {
Result []struct {
Value []json.RawMessage `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
return 0, err
}
var total float64
for _, item := range payload.Data.Result {
if len(item.Value) != 2 {
continue
}
var encoded string
if err := json.Unmarshal(item.Value[1], &encoded); err != nil {
return 0, err
}
value, err := strconv.ParseFloat(encoded, 64)
if err != nil {
return 0, err
}
total += value
}
return total, nil
}
func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error {
payload, err := json.Marshal(body)
if err != nil {
return err
}
request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/json")
if token != "" {
request.Header.Set("Authorization", "Bearer "+token)
}
if idempotencyKey != "" {
request.Header.Set("Idempotency-Key", idempotencyKey)
}
response, err := c.client.Do(request)
if err != nil {
return err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != expected {
return responseError(method+" "+path, response)
}
return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target)
}
func responseError(action string, response *http.Response) error {
body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body))
}
func writeResult(path string, value result) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
data = append(data, '\n')
if path == "" {
_, err = os.Stdout.Write(data)
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}