forked from mtusi2026/mtusi2026lab3
523 lines
16 KiB
Go
523 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type configuration struct {
|
|
baseURL string
|
|
prometheusURL string
|
|
label string
|
|
duration time.Duration
|
|
warmup time.Duration
|
|
concurrency int
|
|
replicas int
|
|
apiCPUs string
|
|
apiMemory string
|
|
dbMaxConns int
|
|
output string
|
|
}
|
|
|
|
type apiClient struct {
|
|
baseURL string
|
|
client *http.Client
|
|
token string
|
|
chatID string
|
|
}
|
|
|
|
type collector struct {
|
|
mu sync.Mutex
|
|
latencies []time.Duration
|
|
errors map[string]int
|
|
instances map[string]struct{}
|
|
}
|
|
|
|
type result struct {
|
|
Label string `json:"label"`
|
|
Timestamp string `json:"timestamp"`
|
|
GOOS string `json:"goos"`
|
|
GOARCH string `json:"goarch"`
|
|
DurationSeconds float64 `json:"duration_seconds"`
|
|
WarmupSeconds float64 `json:"warmup_seconds"`
|
|
Concurrency int `json:"concurrency"`
|
|
Replicas int `json:"replicas"`
|
|
APICPUs string `json:"api_cpus_per_replica"`
|
|
APIMemory string `json:"api_memory_per_replica"`
|
|
DBMaxConns int `json:"db_max_connections_per_replica"`
|
|
SuccessfulRequests int `json:"successful_requests"`
|
|
SuccessfulRPS float64 `json:"successful_rps"`
|
|
Errors map[string]int `json:"errors"`
|
|
LatencyMS latencySummary `json:"latency_ms"`
|
|
ObservedInstances int `json:"observed_instances"`
|
|
Prometheus map[string]float64 `json:"prometheus"`
|
|
}
|
|
|
|
type latencySummary struct {
|
|
P50 float64 `json:"p50"`
|
|
P95 float64 `json:"p95"`
|
|
P99 float64 `json:"p99"`
|
|
}
|
|
|
|
type runtimeSnapshot struct {
|
|
apiCPUSeconds float64
|
|
apiResident float64
|
|
healthyReplicas float64
|
|
nginxRequests float64
|
|
}
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
settings := parseFlags()
|
|
client := &apiClient{
|
|
baseURL: strings.TrimRight(settings.baseURL, "/"),
|
|
client: &http.Client{Transport: &http.Transport{
|
|
MaxIdleConns: settings.concurrency * 2,
|
|
MaxIdleConnsPerHost: settings.concurrency,
|
|
MaxConnsPerHost: settings.concurrency,
|
|
IdleConnTimeout: 30 * time.Second,
|
|
}, Timeout: 10 * time.Second},
|
|
}
|
|
if err := client.waitForReplicas(settings.replicas); err != nil {
|
|
return err
|
|
}
|
|
if err := client.bootstrap(); err != nil {
|
|
return err
|
|
}
|
|
if settings.warmup > 0 {
|
|
warmup := newCollector()
|
|
if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil {
|
|
return fmt.Errorf("warmup: %w", err)
|
|
}
|
|
}
|
|
|
|
var before runtimeSnapshot
|
|
if settings.prometheusURL != "" {
|
|
snapshot, err := collectRuntimeSnapshot()
|
|
if err != nil {
|
|
return fmt.Errorf("collect metrics before load: %w", err)
|
|
}
|
|
before = snapshot
|
|
}
|
|
measurements := newCollector()
|
|
started := time.Now()
|
|
if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil {
|
|
return err
|
|
}
|
|
elapsed := time.Since(started)
|
|
summary := measurements.result(settings, elapsed)
|
|
if settings.prometheusURL != "" {
|
|
after, err := collectRuntimeSnapshot()
|
|
if err != nil {
|
|
return fmt.Errorf("collect metrics after load: %w", err)
|
|
}
|
|
summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return writeResult(settings.output, summary)
|
|
}
|
|
|
|
func (c *apiClient) waitForReplicas(expected int) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
|
|
instances := make(map[string]struct{}, expected)
|
|
for ctx.Err() == nil {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
response, err := c.client.Do(request)
|
|
if err == nil {
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
|
_ = response.Body.Close()
|
|
if response.StatusCode == http.StatusOK {
|
|
if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" {
|
|
instances[instanceID] = struct{}{}
|
|
}
|
|
if len(instances) == expected {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err())
|
|
}
|
|
|
|
func parseFlags() configuration {
|
|
var settings configuration
|
|
flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL")
|
|
flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL")
|
|
flag.StringVar(&settings.label, "label", "manual", "experiment point label")
|
|
flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration")
|
|
flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration")
|
|
flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients")
|
|
flag.IntVar(&settings.replicas, "replicas", 2, "API replica count")
|
|
flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica")
|
|
flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica")
|
|
flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica")
|
|
flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty")
|
|
flag.Parse()
|
|
if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 {
|
|
fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive")
|
|
os.Exit(2)
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func (c *apiClient) bootstrap() error {
|
|
username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
|
|
password := "Synthetic-" + uuid.NewString()
|
|
var user struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{
|
|
"username": username, "display_name": "Scale experiment", "password": password,
|
|
}, &user, http.StatusCreated); err != nil {
|
|
return fmt.Errorf("register: %w", err)
|
|
}
|
|
|
|
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.SetBasicAuth(username, password)
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return fmt.Errorf("login: %w", err)
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusCreated {
|
|
return responseError("login", response)
|
|
}
|
|
var session struct {
|
|
Token string `json:"token"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
|
|
return fmt.Errorf("decode session: %w", err)
|
|
}
|
|
c.token = session.Token
|
|
|
|
var chat struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{
|
|
"title": "Scale experiment", "member_ids": []string{user.ID},
|
|
}, &chat, http.StatusCreated); err != nil {
|
|
return fmt.Errorf("create chat: %w", err)
|
|
}
|
|
c.chatID = chat.ID
|
|
return nil
|
|
}
|
|
|
|
func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), duration)
|
|
defer cancel()
|
|
var sequence atomic.Uint64
|
|
var group sync.WaitGroup
|
|
for range concurrency {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
for ctx.Err() == nil {
|
|
number := sequence.Add(1)
|
|
started := time.Now()
|
|
instanceID, status, err := client.createMessage(ctx, number)
|
|
if err != nil && ctx.Err() != nil {
|
|
return
|
|
}
|
|
measurements.observe(time.Since(started), instanceID, status, err)
|
|
}
|
|
}()
|
|
}
|
|
group.Wait()
|
|
if len(measurements.latencies) == 0 {
|
|
return errors.New("no successful requests")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) {
|
|
payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)})
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
request, err := http.NewRequestWithContext(
|
|
ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload),
|
|
)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
request.Header.Set("Content-Type", "application/json")
|
|
request.Header.Set("Idempotency-Key", uuid.NewString())
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
|
if response.StatusCode != http.StatusCreated {
|
|
return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status")
|
|
}
|
|
return response.Header.Get("X-Instance-Id"), response.StatusCode, nil
|
|
}
|
|
|
|
func newCollector() *collector {
|
|
return &collector{errors: make(map[string]int), instances: make(map[string]struct{})}
|
|
}
|
|
|
|
func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if err != nil {
|
|
key := "network"
|
|
if status != 0 {
|
|
key = strconv.Itoa(status)
|
|
}
|
|
c.errors[key]++
|
|
return
|
|
}
|
|
c.latencies = append(c.latencies, latency)
|
|
if instanceID != "" {
|
|
c.instances[instanceID] = struct{}{}
|
|
}
|
|
}
|
|
|
|
func (c *collector) result(settings configuration, elapsed time.Duration) result {
|
|
sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] })
|
|
return result{
|
|
Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339),
|
|
GOOS: runtime.GOOS, GOARCH: runtime.GOARCH,
|
|
DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(),
|
|
Concurrency: settings.concurrency, Replicas: settings.replicas,
|
|
APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns,
|
|
SuccessfulRequests: len(c.latencies),
|
|
SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors,
|
|
LatencyMS: latencySummary{
|
|
P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99),
|
|
},
|
|
ObservedInstances: len(c.instances),
|
|
}
|
|
}
|
|
|
|
func percentile(values []time.Duration, percentile float64) float64 {
|
|
if len(values) == 0 {
|
|
return 0
|
|
}
|
|
index := int(float64(len(values)-1) * percentile)
|
|
return float64(values[index].Microseconds()) / 1000
|
|
}
|
|
|
|
func collectRuntimeSnapshot() (runtimeSnapshot, error) {
|
|
hosts, err := net.LookupHost("api")
|
|
if err != nil {
|
|
return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err)
|
|
}
|
|
uniqueHosts := make(map[string]struct{}, len(hosts))
|
|
var snapshot runtimeSnapshot
|
|
for _, host := range hosts {
|
|
if _, duplicate := uniqueHosts[host]; duplicate {
|
|
continue
|
|
}
|
|
uniqueHosts[host] = struct{}{}
|
|
metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics")
|
|
if err != nil {
|
|
return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err)
|
|
}
|
|
cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total")
|
|
if err != nil {
|
|
return runtimeSnapshot{}, err
|
|
}
|
|
resident, err := requiredMetric(metrics, "process_resident_memory_bytes")
|
|
if err != nil {
|
|
return runtimeSnapshot{}, err
|
|
}
|
|
snapshot.apiCPUSeconds += cpuSeconds
|
|
snapshot.apiResident += resident
|
|
snapshot.healthyReplicas++
|
|
}
|
|
nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics")
|
|
if err != nil {
|
|
return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err)
|
|
}
|
|
snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total")
|
|
if err != nil {
|
|
return runtimeSnapshot{}, err
|
|
}
|
|
return snapshot, nil
|
|
}
|
|
|
|
func scrapeMetrics(endpoint string) (map[string]float64, error) {
|
|
client := &http.Client{Timeout: 5 * time.Second}
|
|
response, err := client.Get(endpoint)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("status %d", response.StatusCode)
|
|
}
|
|
metrics := make(map[string]float64)
|
|
scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20))
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 2 {
|
|
continue
|
|
}
|
|
name, _, _ := strings.Cut(fields[0], "{")
|
|
value, err := strconv.ParseFloat(fields[len(fields)-1], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse metric %s: %w", name, err)
|
|
}
|
|
metrics[name] += value
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return metrics, nil
|
|
}
|
|
|
|
func requiredMetric(metrics map[string]float64, name string) (float64, error) {
|
|
value, found := metrics[name]
|
|
if !found {
|
|
return 0, fmt.Errorf("metric %s not found", name)
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) {
|
|
postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("query PostgreSQL connections: %w", err)
|
|
}
|
|
seconds := elapsed.Seconds()
|
|
return map[string]float64{
|
|
"api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds,
|
|
"api_resident_bytes": after.apiResident,
|
|
"healthy_replicas": after.healthyReplicas,
|
|
"nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds,
|
|
"postgres_connections": postgresConnections,
|
|
}, nil
|
|
}
|
|
|
|
func prometheusQuery(baseURL, query string) (float64, error) {
|
|
endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query)
|
|
client := &http.Client{Timeout: 5 * time.Second}
|
|
response, err := client.Get(endpoint)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusOK {
|
|
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
|
}
|
|
var payload struct {
|
|
Data struct {
|
|
Result []struct {
|
|
Value []json.RawMessage `json:"value"`
|
|
} `json:"result"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
|
|
return 0, err
|
|
}
|
|
var total float64
|
|
for _, item := range payload.Data.Result {
|
|
if len(item.Value) != 2 {
|
|
continue
|
|
}
|
|
var encoded string
|
|
if err := json.Unmarshal(item.Value[1], &encoded); err != nil {
|
|
return 0, err
|
|
}
|
|
value, err := strconv.ParseFloat(encoded, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
total += value
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error {
|
|
payload, err := json.Marshal(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.Header.Set("Content-Type", "application/json")
|
|
if token != "" {
|
|
request.Header.Set("Authorization", "Bearer "+token)
|
|
}
|
|
if idempotencyKey != "" {
|
|
request.Header.Set("Idempotency-Key", idempotencyKey)
|
|
}
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != expected {
|
|
return responseError(method+" "+path, response)
|
|
}
|
|
return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target)
|
|
}
|
|
|
|
func responseError(action string, response *http.Response) error {
|
|
body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
|
return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body))
|
|
}
|
|
|
|
func writeResult(path string, value result) error {
|
|
data, err := json.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
data = append(data, '\n')
|
|
if path == "" {
|
|
_, err = os.Stdout.Write(data)
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, data, 0o644)
|
|
}
|