forked from mtusi2026/mtusi2026lab3
ops: add balanced replicated stack
This commit is contained in:
@@ -0,0 +1,522 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type configuration struct {
|
||||
baseURL string
|
||||
prometheusURL string
|
||||
label string
|
||||
duration time.Duration
|
||||
warmup time.Duration
|
||||
concurrency int
|
||||
replicas int
|
||||
apiCPUs string
|
||||
apiMemory string
|
||||
dbMaxConns int
|
||||
output string
|
||||
}
|
||||
|
||||
type apiClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
token string
|
||||
chatID string
|
||||
}
|
||||
|
||||
type collector struct {
|
||||
mu sync.Mutex
|
||||
latencies []time.Duration
|
||||
errors map[string]int
|
||||
instances map[string]struct{}
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
GOOS string `json:"goos"`
|
||||
GOARCH string `json:"goarch"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
WarmupSeconds float64 `json:"warmup_seconds"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRequests int `json:"successful_requests"`
|
||||
SuccessfulRPS float64 `json:"successful_rps"`
|
||||
Errors map[string]int `json:"errors"`
|
||||
LatencyMS latencySummary `json:"latency_ms"`
|
||||
ObservedInstances int `json:"observed_instances"`
|
||||
Prometheus map[string]float64 `json:"prometheus"`
|
||||
}
|
||||
|
||||
type latencySummary struct {
|
||||
P50 float64 `json:"p50"`
|
||||
P95 float64 `json:"p95"`
|
||||
P99 float64 `json:"p99"`
|
||||
}
|
||||
|
||||
type runtimeSnapshot struct {
|
||||
apiCPUSeconds float64
|
||||
apiResident float64
|
||||
healthyReplicas float64
|
||||
nginxRequests float64
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
settings := parseFlags()
|
||||
client := &apiClient{
|
||||
baseURL: strings.TrimRight(settings.baseURL, "/"),
|
||||
client: &http.Client{Transport: &http.Transport{
|
||||
MaxIdleConns: settings.concurrency * 2,
|
||||
MaxIdleConnsPerHost: settings.concurrency,
|
||||
MaxConnsPerHost: settings.concurrency,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
}, Timeout: 10 * time.Second},
|
||||
}
|
||||
if err := client.waitForReplicas(settings.replicas); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.bootstrap(); err != nil {
|
||||
return err
|
||||
}
|
||||
if settings.warmup > 0 {
|
||||
warmup := newCollector()
|
||||
if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil {
|
||||
return fmt.Errorf("warmup: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var before runtimeSnapshot
|
||||
if settings.prometheusURL != "" {
|
||||
snapshot, err := collectRuntimeSnapshot()
|
||||
if err != nil {
|
||||
return fmt.Errorf("collect metrics before load: %w", err)
|
||||
}
|
||||
before = snapshot
|
||||
}
|
||||
measurements := newCollector()
|
||||
started := time.Now()
|
||||
if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil {
|
||||
return err
|
||||
}
|
||||
elapsed := time.Since(started)
|
||||
summary := measurements.result(settings, elapsed)
|
||||
if settings.prometheusURL != "" {
|
||||
after, err := collectRuntimeSnapshot()
|
||||
if err != nil {
|
||||
return fmt.Errorf("collect metrics after load: %w", err)
|
||||
}
|
||||
summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeResult(settings.output, summary)
|
||||
}
|
||||
|
||||
func (c *apiClient) waitForReplicas(expected int) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
instances := make(map[string]struct{}, expected)
|
||||
for ctx.Err() == nil {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err == nil {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
||||
_ = response.Body.Close()
|
||||
if response.StatusCode == http.StatusOK {
|
||||
if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" {
|
||||
instances[instanceID] = struct{}{}
|
||||
}
|
||||
if len(instances) == expected {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err())
|
||||
}
|
||||
|
||||
func parseFlags() configuration {
|
||||
var settings configuration
|
||||
flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL")
|
||||
flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL")
|
||||
flag.StringVar(&settings.label, "label", "manual", "experiment point label")
|
||||
flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration")
|
||||
flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration")
|
||||
flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients")
|
||||
flag.IntVar(&settings.replicas, "replicas", 2, "API replica count")
|
||||
flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica")
|
||||
flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica")
|
||||
flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica")
|
||||
flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty")
|
||||
flag.Parse()
|
||||
if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 {
|
||||
fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive")
|
||||
os.Exit(2)
|
||||
}
|
||||
return settings
|
||||
}
|
||||
|
||||
func (c *apiClient) bootstrap() error {
|
||||
username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
|
||||
password := "Synthetic-" + uuid.NewString()
|
||||
var user struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{
|
||||
"username": username, "display_name": "Scale experiment", "password": password,
|
||||
}, &user, http.StatusCreated); err != nil {
|
||||
return fmt.Errorf("register: %w", err)
|
||||
}
|
||||
|
||||
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.SetBasicAuth(username, password)
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("login: %w", err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return responseError("login", response)
|
||||
}
|
||||
var session struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
|
||||
return fmt.Errorf("decode session: %w", err)
|
||||
}
|
||||
c.token = session.Token
|
||||
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{
|
||||
"title": "Scale experiment", "member_ids": []string{user.ID},
|
||||
}, &chat, http.StatusCreated); err != nil {
|
||||
return fmt.Errorf("create chat: %w", err)
|
||||
}
|
||||
c.chatID = chat.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), duration)
|
||||
defer cancel()
|
||||
var sequence atomic.Uint64
|
||||
var group sync.WaitGroup
|
||||
for range concurrency {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for ctx.Err() == nil {
|
||||
number := sequence.Add(1)
|
||||
started := time.Now()
|
||||
instanceID, status, err := client.createMessage(ctx, number)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
measurements.observe(time.Since(started), instanceID, status, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
if len(measurements.latencies) == 0 {
|
||||
return errors.New("no successful requests")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) {
|
||||
payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(
|
||||
ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload),
|
||||
)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Idempotency-Key", uuid.NewString())
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status")
|
||||
}
|
||||
return response.Header.Get("X-Instance-Id"), response.StatusCode, nil
|
||||
}
|
||||
|
||||
func newCollector() *collector {
|
||||
return &collector{errors: make(map[string]int), instances: make(map[string]struct{})}
|
||||
}
|
||||
|
||||
func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if err != nil {
|
||||
key := "network"
|
||||
if status != 0 {
|
||||
key = strconv.Itoa(status)
|
||||
}
|
||||
c.errors[key]++
|
||||
return
|
||||
}
|
||||
c.latencies = append(c.latencies, latency)
|
||||
if instanceID != "" {
|
||||
c.instances[instanceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *collector) result(settings configuration, elapsed time.Duration) result {
|
||||
sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] })
|
||||
return result{
|
||||
Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
GOOS: runtime.GOOS, GOARCH: runtime.GOARCH,
|
||||
DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(),
|
||||
Concurrency: settings.concurrency, Replicas: settings.replicas,
|
||||
APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns,
|
||||
SuccessfulRequests: len(c.latencies),
|
||||
SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors,
|
||||
LatencyMS: latencySummary{
|
||||
P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99),
|
||||
},
|
||||
ObservedInstances: len(c.instances),
|
||||
}
|
||||
}
|
||||
|
||||
func percentile(values []time.Duration, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
index := int(float64(len(values)-1) * percentile)
|
||||
return float64(values[index].Microseconds()) / 1000
|
||||
}
|
||||
|
||||
func collectRuntimeSnapshot() (runtimeSnapshot, error) {
|
||||
hosts, err := net.LookupHost("api")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err)
|
||||
}
|
||||
uniqueHosts := make(map[string]struct{}, len(hosts))
|
||||
var snapshot runtimeSnapshot
|
||||
for _, host := range hosts {
|
||||
if _, duplicate := uniqueHosts[host]; duplicate {
|
||||
continue
|
||||
}
|
||||
uniqueHosts[host] = struct{}{}
|
||||
metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err)
|
||||
}
|
||||
cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, err
|
||||
}
|
||||
resident, err := requiredMetric(metrics, "process_resident_memory_bytes")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, err
|
||||
}
|
||||
snapshot.apiCPUSeconds += cpuSeconds
|
||||
snapshot.apiResident += resident
|
||||
snapshot.healthyReplicas++
|
||||
}
|
||||
nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err)
|
||||
}
|
||||
snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total")
|
||||
if err != nil {
|
||||
return runtimeSnapshot{}, err
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func scrapeMetrics(endpoint string) (map[string]float64, error) {
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
response, err := client.Get(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("status %d", response.StatusCode)
|
||||
}
|
||||
metrics := make(map[string]float64)
|
||||
scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
name, _, _ := strings.Cut(fields[0], "{")
|
||||
value, err := strconv.ParseFloat(fields[len(fields)-1], 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse metric %s: %w", name, err)
|
||||
}
|
||||
metrics[name] += value
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func requiredMetric(metrics map[string]float64, name string) (float64, error) {
|
||||
value, found := metrics[name]
|
||||
if !found {
|
||||
return 0, fmt.Errorf("metric %s not found", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) {
|
||||
postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query PostgreSQL connections: %w", err)
|
||||
}
|
||||
seconds := elapsed.Seconds()
|
||||
return map[string]float64{
|
||||
"api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds,
|
||||
"api_resident_bytes": after.apiResident,
|
||||
"healthy_replicas": after.healthyReplicas,
|
||||
"nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds,
|
||||
"postgres_connections": postgresConnections,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func prometheusQuery(baseURL, query string) (float64, error) {
|
||||
endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query)
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
response, err := client.Get(endpoint)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Value []json.RawMessage `json:"value"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total float64
|
||||
for _, item := range payload.Data.Result {
|
||||
if len(item.Value) != 2 {
|
||||
continue
|
||||
}
|
||||
var encoded string
|
||||
if err := json.Unmarshal(item.Value[1], &encoded); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, err := strconv.ParseFloat(encoded, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += value
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error {
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
if token != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
if idempotencyKey != "" {
|
||||
request.Header.Set("Idempotency-Key", idempotencyKey)
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != expected {
|
||||
return responseError(method+" "+path, response)
|
||||
}
|
||||
return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target)
|
||||
}
|
||||
|
||||
func responseError(action string, response *http.Response) error {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
||||
return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body))
|
||||
}
|
||||
|
||||
func writeResult(path string, value result) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if path == "" {
|
||||
_, err = os.Stdout.Write(data)
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type latency struct {
|
||||
P50 float64 `json:"p50"`
|
||||
P95 float64 `json:"p95"`
|
||||
P99 float64 `json:"p99"`
|
||||
}
|
||||
|
||||
type inputResult struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
WarmupSeconds float64 `json:"warmup_seconds"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRequests int `json:"successful_requests"`
|
||||
SuccessfulRPS float64 `json:"successful_rps"`
|
||||
Errors map[string]int `json:"errors"`
|
||||
LatencyMS latency `json:"latency_ms"`
|
||||
Instances []string `json:"instances"`
|
||||
ObservedInstances int `json:"observed_instances"`
|
||||
Prometheus map[string]float64 `json:"prometheus"`
|
||||
}
|
||||
|
||||
type rawResult struct {
|
||||
Label string `json:"label"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
WarmupSeconds float64 `json:"warmup_seconds"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRequests int `json:"successful_requests"`
|
||||
SuccessfulRPS float64 `json:"successful_rps"`
|
||||
Errors map[string]int `json:"errors"`
|
||||
LatencyMS latency `json:"latency_ms"`
|
||||
ObservedInstances int `json:"observed_instances"`
|
||||
Prometheus map[string]float64 `json:"prometheus"`
|
||||
}
|
||||
|
||||
type aggregate struct {
|
||||
Label string `json:"label"`
|
||||
Repeats int `json:"repeats"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Replicas int `json:"replicas"`
|
||||
APICPUs string `json:"api_cpus_per_replica"`
|
||||
APIMemory string `json:"api_memory_per_replica"`
|
||||
DBMaxConns int `json:"db_max_connections_per_replica"`
|
||||
SuccessfulRPSMedian float64 `json:"successful_rps_median"`
|
||||
SuccessfulRPSMin float64 `json:"successful_rps_min"`
|
||||
SuccessfulRPSMax float64 `json:"successful_rps_max"`
|
||||
LatencyP50MedianMS float64 `json:"latency_p50_median_ms"`
|
||||
LatencyP95MedianMS float64 `json:"latency_p95_median_ms"`
|
||||
LatencyP99MedianMS float64 `json:"latency_p99_median_ms"`
|
||||
TotalErrors int `json:"total_errors"`
|
||||
ObservedInstancesMin int `json:"observed_instances_min"`
|
||||
APICPUCoresMedian float64 `json:"api_cpu_cores_median"`
|
||||
APIResidentBytesMedian float64 `json:"api_resident_bytes_median"`
|
||||
PostgresConnectionsMedian float64 `json:"postgres_connections_median"`
|
||||
PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Method string `json:"method"`
|
||||
Raw []rawResult `json:"raw"`
|
||||
Summary []aggregate `json:"summary"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files")
|
||||
outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output")
|
||||
chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output")
|
||||
flag.Parse()
|
||||
|
||||
results, err := readResults(*inputDirectory)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
aggregates := summarize(results)
|
||||
if err := writeJSON(*outputPath, report{
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Method: "closed-loop HTTP through Nginx; median of three isolated repeats",
|
||||
Raw: sanitize(results),
|
||||
Summary: aggregates,
|
||||
}); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if err := writeChart(*chartPath, aggregates); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func readResults(directory string) ([]inputResult, error) {
|
||||
var paths []string
|
||||
for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} {
|
||||
matches, err := filepath.Glob(filepath.Join(directory, pattern))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths = append(paths, matches...)
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return nil, fmt.Errorf("no experiment results in %s", directory)
|
||||
}
|
||||
|
||||
results := make([]inputResult, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result inputResult
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("decode %s: %w", path, err)
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label })
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func sanitize(results []inputResult) []rawResult {
|
||||
raw := make([]rawResult, 0, len(results))
|
||||
for _, value := range results {
|
||||
raw = append(raw, rawResult{
|
||||
Label: value.Label, Timestamp: value.Timestamp,
|
||||
DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds,
|
||||
Concurrency: value.Concurrency, Replicas: value.Replicas,
|
||||
APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns,
|
||||
SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS,
|
||||
Errors: value.Errors, LatencyMS: value.LatencyMS,
|
||||
ObservedInstances: instanceCount(value), Prometheus: value.Prometheus,
|
||||
})
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func summarize(results []inputResult) []aggregate {
|
||||
groups := make(map[string][]inputResult)
|
||||
for _, value := range results {
|
||||
groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value)
|
||||
}
|
||||
|
||||
labels := make([]string, 0, len(groups))
|
||||
for label := range groups {
|
||||
labels = append(labels, label)
|
||||
}
|
||||
sort.Strings(labels)
|
||||
|
||||
aggregates := make([]aggregate, 0, len(labels))
|
||||
for _, label := range labels {
|
||||
values := groups[label]
|
||||
first := values[0]
|
||||
rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS })
|
||||
p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 })
|
||||
p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 })
|
||||
p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 })
|
||||
cpu := metricNumbers(values, "api_cpu_cores")
|
||||
memory := metricNumbers(values, "api_resident_bytes")
|
||||
connections := metricNumbers(values, "postgres_connections")
|
||||
healthy := metricNumbers(values, "healthy_replicas")
|
||||
aggregate := aggregate{
|
||||
Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas,
|
||||
APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns,
|
||||
SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps),
|
||||
LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99),
|
||||
ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu),
|
||||
APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections),
|
||||
PrometheusHealthyMin: minimum(healthy),
|
||||
}
|
||||
for _, value := range values {
|
||||
if count := instanceCount(value); count < aggregate.ObservedInstancesMin {
|
||||
aggregate.ObservedInstancesMin = count
|
||||
}
|
||||
for _, count := range value.Errors {
|
||||
aggregate.TotalErrors += count
|
||||
}
|
||||
}
|
||||
aggregates = append(aggregates, aggregate)
|
||||
}
|
||||
return aggregates
|
||||
}
|
||||
|
||||
func instanceCount(value inputResult) int {
|
||||
if value.ObservedInstances > 0 {
|
||||
return value.ObservedInstances
|
||||
}
|
||||
return len(value.Instances)
|
||||
}
|
||||
|
||||
func withoutRepeat(label string) string {
|
||||
index := strings.LastIndex(label, "-n")
|
||||
if index == -1 {
|
||||
return label
|
||||
}
|
||||
if _, err := strconv.Atoi(label[index+2:]); err != nil {
|
||||
return label
|
||||
}
|
||||
return label[:index]
|
||||
}
|
||||
|
||||
func numbers(values []inputResult, value func(inputResult) float64) []float64 {
|
||||
result := make([]float64, 0, len(values))
|
||||
for _, item := range values {
|
||||
result = append(result, value(item))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func metricNumbers(values []inputResult, name string) []float64 {
|
||||
result := make([]float64, 0, len(values))
|
||||
for _, item := range values {
|
||||
if value, ok := item.Prometheus[name]; ok {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func median(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
values = append([]float64(nil), values...)
|
||||
sort.Float64s(values)
|
||||
middle := len(values) / 2
|
||||
if len(values)%2 == 1 {
|
||||
return values[middle]
|
||||
}
|
||||
return (values[middle-1] + values[middle]) / 2
|
||||
}
|
||||
|
||||
func minimum(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
minimum := values[0]
|
||||
for _, value := range values[1:] {
|
||||
if value < minimum {
|
||||
minimum = value
|
||||
}
|
||||
}
|
||||
return minimum
|
||||
}
|
||||
|
||||
func maximum(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
maximum := values[0]
|
||||
for _, value := range values[1:] {
|
||||
if value > maximum {
|
||||
maximum = value
|
||||
}
|
||||
}
|
||||
return maximum
|
||||
}
|
||||
|
||||
func writeJSON(path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(data, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func writeChart(path string, values []aggregate) error {
|
||||
const width, height = 960, 560
|
||||
const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0
|
||||
plotWidth := width - left - right
|
||||
plotHeight := height - top - bottom
|
||||
|
||||
var selected []aggregate
|
||||
var maxRPS float64
|
||||
for _, value := range values {
|
||||
if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") {
|
||||
selected = append(selected, value)
|
||||
if value.SuccessfulRPSMedian > maxRPS {
|
||||
maxRPS = value.SuccessfulRPSMedian
|
||||
}
|
||||
}
|
||||
}
|
||||
if maxRPS == 0 {
|
||||
return fmt.Errorf("no scale results for chart")
|
||||
}
|
||||
maxRPS *= 1.1
|
||||
|
||||
var output strings.Builder
|
||||
fmt.Fprintf(&output, `<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">`, width, height, width, height)
|
||||
output.WriteString(`<rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style>`)
|
||||
fmt.Fprintf(&output, `<text x="%g" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text>`, left)
|
||||
|
||||
for step := 0; step <= 5; step++ {
|
||||
y := top + plotHeight*float64(5-step)/5
|
||||
value := maxRPS * float64(step) / 5
|
||||
fmt.Fprintf(&output, `<line class="grid" x1="%g" y1="%g" x2="%g" y2="%g"/><text x="%g" y="%g" text-anchor="end" font-size="12">%.0f</text>`, left, y, left+plotWidth, y, left-10, y+4, value)
|
||||
}
|
||||
fmt.Fprintf(&output, `<line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/><line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/>`, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight)
|
||||
for index, replicas := range []int{1, 2, 4} {
|
||||
x := left + plotWidth*float64(index)/2
|
||||
fmt.Fprintf(&output, `<text x="%g" y="%g" text-anchor="middle" font-size="13">%d</text>`, x, top+plotHeight+25, replicas)
|
||||
}
|
||||
fmt.Fprintf(&output, `<text x="%g" y="%d" text-anchor="middle" font-size="13">API replicas</text>`, left+plotWidth/2, height-15)
|
||||
|
||||
colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"}
|
||||
type series struct {
|
||||
name string
|
||||
values []aggregate
|
||||
}
|
||||
groups := make(map[string][]aggregate)
|
||||
for _, value := range selected {
|
||||
name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency)
|
||||
groups[name] = append(groups[name], value)
|
||||
}
|
||||
var names []string
|
||||
for name := range groups {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
allSeries := make([]series, 0, len(names))
|
||||
for _, name := range names {
|
||||
points := groups[name]
|
||||
sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas })
|
||||
allSeries = append(allSeries, series{name: name, values: points})
|
||||
}
|
||||
for index, current := range allSeries {
|
||||
color := colors[index%len(colors)]
|
||||
var coordinates []string
|
||||
for _, point := range current.values {
|
||||
xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas]
|
||||
x := left + plotWidth*xIndex/2
|
||||
y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS)
|
||||
coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y))
|
||||
fmt.Fprintf(&output, `<circle class="point" cx="%g" cy="%g" r="4" fill="%s"/>`, x, y, color)
|
||||
}
|
||||
fmt.Fprintf(&output, `<polyline class="line" points="%s" stroke="%s"/>`, strings.Join(coordinates, " "), color)
|
||||
legendY := top + 20 + float64(index*28)
|
||||
fmt.Fprintf(&output, `<line x1="%g" y1="%g" x2="%g" y2="%g" stroke="%s" stroke-width="3"/><text x="%g" y="%g" font-size="12">%s</text>`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name))
|
||||
}
|
||||
output.WriteString(`</svg>`)
|
||||
return os.WriteFile(path, []byte(output.String()), 0o644)
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in New Issue
Block a user